Fix: Uncaught exception in downloadConfig

Say where background effects run slowly, and show the wait for the first

.github/workflows/publish-component.yaml

@@ -0,0 +1,135 @@
+name: Build & publish the component package for releases
+
+on:
+  release:
+    types: [published]
+  pull_request:
+    types:
+      - synchronize
+      - opened
+      - labeled
+  push:
+    branches: [main]
+
+# Each job asks for what it needs
+permissions: {}
+
+jobs:
+  versioning:
+    name: Versioning
+    runs-on: ubuntu-latest
+    outputs:
+      DRY_RUN: ${{ steps.dry_run.outputs.DRY_RUN }}
+      PREFIXED_VERSION: ${{ steps.prefixed_version.outputs.PREFIXED_VERSION }}
+      TAG: ${{ steps.tag.outputs.TAG }}
+    steps:
+      - name: Calculate VERSION
+        # Safely store dynamic values in environment variables
+        # to prevent shell injection (template-injection)
+        run: |
+          if [ "$EVENT_NAME" = "release" ]; then
+            echo "VERSION=$RELEASE_TAG" >> "$GITHUB_ENV"
+          else
+            echo "VERSION=v0.0.0-pre.0" >> "$GITHUB_ENV"
+          fi
+        env:
+          RELEASE_TAG: ${{ github.event.release.tag_name }}
+          EVENT_NAME: ${{ github.event_name }}
+      - id: dry_run
+        name: Set DRY_RUN
+        # A dry run for everything but releases, so that a broken package shows up
+        # ahead of a release
+        run: echo "DRY_RUN=${{ github.event_name != 'release' }}" >> "$GITHUB_OUTPUT"
+      - id: prefixed_version
+        name: Set PREFIXED_VERSION
+        run: echo "PREFIXED_VERSION=${VERSION}" >> "$GITHUB_OUTPUT"
+      - id: tag
+        # latest = a proper release
+        # other = anything else
+        name: Set tag
+        run: |
+          if [[ "${VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "TAG=latest" >> "$GITHUB_OUTPUT"
+          elif [[ "${VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+\-rc\.[0-9]+$ ]]; then
+            echo "TAG=rc" >> "$GITHUB_OUTPUT"
+          else
+            echo "TAG=other" >> "$GITHUB_OUTPUT"
+          fi
+
+  publish_npm:
+    needs: versioning
+    name: Publish NPM
+    runs-on: ubuntu-latest
+    outputs:
+      ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
+    permissions:
+      contents: read
+      id-token: write # Allow npm to authenticate as a trusted publisher
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          persist-credentials: false
+      - name: Enable Corepack
+        run: corepack enable
+      - name: Setup node
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        # No dependency cache: a release build must not restore one another workflow could have poisoned
+        with:
+          node-version-file: .node-version
+          registry-url: "https://registry.npmjs.org"
+      # Layered like build-element-call.yaml, so a PR paired with a matrix-js-sdk branch
+      # builds against it; a release pins the SDK, so layered.sh leaves it as locked
+      - name: Install dependencies
+        run: scripts/layered.sh --ignore-pnpmfile
+        env:
+          PR_HEAD_REPO_OWNER: ${{ github.event.pull_request.head.repo.owner.login }}
+      - name: Build component
+        run: pnpm build:component:production
+        env:
+          VITE_APP_VERSION: component-${{ needs.versioning.outputs.PREFIXED_VERSION }}
+          NODE_OPTIONS: "--max-old-space-size=4096"
+
+      # `devEngines` (pnpm only, which npm refuses) and the `prepare` build exist for
+      # git-dependency installs; the published package is already built
+      - name: Publish npm
+        working-directory: component
+        run: |
+          jq 'del(.devEngines, .scripts)' package.json > package.json.tmp && mv package.json.tmp package.json
+          npm version ${NEEDS_VERSIONING_OUTPUTS_PREFIXED_VERSION} --no-git-tag-version
+          echo "ARTIFACT_VERSION=$(jq '.version' --raw-output package.json)" >> "$GITHUB_ENV"
+          npm publish --provenance --access public --tag ${NEEDS_VERSIONING_OUTPUTS_TAG} ${{ needs.versioning.outputs.DRY_RUN == 'true' && '--dry-run' || '' }}
+        env:
+          NEEDS_VERSIONING_OUTPUTS_PREFIXED_VERSION: ${{ needs.versioning.outputs.PREFIXED_VERSION }}
+          NEEDS_VERSIONING_OUTPUTS_TAG: ${{ needs.versioning.outputs.TAG }}
+
+      - id: artifact_version
+        name: Output artifact version
+        run: echo "ARTIFACT_VERSION=${ARTIFACT_VERSION}" >> "$GITHUB_OUTPUT"
+
+  release_notes:
+    needs: [versioning, publish_npm]
+    if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }}
+    name: Update release notes
+    runs-on: ubuntu-latest
+    # Shared with publish-embedded-packages.yaml, which appends to the same release: two
+    # appends at once would each write back the body without the other's section
+    concurrency:
+      group: release-notes-${{ github.event.release.tag_name || github.run_id }}
+      cancel-in-progress: false
+    permissions:
+      contents: write # to update release notes
+    steps:
+      - name: Add release notes
+        uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
+        with:
+          append_body: true
+          body: |
+
+            ## React component (experimental)
+
+            Element Call as a React component, rendered in the host's page instead of in an iframe.
+
+            ```
+            npm install @element-hq/element-call-component@${{ needs.publish_npm.outputs.ARTIFACT_VERSION }}
+            ```

.github/workflows/publish-embedded-packages.yaml

@@ -283,6 +283,11 @@ jobs:
     if: always()
     name: Update release notes
     runs-on: ubuntu-latest
+    # Shared with publish-component.yaml, which appends to the same release: two appends
+    # at once would each write back the body without the other's section
+    concurrency:
+      group: release-notes-${{ github.event.release.tag_name || github.run_id }}
+      cancel-in-progress: false
     permissions:
       contents: write # to update release notes
     steps:

PR4331-merge.md

@@ -0,0 +1,232 @@
+# Merging PR #4331 onto the SDK / Element Call split
+
+How [element-hq/element-call#4331](https://github.com/element-hq/element-call/pull/4331)
+("LiveKit updates": MSC4195 client-server API tokens, `url` transports, delegation
+through the homeserver, a transport dev override) lands on `toger5/use-sdk`, where
+the MatrixRTC logic lives in `sdk/` and Element Call consumes it through
+`sdk/src/api.ts`. The PR was written against `main`, where all of this still sat
+in `src/state/CallViewModel/`. This document goes through the PR commit by commit,
+says whether each one lands in `sdk/` or `src/`, and how; the section
+[Changes to `api.ts`](#changes-to-apits) covers the public interface in full. The
+end state: this branch behaves exactly as the PR does, with the split and the
+`api.ts` seam intact.
+
+The PR's branch is checked out at `agent-workspace/pr4331-merge/worktree` for
+comparison. The `matrix-js-sdk` dependency is pinned to the same `develop`
+commit the PR uses (`4df2f2bf`, 43.0.0), which brings `UnstableLivekitTransport`,
+`CallMembership.backendIdentities`, `_unstable_getLivekitToken` and
+`_unstable_delegateDelayedLeave`. That bump also renamed
+`MatrixRTCSession.leaveRoomSession` to `leave`, which the SDK now calls.
+
+## Where each commit lands
+
+| #   | Commit                                                                    | Lands in      | How                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| --- | ------------------------------------------------------------------------- | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| 1   | Remove stray pause and improve logging for hung e2e tests                 | e2e           | Applied as is: `playwright.config.ts`, `playwright/widget/federated-call.test.ts`.                                                                                                                                                                                                                                                                                                                                                                                                |
+| 2   | Fix typo in the name of the createJWTToken Playwright fixture             | e2e           | Applied as is.                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| 3   | Advertise the loopback address from the dev LiveKit SFUs                  | backend       | Applied as is: `backend/dev_livekit*.yaml`.                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| 4   | Match LiveKit participants against all backend identities                 | sdk           | `RTCMember.backendIdentities` (new, see api.ts). `LivekitBackend.mediaFor` matches `member.backendIdentities.includes(p.identity)`; `RTCParticipation.data$` maps a packet's sender the same way. `mockRtcMembership` in both `sdk/src/utils/test.ts` and `src/utils/test.ts` takes the new 4-argument constructor and a `backendIdentities` override.                                                                                                                            |
+| 5   | Model ConnectionState as a discriminated union                            | sdk           | `Connection.ts` state is `{state: "initialized" \| "authenticating" \| "authenticated" (+ `livekitState`) \| "error" (+ `error`) \| "stopped"}`. Internal to the LiveKit backend: `connectionState.ts` maps it onto the backend-neutral `MediaConnectionState`, so nothing above the backend changes. `DataChannel.ts` and `ConnectionFactory.ts` follow.                                                                                                                         |
+| 6   | Show the whole transport in debug UI and logs                             | sdk + src     | SDK log lines use `JSON.stringify(transport)`. Element Call: `focusUrl$: Behavior<string>` on the member media view models becomes `transport$: Behavior<Transport>` (`TransportMetadata.raw`), carried through `GridTile`, `SpotlightTile`, `MediaView` to `RTCConnectionStats`, which prints the JSON. The old code showed `extractDomain(transport.id)`, which on this branch was already a JSON string, so this also fixes a latent bug.                                      |
+| 7   | Group livekitRoomItems$ by connection                                     | n/a           | `livekitRoomItems$` and `LivekitRoomAudioRenderer` do not exist on this branch: audio renders from `tracks$` (`MemberAudioRenderer`). Nothing to port.                                                                                                                                                                                                                                                                                                                            |
+| 8   | Move SFU authentication into livekit/auth and choose the endpoint by role | sdk           | `sdk/src/media-backend/livekit/auth/{index,openID,csApi,types}.ts` replace `openIDSFU.ts`. Publisher → legacy `/sfu/get`; subscriber → `/get_token`. `slotId` replaces the hard-coded `m.call#ROOM`: `MediaBackendContext.slotId`, taken from the js-sdk session in `RTCParticipation`. The `tokenEndpoint` option of `createLivekitBackend` and the derivation from `MatrixRTCMode` are gone.                                                                                    |
+| 9   | Let the publishing Connection authenticate itself                         | sdk           | Every `Connection` has a `role` and fetches its own token; `existingSFUConfig` is gone. `prepareLocalTransport` no longer fetches a token itself: it starts the publisher connection (through the connection map) and resolves once that connection reports `authenticated`, rejecting with its error otherwise. Since `LocalMembership` joins only after `preparedTransport$` resolves, the join waits for the publisher's token exactly as the PR's `authenticated$` gate does. |
+| 10  | Reduce ConnectionManager to a ConnectionMap                               | sdk           | `ConnectionMap.ts` replaces `ConnectionManager.ts`; `createConnectionMap$` takes a resolved local transport, and `LivekitBackend` wraps it in `mapScoped` over the local transport, as the PR's `CallViewModel` did. `LocalMedia.ts`, `DataChannel.ts` take `connectionMap$`.                                                                                                                                                                                                     |
+| 11  | Use the unstable LiveKit transport type                                   | sdk + src     | `UnstableLivekitTransport` / `isUnstableLivekitTransport` throughout the backend; `src/settings/settings.ts`, `DeveloperSettingsTab`, test helpers. Transport discovery in `sdk/src/matrixrtc/` is now backend neutral (filters by `type`, see commit 15).                                                                                                                                                                                                                        |
+| 12  | Identify transports by server name as well as URLs                        | sdk           | `TransportLocator {transport, serverName}` in `auth/types.ts`; `ConnectionMap.keyFor` keys on `(url, livekit_service_url, serverName)`. The server name is derived from the user id (`serverNameOf`) for remote members and from our own membership identity for the local transport, so neither `api.ts` nor the backend options need it.                                                                                                                                        |
+| 13  | Pin matrix-js-sdk to the member_id branch                                 | superseded    | By commit 19; the lockfile pins the PR's final `develop` commit.                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| 14  | Get SFU tokens and delegate delayed leaves through the MSC4195 CS API     | sdk           | `auth/csApi.ts` (`_unstable_getLivekitToken`, `_unstable_delegateDelayedLeave`), `auth/index.ts` facade with the fallback rules. `LivekitBackend.delegateDelayedLeave` goes through the facade; the delegation probe asks only the service the publisher flow uses. `LivekitBackendOptions.client` picks the two new client methods.                                                                                                                                              |
+| 15  | Let developers override the whole transport                               | sdk + src     | `RTCParticipationOptions.transport` / `fallbackTransport` take a `Transport` instead of `transportUrl` / `fallbackTransportUrl` (see api.ts). Element Call: setting `custom-transport` (JSON) replaces `custom-livekit-url`; `DeveloperSettingsTab` edits JSON and validates through `authenticateWithTransport`; `InCallView` turns `config.livekit.livekit_service_url` into the fallback transport. Locale keys `developer_mode.custom_transport.*`.                           |
+| 16  | Deprecate the livekit.livekit_service_url config option                   | src           | Applied as is in `ConfigOptions.ts`.                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| 17  | Cover url transports end to end                                           | e2e + backend | `playwright/url-transport.spec.ts`, `backend/app-service*.yaml`, `docker-compose-dev.yml` (lk-jwt-service 0.8.0) copied as is.                                                                                                                                                                                                                                                                                                                                                    |
+| 18  | Retain compatibility in compatibility mode                                | superseded    | Reverted by commit 23 inside the PR; only its e2e spec edits survive, in the final spec file.                                                                                                                                                                                                                                                                                                                                                                                     |
+| 19  | Switch JS SDK back to develop                                             | deps          | `pnpm-lock.yaml` pins `4df2f2bf` and `matrix-widget-api` 1.20.0; `package.json` keeps `#develop`.                                                                                                                                                                                                                                                                                                                                                                                 |
+| 20  | Fall back to get_token as subscriber on widget API errors                 | sdk           | `isCsApiUnavailable` in `auth/index.ts`; `doNetworkOperationWithRetry(op, shouldRetry)` in `sdk/src/utils/network.ts` with `isNetworkError` in `csApi.ts`.                                                                                                                                                                                                                                                                                                                        |
+| 21  | Share a connection between homeservers using the same JWT service         | sdk           | `keyFor` ignores the server name for transports without a `url`; `removeDuplicateTransports` keeps the first locator so the local one wins.                                                                                                                                                                                                                                                                                                                                       |
+| 22  | Avoid null in transport dev setting                                       | src           | `transportText()` in `DeveloperSettingsTab.tsx`.                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| 23  | Make the auth choice mode independent again                               | sdk           | `chooseAuthFlow(transport, role)` has no mode parameter; `config.ts` documents that `MatrixRTCMode` decides how the membership is written, not how the transport is authenticated.                                                                                                                                                                                                                                                                                                |
+| 24  | Cover identity matching and delegation across transport shapes            | sdk + e2e     | Unit tests in `LivekitBackend.test.ts` (see below); `playwright/url-transport.spec.ts` final version.                                                                                                                                                                                                                                                                                                                                                                             |
+| 25  | Cover the subscriber token path, widget fallback and server names         | sdk + e2e     | `auth/openID.test.ts` subscriber cases; `playwright/widget/url-transport.test.ts`; the server-name test lives in `LivekitBackend.test.ts` ("connects to a remote member's transport on the member's homeserver").                                                                                                                                                                                                                                                                 |
+
+## Changes to `api.ts`
+
+Three changes to the public SDK interface, all forced by the PR's behaviour.
+
+### `RTCMember.backendIdentities` (new)
+
+```ts
+/** The membership's `rtcBackendIdentity`, `${userId}:${deviceId}`: a key and a debug label. */
+rtcBackendIdentity: string;
+/**
+ * Every identity the member may publish under on its transport: the
+ * participant identity the SFU sees, and the sender of its data packets.
+ * One per authentication flow the member's transport allows, so a transport
+ * with both a `url` and a `livekit_service_url` lists the hashed and the
+ * legacy identity, and the backend matches whichever shows up.
+ */
+backendIdentities: readonly string[];
+```
+
+Commit 4 of the PR. A member on a transport that carries both a `url` and a
+`livekit_service_url` may have authenticated through either flow and so has
+two candidate identities; the js-sdk now lists them on the membership. The
+backend matches media and data packets against the list.
+`rtcBackendIdentity` stays as the stable per-membership key Element Call uses
+for tile ids and the debug overlay, and its doc says that it is no longer
+what the backend matches on. Read off the membership, like the single identity
+before it; the backend will derive both once the identity scheme is its own.
+
+### `RTCParticipationOptions.transport` / `fallbackTransport` replace the url options
+
+```ts
+/**
+ * Use this transport instead of asking the homeserver. Any shape the
+ * backend serves: for LiveKit a `url`, a `livekit_service_url` or both.
+ */
+transport?: Transport;
+/** Use this transport when the homeserver advertises none of the backend's type. */
+fallbackTransport?: Transport;
+```
+
+Commit 15 of the PR. The developer override has to carry a whole transport so
+that a `url`-only or combined shape can be tested against a homeserver that
+does not advertise one yet; a URL cannot say which shape it is. This also
+removes the one piece of LiveKit knowledge that sat outside the backend:
+`LocalTransport.ts` and `TransportAutoDiscovery.ts` no longer build a
+`{type: "livekit", livekit_service_url}` object and filter by `transportType`
+only, which `LocalTransport.ts` had marked as the moment to do this. The
+host turns its config URL into a transport (`InCallView.rtcParticipationOptions`).
+
+### `TransportMetadata` is unchanged, with one documented limit
+
+`TransportMetadata.id` is still the raw transport serialised with sorted keys.
+The LiveKit backend now keys its connections on the transport plus the server
+name where the transport has a `url` (commits 12 and 21), so two homeservers
+advertising the same `url` get two connections but one `TransportMetadata`.
+`connectedTransports$` deduplicates, and `resolved$` reports the first
+connection's resolution. This is diagnostics only; the media and data paths
+go through the backend's own keys. Extending `TransportMetadata` with the
+server name was left out to keep the type transport neutral; a developer panel
+that needs it can be given a `serverName` on `BackendConnection` later.
+
+### Related, outside `api.ts`
+
+- `media-backend/api.ts`: `MediaBackendContext.slotId` (commit 8). The
+  client-server API issues a token for a slot, so the backend needs it;
+  `RTCParticipation` reads it from the js-sdk session.
+- `sdk/index.ts`: `authenticateWithTransport` is now the auth facade's
+  `getSFUConfig({client, membership, transport, serverName, roomId, slotId, role, logger})`
+  (object parameter, whole transport, server name); `AuthenticateWithTransportParams`
+  is exported for it; `TokenEndpoint` is gone.
+- `createLivekitBackend` options: `tokenEndpoint` is gone; `client` now picks
+  `_unstable_getLivekitToken` and `_unstable_delegateDelayedLeave` as well.
+- `config.ts`: the `MatrixRTCMode` doc no longer claims the mode picks the
+  JWT endpoint (commit 23).
+
+## Behaviour, side by side
+
+| Behaviour                              | PR on `main`                                                                                | This branch                                                                                                                 |
+| -------------------------------------- | ------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- |
+| Endpoint choice                        | `chooseAuthFlow` in `src/livekit/auth`                                                      | Same function, `sdk/src/media-backend/livekit/auth/index.ts`                                                                |
+| Join waits for the publisher's token   | `authenticated$` gate in `LocalMember.ts`                                                   | `prepareLocalTransport` resolves on the publisher connection's `authenticated` state; `LocalMembership` joins after it      |
+| Delegation probe                       | The flow's service only, in `LocalMember.joinParams$`                                       | The flow's service only, in `prepareLocalTransport`                                                                         |
+| Delegation handover                    | `delegateDelayedLeave` facade from `LocalMember`                                            | Same facade from `LivekitBackend.delegateDelayedLeave`, with the delay timeout from the session timings instead of `Config` |
+| Transport keys and connection sharing  | `ConnectionMap.keyFor` on `TransportLocator`                                                | Same, moved                                                                                                                 |
+| Server names                           | `membershipsAndTransports$` from the user id; `getLocalTransport` from `client.getDomain()` | `serverNameOf(userId)` for members; our own membership identity's user id for the local transport                           |
+| Identity matching                      | `membership.backendIdentities.includes(p.identity)`                                         | `member.backendIdentities.includes(p.identity)` in `mediaFor` and for data packets                                          |
+| Dev transport override                 | `customTransport` setting → `getLocalTransport`                                             | `customTransport` setting → `RTCParticipationOptions.transport`                                                             |
+| Config fallback                        | `RtcTransportAutoDiscovery.tryConfigTransport`                                              | Host builds the transport, `RTCParticipationOptions.fallbackTransport`                                                      |
+| Error for a `url` host without MSC4195 | `NoMatrix2AuthorizationService` from the publisher connection                               | Same error, surfaced through `prepareLocalTransport` → `state$ failed` → `fromMatrixRTCError`                               |
+
+One deliberate difference: on `main` the publisher connection fetches its token
+on every (re)creation and nothing else does; here it is the same, since the
+up-front token fetch in `prepareLocalTransport` is gone. There is no second
+token request per join.
+
+## Where the PR's new tests went
+
+| PR test                                                                               | Here                                                                                                                                                                                                                                                                                                 |
+| ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `src/livekit/auth/index.test.ts`                                                      | `sdk/src/media-backend/livekit/auth/index.test.ts`. Rewritten on the network layer (fetch-mock for the JWT service, mocked client methods) because a `vi.mock` of the sibling `openID` module does not reach the facade's own import under `sdk/`. Same cases, plus a table for `chooseAuthFlow`.    |
+| `src/livekit/auth/openID.test.ts`                                                     | `sdk/src/media-backend/livekit/auth/openID.test.ts`, replacing `openIDSFU.test.ts`                                                                                                                                                                                                                   |
+| `ConnectionMap.test.ts` (incl. commit 21's sharing tests)                             | `sdk/src/media-backend/livekit/ConnectionMap.test.ts`, replacing `ConnectionManager.test.ts`; plus a role-tagging test                                                                                                                                                                               |
+| `Connection.test.ts` changes                                                          | `sdk/src/media-backend/livekit/Connection.test.ts`; plus a publisher-through-the-JWT-service case                                                                                                                                                                                                    |
+| `LocalMember.test.ts` "joins session with %s delegation support" (by transport shape) | `LivekitBackend.test.ts` "prepares the local transport and asks %s about delegation", "waits for the publisher connection to hold its token before the join", "fails the preparation with the publisher connection's error", "delegates the leave through the flow that authenticated the publisher" |
+| `MatrixLivekitMembers.test.ts` two-candidate-identity tests                           | `LivekitBackend.test.ts` › `mediaFor`                                                                                                                                                                                                                                                                |
+| `SessionBehaviors.test.ts` server-name test                                           | `LivekitBackend.test.ts` "connects to a remote member's transport on the member's homeserver"                                                                                                                                                                                                        |
+| `LocalTransport.test.ts` / `RtcTransportAutoDiscovery.test.ts` changes                | `sdk/src/matrixrtc/TransportAutoDiscovery.test.ts` (fallback is a transport)                                                                                                                                                                                                                         |
+| `DeveloperSettingsTab.test.tsx` custom transport cases                                | `src/settings/DeveloperSettingsTab.test.tsx`, snapshot updated for the label                                                                                                                                                                                                                         |
+| `DataChannel`, `LivekitBackend` registry tests                                        | Adapted to the state union and `TransportLocator`                                                                                                                                                                                                                                                    |
+| `playwright/url-transport.spec.ts`, `playwright/widget/url-transport.test.ts`         | Copied as is: they drive the app, which behaves the same                                                                                                                                                                                                                                             |
+
+The PR's `CallViewModel.test.ts`, `CallViewModelTestUtils.ts`, `LocalMember.test.ts`
+(the rest) and `MatrixAudioRenderer.test.tsx` edits have no counterpart: those
+modules were replaced by the SDK before this merge.
+
+## One fix the e2e run forced
+
+The bumped js-sdk sets `rtcBackendIdentity` to the legacy `${userId}:${deviceId}`
+identity for every membership (it is deprecated there; the hashed one now only
+appears in `backendIdentities`). On this branch the call view model keyed its
+tiles by `[dup, mediaId, rtcBackendIdentity]`, so a device that rejoined after an
+improper leave, which has two memberships until the stale one expires, produced
+duplicate keys and crashed the host's page with `UNKNOWN_ERROR`
+(`spa-call-sticky.spec.ts`, "rejoin after improper leave"). The key is now the
+membership's own `memberId`, and `MemberAudioRenderer` and the SDK harness key
+by it too. The PR on `main` keyed by the participant object as well and did not
+hit this.
+
+## Coverage added on the package branch, and one bug it found
+
+The PR's tests mostly covered the auth layer; the SDK modules under them had
+little or none. Added on the package branch, which this branch is rebased onto:
+
+| Module                                       | Test                                                                                                                                                                                  |
+| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `media-backend/livekit/Publisher.ts`         | `Publisher.test.ts`: the cases from `main`'s `localMember/Publisher.test.ts` on a real `LocalParticipant` with faked device and signalling calls, plus publish, unpublish and destroy |
+| `media-backend/livekit/LocalMedia.ts`        | `LocalMedia.test.ts`: on a fake connection map; publish before and after the connection, republish on a new connection, mutes remembered, first publish error, output device          |
+| `RTCParticipation.ts`                        | `RTCParticipation.test.ts`: on a fake media backend; join options, state machine, publishing only while joined, delegation, member ids per mode, key replay, data sender check        |
+| `matrixrtc/LocalTransport.ts`                | `LocalTransport.test.ts`: override, homeserver, fallback, none                                                                                                                        |
+| `matrixrtc/joinJsSdkSession.ts`              | `joinJsSdkSession.test.ts`: the js-sdk join options, the retry budget, the intent and sticky events                                                                                   |
+| `media-backend/livekit/publishOptions.ts`    | `publishOptions.test.ts`                                                                                                                                                              |
+| `media-backend/livekit/ConnectionFactory.ts` | `ConnectionFactory.test.ts`: the room options from the requests and media quality (the cases from `main`'s `ECConnectionFactory.test.ts`), `resolved$`                                |
+| `LivekitBackend.ts` › `mediaFor`             | A member on another transport, a member changing transport, a member without one (from `main`'s `MatrixLivekitMembers.test.ts`)                                                       |
+
+`ConnectionFactory.test.ts` reads the options off the real LiveKit room the
+factory builds instead of mocking `livekit-client`: a `vi.mock` of the package
+reached the factory's import on the package branch but not on this one.
+
+The `LocalMedia` tests found a race in `ObservableScope.reconcile`: the loop was
+marked as running until the promise of its last run had settled, a few
+microtasks after it had last checked for new values, so a value arriving in
+that window (a connection appearing at just that moment) was never processed.
+Fixed in `[SDK] Keep a value that reaches reconcile as its loop winds down`,
+with a test that drives the window microtask by microtask. Element Call's
+`src/state/ObservableScope.ts` re-exports the SDK's class, so the fix applies
+to the app as well.
+
+## Checked
+
+- `pnpm lint` (tsc, oxlint, knip, externals), `pnpm format:check`, `pnpm i18n:check`.
+- `pnpm test`: 119 unit files, 950 tests; 4 storybook files, 45 tests (after the coverage above).
+- Playwright on Chromium against the local Playwright backend, one worker:
+  - `playwright/url-transport.spec.ts`: 10 of 10 pass, including delegation
+    through the homeserver and the JWT service, the subscriber fallback, the
+    publisher error and the federated token.
+  - `playwright/widget/url-transport.test.ts`: passes.
+  - The rest of `playwright/*.spec.ts`, `playwright/component`, `playwright/sdk`:
+    48 pass, 1 skipped, 1 fails (`reconnect.spec.ts`, which fails on `main`
+    locally too because the local JWT service is slower than the spec's wait).
+  - `playwright/widget`: 7 of 10 pass, including `federated-call`. `huddle-call`
+    fails before any call code runs (the Element Web develop image's room list no
+    longer matches the helper's "Open room" selector), and `screen-share` looks
+    for `video[data-lk-source="screen_share"]`, an attribute the LiveKit React
+    components set and the SDK's plain `<video>` elements do not: a failure of this
+    branch that predates the PR.
+  - Local environment work needed for that: the othersite JWT service and nginx
+    containers had to be recreated from the compose file, two stale containers held
+    port 16080, and the widget helper's hard-coded `https://localhost:3000` was
+    pointed at 3100 for the run only, since port 3000 is taken on this machine.
+- Standalone, widget and component all go through `InCallView.rtcParticipationOptions`
+  and the SDK, so the change is the same in all three modes; the widget-specific
+  fallback (a host without MSC4533) is in `isCsApiUnavailable`.

README.md

@@ -289,8 +289,14 @@ callback; with one, it asks the host to unmount it instead. The
 [global JS controls](./docs/controls.md) on `window` are unchanged and remain
 page-wide, so with several instances on one page they apply to all of them.
 
-The package is not published yet. A host installs it as a git dependency on the
-`component` directory of this repository,
+Each release publishes the package to npm, with the same version as the release:
+
+```sh
+npm install @element-hq/element-call-component
+```
+
+To use an unreleased commit, a host can instead install it as a git dependency
+on the `component` directory of this repository,
 
 ```json
 "@element-hq/element-call-component": "github:element-hq/element-call#main&path:/component"

codecov.yml

@@ -0,0 +1,20 @@
+# Coverage comes from a single vitest run (see .github/workflows/test.yaml), so
+# the app and the SDK are split by path into codecov components rather than
+# uploaded under separate flags. Each component gets its own coverage figure in
+# the PR comment and on codecov.io.
+#
+# Validate with: curl --data-binary @codecov.yml https://codecov.io/validate
+
+comment:
+  layout: "reach, diff, flags, components, files"
+
+component_management:
+  individual_components:
+    - component_id: app
+      name: app
+      paths:
+        - src/**
+    - component_id: sdk
+      name: sdk
+      paths:
+        - sdk/**

component/LICENSE-AGPL-3.0

@@ -0,0 +1,661 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

component/LICENSE-COMMERCIAL

@@ -0,0 +1,6 @@
+Licensees holding a valid commercial license with Element may use this
+software in accordance with the terms contained in a written agreement
+between you and Element.
+
+To purchase a commercial license please contact our sales team at
+licensing@element.io

component/README.md

@@ -0,0 +1,7 @@
+# @element-hq/element-call-component
+
+Element Call as a React component, rendered in the host application's page
+instead of in an iframe. Experimental.
+
+See [Element Call as a component](https://github.com/element-hq/element-call#element-call-as-a-component-experimental)
+in the Element Call README for usage.

component/build/scopeStylesToRoot.test.ts

@@ -109,4 +109,12 @@ describe("scopeStylesToRoot", () => {
       ".a .b_abc12:hover { border: 0 }",
     );
   });
+
+  it("leaves an excluded stylesheet as it is", async () => {
+    const css = "body { margin: 0 } .menu { color: red }";
+    const result = await postcss([
+      scopeStylesToRoot({ exclude: [/host\.css$/] }),
+    ]).process(css, { from: "/app/host.css" });
+    expect(result.css).toBe(css);
+  });
 });

component/build/scopeStylesToRoot.ts

@@ -60,12 +60,18 @@ const IS_ROOT = `:where(${ROOT_SELECTOR})`;
  * tokens, the component styles, the fonts — is not in this build at all: the
  * host supplies it, unscoped, along with its copy of Compound.
  */
-export function scopeStylesToRoot(): Plugin {
+interface Options {
+  /** Stylesheets left as they are: a host's own, which it doesn't scope. */
+  exclude?: RegExp[];
+}
+
+export function scopeStylesToRoot({ exclude = [] }: Options = {}): Plugin {
   return {
     postcssPlugin: "element-call-scope-styles-to-root",
     Once(root) {
-      const isModule =
-        root.source?.input.file?.endsWith(".module.css") ?? false;
+      const file = root.source?.input.file ?? "";
+      if (exclude.some((pattern) => pattern.test(file))) return;
+      const isModule = file.endsWith(".module.css");
       root.walkRules((rule) => {
         if (isRelative(rule)) return;
         rule.selector = (isModule ? scopeBare : scopeAll).processSync(

component/index.tsx

@@ -72,9 +72,9 @@ import {
   useUrlParams,
 } from "../src/UrlParams";
 import { MediaDevicesContext } from "../src/MediaDevicesContext";
-import { MediaDevices } from "../src/state/MediaDevices";
+import { AppViewModel } from "../src/state/AppViewModel";
 import { ObservableScope } from "../src/state/ObservableScope";
-import { ProcessorProvider } from "../src/livekit/TrackProcessorContext";
+import { BackgroundEffectsProvider } from "../src/tracks/BackgroundEffectsContext";
 import { Config } from "../src/config/Config";
 import { type ConfigOptions } from "../src/config/ConfigOptions";
 import { i18n } from "../src/utils/i18n";
@@ -194,14 +194,12 @@ export const ElementCall: FC<ElementCallProps> = ({
   // observers running for the rest of the page's life. Null until then, which
   // is one render.
   const { controlledAudioDevices, callIntent } = params;
-  const [mediaDevices, setMediaDevices] = useState<MediaDevices | null>(null);
+  const [vm, setVm] = useState<AppViewModel | null>(null);
   useEffect(() => {
     const scope = new ObservableScope();
-    setMediaDevices(
-      new MediaDevices(scope, { controlledAudioDevices, callIntent }),
-    );
+    setVm(new AppViewModel(scope, { controlledAudioDevices, callIntent }));
     return (): void => {
-      setMediaDevices(null);
+      setVm(null);
       scope.end();
     };
   }, [controlledAudioDevices, callIntent]);
@@ -222,8 +220,7 @@ export const ElementCall: FC<ElementCallProps> = ({
   // is told that Element Call has loaded, as the widget tells its client once
   // its own initialisation is over. Once per mount, however often the pieces
   // are later swapped out.
-  const ready =
-    container !== null && rtcSession !== null && mediaDevices !== null;
+  const ready = container !== null && rtcSession !== null && vm !== null;
   const announcedLoaded = useRef(false);
   useEffect(() => {
     if (!ready || announcedLoaded.current) return;
@@ -254,8 +251,10 @@ export const ElementCall: FC<ElementCallProps> = ({
                       <Decoration>
                         <TooltipProvider>
                           <ClientProvider client={client}>
-                            <MediaDevicesContext value={mediaDevices}>
-                              <ProcessorProvider>
+                            <MediaDevicesContext value={vm.mediaDevices}>
+                              <BackgroundEffectsProvider
+                                effects={vm.backgroundEffects}
+                              >
                                 <CallView
                                   client={client}
                                   rtcSession={rtcSession}
@@ -264,7 +263,7 @@ export const ElementCall: FC<ElementCallProps> = ({
                                   preload={params.preload}
                                   skipLobby={params.skipLobby}
                                 />
-                              </ProcessorProvider>
+                              </BackgroundEffectsProvider>
                             </MediaDevicesContext>
                           </ClientProvider>
                         </TooltipProvider>

component/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@element-hq/element-call-component",
   "version": "0.0.0",
-  "description": "Element Call as a React component. Consumed straight from the repository as a git dependency (github:element-hq/element-call#<ref>&path:/component): the host's package manager runs `prepare`, which builds `dist/`.",
-  "license": "SEE LICENSE IN ../README.md",
+  "description": "Element Call as a React component.",
+  "license": "AGPL-3.0-only OR LicenseRef-Element-Commercial",
   "repository": {
     "type": "git",
     "url": "https://github.com/element-hq/element-call",
@@ -15,6 +15,8 @@
     }
   },
   "files": [
+    "LICENSE-AGPL-3.0",
+    "LICENSE-COMMERCIAL",
     "dist"
   ],
   "main": "./dist/element-call.js",

docker-compose-playwright.yml

@@ -19,7 +19,7 @@ services:
       - ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
   element-web:
     # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
-    image: ghcr.io/element-hq/element-web:develop@sha256:e464b2aca73e248195b24b8c6f62cd331100302284d60cc735405807fb2730be
+    image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7
   element-web-1:
     # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
-    image: ghcr.io/element-hq/element-web:develop@sha256:e464b2aca73e248195b24b8c6f62cd331100302284d60cc735405807fb2730be
+    image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7

embedded/android/gradle/libs.versions.toml

@@ -9,4 +9,4 @@ android_gradle_plugin = { module = "com.android.tools.build:gradle", version.ref
 
 [plugins]
 android_library = { id = "com.android.library", version.ref = "android_gradle_plugin" }
-maven_publish = { id = "com.vanniktech.maven.publish", version = "0.36.0" }
+maven_publish = { id = "com.vanniktech.maven.publish", version = "0.37.0" }

embedded/android/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.5-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.6-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

localazy.json

@@ -7,13 +7,13 @@
     "features": ["plural_postfix_us", "filter_untranslated"],
     "files": [
       {
-        "pattern": "locales/en/*.json",
+        "pattern": "locales/en/app.json",
         "lang": "inherited"
       },
       {
         "group": "existing",
-        "pattern": "locales/*/*.json",
-        "excludes": ["locales/en/*.json"],
+        "pattern": "locales/*/app.json",
+        "excludes": ["locales/en/app.json"],
         "lang": "${autodetectLang}"
       }
     ]
@@ -22,6 +22,7 @@
   "download": {
     "files": [
       {
+        "conditions": ["equals: ${file}, app.json"],
         "output": "locales/${langLsrDash}/${file}"
       }
     ],

locales/en/app.json

@@ -3,7 +3,6 @@
     "user_menu": "User menu"
   },
   "action": {
-    "blur_background": "Blur background",
     "close": "Close",
     "copy_link": "Copy link",
     "edit": "Edit",
@@ -23,6 +22,13 @@
     "upload_file": "Upload file"
   },
   "analytics_notice": "By participating in this beta, you consent to the collection of anonymous data, which we use to improve the product. You can find more information about which data we track in our <2>Privacy Policy</2> and our <6>Cookie Policy</6>.",
+  "background_effects": {
+    "blur": "Blur",
+    "none": "None",
+    "numbered": "Background {{n}}",
+    "slow": "Background effects run slowly on this platform, which may cause your video to stutter.",
+    "unavailable": "Background effects are not supported on this platform."
+  },
   "call_ended_view": {
     "create_account_button": "Create account",
     "create_account_prompt": "<0>Why not finish by setting up a password to keep your account?</0><1>You'll be able to keep your name and set an avatar for use on future calls</1>",
@@ -106,7 +112,7 @@
     "insufficient_capacity_description": "The server has reached its maximum capacity and you cannot join the call at this time. Try again later, or contact your server admin if the problem persists.",
     "livekit_connection_error": "Failed to connect to Livekit server",
     "livekit_connection_error_description": "An error occurred while connecting to the Livekit server (<1>Reason:</1> <2>{{ reason }}</2>).",
-    "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
+    "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}} and no other call participant provides a usable transport. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
     "membership_manager": "Membership Manager Error",
     "membership_manager_description": "The Membership Manager had to shut down. This is caused by many consecutive failed network requests.",
     "no_matrix_2_authorization_service": "The authorization service for your media server (SFU) is out of date.",
@@ -212,6 +218,7 @@
     "auto_gain_control_label": "Automatic gain control",
     "background_blur_header": "Background",
     "background_blur_label": "Blur the background of the video",
+    "background_effects_header": "Background effects",
     "bitrate_label": "Bitrate",
     "blur_not_supported_by_browser": "(Background blur is not supported by this device.)",
     "camera_header": "Camera quality",

package.json

@@ -57,7 +57,6 @@
     "@formatjs/intl-durationformat": "^0.10.0",
     "@formatjs/intl-segmenter": "^11.7.3",
     "@livekit/components-core": "^0.12.0",
-    "@livekit/components-react": "^2.0.0",
     "@livekit/track-processors": "^0.7.1",
     "@mediapipe/tasks-vision": "^0.10.18",
     "@playwright/test": "^1.60.0",
@@ -88,7 +87,7 @@
     "@typescript-eslint/utils": "^8.61.0",
     "@use-gesture/react": "^10.2.11",
     "@vector-im/compound-design-tokens": "^11.0.0",
-    "@vector-im/compound-web": "^10.4.0",
+    "@vector-im/compound-web": "^10.5.0",
     "@vitejs/plugin-react": "^6.0.2",
     "@vitest/browser-playwright": "^4.1.5",
     "@vitest/coverage-v8": "^4.0.18",
@@ -107,7 +106,7 @@
     "livekit-client": "^2.18.1",
     "lodash-es": "^4.17.21",
     "loglevel": "^1.9.1",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01",
     "matrix-widget-api": "^1.18.0",
     "node-stdlib-browser": "^1.3.1",
     "normalize.css": "^8.0.1",

playwright/background-effects.spec.ts

@@ -0,0 +1,126 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { expect, test, type Locator, type Page } from "@playwright/test";
+
+import { SpaHelpers } from "./spa-helpers.ts";
+
+// Background effects need WebGL2, and headless Firefox on a CI runner has none,
+// so it rightly offers none of them.
+test.skip(
+  ({ browserName }) => browserName === "firefox",
+  "Background effects need WebGL2, which headless Firefox on CI does not have",
+);
+
+type Colour = [number, number, number];
+
+test.describe("background effects", () => {
+  test("pre-join preview shows the chosen effect", async ({ page }) => {
+    test.slow();
+    await page.goto("/");
+    await SpaHelpers.createCall(page, "Effect user", "Background effects");
+    const preview = page.locator("video").first();
+    const camera = await averageColour(preview);
+
+    await page.getByRole("button", { name: "Camera", exact: true }).click();
+    const tile = page
+      .getByRole("group", { name: "Background effects" })
+      .getByRole("menuitemradio", { name: "Background 1" });
+    await tile.click();
+    await expect(tile).toHaveAttribute("aria-checked", "true");
+
+    // The fake camera shows no person, so the whole frame is background: the
+    // preview takes on the picture's colours and leaves the camera's.
+    const picture = await averageColour(tile.locator("img"));
+    // Long enough for the first effect of a session to be built.
+    await expect
+      .poll(async () => distance(await averageColour(preview), picture), {
+        timeout: 60_000,
+      })
+      .toBeLessThan(distance(camera, picture) / 2);
+  });
+});
+
+test.describe("no effect", () => {
+  test("draws nothing, even after blurring", async ({ page }) => {
+    test.setTimeout(120_000);
+    await countDraws(page);
+    await page.goto("/");
+    await SpaHelpers.createCall(page, "Idle user", "No effect after blur");
+
+    await page.getByRole("button", { name: "Camera", exact: true }).click();
+    const effects = page.getByRole("group", { name: "Background effects" });
+    await effects.getByRole("menuitemradio", { name: "Blur" }).click();
+    // Long enough for the first effect of a session to be built.
+    await expect
+      .poll(async () => drawsPerSecond(page), { timeout: 60_000 })
+      .toBeGreaterThan(0);
+    const blurred = await drawsPerSecond(page);
+
+    await effects.getByRole("menuitemradio", { name: "None" }).click();
+    // A frame already on its way may still be drawn.
+    await expect.poll(async () => drawsPerSecond(page)).toBe(0);
+    test.info().annotations.push({
+      type: "WebGL draws per second",
+      description: `blurred ${blurred}, none 0`,
+    });
+  });
+});
+
+/** Counts every WebGL draw on the page, the segmenter's and the effect's. */
+async function countDraws(page: Page): Promise<void> {
+  await page.addInitScript(() => {
+    const counter = { draws: 0 };
+    (window as unknown as { webglDraws: typeof counter }).webglDraws = counter;
+    for (const name of ["drawArrays", "drawElements"] as const) {
+      const draw = WebGL2RenderingContext.prototype[name] as (
+        ...args: unknown[]
+      ) => void;
+      WebGL2RenderingContext.prototype[name] = function (
+        this: WebGL2RenderingContext,
+        ...args: unknown[]
+      ): void {
+        counter.draws += 1;
+        draw.apply(this, args);
+      } as never;
+    }
+  });
+}
+
+/** WebGL draws over the next second. */
+async function drawsPerSecond(page: Page): Promise<number> {
+  return page.evaluate(async () => {
+    const counter = (window as unknown as { webglDraws: { draws: number } })
+      .webglDraws;
+    const before = counter.draws;
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    return counter.draws - before;
+  });
+}
+
+/** The mean colour of a video's current frame or an image, drawn small. */
+async function averageColour(element: Locator): Promise<Colour> {
+  return element.evaluate(
+    async (source: HTMLVideoElement | HTMLImageElement) => {
+      if (source instanceof HTMLImageElement) await source.decode();
+      const canvas = document.createElement("canvas");
+      canvas.width = canvas.height = 32;
+      const context = canvas.getContext("2d")!;
+      context.drawImage(source, 0, 0, 32, 32);
+      const { data } = context.getImageData(0, 0, 32, 32);
+      const sum = [0, 0, 0];
+      for (let i = 0; i < data.length; i += 4)
+        for (let c = 0; c < 3; c++) sum[c] += data[i + c];
+      const pixels = data.length / 4;
+      return sum.map((s) => s / pixels) as Colour;
+    },
+  );
+}
+
+function distance(a: Colour, b: Colour): number {
+  return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]);
+}

playwright/component/component-call.spec.ts

@@ -125,25 +125,24 @@ test("leaves the host's own page unstyled", async ({ page }) => {
     timeout: 60_000,
   });
 
-  // Element Call's stylesheet is written for a page of its own: normalize.css
-  // gives `html` a line height, Compound gives `body` its font and feature
-  // settings, and the design tokens live on `:root`. None of that may reach the
-  // host's document — the harness sets none of these itself, so anything other
-  // than the browser's defaults here came from us.
+  // Element Call's page-wide CSS (normalize.css and the tokens in base.css)
+  // should not reach the host's page. Compound CSS that gets loaded to EW (host)
+  // on the other hand should impact the host. So probe with styles only Element Call sets:
+  // - A fresh <sub> keeps the browser's `vertical-align: sub` unless
+  //   normalize.css leaked; it turns that into `baseline`.
+  // - `--font-scale` is Element Call's own token, so `:root` must not have it.
   const host = await page.evaluate(() => {
-    const html = getComputedStyle(document.documentElement);
-    const body = getComputedStyle(document.body);
+    const sub = document.body.appendChild(document.createElement("sub"));
+    const verticalAlign = getComputedStyle(sub).verticalAlign;
+    sub.remove();
     return {
-      lineHeight: html.lineHeight,
-      fontFeatureSettings: body.fontFeatureSettings,
-      token: html.getPropertyValue("--cpd-color-text-primary"),
+      verticalAlign,
+      token: getComputedStyle(document.documentElement).getPropertyValue(
+        "--font-scale",
+      ),
     };
   });
-  expect(host).toEqual({
-    lineHeight: "normal",
-    fontFeatureSettings: "normal",
-    token: "",
-  });
+  expect(host).toEqual({ verticalAlign: "sub", token: "" });
 
   // While inside the container, the same rules do apply
   const root = panes.first().locator("[data-element-call-root]");

playwright/spa-helpers.ts

@@ -133,7 +133,7 @@ async function disableLeaveDelegation(page: Page): Promise<void> {
  * Expect a certain number of video tiles to be present and visible.
  */
 async function expectVideoTilesCount(page: Page, count: number): Promise<void> {
-  await expect(page.getByTestId("videoTile")).toHaveCount(2);
+  await expect(page.getByTestId("videoTile")).toHaveCount(count);
 
   // No one should be waiting for media
   await expect(page.getByText("Waiting for media...")).not.toBeVisible({

playwright/url-transport.spec.ts

@@ -31,7 +31,11 @@ interface Transport {
  */
 const shapes = {
   url: { type: "livekit", url: sfuUrl },
-  both: { type: "livekit", url: sfuUrl, livekit_service_url: serviceUrl },
+  "url+legacy": {
+    type: "livekit",
+    url: sfuUrl,
+    livekit_service_url: serviceUrl,
+  },
   legacy: null,
 } satisfies Record<string, Transport | null>;
 type Shape = keyof typeof shapes;
@@ -56,15 +60,15 @@ const cases: {
   {
     // Publishers keep the legacy participant identity that clients predating
     // MSC4195 expect, which only the JWT service assigns.
-    hostShape: "both",
-    guestShape: "both",
+    hostShape: "url+legacy",
+    guestShape: "url+legacy",
     hostFlows: ["service-legacy"],
     guestFlows: ["service-legacy"],
   },
   {
     // The MatrixRTC mode has no bearing on the choice.
-    hostShape: "both",
-    guestShape: "both",
+    hostShape: "url+legacy",
+    guestShape: "url+legacy",
     mode: "2_0",
     hostFlows: ["service-legacy"],
     guestFlows: ["service-legacy"],
@@ -75,6 +79,12 @@ const cases: {
     hostFlows: ["cs-api", "service-default"],
     guestFlows: ["service-legacy", "cs-api"],
   },
+  {
+    hostShape: "url",
+    guestShape: "url+legacy",
+    hostFlows: ["cs-api", "service-default"],
+    guestFlows: ["service-legacy", "cs-api"],
+  },
 ];
 
 for (const { hostShape, guestShape, mode, hostFlows, guestFlows } of cases) {
@@ -126,47 +136,6 @@ for (const { hostShape, guestShape, mode, hostFlows, guestFlows } of cases) {
   });
 }
 
-test("Subscriber falls back to the JWT service when the homeserver lacks MSC4195", async ({
-  browser,
-  page,
-  browserName,
-}) => {
-  skipOnFirefox(browserName);
-
-  // The host starts the call with both transport shapes.
-  await advertiseTransport(page, shapes.both);
-  await page.goto("/");
-  await SpaHelpers.createCall(page, "Androl", "HelloCall", true);
-  const inviteLink = await SpaHelpers.getCallInviteLink(page);
-
-  // The guest joins but their C-S requests fail.
-  const guestContext = await browser.newContext({ reducedMotion: "reduce" });
-  const guestPage = await guestContext.newPage();
-  await rejectCsApiTokens(guestPage);
-  const guestRequests = recordAuthRequests(guestPage);
-  await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink, "Pevara");
-
-  // Wait for the call to connect and render.
-  await SpaHelpers.expectVideoTilesCount(page, 2);
-  await SpaHelpers.expectVideoTilesCount(guestPage, 2);
-
-  // The guest ran through all flows.
-  expect(flowsOf(guestRequests)).toEqual(
-    new Set<Flow>(["cs-api", "service-legacy", "service-default"]),
-  );
-
-  // The C-S token requests failed.
-  for (const request of guestRequests) {
-    expect(request.status).toBe(request.flow === "cs-api" ? 404 : 200);
-  }
-
-  // The C-S token requests preceeded the fallback request.
-  const flows = guestRequests.map((r) => r.flow);
-  expect(flows.indexOf("service-default")).toBeGreaterThan(
-    flows.indexOf("cs-api"),
-  );
-});
-
 test("Publisher shows an error when the homeserver lacks MSC4195", async ({
   page,
   browserName,
@@ -191,6 +160,38 @@ test("Publisher shows an error when the homeserver lacks MSC4195", async ({
   ).toBeVisible();
 });
 
+test("Group call with all possible transport shapes at once", async ({
+  browser,
+  page,
+  browserName,
+}) => {
+  test.slow();
+  skipOnFirefox(browserName);
+
+  // The host starts the call with a url transport.
+  await advertiseTransport(page, shapes.url);
+  await page.goto("/");
+  await SpaHelpers.createCall(page, "Alice", "HelloCall", true);
+  const inviteLink = await SpaHelpers.getCallInviteLink(page);
+
+  // The guest with a legacy transport joins.
+  const bobContext = await browser.newContext({ reducedMotion: "reduce" });
+  const bobPage = await bobContext.newPage();
+  await advertiseTransport(bobPage, shapes.legacy);
+  await SpaHelpers.joinCallFromInviteLink(bobPage, inviteLink, "Bob");
+
+  // The guest with a url+legacy transport joins.
+  const carolContext = await browser.newContext({ reducedMotion: "reduce" });
+  const carolPage = await carolContext.newPage();
+  await advertiseTransport(carolPage, shapes["url+legacy"]);
+  await SpaHelpers.joinCallFromInviteLink(carolPage, inviteLink, "Carol");
+
+  // Wait for the call to connect and render.
+  await SpaHelpers.expectVideoTilesCount(page, 3);
+  await SpaHelpers.expectVideoTilesCount(bobPage, 3);
+  await SpaHelpers.expectVideoTilesCount(carolPage, 3);
+});
+
 /**
  * Delegation follows the flow that authenticated the publisher: the homeserver
  * for a `url`-only transport, the JWT service wherever there is one.
@@ -198,7 +199,7 @@ test("Publisher shows an error when the homeserver lacks MSC4195", async ({
 const delegationCases: { shape: Shape; via: Delegation["via"] }[] = [
   { shape: "url", via: "homeserver" },
   { shape: "legacy", via: "service" },
-  { shape: "both", via: "service" },
+  { shape: "url+legacy", via: "service" },
 ];
 
 for (const { shape, via } of delegationCases) {

playwright/widget/screen-share.test.ts

@@ -76,9 +76,9 @@ widgetTest("Sharing screen in group call", async ({ addUser, browserName }) => {
       timeout: 5000,
     });
 
-    await expect(
-      frame.locator('video[data-lk-source="screen_share"]'),
-    ).toHaveCount(1);
+    await expect(frame.locator('video[data-source="screenShare"]')).toHaveCount(
+      1,
+    );
   }
 
   // Alice should be in grid mode as she is local sharing
@@ -117,9 +117,9 @@ widgetTest("Sharing screen in group call", async ({ addUser, browserName }) => {
       timeout: 5000,
     });
 
-    await expect(
-      frame.locator('video[data-lk-source="screen_share"]'),
-    ).toHaveCount(2);
+    await expect(frame.locator('video[data-source="screenShare"]')).toHaveCount(
+      2,
+    );
 
     // Expect 2 indicators at the bottom
     await expect(frame.getByTestId("screenshare-indicator")).toHaveCount(2);

playwright/widget/test-helpers.ts

@@ -159,7 +159,7 @@ export class TestHelpers {
   public static async dismissStartupToasts(page: Page): Promise<void> {
     const expectedToasts = [
       { title: "Failed to load service worker", button: "OK" },
-      { title: "Back up your chats", button: "Dismiss" },
+      { title: "Back up your chats", button: "Not now" },
       { title: "Turn on key storage", button: "Dismiss" },
       { title: "Element does not support this browser", button: "Dismiss" },
     ];
@@ -178,7 +178,11 @@ export class TestHelpers {
         );
 
         if (toastConfig) {
-          await toast.getByRole("button", { name: toastConfig.button }).click();
+          // Bounded so that a renamed button leaves the toast in place and the
+          // next step reports it, instead of hanging until the test timeout.
+          await toast
+            .getByRole("button", { name: toastConfig.button })
+            .click({ timeout: 5000 });
         } else {
           // Unknown toast. We don't want to act on unknown toasts
           break;

playwright/widget/url-transport.test.ts

@@ -11,13 +11,12 @@ import { widgetTest } from "../fixtures/widget-user.ts";
 import { HOST1, TestHelpers } from "./test-helpers.ts";
 
 const sfuUrl = "wss://matrix-rtc.m.localhost/livekit/sfu";
-const serviceUrl = "https://matrix-rtc.m.localhost/livekit/jwt";
 
 /** Which endpoint a page asks for its SFU tokens. */
 type Flow = "cs-api" | "service-legacy" | "service-default";
 
 widgetTest(
-  "Call between a url + livekit_service_url transport and a legacy one",
+  "Call between a url transport and a legacy one",
   async ({ addUser, browserName }) => {
     test.skip(
       browserName === "firefox",
@@ -30,12 +29,10 @@ widgetTest(
       addUser("Pevara", HOST1),
     ]);
 
-    // The host's homeserver has started advertising a `url` next to its JWT
-    // service; the guest's still advertises the JWT service alone.
+    // Host advertises a url transport, guest still advertises a legacy one
     await advertiseTransport(host.page, {
       type: "livekit",
       url: sfuUrl,
-      livekit_service_url: serviceUrl,
     });
     // Element Web answers the widget from a cache it filled at login, so make
     // it fetch the transports again now that they are rerouted
@@ -66,13 +63,9 @@ widgetTest(
       await TestHelpers.expectVisibleVideoCount(frame, 2);
     }
 
-    // Element Web does not expose the MSC4195 endpoints to widgets yet, so
-    // neither side reached the CS API. The host published through the JWT
-    // service's legacy endpoint and subscribed to the guest through its
-    // default one; the guest published the same way and fell back to the
-    // default endpoint for the host's transport.
-    expect(hostFlows).toEqual(new Set(["service-legacy", "service-default"]));
-    expect(guestFlows).toEqual(new Set(["service-legacy", "service-default"]));
+    // Element Web used for Playwright does support MSC4195 for widgets.
+    expect(hostFlows).toEqual(new Set(["cs-api", "service-default"]));
+    expect(guestFlows).toEqual(new Set(["service-legacy", "cs-api"]));
   },
 );
 

scripts/check-component-externals.mjs

@@ -44,8 +44,9 @@ const TARGETS = [
   {
     config: "vite-component.config.ts",
     what: "the component build",
-    sources: ["src", "component"],
-    excluded: ["component/dev"],
+    // `sdk` too: the build resolves `@element-hq/matrixrtc-sdk` to its source
+    sources: ["src", "component", "sdk"],
+    excluded: ["component/dev", "sdk/dev"],
     mustBeExternal: [
       "react",
       "react-dom",

sdk-target-based-on-call-view-model/README.md

@@ -1,8 +1,10 @@
-# SDK mode (EXPERIMENTAL, superseded)
+# SDK mode (LEGACY)
 
 > Superseded by the `sdk/` package (`@element-hq/matrixrtc-sdk`) and its harness in
-> `sdk/dev`; see `sdk-plan.md`. This folder still builds, with its own script, until
-> that package can do what this does.
+> `sdk/dev`; see `sdk/SdkArchitecture.md`. This bundle is in production use, so it
+> keeps building as a thin layer over that package until its consumers have moved
+> to the package directly. It will then be removed. Its members are the package's
+> `RTCMember`s now, in place of its own wrapper around the raw membership.
 
 EC can be build in sdk mode. This will result in a compiled js file that can be imported in very simple webapps.
 

sdk-target-based-on-call-view-model/helper.ts

@@ -1,51 +0,0 @@
-/*
-Copyright 2025 Element Creations Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-/**
- * This file contains helper functions and types for the MatrixRTC SDK.
- */
-
-import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
-import { scan } from "rxjs";
-
-import { type WidgetHelpers } from "../src/widget";
-import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel";
-
-export const tryMakeSticky = (widget: WidgetHelpers): void => {
-  const logger = rootLogger.getChild("[MatrixRTCSdk]");
-  logger.info("try making sticky MatrixRTCSdk");
-  void widget.api
-    .setAlwaysOnScreen(true)
-    .then(() => {
-      logger.info("sticky MatrixRTCSdk");
-    })
-    .catch((error) => {
-      logger.error("failed to make sticky MatrixRTCSdk", error);
-    });
-};
-export const TEXT_LK_TOPIC = "matrixRTC";
-/**
- * simple helper operator to combine the last emitted and the current emitted value of a rxjs observable
- *
- * I think there should be a builtin for this but i did not find it...
- */
-export const currentAndPrev = scan<
-  LivekitRoomItem[],
-  {
-    prev: LivekitRoomItem[];
-    current: LivekitRoomItem[];
-  }
->(
-  ({ current: lastCurrentVal }, items) => ({
-    prev: lastCurrentVal,
-    current: items,
-  }),
-  {
-    prev: [],
-    current: [],
-  },
-);

sdk-target-based-on-call-view-model/index.html

@@ -30,12 +30,12 @@
         div.appendChild(child);
       });
 
-      sdk.members$.subscribe((memberObjects) => {
+      sdk.remoteMembers$.subscribe((memberObjects) => {
         const div = document.getElementById("members");
         div.innerHTML = "<h3>Members:</h3>";
 
         // Create member list
-        const members = memberObjects.map((member) => member.membership.sender);
+        const members = memberObjects.map((member) => member.userId);
         console.info("members changed", members);
         for (const m of members) {
           console.info("member", m);

sdk-target-based-on-call-view-model/main.ts

@@ -6,131 +6,82 @@ Please see LICENSE in the repository root for full details.
 */
 
 /**
- * EXPERIMENTAL
+ * LEGACY, in production use
  *
- * This file is the entrypoint for the sdk build of element call:
- * `pnpm build:sdk-target-based-on-call-view-model`
- * use in widgets.
- * It exposes the `createMatrixRTCSdk` which creates the `MatrixRTCSdk` interface (see below) that
- * can be used to join a rtc session and exchange realtime data.
- * It takes care of all the tricky bits:
- *  - sending delayed events
- *  - finding the right sfu
- *  - handling the media stream
- *  - sending join/leave state or sticky events
- *  - setting up encryption and scharing keys
+ * The entrypoint of `pnpm build:sdk-target-based-on-call-view-model`, a
+ * bundle for widgets that exposes `createMatrixRTCSdk`: join a slot and
+ * exchange realtime data, with the `@element-hq/matrixrtc-sdk` package and
+ * Element Call's call view model doing the work underneath. New consumers use
+ * the package directly; this bundle goes away once the existing ones have.
  */
 
+import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
+import { EMPTY, filter, map, type Observable, of, switchMap, take } from "rxjs";
 import {
-  combineLatest,
-  map,
-  type Observable,
-  of,
-  shareReplay,
-  Subject,
-  switchMap,
-  tap,
-} from "rxjs";
-import {
-  type CallMembership,
-  MatrixRTCSessionEvent,
-  MatrixRTCSessionManager,
-} from "matrix-js-sdk/lib/matrixrtc";
-import {
-  type Room as LivekitRoom,
-  type TextStreamReader,
-  type LocalParticipant,
-  type RemoteParticipant,
-} from "livekit-client";
+  type Behavior,
+  createRTCSlot,
+  E2eeType,
+  type LocalRTCMember,
+  ObservableScope,
+  type RemoteRTCMember,
+  type RTCParticipation,
+} from "@element-hq/matrixrtc-sdk";
 
-// TODO how can this get fixed? to just be part of `livekit-client`
-// Can this be done in the tsconfig.json
-import { type TextStreamInfo } from "../node_modules/livekit-client/dist/src/room/types";
-import { type Behavior, constant } from "../src/state/Behavior";
 import {
   callViewModelOptionsFromParams,
+  type CallViewModel,
   createCallViewModel$,
+  initialPublishRequests,
 } from "../src/state/CallViewModel/CallViewModel";
-import { ObservableScope } from "../src/state/ObservableScope";
-import { getUrlParams } from "../src/UrlParams";
-import { MuteStates } from "../src/state/MuteStates";
-import { MediaDevices } from "../src/state/MediaDevices";
-import { E2eeType } from "../src/e2ee/e2eeType";
-import { currentAndPrev, TEXT_LK_TOPIC, tryMakeSticky } from "./helper";
-import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
-import { initializeWidget } from "../src/widget";
-import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection";
+import {
+  captureSettings,
+  rtcParticipationOptions,
+  rtcSlotOptions,
+} from "../src/room/InCallView";
 import { createWidgetHostBridge } from "../src/HostBridge";
+import { MediaDevices } from "../src/state/MediaDevices";
+import { MuteStates } from "../src/state/MuteStates";
+import { getUrlParams } from "../src/UrlParams";
 import { observeElementSize$ } from "../src/utils/elementSize";
+import { initializeWidget } from "../src/widget";
 
-interface MatrixRTCSdk {
-  /**
-   * observe connected$ to track the state.
-   * @returns
-   */
+/** The data channel topic this bundle has always used, so that old and new builds talk. */
+const TOPIC = "matrixRTC";
+
+export interface MatrixRTCSdk {
+  /** Joins the slot; `connected$` tracks the progress. A second call does nothing. */
   join: () => void;
-  /** @throws on leave errors */
   leave: () => void;
-  /**
-   * Ends the rtc sdk. This will unsubscribe any event listeners. And end the associated scope.
-   * No updates can be received from the rtc sdk. The sdk cannot be restarted after.
-   * A new sdk needs to be created via createMatrixRTCSdk.
-   */
+  /** Ends everything. The sdk cannot be restarted; create a new one. */
   stop: () => void;
-  data$: Observable<{ rtcBackendIdentity: string; data: string }>;
-  /**
-   * flattened list of remote members
-   */
-  remoteMembers$: Behavior<
-    {
-      connection: Connection | null;
-      membership: CallMembership;
-      participant: LocalParticipant | RemoteParticipant | null;
-    }[]
-  >;
-  /**
-   * flattened local member
-   */
-  localMember$: Behavior<{
-    connection: Connection | null;
-    membership: CallMembership;
-    participant: LocalParticipant | null;
-  } | null>;
-  /** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state  */
   connected$: Behavior<boolean>;
-  sendData?: (data: unknown) => Promise<void>;
-  sendRoomMessage?: (message: string) => Promise<void>;
+  /** Our own member, null until our membership has been seen in the room. */
+  localMember$: Behavior<LocalRTCMember | null>;
+  /** The other members of the slot, with the media the participation carries for them. */
+  remoteMembers$: Behavior<RemoteRTCMember[]>;
+  /** What other members sent with `sendData`. */
+  data$: Observable<{ rtcBackendIdentity: string; data: string }>;
+  /** Sends to every member on our transport. Rejects while not joined. */
+  sendData: (data: unknown) => Promise<void>;
+  sendRoomMessage: (message: string) => Promise<void>;
 }
 
 export async function createMatrixRTCSdk(
-  application: string = "m.call",
-  id: string = "",
-  sticky: boolean = false,
+  application = "m.call",
+  id = "",
+  sticky = false,
 ): Promise<MatrixRTCSdk> {
   const logger = rootLogger.getChild("[MatrixRTCSdk]");
   const scope = new ObservableScope();
-
-  // widget client
   const widget = initializeWidget(application, true);
-  if (!widget) throw Error("No widget. This webapp can only start as a widget");
+  if (!widget) throw new Error("No widget: this bundle only runs as a widget");
   const client = await widget.client;
   const hostBridge = createWidgetHostBridge(widget);
-  logger.info("client created");
-
-  // url params
   const urlParams = getUrlParams();
   const { roomId, controlledAudioDevices, callIntent } = urlParams;
-  if (roomId === null) throw Error("could not get roomId from url params");
-  const room = client.getRoom(roomId);
-  if (room === null) throw Error("could not get room from client");
+  const room = roomId === null ? null : client.getRoom(roomId);
+  if (room === null) throw new Error("No room: the url needs a joined roomId");
 
-  // rtc session
-  const slot = { application, id };
-  const rtcSessionManager = new MatrixRTCSessionManager(logger, client, slot);
-  rtcSessionManager.start();
-  const rtcSession = rtcSessionManager.getRoomSession(room);
-
-  // media devices
   const mediaDevices = new MediaDevices(scope, {
     controlledAudioDevices,
     callIntent,
@@ -141,243 +92,100 @@ export async function createMatrixRTCSdk(
     { audioEnabled: false, videoEnabled: false },
     hostBridge,
   );
+  const encryptionSystem = { kind: E2eeType.PER_PARTICIPANT } as const;
+  const capture = captureSettings();
+  const slot = createRTCSlot(scope, client, room, {
+    ...rtcSlotOptions(encryptionSystem),
+    application,
+    id,
+  });
+  let vm: CallViewModel | undefined;
+  const failed =
+    (what: string) =>
+    (e: unknown): void =>
+      logger.error(what, e);
 
-  // call view model
-  const callViewModel = createCallViewModel$(
-    scope,
-    rtcSession,
-    room,
-    mediaDevices,
-    muteStates,
-    {
-      ...callViewModelOptionsFromParams(urlParams),
-      encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
-      hostBridge,
-      // The SDK owns its page, so the body is the space it has
-      windowSize$: scope.behavior(observeElementSize$(document.body)),
-    },
-    of({}),
-    of({}),
-    constant({ supported: false, processor: undefined }),
-  );
-  logger.info("CallViewModelCreated");
-
-  // create data listener
-  const data$ = new Subject<{ rtcBackendIdentity: string; data: string }>();
-
-  const lkTextStreamHandlerFunction = async (
-    reader: TextStreamReader,
-    participantInfo: { identity: string },
-    livekitRoom: LivekitRoom,
-  ): Promise<void> => {
-    const info = reader.info;
-    logger.info(
-      `Received text stream from ${participantInfo.identity}\n` +
-        `  Topic: ${info.topic}\n` +
-        `  Timestamp: ${info.timestamp}\n` +
-        `  ID: ${info.id}\n` +
-        `  Size: ${info.size}`, // Optional, only available if the stream was sent with `sendText`
-    );
-
-    const participants = callViewModel.livekitRoomItems$.value.find(
-      (i) => i.livekitRoom === livekitRoom,
-    )?.participants;
-    if (participants && participants.includes(participantInfo.identity)) {
-      const text = await reader.readAll();
-      logger.info(`Received text: ${text}`);
-      data$.next({ rtcBackendIdentity: participantInfo.identity, data: text });
-    } else {
-      logger.warn(
-        "Received text from unknown participant",
-        participantInfo.identity,
-      );
-    }
-  };
-
-  const livekitRoomItemsSub = callViewModel.livekitRoomItems$
-    .pipe(
-      tap((beforecurrentAndPrev) => {
-        logger.info(
-          `LiveKit room items updated: ${beforecurrentAndPrev.length}`,
-          beforecurrentAndPrev,
-        );
-      }),
-      currentAndPrev,
-      tap((aftercurrentAndPrev) => {
-        logger.info(
-          `LiveKit room items updated: ${aftercurrentAndPrev.current.length}, ${aftercurrentAndPrev.prev.length}`,
-          aftercurrentAndPrev,
-        );
-      }),
-    )
-    .subscribe({
-      next: ({ prev, current }) => {
-        const prevRooms = prev.map((i) => i.livekitRoom);
-        const currentRooms = current.map((i) => i.livekitRoom);
-        const addedRooms = currentRooms.filter((r) => !prevRooms.includes(r));
-        const removedRooms = prevRooms.filter((r) => !currentRooms.includes(r));
-        addedRooms.forEach((r) => {
-          logger.info(`Registering text stream handler for room `);
-          r.registerTextStreamHandler(
-            TEXT_LK_TOPIC,
-            (reader, participantInfo) =>
-              void lkTextStreamHandlerFunction(reader, participantInfo, r),
-          );
-        });
-        removedRooms.forEach((r) => {
-          logger.info(`Unregistering text stream handler for room `);
-          r.unregisterTextStreamHandler(TEXT_LK_TOPIC);
-        });
-      },
-      complete: () => {
-        logger.info("Livekit room items subscription completed");
-        for (const item of callViewModel.livekitRoomItems$.value) {
-          logger.info("unregistering room item from room", item.transport);
-          item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC);
-        }
-      },
-    });
-
-  // create sendData function
-  const sendFn: Behavior<(data: string) => Promise<TextStreamInfo>> =
+  const over = <T>(
+    pick: (participation: RTCParticipation) => Observable<T>,
+    fallback: T,
+  ): Behavior<T> =>
     scope.behavior(
-      callViewModel.localMatrixLivekitMember$.pipe(
-        switchMap((m) => {
-          if (!m)
-            return of((data: string): never => {
-              throw Error("local membership not yet ready.");
-            });
-          return m.participant.value$.pipe(
-            map((p) => {
-              if (p === null) {
-                return (data: string): never => {
-                  throw Error("local participant not yet ready to send data.");
-                };
-              } else {
-                return async (data: string): Promise<TextStreamInfo> =>
-                  p.sendText(data, { topic: TEXT_LK_TOPIC });
-              }
-            }),
-          );
-        }),
+      slot.participation$.pipe(
+        switchMap((p) => (p === null ? of(fallback) : pick(p))),
       ),
     );
 
-  const sendData = async (data: unknown): Promise<void> => {
-    const dataString = JSON.stringify(data);
-    logger.info("try sending: ", dataString);
-    try {
-      await Promise.resolve();
-      const info = await sendFn.value(dataString);
-      logger.info(`Sent text with stream ID: ${info.id}`);
-    } catch (e) {
-      logger.error("failed sending: ", dataString, e);
-    }
-  };
-
-  const sendRoomMessage = async (message: string): Promise<void> => {
-    const messageString = JSON.stringify(message);
-    logger.info("try sending to room: ", messageString);
-    try {
-      await client.sendTextMessage(room.roomId, message);
-    } catch (e) {
-      logger.error("failed sending to room: ", messageString, e);
-    }
-  };
-
-  // after hangup gets called
-  const leaveSubs = callViewModel.leave$.subscribe(() => {
-    const scheduleWidgetCloseOnLeave = async (): Promise<void> => {
-      const leaveResolver = Promise.withResolvers<void>();
-      logger.info("waiting for RTC leave");
-      rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, (isJoined) => {
-        logger.info("received RTC join update: ", isJoined);
-        if (!isJoined) leaveResolver.resolve();
-      });
-      await leaveResolver.promise;
-      logger.info("send Unstick");
-      await hostBridge
-        .setAlwaysOnScreen(false)
-        .catch((e: unknown) =>
-          logger.error("Failed to set `alwaysOnScreen` to false", e),
-        );
-      logger.info("send Close");
-      await hostBridge
-        .close?.()
-        .catch((e: unknown) =>
-          logger.error("Failed to ask the host to close", e),
-        );
-    };
-
-    // schedule close first and then leave (scope.end)
-    void scheduleWidgetCloseOnLeave();
-  });
-
-  logger.info("createMatrixRTCSdk done");
-
   return {
     join: (): void => {
-      // first lets try making the widget sticky
-      if (sticky) tryMakeSticky(widget);
-      callViewModel.join();
-    },
-    leave: (): void => {
-      callViewModel.leave();
-    },
-    stop: (): void => {
-      leaveSubs.unsubscribe();
-      livekitRoomItemsSub.unsubscribe();
-      scope.end();
-    },
-    data$,
-    localMember$: scope.behavior(
-      callViewModel.localMatrixLivekitMember$.pipe(
-        tap((member) =>
-          logger.info("localMatrixLivekitMember$ next: ", member),
-        ),
-        switchMap((member) => {
-          if (member === null) return of(null);
-          return combineLatest([
-            member.connection$,
-            member.membership$,
-            member.participant.value$,
-          ]).pipe(
-            map(([connection, membership, participant]) => ({
-              connection,
-              membership,
-              participant,
-            })),
-          );
-        }),
-        tap((member) => logger.info("localMember$ next: ", member)),
-      ),
-    ),
-    connected$: callViewModel.connected$,
-    remoteMembers$: scope.behavior(
-      callViewModel.remoteMatrixLivekitMembers$.pipe(
-        switchMap((members) => {
-          const listOfMemberObservables = members.map((member) =>
-            combineLatest([
-              member.connection$,
-              member.membership$,
-              member.participant.value$,
-            ]).pipe(
-              map(([connection, membership, participant]) => ({
-                connection,
-                membership,
-                participant,
-              })),
-              // using shareReplay instead of a Behavior here because the behavior would need
-              // a tricky scope.end() setup.
-              shareReplay({ bufferSize: 1, refCount: true }),
+      if (slot.participation$.value !== null) return;
+      if (sticky)
+        widget.api
+          .setAlwaysOnScreen(true)
+          .catch(failed("Failed to make the widget sticky"));
+      const participation = slot.join({
+        ...rtcParticipationOptions(urlParams),
+        publish: initialPublishRequests(muteStates, mediaDevices, capture),
+      });
+      vm = createCallViewModel$(
+        scope,
+        participation,
+        room,
+        mediaDevices,
+        muteStates,
+        {
+          ...callViewModelOptionsFromParams(urlParams),
+          encryptionSystem,
+          capture,
+          hostBridge,
+          // The bundle owns its page, so the body is the space it has
+          windowSize$: scope.behavior(observeElementSize$(document.body)),
+        },
+        of({}),
+        of({}),
+      );
+      // Once the hang-up has gone through, unstick and ask the host to close us
+      vm.leave$
+        .pipe(
+          switchMap(() =>
+            participation.state$.pipe(
+              filter((state) => state.kind === "left"),
+              take(1),
             ),
-          );
-          return combineLatest(listOfMemberObservables);
-        }),
-      ),
-      [],
+          ),
+          scope.bind(),
+        )
+        .subscribe(() => {
+          hostBridge
+            .setAlwaysOnScreen(false)
+            .catch(failed("Failed to unstick the widget"))
+            .then(() => hostBridge.close?.())
+            .catch(failed("Failed to ask the host to close"));
+        });
+    },
+    leave: (): void => vm?.leave(),
+    stop: (): void => scope.end(),
+    connected$: over(
+      (p) => p.state$.pipe(map((state) => state.kind === "connected")),
+      false,
     ),
-    sendData,
-    sendRoomMessage,
+    localMember$: over((p) => p.localMember$, null),
+    remoteMembers$: over((p) => p.remoteMembers$, []),
+    // Attested members only: the SDK has matched the sender
+    data$: slot.participation$.pipe(
+      switchMap((p) => p?.data$ ?? EMPTY),
+      filter(({ topic }) => topic === TOPIC),
+      map(({ member, text }) => ({
+        rtcBackendIdentity: member.rtcBackendIdentity,
+        data: text,
+      })),
+    ),
+    sendData: async (data): Promise<void> => {
+      const participation = slot.participation$.value;
+      if (participation === null) throw new Error("Not joined");
+      await participation.sendData(TOPIC, JSON.stringify(data));
+    },
+    sendRoomMessage: async (message): Promise<void> => {
+      await client.sendTextMessage(room.roomId, message);
+    },
   };
 }

sdk/SdkArchitecture.md

@@ -101,6 +101,8 @@ LiveKit participants.
   │  create ───▶│                                       │                         │
   │             │ GET rtc/transports (session) ────────▶│                         │
   │             │◀── first transport of the backend's type                        │
+  │             │   (none, LiveKit only: a member's SFU │                         │
+  │             │    that lets us publish, get_token) ─▶│                         │
   │             │ prepareLocalTransport (backend):      │                         │
   │             │   publisher token: CS API get_token,  │                         │
   │             │   or OpenID token then /sfu/get ─────▶│                         │
@@ -124,10 +126,15 @@ probe that decides whether the service that issued the token can take over the
 delayed leave, which picks the delayed leave timings the join uses. The join
 waits for the publisher to hold its token, so that the LiveKit room exists
 before a subscriber, who lacks the permission to create it, asks for a token
-of its own. Until `join()` the tracks exist but their upstream is paused: a host
-can show a preview without anyone hearing it. The upstream flows exactly while the
-membership is `joined`: a sync outage pauses it, and `leave()` pauses it for good
-and sends the leave.
+of its own. The tracks are created once the membership is `joined`, with their
+upstream paused until then. A lobby that wants them earlier captures them with
+`createLocalMedia`, on the browser's devices alone so that any backend fits, and
+hands the media to `join()` as `publish`: the tracks are moved, not shared — they
+leave the lobby's `tracks$`, their objects are finished, and the backend
+publishes the very same device tracks and the processor already running on them,
+so nothing is captured or initialised again at the join. The upstream flows
+exactly while the membership is `joined`: a sync outage pauses it, and `leave()`
+pauses it for good and sends the leave.
 
 ## Vocabulary
 
@@ -536,7 +543,7 @@ backend is handed the whole session's view and gives back media:
 type MemberMediaFields = Pick<RTCMember, "tracks$" | "encryptionError$">;
 
 export interface MediaBackend {
-  readonly transportType: string; // "livekit"
+  readonly transportTypes: readonly string[]; // ["m.livekit", "livekit"]
   /** Authenticate with the transport the client found; the probe for delegation runs here. */
   prepareLocalTransport(
     transport: Transport,
@@ -556,7 +563,10 @@ export interface MediaBackend {
 ```
 
 What stays on the client's side: discovery of the local transport from the
-homeserver's list (a homeserver endpoint, so not a backend's), the membership
+homeserver's list (a homeserver endpoint, so not a backend's) with, for the
+LiveKit backend only, a last resort of another member's SFU that lets us
+publish (`findRemotePublishLivekitTransport`, wired in by `RTCParticipation` rather
+than by the backend interface, since it is an MSC4195 matter), the membership
 state machine in `LocalMembership.ts` and its combination with
 `local.connectionState$` into `state$`, the choice of delayed leave timings,
 mapping a data packet's sender to a member, and the transport registry, which keys on the raw

sdk/index.ts

@@ -61,8 +61,13 @@ export {
   createLivekitBackend,
   type LivekitBackendOptions,
 } from "./src/media-backend/livekit/LivekitBackend";
-// For a host that creates LiveKit tracks itself, such as a preview before joining
-export { convertToLivekitProcessor } from "./src/media-backend/livekit/videoProcessor";
+// Tracks for a lobby, captured ahead of the join and handed to it
+export { createLocalMedia } from "./src/LocalMedia";
+// For a host whose processor is a livekit-client one, such as @livekit/track-processors
+export {
+  fromLivekitProcessor,
+  toLivekitProcessor,
+} from "./src/media-backend/livekit/videoProcessor";
 // For a developer panel to check a transport before using it
 export {
   getSFUConfig as authenticateWithTransport,

sdk/pnpm-lock.yaml

@@ -0,0 +1,9 @@
+lockfileVersion: '9.0'
+
+settings:
+  autoInstallPeers: false
+  excludeLinksFromLockfile: false
+
+importers:
+
+  .: {}

sdk/src/LocalMedia.test.ts

@@ -0,0 +1,237 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { beforeAll, describe, expect, it, vi } from "vitest";
+
+import { createLocalMedia, type GetUserMedia } from "./LocalMedia";
+import { type LocalVideoMediaTrack, type VideoProcessor } from "./media-api";
+
+describe("createLocalMedia", () => {
+  it("opens the microphone and the camera in one request", async () => {
+    const getUserMedia = fakeGetUserMedia();
+    const media = createLocalMedia(getUserMedia);
+
+    const tracks = await media.capture([
+      { source: "microphone", deviceId: "mic" },
+      {
+        source: "camera",
+        capture: { resolution: { width: 640, height: 480 } },
+      },
+    ]);
+
+    expect(getUserMedia).toHaveBeenCalledExactlyOnceWith({
+      audio: { deviceId: "mic" },
+      video: { width: 640, height: 480, frameRate: undefined },
+    });
+    expect(tracks.map((t) => t.source)).toEqual(["microphone", "camera"]);
+    expect(media.tracks$.value).toEqual(tracks);
+    expect(media.requests.map((r) => r.source)).toEqual([
+      "microphone",
+      "camera",
+    ]);
+  });
+
+  it("joins an identical capture still in flight rather than opening the devices again", async () => {
+    const getUserMedia = fakeGetUserMedia();
+    const media = createLocalMedia(getUserMedia);
+    const requests = [{ source: "camera" as const }];
+
+    const [first, second] = await Promise.all([
+      media.capture(requests),
+      media.capture(requests),
+    ]);
+
+    expect(getUserMedia).toHaveBeenCalledOnce();
+    expect(second).toEqual(first);
+    expect(media.tracks$.value).toEqual(first);
+  });
+
+  it("shows the processor's output, initialised once", async () => {
+    const processor = fakeProcessor();
+    const media = createLocalMedia(fakeGetUserMedia());
+    const [camera] = await media.capture([{ source: "camera", processor }]);
+    const element = document.createElement("video");
+
+    camera.attach(element);
+
+    expect(processor.init).toHaveBeenCalledOnce();
+    expect(tracksOn(element)).toEqual([processor.output]);
+  });
+
+  it("hands the tracks over as they are: no second capture, no second init", async () => {
+    const getUserMedia = fakeGetUserMedia();
+    const processor = fakeProcessor();
+    const media = createLocalMedia(getUserMedia);
+    const [microphone, camera] = await media.capture([
+      { source: "microphone" },
+      { source: "camera", processor },
+    ]);
+
+    const taken = media.takeTracks();
+
+    expect(taken.get("microphone")?.track).toBe(trackBehind(microphone));
+    expect(taken.get("camera")).toEqual({
+      track: trackBehind(camera),
+      constraints: {},
+      muted: false,
+      processor: { processor, processedTrack: processor.output },
+    });
+    expect(media.tracks$.value).toEqual([]);
+    expect(getUserMedia).toHaveBeenCalledOnce();
+    expect(processor.init).toHaveBeenCalledOnce();
+    expect(processor.destroy).not.toHaveBeenCalled();
+    media.stop();
+    expect(taken.get("camera")!.track.stop).not.toHaveBeenCalled();
+  });
+
+  it("finishes a handed-over track", async () => {
+    const media = createLocalMedia(fakeGetUserMedia());
+    const [camera] = await media.capture([{ source: "camera" }]);
+    const element = document.createElement("video");
+    camera.attach(element);
+
+    media.takeTracks();
+
+    expect(element.srcObject).toBeNull();
+    await expect(camera.setEnabled(false)).rejects.toThrow("handed over");
+    expect(() => camera.attach(element)).toThrow("handed over");
+  });
+
+  it("mutes by disabling the track and hands the mute over", async () => {
+    const media = createLocalMedia(fakeGetUserMedia());
+    const [microphone] = await media.capture([{ source: "microphone" }]);
+
+    await expect(microphone.setEnabled(false)).resolves.toBe(false);
+
+    expect(microphone.muted$.value).toBe(true);
+    expect(trackBehind(microphone).enabled).toBe(false);
+    expect(media.takeTracks().get("microphone")?.muted).toBe(true);
+  });
+
+  it("switches the device by opening it anew", async () => {
+    const getUserMedia = fakeGetUserMedia();
+    const processor = fakeProcessor();
+    const media = createLocalMedia(getUserMedia);
+    const [camera] = await media.capture([{ source: "camera", processor }]);
+    const first = trackBehind(camera);
+    const element = document.createElement("video");
+    camera.attach(element);
+
+    await camera.setDevice("other");
+
+    expect(getUserMedia).toHaveBeenLastCalledWith({
+      video: { deviceId: "other" },
+    });
+    expect(first.stop).toHaveBeenCalled();
+    expect(processor.restart).toHaveBeenCalledOnce();
+    const [restartedOn, input] = vi.mocked(processor.restart).mock.calls[0];
+    expect(restartedOn).not.toBe(first);
+    expect(input).toBe(vi.mocked(processor.init).mock.calls[0][1]);
+    expect(tracksOn(element)).toEqual([processor.output]);
+    expect(media.takeTracks().get("camera")?.constraints).toEqual({
+      deviceId: "other",
+    });
+  });
+
+  it("reports the facing mode and switches it", async () => {
+    const getUserMedia = fakeGetUserMedia({ facingMode: "user" });
+    const media = createLocalMedia(getUserMedia);
+    const [camera] = (await media.capture([{ source: "camera" }])) as [
+      LocalVideoMediaTrack,
+    ];
+    expect(camera.facingMode$.value).toBe("user");
+
+    await camera.switchFacingMode();
+
+    expect(getUserMedia).toHaveBeenLastCalledWith({
+      video: { facingMode: "environment" },
+    });
+  });
+
+  it("releases one device and stops the rest", async () => {
+    const media = createLocalMedia(fakeGetUserMedia());
+    const [microphone, camera] = await media.capture([
+      { source: "microphone" },
+      { source: "camera" },
+    ]);
+
+    media.release("camera");
+    expect(trackBehind(camera).stop).toHaveBeenCalled();
+    expect(media.tracks$.value).toEqual([microphone]);
+
+    media.stop();
+    expect(trackBehind(microphone).stop).toHaveBeenCalled();
+    expect(media.tracks$.value).toEqual([]);
+  });
+
+  it("refuses a screen share", async () => {
+    const media = createLocalMedia(fakeGetUserMedia());
+    await expect(media.capture([{ source: "screenShare" }])).rejects.toThrow(
+      "screen share",
+    );
+  });
+});
+
+class FakeMediaStream {
+  public constructor(private readonly tracks: MediaStreamTrack[]) {}
+  public getTracks(): MediaStreamTrack[] {
+    return this.tracks;
+  }
+}
+
+beforeAll(() => {
+  globalThis.MediaStream = FakeMediaStream as unknown as typeof MediaStream;
+});
+
+let nextId = 0;
+const tracksById = new Map<string, MediaStreamTrack>();
+
+function fakeTrack(
+  kind: "audio" | "video",
+  settings: MediaTrackSettings = {},
+): MediaStreamTrack {
+  const track = {
+    kind,
+    id: `${kind}-${nextId++}`,
+    enabled: true,
+    stop: vi.fn(),
+    getSettings: () => settings,
+  } as unknown as MediaStreamTrack;
+  tracksById.set(track.id, track);
+  return track;
+}
+
+function fakeGetUserMedia(videoSettings?: MediaTrackSettings): GetUserMedia {
+  return vi.fn(async (constraints: MediaStreamConstraints) => {
+    const tracks: MediaStreamTrack[] = [];
+    if (constraints.audio) tracks.push(fakeTrack("audio"));
+    if (constraints.video) tracks.push(fakeTrack("video", videoSettings));
+    return Promise.resolve(
+      new FakeMediaStream(tracks) as unknown as MediaStream,
+    );
+  });
+}
+
+function fakeProcessor(): VideoProcessor & { output: MediaStreamTrack } {
+  const output = fakeTrack("video");
+  return {
+    name: "blur",
+    output,
+    init: vi.fn(async () => Promise.resolve(output)),
+    restart: vi.fn(async () => Promise.resolve(output)),
+    destroy: vi.fn(async () => Promise.resolve()),
+  };
+}
+
+/** The device's track, which a media track was created on and keeps the id of. */
+function trackBehind(mediaTrack: { id: string }): MediaStreamTrack {
+  return tracksById.get(mediaTrack.id)!;
+}
+
+function tracksOn(element: HTMLMediaElement): MediaStreamTrack[] {
+  return (element.srcObject as unknown as FakeMediaStream).getTracks();
+}

sdk/src/LocalMedia.ts

@@ -0,0 +1,335 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { BehaviorSubject, type Observable, of } from "rxjs";
+
+import { MatrixRTCError } from "./errors";
+import {
+  type HandedOverTrack,
+  type LocalAudioMediaTrack,
+  type LocalMedia,
+  type LocalVideoMediaTrack,
+  type MediaSource,
+  type MediaStreamStats,
+  type PublishRequest,
+  type VideoProcessor,
+} from "./media-api";
+import { constant } from "./reactive/Behavior";
+import { inTurn } from "./utils/processorTurns";
+
+type Source = "microphone" | "camera";
+type Request = Extract<PublishRequest, { source: Source }>;
+export type GetUserMedia = (
+  constraints: MediaStreamConstraints,
+) => Promise<MediaStream>;
+
+/**
+ * Local media on the browser's devices alone, for a lobby ahead of a join:
+ * no backend is involved, so whichever backend joins can take the tracks
+ * over. Activity and level are not measured; stats are none.
+ */
+export function createLocalMedia(
+  getUserMedia: GetUserMedia = async (constraints) =>
+    navigator.mediaDevices.getUserMedia(constraints),
+): LocalMedia {
+  const capturedTracks = new Map<Source, CapturedTrack>();
+  const requests = new Map<Source, Request>();
+  const tracks$ = new BehaviorSubject<
+    (LocalAudioMediaTrack | LocalVideoMediaTrack)[]
+  >([]);
+  const publishCapturedTracks = (): void =>
+    tracks$.next(
+      [...capturedTracks.values()].map(({ mediaTrack }) => mediaTrack),
+    );
+
+  // A second identical capture while the first is open joins it: a host that
+  // mounts twice (React's strict mode) must not open the devices twice and
+  // replace the tracks it is already showing
+  const pending = new Map<
+    string,
+    Promise<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]>
+  >();
+  const capture = async (
+    wanted: PublishRequest[],
+  ): Promise<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]> => {
+    const key = JSON.stringify(
+      wanted.map((request) => ({ ...request, processor: undefined })),
+    );
+    const running = pending.get(key);
+    if (running) return running;
+    const started = captureNow(wanted).finally(() => pending.delete(key));
+    pending.set(key, started);
+    return started;
+  };
+
+  const captureNow = async (
+    wanted: PublishRequest[],
+  ): Promise<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]> => {
+    const bySource = new Map<Source, Request>();
+    for (const request of wanted) {
+      if (request.source === "screenShare")
+        throw new MatrixRTCError(
+          "A screen share cannot be captured ahead of a join",
+        );
+      bySource.set(request.source, request);
+    }
+    const constraints: MediaStreamConstraints = {};
+    for (const request of bySource.values())
+      constraints[request.source === "microphone" ? "audio" : "video"] =
+        trackConstraints(request);
+    const stream = await getUserMedia(constraints);
+    const captured: CapturedTrack[] = [];
+    for (const track of stream.getTracks()) {
+      const source: Source = track.kind === "audio" ? "microphone" : "camera";
+      const request = bySource.get(source)!;
+      capturedTracks.get(source)?.stop();
+      const captured_ = new CapturedTrack(
+        source,
+        track,
+        trackConstraints(request),
+        getUserMedia,
+      );
+      capturedTracks.set(source, captured_);
+      requests.set(source, request);
+      if (request.source === "camera" && request.processor)
+        await captured_.setProcessor(request.processor);
+      captured.push(captured_);
+    }
+    publishCapturedTracks();
+    return captured.map(({ mediaTrack }) => mediaTrack);
+  };
+
+  const release = (source: MediaSource): void => {
+    if (source !== "microphone" && source !== "camera") return;
+    capturedTracks.get(source)?.stop();
+    capturedTracks.delete(source);
+    publishCapturedTracks();
+  };
+
+  const stop = (): void => {
+    for (const track of capturedTracks.values()) track.stop();
+    capturedTracks.clear();
+    publishCapturedTracks();
+  };
+
+  const takeTracks = (): Map<Source, HandedOverTrack> => {
+    const taken = new Map<Source, HandedOverTrack>();
+    for (const [source, track] of capturedTracks)
+      taken.set(source, track.handOver());
+    capturedTracks.clear();
+    publishCapturedTracks();
+    return taken;
+  };
+
+  return {
+    get requests(): PublishRequest[] {
+      return [...requests.values()];
+    },
+    tracks$,
+    capture,
+    release,
+    stop,
+    takeTracks,
+  };
+}
+
+/** The constraints a request means to the browser. */
+function trackConstraints(request: Request): MediaTrackConstraints {
+  const device = request.deviceId && { deviceId: request.deviceId };
+  if (request.source === "microphone") return { ...device, ...request.capture };
+  const resolution = request.capture?.resolution;
+  return {
+    ...device,
+    ...(resolution && {
+      width: resolution.width,
+      height: resolution.height,
+      frameRate: resolution.frameRate,
+    }),
+  };
+}
+
+/**
+ * Internal Class that is the owner of the tracks.
+ * Will get stored in `capturedTracks` of `createLocalMedia`.
+ * To the outside this class will return the `LocalMedia` interface.
+ */
+class CapturedTrack {
+  public readonly mediaTrack: LocalAudioMediaTrack | LocalVideoMediaTrack;
+  private readonly muted$ = new BehaviorSubject(false);
+  private readonly facingMode$: BehaviorSubject<
+    "user" | "environment" | undefined
+  >;
+  private readonly elements = new Set<HTMLMediaElement>();
+  private processor: HandedOverTrack["processor"];
+  /** What the processor reads frames from; a processor needs a playing element. */
+  private processorInput?: HTMLVideoElement;
+  private finished = false;
+
+  public constructor(
+    public readonly source: Source,
+    private track: MediaStreamTrack,
+    private constraints: MediaTrackConstraints,
+    private readonly getUserMedia: GetUserMedia,
+  ) {
+    this.facingMode$ = new BehaviorSubject(facingModeOf(track));
+    this.mediaTrack = this.asMediaTrack();
+  }
+
+  public handOver(): HandedOverTrack {
+    const { track, processor } = this;
+    // The device actually in use, so that a backend asked for it again
+    // recognises it rather than restarting the track
+    const { deviceId } = track.getSettings();
+    const constraints = { ...this.constraints, ...(deviceId && { deviceId }) };
+    this.finish();
+    return { track, constraints, muted: this.muted$.value, processor };
+  }
+
+  public stop(): void {
+    const { processor } = this;
+    if (processor)
+      void inTurn(processor.processor, async () =>
+        processor.processor.destroy(),
+      );
+    this.processor?.processedTrack.stop();
+    this.track.stop();
+    this.finish();
+  }
+
+  public async setProcessor(
+    processor: VideoProcessor | undefined,
+  ): Promise<void> {
+    this.live();
+    if (processor === this.processor?.processor) return;
+    const previous = this.processor?.processor;
+    if (previous) await inTurn(previous, async () => previous.destroy());
+    this.processor = processor && {
+      processor,
+      processedTrack: await inTurn(processor, async () =>
+        processor.init(this.track, this.input()),
+      ),
+    };
+    this.show();
+  }
+
+  private input(): HTMLVideoElement {
+    this.processorInput ??= document.createElement("video");
+    this.processorInput.muted = true;
+    this.processorInput.playsInline = true;
+    play(this.processorInput, this.track);
+    return this.processorInput;
+  }
+
+  private finish(): void {
+    this.finished = true;
+    for (const element of this.elements) element.srcObject = null;
+    this.elements.clear();
+  }
+
+  private live(): void {
+    if (this.finished)
+      throw new MatrixRTCError(
+        `The ${this.source} was handed over to a participation`,
+      );
+  }
+
+  private shown(): MediaStreamTrack {
+    return this.processor?.processedTrack ?? this.track;
+  }
+
+  private show(): void {
+    for (const element of this.elements) play(element, this.shown());
+  }
+
+  private async reacquire(constraints: MediaTrackConstraints): Promise<void> {
+    this.live();
+    this.constraints = { ...this.constraints, ...constraints };
+    const stream = await this.getUserMedia({
+      [this.track.kind]: this.constraints,
+    });
+    this.track.stop();
+    this.track = stream.getTracks()[0];
+    this.track.enabled = !this.muted$.value;
+    const { processor } = this;
+    if (processor)
+      processor.processedTrack = await inTurn(processor.processor, async () =>
+        processor.processor.restart(this.track, this.input()),
+      );
+    this.facingMode$.next(facingModeOf(this.track));
+    this.show();
+  }
+
+  private asMediaTrack(): LocalAudioMediaTrack | LocalVideoMediaTrack {
+    const base = {
+      source: this.source,
+      id: this.track.id,
+      muted$: this.muted$,
+      // Nothing leaves the device before the track is published, so there is
+      // nothing unencrypted to report
+      encrypted: true,
+      stats$: (): Observable<MediaStreamStats> => of(undefined),
+      attach: (element: HTMLMediaElement): void => {
+        this.live();
+        this.elements.add(element);
+        play(element, this.shown());
+      },
+      detach: (element: HTMLMediaElement): void => {
+        if (this.elements.delete(element)) element.srcObject = null;
+      },
+      setEnabled: async (enabled: boolean): Promise<boolean> => {
+        this.live();
+        this.track.enabled = enabled;
+        this.muted$.next(!enabled);
+        return Promise.resolve(enabled);
+      },
+      setDevice: async (deviceId: string): Promise<void> => {
+        if (this.track.getSettings().deviceId === deviceId) return;
+        await this.reacquire({ deviceId });
+      },
+    };
+    if (this.source === "microphone")
+      return {
+        ...base,
+        kind: "audio",
+        isActive$: constant(false),
+        audioLevel$: constant(0),
+        setAudioContext: () => {},
+        setVolume: () => {},
+      };
+    return {
+      ...base,
+      kind: "video",
+      facingMode$: this.facingMode$,
+      switchFacingMode: async () => {
+        const facingMode = this.facingMode$.value;
+        if (facingMode === undefined) return;
+        await this.reacquire({
+          facingMode: facingMode === "user" ? "environment" : "user",
+        });
+        return this.track.getSettings().deviceId;
+      },
+      setProcessor: async (processor) => this.setProcessor(processor),
+    };
+  }
+}
+
+function play(element: HTMLMediaElement, track: MediaStreamTrack): void {
+  element.srcObject = new MediaStream([track]);
+  // Autoplay is not a given for a stream set after mount; a test DOM returns
+  // nothing from play
+  const playing = element.play() as Promise<void> | undefined;
+  playing?.catch(() => undefined);
+}
+
+function facingModeOf(
+  track: MediaStreamTrack,
+): "user" | "environment" | undefined {
+  const { facingMode } = track.getSettings();
+  return facingMode === "user" || facingMode === "environment"
+    ? facingMode
+    : undefined;
+}

sdk/src/RTCParticipation.test.ts

@@ -8,9 +8,11 @@ Please see LICENSE in the repository root for full details.
 import { type MatrixClient, type Room, SyncState } from "matrix-js-sdk";
 import { logger } from "matrix-js-sdk/lib/logger";
 import {
+  type CallMembership,
   MatrixRTCSessionEvent,
   MembershipManagerEvent,
   type Transport,
+  type UnstableLivekitTransport,
 } from "matrix-js-sdk/lib/matrixrtc";
 import { BehaviorSubject, EMPTY, Subject } from "rxjs";
 import { describe, expect, it, type Mock, vi } from "vitest";
@@ -38,13 +40,32 @@ import { constant } from "./reactive/Behavior";
 import { ObservableScope } from "./reactive/ObservableScope";
 import { createRTCParticipation } from "./RTCParticipation";
 import { transportId } from "./matrixrtc/Transports";
-import { flushPromises, mockEmitter, MockRTCSession } from "./utils/test";
+import {
+  flushPromises,
+  mockEmitter,
+  mockRtcMembership,
+  MockRTCSession,
+} from "./utils/test";
 
-const transport: Transport = {
+const transport: UnstableLivekitTransport = {
   type: "livekit",
   livekit_service_url: "https://lk.example.org",
 };
+
+const transportWithAlias: Transport = {
+  livekit_alias: "!room:example.org",
+  ...transport,
+};
+
 const userId = "@alice:example.org";
+/** A token with the publish grant, as a homeserver issues it for its own SFU. */
+const publisherJWT = [
+  {},
+  { sub: "identity", video: { room: "room", canPublish: true } },
+  {},
+]
+  .map((d) => global.btoa(JSON.stringify(d)))
+  .join(".");
 const deviceId = "AAAA";
 const legacyMemberId = `${userId}:${deviceId}`;
 
@@ -91,6 +112,13 @@ interface FakeBackend {
 function setup(
   options: Partial<RTCParticipationOptions> = {},
   slotOptions: { matrixRTCMode?: MatrixRTCMode } = {},
+  {
+    homeserverTransports = [transport],
+    memberships = [],
+  }: {
+    homeserverTransports?: Transport[];
+    memberships?: CallMembership[];
+  } = {},
 ): {
   scope: ObservableScope;
   participation: ReturnType<typeof createRTCParticipation>["participation"];
@@ -103,7 +131,7 @@ function setup(
 } {
   const scope = new ObservableScope();
   const room = { roomId: "!room:example.org" } as Room;
-  const session = new MockRTCSession(room);
+  const session = new MockRTCSession(room, memberships);
   const mockedSession = session.asMockedSession();
   const join = vi.spyOn(session, "join");
   const client = {
@@ -112,7 +140,8 @@ function setup(
     getDeviceId: () => deviceId,
     getDomain: () => "example.org",
     getSyncState: () => SyncState.Syncing,
-    _unstable_getRTCTransports: vi.fn().mockResolvedValue([transport]),
+    _unstable_getRTCTransports: vi.fn().mockResolvedValue(homeserverTransports),
+    _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: publisherJWT }),
   } as unknown as MatrixClient;
   const members$ = new BehaviorSubject<RTCMember[]>([]);
 
@@ -135,7 +164,7 @@ function setup(
     context: MediaBackendContext,
   ): MediaBackend => {
     const backend: MediaBackend = {
-      transportType: "livekit",
+      transportTypes: ["m.livekit", "livekit"],
       prepareLocalTransport: async () => fake.prepared.promise,
       delegateDelayedLeave: fake.delegateDelayedLeave,
       local: {
@@ -201,7 +230,7 @@ describe("createRTCParticipation", () => {
     await flushPromises();
     expect(join).toHaveBeenCalledWith(
       { userId, deviceId, memberId: legacyMemberId },
-      [transport],
+      [transportWithAlias],
       expect.objectContaining({
         manageMediaKeys: true,
         unstableSendStickyEvents: false,
@@ -220,6 +249,32 @@ describe("createRTCParticipation", () => {
     });
   });
 
+  it("publishes on another member's SFU where the homeserver advertises none", async () => {
+    const bobOnSfu = mockRtcMembership("@bob:example.org", "BOB", {
+      fociPreferred: [{ type: "livekit", url: "wss://sfu.example.org" }],
+    });
+    const { join, fake } = setup(
+      {},
+      {},
+      { homeserverTransports: [], memberships: [bobOnSfu] },
+    );
+    fake.prepared.resolve({ canDelegateDelayedLeave: false });
+    await flushPromises();
+    await flushPromises();
+
+    expect(join).toHaveBeenCalledWith(
+      expect.anything(),
+      [
+        {
+          type: "livekit",
+          url: "wss://sfu.example.org",
+          server_name: "example.org",
+        },
+      ],
+      expect.anything(),
+    );
+  });
+
   it("publishes only while joined", async () => {
     const { participation, session, fake, scope } = setup();
     const ended = vi.fn();
@@ -271,7 +326,7 @@ describe("createRTCParticipation", () => {
     await flushPromises();
     expect(join).toHaveBeenCalledWith(
       expect.anything(),
-      [transport],
+      [transportWithAlias],
       expect.objectContaining({ delayedLeaveEventDelayMs: 7_200_000 }),
     );
     expect(fake.delegateDelayedLeave).not.toHaveBeenCalled();

sdk/src/RTCParticipation.ts

@@ -11,6 +11,7 @@ import {
   type MatrixRTCSession as JsSdkRTCSession,
   MatrixRTCSessionEvent,
   MembershipManagerEvent,
+  type Transport,
 } from "matrix-js-sdk/lib/matrixrtc";
 import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager";
 import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
@@ -50,6 +51,7 @@ import {
   type MediaKey,
 } from "./media-backend/api";
 import { createLivekitBackend } from "./media-backend/livekit/LivekitBackend";
+import { findRemotePublishLivekitTransport } from "./media-backend/livekit/RemotePublishTransport";
 import { createHomeserverConnected$ } from "./matrixrtc/HomeserverConnected";
 import { joinJsSdkSession } from "./matrixrtc/joinJsSdkSession";
 import {
@@ -130,13 +132,35 @@ export function createRTCParticipation(
     logger,
   });
 
+  // If the backend is LiveKit, we check it for transports as well, as a
+  // fallback: another member's SFU may let us publish (MSC4195). LiveKit only,
+  // so it stays out of the backend interface
+  const transportFromBackends = async (): Promise<Transport | null> =>
+    backend.transportTypes.includes("m.livekit") ||
+    backend.transportTypes.includes("livekit")
+      ? findRemotePublishLivekitTransport({
+          client,
+          // A snapshot: if nobody with a usable SFU is here yet, the user has
+          // to come back later rather than wait in a half-working lobby
+          memberships: jsSdkSession.memberships,
+          ownMembershipIdentity,
+          roomId: room.roomId,
+          slotId: jsSdkSession.slotId,
+          logger,
+        })
+      : null;
+
   const preparedTransport$ = from(
-    discoverLocalTransport(client, backend.transportType, options, logger).then(
-      async (transport): Promise<PreparedTransport> => ({
-        transport,
-        ...(await backend.prepareLocalTransport(transport)),
-      }),
-    ),
+    discoverLocalTransport(
+      client,
+      backend.transportTypes,
+      options,
+      transportFromBackends,
+      logger,
+    ).then(async (transport): Promise<PreparedTransport> => ({
+      transport,
+      ...(await backend.prepareLocalTransport(transport)),
+    })),
   );
 
   const localMembership = createLocalMembership$({
@@ -151,14 +175,20 @@ export function createRTCParticipation(
       timings.syncDisconnectGracePeriodMs,
     ),
     joinMatrixRTC: (transport, delayedLeave) =>
-      joinJsSdkSession(jsSdkSession, ownMembershipIdentity, transport, {
-        encryptMedia: encryptionSystem.kind !== E2eeType.NONE,
-        matrixRTCMode,
-        sendNotificationType: options.sendNotificationType,
-        applicationData: options.applicationData,
-        timings,
-        delayedLeave,
-      }),
+      joinJsSdkSession(
+        room.roomId,
+        jsSdkSession,
+        ownMembershipIdentity,
+        transport,
+        {
+          encryptMedia: encryptionSystem.kind !== E2eeType.NONE,
+          matrixRTCMode,
+          sendNotificationType: options.sendNotificationType,
+          applicationData: options.applicationData,
+          timings,
+          delayedLeave,
+        },
+      ),
     membershipManagerError$: fromEvent(
       jsSdkSession,
       MatrixRTCSessionEvent.MembershipManagerError,

sdk/src/api.ts

@@ -20,6 +20,7 @@ import { type Observable } from "rxjs";
 import { type Behavior } from "./reactive/Behavior";
 import {
   type LocalAudioMediaTrack,
+  type LocalMedia,
   type LocalMemberMedia,
   type LocalVideoMediaTrack,
   type MediaSource,
@@ -84,12 +85,15 @@ export interface RTCSlot {
 
 export interface RTCParticipationOptions {
   /**
-   * Published at the join; `publish` on the participation adds to it from then
-   * on. Kept as an option, rather than only the method, because the LiveKit
-   * backend reads the initial requests to build the room's default capture
-   * options before the first connection exists.
+   * What the local member publishes at the join: requests the SDK captures
+   * itself, or the media a lobby captured ahead of the join, which the
+   * participation takes over (its tracks leave `media.tracks$`) and publishes
+   * in place of capturing. `publish` on the participation adds to either from
+   * then on. Kept as an option, rather than only the method, because the
+   * LiveKit backend reads it to build the room's default capture options
+   * before the first connection exists.
    */
-  publish: PublishRequest[];
+  publish: PublishRequest[] | LocalMedia;
   /**
    * LEGACY
    *
@@ -230,7 +234,7 @@ export interface DataMessage {
  * backend-specific developer panel and for connection diagnostics.
  */
 export interface TransportMetadata {
-  /** `"livekit"` today. */
+  /** `"m.livekit"` or `"livekit"` today. */
   type: string;
   /** Stable key, unique per transport in the slot: the raw transport, serialised with sorted keys. */
   id: string;

sdk/src/matrixrtc/LocalTransport.test.ts

@@ -20,6 +20,11 @@ const hostTransport: Transport = {
   type: "livekit",
   livekit_service_url: "https://host.example.org",
 };
+const bobsTransport: Transport = {
+  type: "livekit",
+  url: "wss://sfu.bob.com",
+  server_name: "bob.com",
+};
 
 function client(
   advertised: Transport[],
@@ -30,6 +35,12 @@ function client(
   };
 }
 
+function fromBackends(
+  transport: Transport | null = null,
+): ReturnType<typeof vi.fn<() => Promise<Transport | null>>> {
+  return vi.fn(async () => Promise.resolve(transport));
+}
+
 describe("discoverLocalTransport", () => {
   it("takes the transport the host names without asking the homeserver", async () => {
     const c = client([homeserverTransport]);
@@ -37,8 +48,9 @@ describe("discoverLocalTransport", () => {
     await expect(
       discoverLocalTransport(
         c,
-        "livekit",
+        ["m.livekit", "livekit"],
         { transport: hostTransport },
+        fromBackends(),
         logger,
       ),
     ).resolves.toBe(hostTransport);
@@ -49,8 +61,9 @@ describe("discoverLocalTransport", () => {
     await expect(
       discoverLocalTransport(
         client([homeserverTransport]),
-        "livekit",
+        ["m.livekit", "livekit"],
         { fallbackTransport: hostTransport },
+        fromBackends(),
         logger,
       ),
     ).resolves.toEqual(homeserverTransport);
@@ -60,15 +73,37 @@ describe("discoverLocalTransport", () => {
     await expect(
       discoverLocalTransport(
         client([{ type: "not_livekit" }]),
-        "livekit",
+        ["m.livekit", "livekit"],
         { fallbackTransport: hostTransport },
+        fromBackends(),
         logger,
       ),
     ).resolves.toEqual(hostTransport);
   });
 
+  it("prefers a local transport over the backends' without asking them", async () => {
+    const backends = fromBackends(bobsTransport);
+
+    await expect(
+      discoverLocalTransport(
+        client([homeserverTransport]),
+        ["m.livekit", "livekit"],
+        {},
+        backends,
+        logger,
+      ),
+    ).resolves.toEqual(homeserverTransport);
+    expect(backends).not.toHaveBeenCalled();
+  });
+
   it("fails naming the homeserver where there is no transport at all", async () => {
-    const promise = discoverLocalTransport(client([]), "livekit", {}, logger);
+    const promise = discoverLocalTransport(
+      client([]),
+      ["m.livekit", "livekit"],
+      {},
+      fromBackends(),
+      logger,
+    );
 
     await expect(promise).rejects.toBeInstanceOf(
       MatrixRTCTransportMissingError,

sdk/src/matrixrtc/LocalTransport.ts

@@ -15,25 +15,32 @@ import { TransportAutoDiscovery } from "./TransportAutoDiscovery";
 
 /**
  * The transport the local member advertises: the one the host names, else the
- * homeserver's first of the type the backend serves, else the host's fallback.
+ * homeserver's first of the most preferred type the backend serves, else the
+ * host's fallback, else the first the backends can provide, which are asked
+ * only then.
+ *
+ * @throws {MatrixRTCTransportMissingError} where none of these exists.
  */
 export async function discoverLocalTransport(
   client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports">,
-  transportType: string,
+  transportTypes: readonly string[],
   {
     transport: override,
     fallbackTransport,
   }: Pick<RTCParticipationOptions, "transport" | "fallbackTransport">,
+  transportFromBackends: () => Promise<Transport | null>,
   logger: Logger,
 ): Promise<Transport> {
   const transport =
     override ??
     (await new TransportAutoDiscovery({
       client,
-      transportType,
+      transportTypes,
       fallbackTransport,
       logger,
-    }).discoverPreferredTransport());
+    }).discoverPreferredTransport()) ??
+    (await transportFromBackends()) ??
+    null;
   if (transport === null)
     throw new MatrixRTCTransportMissingError(client.getDomain() ?? "");
   return transport;

sdk/src/matrixrtc/TransportAutoDiscovery.test.ts

@@ -61,7 +61,7 @@ describe("TransportAutoDiscovery", () => {
 
       const discovery = new TransportAutoDiscovery({
         client,
-        transportType: "livekit",
+        transportTypes: ["m.livekit", "livekit"],
         fallbackTransport: configTransport,
         logger: rootLogger,
       });
@@ -75,6 +75,29 @@ describe("TransportAutoDiscovery", () => {
     },
   );
 
+  it("prefers a transport of an earlier type over one listed before it", async () => {
+    const mLivekit: Transport = {
+      type: "m.livekit",
+      url: "wss://sfu.example.org",
+    };
+    const client = makeClient();
+    client._unstable_getRTCTransports.mockResolvedValue([
+      backendTransport,
+      mLivekit,
+    ]);
+
+    const discovery = new TransportAutoDiscovery({
+      client,
+      transportTypes: ["m.livekit", "livekit"],
+      fallbackTransport: configTransport,
+      logger: rootLogger,
+    });
+
+    await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual(
+      mLivekit,
+    );
+  });
+
   it("Retries limit_exceeded backend transport", async () => {
     const client = makeClient();
     client._unstable_getRTCTransports
@@ -92,7 +115,7 @@ describe("TransportAutoDiscovery", () => {
 
     const discovery = new TransportAutoDiscovery({
       client,
-      transportType: "livekit",
+      transportTypes: ["m.livekit", "livekit"],
       fallbackTransport: configTransport,
       logger: rootLogger,
     });
@@ -116,7 +139,7 @@ describe("TransportAutoDiscovery", () => {
 
       const discovery = new TransportAutoDiscovery({
         client,
-        transportType: "livekit",
+        transportTypes: ["m.livekit", "livekit"],
         fallbackTransport: configTransport,
         logger: rootLogger,
       });
@@ -133,7 +156,7 @@ describe("TransportAutoDiscovery", () => {
 
     const discovery = new TransportAutoDiscovery({
       client,
-      transportType: "livekit",
+      transportTypes: ["m.livekit", "livekit"],
       fallbackTransport: undefined,
       logger: rootLogger,
     });

sdk/src/matrixrtc/TransportAutoDiscovery.ts

@@ -17,27 +17,27 @@ type TransportDiscoveryClient = Pick<
 
 export interface TransportAutoDiscoveryProps {
   client: TransportDiscoveryClient;
-  /** The transport type the media backend serves. */
-  transportType: string;
-  /** A transport to use when the homeserver advertises none of the type. */
+  /** The transport types we want to discover. Those are the ones, the local media backend supports. most preferred first. */
+  transportTypes: readonly string[];
+  /** A transport to use when the homeserver advertises none of the types. */
   fallbackTransport?: Transport;
   logger: Logger;
 }
 
 export class TransportAutoDiscovery {
   private readonly client: TransportDiscoveryClient;
-  private readonly transportType: string;
+  private readonly transportTypes: readonly string[];
   private readonly fallbackTransport: Transport | undefined;
   private readonly logger: Logger;
 
   public constructor({
     client,
-    transportType,
+    transportTypes,
     fallbackTransport,
     logger,
   }: TransportAutoDiscoveryProps) {
     this.client = client;
-    this.transportType = transportType;
+    this.transportTypes = transportTypes;
     this.fallbackTransport = fallbackTransport;
     this.logger = logger.getChild("[TransportAutoDiscovery]");
   }
@@ -64,7 +64,7 @@ export class TransportAutoDiscovery {
   }
 
   /**
-   * Fetches the first transport of the backend's type from the homeserver.
+   * Fetches the first transport of the backend's most preferred type from the homeserver.
    * This will not throw errors, but instead just log them and return null if the expected config is not found or malformed.
    * @private
    */
@@ -76,17 +76,16 @@ export class TransportAutoDiscovery {
       const transportList = await doNetworkOperationWithRetry(async () =>
         client._unstable_getRTCTransports(),
       );
-      const first = transportList.find(
-        (transport) => transport.type === this.transportType,
-      );
-      if (first) {
-        return first;
-      } else {
-        this.logger.info(
-          `No ${this.transportType} transport found in getRTCTransports end point`,
-          transportList,
+      for (const type of this.transportTypes) {
+        const first = transportList.find(
+          (transport) => transport.type === type,
         );
+        if (first) return first;
       }
+      this.logger.info(
+        `No ${this.transportTypes.join(" or ")} transport found in getRTCTransports end point`,
+        transportList,
+      );
     } catch (ex) {
       this.logger.info(`Failed to use getRTCTransports end point: ${ex}`);
     }

sdk/src/matrixrtc/joinJsSdkSession.test.ts

@@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
+  type UnstableLivekitTransport,
   type MatrixRTCSession,
   type Transport,
 } from "matrix-js-sdk/lib/matrixrtc";
@@ -15,11 +16,16 @@ import { defaultSessionTimings, MatrixRTCMode } from "../config";
 import { ownMemberMock } from "../utils/test";
 import { joinJsSdkSession } from "./joinJsSdkSession";
 
-const transport: Transport = {
+const transport: UnstableLivekitTransport = {
   type: "livekit",
   livekit_service_url: "https://lk.example.org",
 };
 
+const transportWithAlias: Transport = {
+  livekit_alias: "!room:example.org",
+  ...transport,
+};
+
 /** A session that only records the join; nothing else of it is touched. */
 function session(): MatrixRTCSession & {
   join: Mock<MatrixRTCSession["join"]>;
@@ -39,7 +45,7 @@ describe("joinJsSdkSession", () => {
       keyRotationParticipantLimit: 20,
     };
 
-    joinJsSdkSession(s, ownMemberMock, transport, {
+    joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
       encryptMedia: true,
       matrixRTCMode: MatrixRTCMode.Compatibility,
       sendNotificationType: "ring",
@@ -51,7 +57,7 @@ describe("joinJsSdkSession", () => {
       },
     });
 
-    expect(s.join).toHaveBeenCalledWith(ownMemberMock, [transport], {
+    expect(s.join).toHaveBeenCalledWith(ownMemberMock, [transportWithAlias], {
       notificationType: "ring",
       callIntent: undefined,
       applicationData: {},
@@ -72,7 +78,7 @@ describe("joinJsSdkSession", () => {
   it("gives up on the network once the delayed leave has probably been sent", () => {
     const s = session();
 
-    joinJsSdkSession(s, ownMemberMock, transport, {
+    joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
       encryptMedia: false,
       matrixRTCMode: MatrixRTCMode.Compatibility,
       timings: { ...defaultSessionTimings, networkErrorRetryMs: 4_000 },
@@ -81,7 +87,7 @@ describe("joinJsSdkSession", () => {
 
     expect(s.join).toHaveBeenCalledWith(
       ownMemberMock,
-      [transport],
+      [transportWithAlias],
       expect.objectContaining({
         manageMediaKeys: false,
         maximumNetworkErrorRetryCount: 4,
@@ -92,7 +98,30 @@ describe("joinJsSdkSession", () => {
   it("lifts the call intent out of the application data and sends sticky events in Matrix 2.0 mode", () => {
     const s = session();
 
-    joinJsSdkSession(s, ownMemberMock, transport, {
+    joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
+      encryptMedia: true,
+      matrixRTCMode: MatrixRTCMode.Matrix_2_0,
+      applicationData: { "m.call.intent": "audio", custom: 1 },
+      timings: defaultSessionTimings,
+      delayedLeave: defaultSessionTimings.delayedLeave,
+    });
+
+    expect(s.join).toHaveBeenCalledWith(
+      ownMemberMock,
+      [transportWithAlias],
+      expect.objectContaining({
+        callIntent: "audio",
+        applicationData: { custom: 1 },
+        unstableSendStickyEvents: true,
+      }),
+    );
+  });
+
+  it("does not add livekit_alias to newer transport format", () => {
+    const s = session();
+    const transport = { type: "livekit", url: "wss://lk.example.org" };
+
+    joinJsSdkSession("!room:example.org", s, ownMemberMock, transport, {
       encryptMedia: true,
       matrixRTCMode: MatrixRTCMode.Matrix_2_0,
       applicationData: { "m.call.intent": "audio", custom: 1 },

sdk/src/matrixrtc/joinJsSdkSession.ts

@@ -35,6 +35,7 @@ interface Options {
  * @throws If the join could not be started.
  */
 export function joinJsSdkSession(
+  roomId: string,
   session: JsSdkRTCSession,
   ownMembershipIdentity: CallMembershipIdentityParts,
   transport: Transport,
@@ -58,20 +59,37 @@ export function joinJsSdkSession(
     timings.syncDisconnectGracePeriodMs,
     delayedLeave.delay_ms,
   );
-  session.join(ownMembershipIdentity, [transport], {
-    notificationType: sendNotificationType,
-    callIntent,
-    applicationData: rest,
-    manageMediaKeys: encryptMedia,
-    delayedLeaveEventRestartMs: delayedLeave.restart_ms,
-    delayedLeaveEventDelayMs: delayedLeave.delay_ms,
-    delayedLeaveEventRestartLocalTimeoutMs: delayedLeave.restart_timeout_ms,
-    networkErrorRetryMs: timings.networkErrorRetryMs,
-    makeKeyDelay: timings.waitForKeyRotationMs,
-    membershipEventExpiryMs: timings.membershipEventExpiryMs,
-    keyRotationParticipantLimit: timings.keyRotationParticipantLimit,
-    unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
-    maximumNetworkErrorRetryCount:
-      Math.ceil(maxWaitMs / timings.networkErrorRetryMs) + 1,
-  });
+
+  session.join(
+    ownMembershipIdentity,
+    // TODO: `livekit_alias` should be unused these days, but some Ruma-based
+    // clients still expect it to be present. Remove from `publishedTransports`
+    // as soon as https://github.com/ruma/ruma/pull/2642 and
+    // https://github.com/matrix-org/matrix-rust-sdk/pull/7335 are widely
+    // deployed.
+    [
+      {
+        ...("livekit_service_url" in transport
+          ? { livekit_alias: roomId }
+          : {}),
+        ...transport,
+      },
+    ],
+    {
+      notificationType: sendNotificationType,
+      callIntent,
+      applicationData: rest,
+      manageMediaKeys: encryptMedia,
+      delayedLeaveEventRestartMs: delayedLeave.restart_ms,
+      delayedLeaveEventDelayMs: delayedLeave.delay_ms,
+      delayedLeaveEventRestartLocalTimeoutMs: delayedLeave.restart_timeout_ms,
+      networkErrorRetryMs: timings.networkErrorRetryMs,
+      makeKeyDelay: timings.waitForKeyRotationMs,
+      membershipEventExpiryMs: timings.membershipEventExpiryMs,
+      keyRotationParticipantLimit: timings.keyRotationParticipantLimit,
+      unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
+      maximumNetworkErrorRetryCount:
+        Math.ceil(maxWaitMs / timings.networkErrorRetryMs) + 1,
+    },
+  );
 }

sdk/src/media-api.ts

@@ -185,3 +185,45 @@ export interface MemberMedia {
 export interface LocalMemberMedia extends MemberMedia {
   tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>;
 }
+
+/**
+ * Tracks captured ahead of a join, for a lobby: the same local tracks a
+ * participation hands out, on nothing but the browser's devices, so they fit
+ * whichever backend joins. Handed to `join` as `publish`, they are moved once
+ * the participation publishes them: they leave `tracks$`, their objects are
+ * finished (controls reject, elements detached), and the same devices come
+ * back under `localMember$.tracks$` as new objects. `stop` releases only what
+ * was not moved; `capture` works again afterwards.
+ */
+export interface LocalMedia {
+  /** What was captured with, one per source; a join reads its defaults from here. */
+  readonly requests: PublishRequest[];
+  tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]>;
+  /**
+   * Opens the devices, in one permission prompt where the browser asks for
+   * one. Rejects where a device cannot be opened, having opened none of them.
+   * A source captured already is replaced.
+   */
+  capture(
+    requests: PublishRequest[],
+  ): Promise<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]>;
+  /** Closes the device behind a source; nothing where it was not captured. */
+  release(source: MediaSource): void;
+  /** Closes every device still held. */
+  stop(): void;
+  /**
+   * For a media backend, as it starts publishing: moves every track out,
+   * finishing its object here. Not for a host; the participation calls it.
+   */
+  takeTracks(): Map<"microphone" | "camera", HandedOverTrack>;
+}
+
+/** One track as a backend receives it: the device's own track and what runs on it. */
+export interface HandedOverTrack {
+  track: MediaStreamTrack;
+  /** What it was opened with, for a backend that restarts it. */
+  constraints: MediaTrackConstraints;
+  muted: boolean;
+  /** Where a processor runs: it and its output, to adopt rather than initialise again. */
+  processor?: { processor: VideoProcessor; processedTrack: MediaStreamTrack };
+}

sdk/src/media-backend/api.ts

@@ -20,6 +20,7 @@ import { type Observable } from "rxjs";
 import { type ResolvedTransport, type RTCMember } from "../api";
 import {
   type LocalAudioMediaTrack,
+  type LocalMedia,
   type LocalMemberMedia,
   type LocalVideoMediaTrack,
   type MediaSource,
@@ -56,8 +57,8 @@ export interface MediaBackendContext {
   /** The slot the session is in, `application#id`; a token is issued for it. */
   slotId: string;
   ownMembershipIdentity: CallMembershipIdentityParts;
-  /** What the local member publishes at the join. */
-  publish: PublishRequest[];
+  /** What the local member publishes at the join: requests, or a lobby's media. */
+  publish: PublishRequest[] | LocalMedia;
   encryptionSystem: EncryptionSystem;
   /**
    * Every media key the session knows: the current ones replayed on
@@ -113,8 +114,11 @@ export interface DataPacket {
 }
 
 export interface MediaBackend {
-  /** The membership transport type this backend serves: "livekit". */
-  readonly transportType: string;
+  /**
+   * The membership transport types this backend serves, ordered decreasingly
+   * by preference.
+   */
+  readonly transportTypes: readonly string[];
 
   /**
    * Makes `transport` usable as the one the local member publishes on. The

sdk/src/media-backend/livekit/Connection.test.ts

@@ -16,6 +16,7 @@ import {
   vi,
 } from "vitest";
 import {
+  ConnectionError,
   type LocalParticipant,
   type RemoteParticipant,
   type RemoteTrack,
@@ -41,7 +42,7 @@ import {
 } from "./Connection.ts";
 import { ObservableScope } from "../../reactive/ObservableScope";
 import { type ClientGetTokenParts, type ClientOpenIDParts } from "./auth";
-import { MatrixRTCError } from "../../errors";
+import { LivekitConnectionError, MatrixRTCError } from "../../errors";
 import { testJWTToken } from "../../utils/test-fixtures";
 import { ownMemberMock } from "../../utils/test";
 import { mockRemoteParticipant } from "./test";
@@ -57,6 +58,10 @@ let fakeLocalParticipant: MockedObject<LocalParticipant>;
 
 const ROOM_ID = "!roomID:example.org";
 const SLOT_ID = "m.call#ROOM";
+const FIREFOX_USER_AGENT =
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:147.0) Gecko/20100101 Firefox/147.0";
+const CHROME_USER_AGENT =
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36";
 
 const transport: UnstableLivekitTransport = {
   type: "livekit",
@@ -147,6 +152,13 @@ function setupRemoteConnection(): Connection {
   return new Connection(opts, logger);
 }
 
+function mockUserAgent(userAgent: string): void {
+  const spy = vi
+    .spyOn(navigator, "userAgent", "get")
+    .mockReturnValue(userAgent);
+  onTestFinished(() => spy.mockRestore());
+}
+
 afterEach(() => {
   vi.useRealTimers();
   vi.clearAllMocks();
@@ -375,6 +387,89 @@ describe("Start connection states", () => {
     expect(connection.livekitAlias).toBe("!example_room_id");
   });
 
+  it("retries a join that failed on the WebSocket in Firefox", async () => {
+    setupTest();
+    mockUserAgent(FIREFOX_USER_AGENT);
+    const connection = setupRemoteConnection();
+    const capturedStates: ConnectionState[] = [];
+    const s = connection.state$.subscribe((value) => {
+      capturedStates.push(value);
+    });
+    onTestFinished(() => s.unsubscribe());
+    const firstJoin = Promise.withResolvers<void>();
+    fakeLivekitRoom.connect.mockReturnValueOnce(firstJoin.promise);
+
+    const started = connection.start();
+    await vi.waitFor(() => expect(fakeLivekitRoom.connect).toHaveBeenCalled());
+    expect(fakeLivekitRoom.connect).toHaveBeenCalledTimes(1);
+    firstJoin.reject(ConnectionError.websocket("WS closed unexpectedly"));
+    await started;
+
+    expect(fakeLivekitRoom.connect).toHaveBeenCalledTimes(2);
+    expect(capturedStates.at(-1)).toEqual({
+      state: "authenticated",
+      livekitState: LivekitConnectionState.Connected,
+    });
+    expect(capturedStates.some((st) => st.state === "error")).toBe(false);
+  });
+
+  it("retries a join in Firefox only once", async () => {
+    setupTest();
+    mockUserAgent(FIREFOX_USER_AGENT);
+    const connection = setupRemoteConnection();
+    fakeLivekitRoom.connect.mockRejectedValue(
+      ConnectionError.websocket("WS closed unexpectedly"),
+    );
+
+    await expect(connection.start()).rejects.toThrow(LivekitConnectionError);
+
+    expect(fakeLivekitRoom.connect).toHaveBeenCalledTimes(2);
+  });
+
+  it("does not retry a join that failed for another reason in Firefox", async () => {
+    setupTest();
+    mockUserAgent(FIREFOX_USER_AGENT);
+    const connection = setupRemoteConnection();
+    fakeLivekitRoom.connect.mockRejectedValueOnce(
+      ConnectionError.serverUnreachable("server was not reachable"),
+    );
+
+    await expect(connection.start()).rejects.toThrow(LivekitConnectionError);
+
+    expect(fakeLivekitRoom.connect).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not retry a join that failed on the WebSocket in other browsers", async () => {
+    setupTest();
+    mockUserAgent(CHROME_USER_AGENT);
+    const connection = setupRemoteConnection();
+    fakeLivekitRoom.connect.mockRejectedValueOnce(
+      ConnectionError.websocket("WS closed unexpectedly"),
+    );
+
+    await expect(connection.start()).rejects.toThrow(LivekitConnectionError);
+
+    expect(fakeLivekitRoom.connect).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not retry a join in Firefox once stopped", async () => {
+    setupTest();
+    mockUserAgent(FIREFOX_USER_AGENT);
+    const connection = setupRemoteConnection();
+    const firstJoin = Promise.withResolvers<void>();
+    fakeLivekitRoom.connect.mockReturnValueOnce(firstJoin.promise);
+    fakeLivekitRoom.disconnect.mockResolvedValue(undefined);
+
+    const started = connection.start();
+    await vi.waitFor(() => expect(fakeLivekitRoom.connect).toHaveBeenCalled());
+    const stopping = connection.stop();
+    firstJoin.reject(ConnectionError.websocket("WS closed unexpectedly"));
+    await stopping;
+
+    await expect(started).resolves.toBeUndefined();
+    expect(fakeLivekitRoom.connect).toHaveBeenCalledTimes(1);
+  });
+
   it("stopping while connecting does not report an error", async () => {
     setupTest();
     const connection = setupRemoteConnection();

sdk/src/media-backend/livekit/Connection.ts

@@ -14,6 +14,7 @@ import {
   ConnectionError,
   ConnectionErrorReason,
   type ConnectionState as LivekitConnectionState,
+  getBrowser,
   type Participant,
   type RemoteParticipant,
   type RemoteTrackPublication,
@@ -322,7 +323,7 @@ export class Connection {
 
       try {
         this.logger.info(`livekitRoom.connect ${url}`);
-        await this.livekitRoom.connect(url, jwt);
+        await this.connectLivekitRoom(url, jwt);
         this.logger.info(`livekitRoom.connect SUCCESS ${url}`);
       } catch (e) {
         this.logger.info(`livekitRoom.connect FAILED ${url}`, e);
@@ -382,6 +383,23 @@ export class Connection {
     }
   }
 
+  private async connectLivekitRoom(url: string, jwt: string): Promise<void> {
+    try {
+      await this.livekitRoom.connect(url, jwt);
+    } catch (e) {
+      // Behind a proxy, Firefox fails the first WebSocket to a host after trying
+      // it over HTTP/2, so a join that fails on the WebSocket gets a second try.
+      // https://bugzilla.mozilla.org/show_bug.cgi?id=2015411
+      const failedOnFirefoxWebSocket =
+        getBrowser()?.name === "Firefox" &&
+        e instanceof ConnectionError &&
+        e.reason === ConnectionErrorReason.WebSocket;
+      if (this.stopped || !failedOnFirefoxWebSocket) throw e;
+      this.logger.info(`livekitRoom.connect RETRY ${url}`, e);
+      await this.livekitRoom.connect(url, jwt);
+    }
+  }
+
   protected async getSFUConfig(): Promise<SFUConfig> {
     return await getSFUConfig({
       client: this.client,

sdk/src/media-backend/livekit/ConnectionMap.test.ts

@@ -24,7 +24,7 @@ import {
 import { type ConnectionFactory } from "./ConnectionFactory.ts";
 import { type Connection } from "./Connection.ts";
 import { ownMemberMock, withTestScheduler } from "../../utils/test";
-import { type Behavior } from "../../reactive/Behavior";
+import { constant, type Behavior } from "../../reactive/Behavior";
 import { type TransportLocator } from "./auth";
 
 const TRANSPORT_1: TransportLocator = {
@@ -207,7 +207,7 @@ describe("connections$ stream", () => {
     });
   });
 
-  test("Keeps one connection per homeserver for transports with a url", () => {
+  test("Keeps one connection per homeserver for url transports", () => {
     withTestScheduler(({ behavior, expectObservable }) => {
       const urlTransport: TransportLocator = {
         transport: { type: "livekit", url: "wss://sfu.example.org" },
@@ -243,6 +243,94 @@ describe("connections$ stream", () => {
     });
   });
 
+  test("Shares connection between local livekit_service_url transport and remote url + livekit_service_url transport", () => {
+    withTestScheduler(({ behavior, expectObservable, scope }) => {
+      const serviceUrlTransport: TransportLocator = {
+        transport: {
+          type: "livekit",
+          livekit_service_url: "https://jwt.example.org",
+        },
+        serverName: "example.org",
+      };
+      const urlAndServiceUrlTransport: TransportLocator = {
+        transport: {
+          type: "livekit",
+          livekit_service_url: "https://jwt.example.org",
+          url: "wss://sfu.elsewhere.org",
+        },
+        // Should pass even if local server != remote server
+        serverName: "elsewhere.org",
+      };
+      const connectionMap$ = createConnectionMap$({
+        scope,
+        connectionFactory: fakeConnectionFactory,
+        localTransport: serviceUrlTransport,
+        remoteTransports$: constant(new Epoch([urlAndServiceUrlTransport])),
+        logger,
+        ownMembershipIdentity: ownMemberMock,
+      });
+
+      expectObservable(
+        connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
+      ).toBe("a", {
+        a: expect.toSatisfy(({ value: cs }: Epoch<Connection[]>) => {
+          // One connection total, since local and remote connections are shared
+          expect(cs.length).toBe(1);
+          expect(findConnection(serviceUrlTransport, cs)).toBeDefined();
+          expect(findConnection(serviceUrlTransport, cs)).toBe(
+            findConnection(urlAndServiceUrlTransport, cs),
+          );
+          return true;
+        }),
+      });
+    });
+  });
+
+  test("Shares connection between remote livekit_service_url transport and remote url + livekit_service_url transport", () => {
+    withTestScheduler(({ behavior, expectObservable, scope }) => {
+      const serviceUrlTransport: TransportLocator = {
+        transport: {
+          type: "livekit",
+          livekit_service_url: "https://jwt.example.org",
+        },
+        serverName: "example.org",
+      };
+      const urlAndServiceUrlTransport: TransportLocator = {
+        transport: {
+          type: "livekit",
+          livekit_service_url: "https://jwt.example.org",
+          url: "wss://sfu.elsewhere.org",
+        },
+        // Should pass even if local server != remote server
+        serverName: "elsewhere.org",
+      };
+      const connectionMap$ = createConnectionMap$({
+        scope,
+        connectionFactory: fakeConnectionFactory,
+        localTransport: TRANSPORT_2,
+        remoteTransports$: constant(
+          new Epoch([serviceUrlTransport, urlAndServiceUrlTransport]),
+        ),
+        logger,
+        ownMembershipIdentity: ownMemberMock,
+      });
+
+      expectObservable(
+        connectionMap$.pipe(mapEpoch((d) => d.getConnections())),
+      ).toBe("a", {
+        a: expect.toSatisfy(({ value: cs }: Epoch<Connection[]>) => {
+          // Two connections total: a local connection and a shared remote connection
+          expect(cs.length).toBe(2);
+          expect(findConnection(serviceUrlTransport, cs)).toBeDefined();
+          expect(findConnection(serviceUrlTransport, cs)).toBe(
+            findConnection(urlAndServiceUrlTransport, cs),
+          );
+          return true;
+        }),
+      });
+    });
+  });
+
   test("Should start connection only once", () => {
     withTestScheduler(({ behavior, expectObservable }) => {
       const connectionMap$ = createConnectionMap$({
@@ -344,7 +432,7 @@ describe("ConnectionMap", () => {
     data.add({ ...connection } as Connection, [p("b")]);
     expect(warn).toHaveBeenCalledWith(
       expect.stringContaining(
-        'second connection to [null,"https://lk.example.org",null]: existing [a], adding [b]',
+        'second connection to ["openID","https://lk.example.org"]: existing [a], adding [b]',
       ),
     );
     expect(data.getParticipantsForTransport(TRANSPORT_1)).toHaveLength(2);

sdk/src/media-backend/livekit/ConnectionMap.ts

@@ -18,7 +18,11 @@ import {
   type ObservableScope,
 } from "../../reactive/ObservableScope";
 import { generateItemsWithEpoch } from "../../reactive/observable";
-import { type ConnectionRole, type TransportLocator } from "./auth";
+import {
+  chooseAuthFlow,
+  type ConnectionRole,
+  type TransportLocator,
+} from "./auth";
 import { type Connection } from "./Connection";
 import { type ConnectionFactory } from "./ConnectionFactory";
 
@@ -26,22 +30,25 @@ import { type ConnectionFactory } from "./ConnectionFactory";
 export type TransportKey = string & { __transportKey: true };
 
 /**
- * The key a locator is identified by in a map or set. Locators with the same
- * key share a connection: the server name only takes part in authentication
- * for a transport with a `url`, where our homeserver asks that server for the
- * token. A JWT service is reached directly, so members of several homeservers
- * advertising the same one share a connection rather than each get their own.
+ * Converts a {@link TransportLocator} to a key suitable for identifying it in a
+ * map of connections.
  */
 export function keyFor({
   transport,
   serverName,
 }: TransportLocator): TransportKey {
-  const hasUrl = "url" in transport;
+  // Base the key on the auth flow we would use, so that we never create two
+  // connections that share the same auth details. (That would be bad, as we
+  // might then try to connect to the same SFU twice with the same participant
+  // ID, resulting in only one working connection.)
+  // TODO: This strategy will not work once `chooseAuthFlow` begins allowing for
+  // falling back from the CS API flow to the OpenID flow.
+  const flow = chooseAuthFlow(transport);
   // Serialize the fields in a consistent order
   return JSON.stringify([
-    hasUrl ? transport.url : null,
-    "livekit_service_url" in transport ? transport.livekit_service_url : null,
-    hasUrl ? serverName : null,
+    flow.kind,
+    // With the CS API flow, we additionally need to distinguish server names
+    ...(flow.kind === "csApi" ? [flow.url, serverName] : [flow.serviceUrl]),
   ]) as TransportKey;
 }
 

sdk/src/media-backend/livekit/LivekitBackend.test.ts

@@ -258,6 +258,23 @@ describe("createLivekitBackend", () => {
     },
   );
 
+  it("prepares a remote SFU's transport on its homeserver, where delegation is not possible", async () => {
+    // Our own homeserver would delegate, so only the SFU being remote can
+    // explain delegation being skipped
+    fetchMock.post(homeserverEndpoint, 401);
+    const remoteTransport: UnstableLivekitTransport = {
+      type: "livekit",
+      url: "wss://sfu.sample.com",
+      server_name: "sample.com",
+    };
+
+    const capabilities = await backend().prepareLocalTransport(remoteTransport);
+
+    expect(capabilities).toEqual({ canDelegateDelayedLeave: false });
+    expect(connections[0].connection.role).toBe("publisher");
+    expect(connections[0].connection.serverName).toBe("sample.com");
+  });
+
   it("waits for the publisher connection to hold its token before the join", async () => {
     fetchMock.post(homeserverEndpoint, 404);
     fetchMock.post(`${exampleServiceUrl}/delegate_delayed_leave`, 404);
@@ -488,6 +505,27 @@ describe("createLivekitBackend", () => {
       expect(remote?.connection.transport).toEqual(carlsTransport);
       expect(remote?.connection.serverName).toBe("sample.com");
     });
+
+    it("connects to a remote publisher's transport on the SFU's homeserver", async () => {
+      const b = await prepared();
+      // Dave has no SFU of his own and publishes on sample.com's
+      const remoteTransport: UnstableLivekitTransport = {
+        type: "livekit",
+        url: "wss://sfu.sample.com",
+        server_name: "sample.com",
+      };
+      b.mediaFor(testScope(), {
+        ...bob(remoteTransport, ["dave-hashed"]),
+        userId: "@dave:example.org",
+      });
+      await flushPromises();
+
+      const remote = connections.find(
+        (c) => c.connection.role === "subscriber",
+      );
+      expect(remote?.connection.transport).toEqual(remoteTransport);
+      expect(remote?.connection.serverName).toBe("sample.com");
+    });
   });
 });
 

sdk/src/media-backend/livekit/LivekitBackend.ts

@@ -34,7 +34,7 @@ import {
   trackEpoch,
 } from "../../reactive/ObservableScope";
 import { mapScoped } from "../../utils/mapScoped";
-import { serverNameOf } from "../../utils/serverName";
+import { serverNameOf, serverNameOfTransport } from "../../utils/serverName";
 import {
   type BackendConnection,
   type MediaBackend,
@@ -60,7 +60,10 @@ import {
 import { toMediaConnectionState } from "./connectionState";
 import { createDataChannel$ } from "./DataChannel";
 import { createKeyProvider } from "./KeyProvider";
-import { createLivekitMemberMedia } from "./LivekitMemberMedia";
+import {
+  createLivekitMemberMedia,
+  participantSourcesEqual,
+} from "./LivekitMemberMedia";
 import { createLivekitLocalMedia } from "./LocalMedia";
 
 export interface LivekitBackendOptions {
@@ -89,7 +92,7 @@ export function createLivekitBackend(
     roomId,
     slotId,
     ownMembershipIdentity,
-    publish,
+    publish: requestsOrMedia,
     encryptionSystem,
     mediaKeys$,
     timings,
@@ -98,6 +101,9 @@ export function createLivekitBackend(
   { client, mediaQuality }: LivekitBackendOptions,
 ): MediaBackend {
   const logger = parentLogger.getChild("[LivekitBackend]");
+  const [publish, media] = Array.isArray(requestsOrMedia)
+    ? [requestsOrMedia, undefined]
+    : [requestsOrMedia.requests, requestsOrMedia];
   const keyProvider = createKeyProvider(encryptionSystem, mediaKeys$, logger);
   const transports = createTransportRegistry(scope);
   const localTransport$ = new BehaviorSubject<TransportLocator | null>(null);
@@ -134,6 +140,7 @@ export function createLivekitBackend(
     connectionMap$,
     localTransport$,
     publish,
+    media,
     audioOutputDeviceId$,
     encryptionSystem,
     logger,
@@ -146,26 +153,37 @@ export function createLivekitBackend(
     logger,
   );
 
+  const ownServerName = serverNameOf(ownMembershipIdentity.userId);
+
   const prepareLocalTransport = async (
     transport: Transport,
   ): Promise<TransportCapabilities> => {
     if (!isUnstableLivekitTransport(transport))
       throw new MatrixRTCError(`Not a LiveKit transport: ${transport.type}`);
-    const locator: TransportLocator = {
+    const serverName = serverNameOfTransport(
       transport,
-      serverName: serverNameOf(ownMembershipIdentity.userId),
-    };
-    // Delegation goes through whichever service authenticates us as a
-    // publisher, so only that service's support counts
-    const flow = chooseAuthFlow(transport, "publisher");
-    const canDelegate =
-      flow.kind === "openID"
-        ? supportsDelegation(
-            `${flow.serviceUrl}/delegate_delayed_leave`,
-            `transport ${flow.serviceUrl}`,
-            logger,
-          )
-        : homeserverSupportsDelegation;
+      ownMembershipIdentity.userId,
+    );
+    const locator: TransportLocator = { transport, serverName };
+    let canDelegate: Promise<boolean>;
+    if (serverName !== ownServerName) {
+      // Our homeserver only takes over leaves for its own SFUs
+      logger.info(
+        `Publishing on a remote SFU of ${serverName}, delegation is not possible`,
+      );
+      canDelegate = Promise.resolve(false);
+    } else {
+      // Check for support using the same auth flow that we intend to use later
+      const flow = chooseAuthFlow(transport);
+      canDelegate =
+        flow.kind === "openID"
+          ? supportsDelegation(
+              `${flow.serviceUrl}/delegate_delayed_leave`,
+              `transport ${flow.serviceUrl}`,
+              logger,
+            )
+          : homeserverSupportsDelegation;
+    }
     localTransport$.next(locator);
     // The membership is only sent once the publisher connection holds its
     // token, so that the LiveKit room exists before anyone subscribes to it:
@@ -204,7 +222,10 @@ export function createLivekitBackend(
         map((transport): TransportLocator | undefined => {
           const raw = transport?.raw;
           return isUnstableLivekitTransport(raw)
-            ? { transport: raw, serverName: serverNameOf(member.userId) }
+            ? {
+                transport: raw,
+                serverName: serverNameOfTransport(raw, member.userId),
+              }
             : undefined;
         }),
         distinctUntilChanged((a, b) => (a && keyFor(a)) === (b && keyFor(b))),
@@ -219,22 +240,28 @@ export function createLivekitBackend(
           const connection = connections.getConnectionForTransport(locator);
           // A member advertising both a `url` and a `livekit_service_url` may
           // have authenticated through either flow, under either identity
-          const participant = connections
+          const participants = connections
             .getParticipantsForTransport(locator)
-            .find((p) => backendIdentities.includes(p.identity));
-          return connection && participant
-            ? { participant, room: connection.livekitRoom }
+            .filter((p) => backendIdentities.includes(p.identity));
+          return connection && participants.length
+            ? { participants, room: connection.livekitRoom }
             : null;
         }),
-        distinctUntilChanged(
-          (a, b) => a?.participant === b?.participant && a?.room === b?.room,
-        ),
+        distinctUntilChanged(participantSourcesEqual),
       ),
     );
     source$.pipe(memberScope.bind()).subscribe((source) => {
-      logger.info(
-        `${member.rtcBackendIdentity}: LiveKit participant ${source ? `matched (${source.participant.sid})` : "missing"} on ${locator$.value ? JSON.stringify(locator$.value.transport) : "no transport"}`,
-      );
+      const transportString = locator$.value
+        ? JSON.stringify(locator$.value.transport)
+        : "no transport";
+      if (source)
+        logger.info(
+          `${member.rtcBackendIdentity}: ${source.participants.length} LiveKit participant(s) matched (${source.participants.map((p) => p.sid).join(", ")}) on ${transportString}`,
+        );
+      else
+        logger.info(
+          `${member.rtcBackendIdentity}: LiveKit participant missing on ${transportString}`,
+        );
     });
     return createLivekitMemberMedia(memberScope, source$, encryptionSystem);
   };
@@ -256,7 +283,7 @@ export function createLivekitBackend(
   });
 
   return {
-    transportType: "livekit",
+    transportTypes: ["m.livekit", "livekit"],
     prepareLocalTransport,
     delegateDelayedLeave: delegateLeave,
     local: localMedia_,

sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
 import { EventEmitter } from "events";
 import {
   type LocalParticipant,
+  LocalVideoTrack,
   ParticipantEvent,
   RemoteAudioTrack,
   type RemoteParticipant,
@@ -17,10 +18,22 @@ import {
   TrackEvent,
   type TrackPublication,
 } from "livekit-client";
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  onTestFinished,
+  vi,
+} from "vitest";
 
 import { ObservableScope } from "../../reactive/ObservableScope";
-import { type AudioMediaTrack } from "../../media-api";
+import {
+  type AudioMediaTrack,
+  type LocalVideoMediaTrack,
+  type VideoProcessor,
+} from "../../media-api";
 import {
   createLivekitMediaTrack,
   createLocalLivekitMediaTrack,
@@ -205,8 +218,55 @@ describe("createLivekitMediaTrack", () => {
     await expect(track.setEnabled(false)).resolves.toBe(false);
     expect(setEnabled).toHaveBeenCalledWith("microphone", false);
     await track.setDevice("mic-2");
-    expect(switchActiveDevice).toHaveBeenCalledWith("audioinput", "mic-2");
+    expect(switchActiveDevice).toHaveBeenCalledWith(
+      "audioinput",
+      "mic-2",
+      false,
+    );
   });
+
+  // LiveKit stops a camera that is turned off; the processor is built when it
+  // is turned on again
+  it.each([
+    { state: "is off", isMuted: true, takes: true },
+    { state: "has stopped for good", isMuted: false, takes: false },
+  ])(
+    "gives a processor to a camera that $state: $takes",
+    async ({ isMuted, takes }) => {
+      // jsdom has none; LiveKit's facing mode checks against it
+      vi.stubGlobal("MediaStreamTrack", class {});
+      onTestFinished(() => {
+        vi.unstubAllGlobals();
+      });
+      const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
+        isMuted,
+        _mediaStreamTrack: {
+          readyState: "ended",
+          label: "",
+          getSettings: () => ({}),
+        },
+        getProcessor: () => undefined,
+        setProcessor: vi.fn().mockResolvedValue(undefined),
+      }) as LocalVideoTrack;
+      const { participant, publication, room } = fakes({
+        track: camera as unknown as FakeTrack,
+      });
+      const track = createLocalLivekitMediaTrack(
+        scope,
+        Object.assign(participant, {
+          isLocal: true,
+        }) as unknown as LocalParticipant,
+        publication as unknown as TrackPublication,
+        room,
+        vi.fn(),
+      ) as LocalVideoMediaTrack;
+
+      await track.setProcessor({ name: "blur" } as VideoProcessor);
+      expect(vi.mocked(camera.setProcessor).mock.calls.length).toBe(
+        takes ? 1 : 0,
+      );
+    },
+  );
 });
 
 type FakeTrack = Track & {

sdk/src/media-backend/livekit/LivekitMediaTrack.ts

@@ -50,7 +50,7 @@ import {
   type VideoMediaTrack,
 } from "../../media-api";
 import { MatrixRTCError } from "../../errors";
-import { convertToLivekitProcessor } from "./videoProcessor";
+import { toLivekitProcessor, type LivekitProcessor } from "./videoProcessor";
 
 export const mediaSources: Record<Track.Source, MediaSource> = {
   [Track.Source.Microphone]: "microphone",
@@ -197,9 +197,19 @@ export function createLocalLivekitMediaTrack(
     setDevice: async (deviceId: string): Promise<void> => {
       if (base.source !== "microphone" && base.source !== "camera")
         throw new MatrixRTCError(`A ${base.source} track has no device`);
+      // LiveKit decides whether to restart by the constraints it keeps, which
+      // a browser that does not echo the device id back (Firefox) leaves
+      // without one; a track already on the device is left alone here
+      const current = publication.track;
+      if (
+        current instanceof LocalTrack &&
+        current.mediaStreamTrack.getSettings().deviceId === deviceId
+      )
+        return;
       await room.switchActiveDevice(
         base.kind === "audio" ? "audioinput" : "videoinput",
         deviceId,
+        false,
       );
     },
   };
@@ -228,11 +238,17 @@ export function createLocalLivekitMediaTrack(
     },
     setProcessor: async (processor) => {
       const track = videoTrack();
-      // A processor cannot be built on a track that has already ended
-      if (!track || track.mediaStreamTrack.readyState === "ended") return;
-      if (processor)
-        await track.setProcessor(convertToLivekitProcessor(processor));
-      else if (track.getProcessor()) await track.stopProcessor();
+      // A camera that is off has a stopped track and takes the processor on
+      // turning on again; one stopped for good takes none
+      if (
+        !track ||
+        (track.mediaStreamTrack.readyState === "ended" && !track.isMuted)
+      )
+        return;
+      const current = track.getProcessor() as LivekitProcessor | undefined;
+      if (processor === current?.source) return;
+      if (processor) await track.setProcessor(toLivekitProcessor(processor));
+      else if (current) await track.stopProcessor();
     },
   };
 }

sdk/src/media-backend/livekit/LivekitMemberMedia.test.ts

@@ -25,7 +25,7 @@ describe("createLivekitMemberMedia", () => {
     const { participant, publications, room, emitter } = fakes();
     const media = createLivekitMemberMedia(
       testScope(),
-      constant({ participant, room }),
+      constant({ participants: [participant], room }),
       { kind: E2eeType.NONE },
     );
     expect(media.tracks$.value).toEqual([]);
@@ -52,7 +52,7 @@ describe("createLivekitMemberMedia", () => {
     const { participant, publications, room, emitter } = fakes();
     const media = createLivekitMemberMedia(
       testScope(),
-      constant({ participant, room }),
+      constant({ participants: [participant], room }),
       { kind: E2eeType.NONE },
     );
     emitter.emit(

sdk/src/media-backend/livekit/LivekitMemberMedia.ts

@@ -16,13 +16,20 @@ import {
   RoomEvent,
   type TrackPublication,
 } from "livekit-client";
-import { EMPTY, filter, map, type Observable, of, switchMap } from "rxjs";
+import {
+  EMPTY,
+  filter,
+  map,
+  type Observable,
+  of,
+  switchMap,
+  combineLatest,
+} from "rxjs";
 
 import { type ObservableScope } from "../../reactive/ObservableScope";
 import { generateItems } from "../../reactive/observable";
 import { E2eeType, type EncryptionSystem } from "../../encryption";
 import { type Behavior } from "../../reactive/Behavior";
-import { mapScoped } from "../../utils/mapScoped";
 import {
   type EncryptionError,
   type LocalMemberMedia,
@@ -33,13 +40,27 @@ import {
   createLivekitMediaTrack,
   createLocalLivekitMediaTrack,
 } from "./LivekitMediaTrack";
+import { shallowEquals as arrayShallowEquals } from "../../utils/array";
 
 /** Where a member's media comes from: its participant on a room, null while it has none. */
 export type ParticipantSource<P extends Participant = Participant> = {
-  participant: P;
+  participants: P[];
   room: LivekitRoom;
 } | null;
 
+export function participantSourcesEqual<P extends Participant = Participant>(
+  a: ParticipantSource<P>,
+  b: ParticipantSource<P>,
+): boolean {
+  return (
+    a === b ||
+    (a !== null &&
+      b !== null &&
+      arrayShallowEquals(a.participants, b.participants) &&
+      a.room === b.room)
+  );
+}
+
 /**
  * A member's media from its participant. The only place, with
  * `LivekitMediaTrack`, that reads a LiveKit participant; everything above
@@ -99,21 +120,28 @@ function memberTracks$<P extends Participant, T>(
   ) => T,
 ): Behavior<T[] | null> {
   return scope.behavior(
-    mapScoped(scope, source$, (participantScope, { participant, room }) =>
-      participantScope.behavior(
-        observeParticipantMedia(participant).pipe(
-          generateItems(
-            `${participant.identity} tracks$`,
-            function* () {
-              for (const publication of participant.trackPublications.values())
-                yield { keys: [publication.trackSid], data: publication };
-            },
-            (trackScope, publication$) =>
-              factory(trackScope, participant, publication$.value, room),
+    source$.pipe(
+      switchMap((source) => {
+        if (source === null) return of(null);
+        const { participants, room } = source;
+        return combineLatest(
+          participants.map((participant) =>
+            observeParticipantMedia(participant).pipe(
+              generateItems(
+                `${participant.identity} tracks$`,
+                function* () {
+                  for (const publication of participant.trackPublications.values())
+                    yield { keys: [publication.trackSid], data: publication };
+                },
+                (trackScope, publication$) =>
+                  factory(trackScope, participant, publication$.value, room),
+              ),
+            ),
           ),
-        ),
-      ),
-    ).pipe(switchMap((tracks$) => tracks$ ?? of(null))),
+          (...tracks) => tracks.flat(1),
+        );
+      }),
+    ),
   );
 }
 
@@ -133,7 +161,7 @@ function encryptionErrors$(
             filter(
               (message) =>
                 encryptionSystem.kind === E2eeType.SHARED_KEY ||
-                message.includes(source.participant.identity),
+                source.participants.some((p) => message.includes(p.identity)),
             ),
             map((message): EncryptionError | undefined =>
               message.includes("MissingKey")

sdk/src/media-backend/livekit/LocalMedia.ts

@@ -10,7 +10,6 @@ import {
   type LocalParticipant,
   type LocalTrackPublication,
   ParticipantEvent,
-  type Room as LivekitRoom,
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";
 import {
@@ -28,6 +27,7 @@ import {
 
 import {
   type LocalAudioMediaTrack,
+  type LocalMedia,
   type LocalVideoMediaTrack,
   type MediaSource,
   type PublishRequest,
@@ -43,7 +43,11 @@ import { type LocalMediaBackend, MediaConnectionState } from "../api";
 import { type TransportLocator } from "./auth";
 import { type Connection } from "./Connection";
 import { type ConnectionMap } from "./ConnectionMap";
-import { createLocalLivekitMemberMedia } from "./LivekitMemberMedia";
+import {
+  createLocalLivekitMemberMedia,
+  type ParticipantSource,
+  participantSourcesEqual,
+} from "./LivekitMemberMedia";
 import { livekitSources } from "./LivekitMediaTrack";
 import { type DesiredMedia, Publisher } from "./Publisher";
 import { toMediaConnectionState } from "./connectionState";
@@ -54,6 +58,8 @@ interface Props {
   /** The transport we publish on; null until the session has found it. */
   localTransport$: Behavior<TransportLocator | null>;
   publish: PublishRequest[];
+  /** Tracks the host captured ahead of the join; the publisher takes them over. */
+  media?: LocalMedia;
   /** Read by the connection factory for the rooms it creates from now on. */
   audioOutputDeviceId$: BehaviorSubject<string | undefined>;
   encryptionSystem: EncryptionSystem;
@@ -76,6 +82,7 @@ export function createLivekitLocalMedia({
   connectionMap$,
   localTransport$,
   publish: initialRequests,
+  media,
   audioOutputDeviceId$,
   encryptionSystem,
   logger: parentLogger,
@@ -116,6 +123,7 @@ export function createLivekitLocalMedia({
       connection.livekitRoom,
       desired,
       logger.getChild(`[Publisher ${JSON.stringify(connection.transport)}]`),
+      media,
     );
     publisher$.next(publisher);
     return Promise.resolve(async (): Promise<void> => {
@@ -217,17 +225,15 @@ function participantAndRoom$(
   scope: ObservableScope,
   participant$: Behavior<LocalParticipant | null>,
   connection$: Behavior<Connection | null>,
-): Behavior<{ participant: LocalParticipant; room: LivekitRoom } | null> {
+): Behavior<ParticipantSource<LocalParticipant>> {
   return scope.behavior(
     combineLatest([participant$, connection$]).pipe(
       map(([participant, connection]) =>
         participant && connection
-          ? { participant, room: connection.livekitRoom }
+          ? { participants: [participant], room: connection.livekitRoom }
           : null,
       ),
-      distinctUntilChanged(
-        (a, b) => a?.participant === b?.participant && a?.room === b?.room,
-      ),
+      distinctUntilChanged(participantSourcesEqual),
     ),
   );
 }

sdk/src/media-backend/livekit/Publisher.test.ts

@@ -11,16 +11,32 @@ import {
   LocalParticipant,
   type LocalTrack,
   type LocalTrackPublication,
+  LocalVideoTrack,
   ParticipantEvent,
   type Room as LivekitRoom,
   Track,
+  type TrackPublishOptions,
 } from "livekit-client";
 import { logger } from "matrix-js-sdk/lib/logger";
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  afterEach,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  vi,
+} from "vitest";
 
-import { type PublishRequest } from "../../media-api";
+import {
+  type HandedOverTrack,
+  type LocalMedia,
+  type PublishRequest,
+  type VideoProcessor,
+} from "../../media-api";
 import { flushPromises } from "../../utils/test";
 import { type DesiredMedia, Publisher } from "./Publisher";
+import { type LivekitProcessor } from "./videoProcessor";
 
 describe("Publisher", () => {
   describe("setAudioOutputDevice", () => {
@@ -104,6 +120,24 @@ describe("Publisher", () => {
       expect(participant.createTracks).not.toHaveBeenCalled();
     });
 
+    it("creates both with the options of requests made since the join", async () => {
+      const processor: VideoProcessor = {
+        name: "effect",
+        init: vi.fn(),
+        restart: vi.fn(),
+        destroy: vi.fn(),
+      };
+      want(microphone, { source: "camera", deviceId: "front", processor });
+
+      publisher.start();
+      await flushPromises();
+
+      expect(room.options.videoCaptureDefaults).toMatchObject({
+        deviceId: "front",
+        processor: expect.objectContaining({ source: processor }),
+      });
+    });
+
     it("asks for the camera and the microphone in one call", async () => {
       const enableBoth = vi.spyOn(participant, "enableCameraAndMicrophone");
       want(microphone, camera);
@@ -327,6 +361,143 @@ function fakeRoom(
   return { room, switchActiveDevice };
 }
 
+// A lobby's tracks come as the device's own MediaStreamTracks; LiveKit's
+// track classes are real here so that what is published is what LiveKit will
+// restart and stop as its own
+describe("adopting a lobby's tracks", () => {
+  // LiveKit wraps every track in a MediaStream, which jsdom does not have
+  beforeAll(() => {
+    globalThis.MediaStream = class {
+      public constructor(private readonly tracks: MediaStreamTrack[]) {}
+      public getTracks(): MediaStreamTrack[] {
+        return this.tracks;
+      }
+    } as unknown as typeof MediaStream;
+  });
+
+  function setup(
+    source: "microphone" | "camera",
+    handed: HandedOverTrack,
+  ): { participant: LocalParticipant; publisher: Publisher } {
+    const participant = fakeLocalParticipant(async () => Promise.resolve());
+    const room = {
+      options: {},
+      setE2EEEnabled: () => undefined,
+      localParticipant: participant,
+    } as unknown as LivekitRoom;
+    const desired: DesiredMedia = new Map([
+      [source, { request: { source }, enabled: true }],
+    ]);
+    const media = {
+      takeTracks: () => new Map([[source, handed]]),
+    } as unknown as LocalMedia;
+    const publisher = new Publisher(room, desired, logger, media);
+    return { participant, publisher };
+  }
+
+  it("publishes the lobby's track as its own rather than capturing", async () => {
+    const handed = handedOver("video");
+    const { participant, publisher } = setup("camera", handed);
+
+    publisher.start();
+    await flushPromises();
+
+    expect(participant.createTracks).not.toHaveBeenCalled();
+    const [published, options] = vi.mocked(participant.publishTrack).mock
+      .calls[0] as [LocalVideoTrack, TrackPublishOptions];
+    expect(published).toBeInstanceOf(LocalVideoTrack);
+    expect(published.mediaStreamTrack).toBe(handed.track);
+    expect(published.isUserProvided).toBe(false);
+    expect(options.source).toBe(Track.Source.Camera);
+  });
+
+  it("adopts the processor's output without initialising it again", async () => {
+    const setProcessor = vi
+      .spyOn(LocalVideoTrack.prototype, "setProcessor")
+      .mockResolvedValue(undefined);
+    const processor = {
+      name: "blur",
+      init: vi.fn(),
+      restart: vi.fn(),
+      destroy: vi.fn(),
+    } as unknown as VideoProcessor;
+    const processedTrack = fakeMediaStreamTrack("video");
+    const handed = handedOver("video", {
+      processor: { processor, processedTrack },
+    });
+    const { publisher } = setup("camera", handed);
+
+    publisher.start();
+    await flushPromises();
+
+    const adapter = setProcessor.mock.calls[0][0] as LivekitProcessor;
+    await adapter.init({ kind: Track.Kind.Video, track: handed.track });
+    expect(adapter.processedTrack).toBe(processedTrack);
+    expect(adapter.source).toBe(processor);
+    expect(processor.init).not.toHaveBeenCalled();
+    setProcessor.mockRestore();
+  });
+
+  // A development build mounts, unmounts and mounts the call again: the first
+  // publisher must not have taken the tracks the second one needs
+  it("leaves the lobby's tracks alone until it starts", async () => {
+    const participant = fakeLocalParticipant(async () => Promise.resolve());
+    const room = {
+      options: {},
+      setE2EEEnabled: () => undefined,
+      localParticipant: participant,
+    } as unknown as LivekitRoom;
+    const takeTracks = vi.fn(() => new Map());
+    const media = { takeTracks } as unknown as LocalMedia;
+
+    const publisher = new Publisher(room, new Map(), logger, media);
+    await publisher.destroy();
+
+    expect(takeTracks).not.toHaveBeenCalled();
+  });
+
+  it("joins muted where the lobby muted", async () => {
+    const { participant, publisher } = setup(
+      "microphone",
+      handedOver("audio", { muted: true }),
+    );
+
+    publisher.start();
+    await flushPromises();
+
+    const [published] = vi.mocked(participant.publishTrack).mock.calls[0];
+    expect((published as LocalTrack).isMuted).toBe(true);
+  });
+});
+
+function fakeMediaStreamTrack(kind: "audio" | "video"): MediaStreamTrack {
+  return {
+    kind,
+    id: `${kind}-track`,
+    label: "",
+    enabled: true,
+    readyState: "live",
+    getSettings: () => ({}),
+    getConstraints: () => ({}),
+    getCapabilities: () => ({}),
+    addEventListener: vi.fn(),
+    removeEventListener: vi.fn(),
+    stop: vi.fn(),
+  } as unknown as MediaStreamTrack;
+}
+
+function handedOver(
+  kind: "audio" | "video",
+  extra: Partial<HandedOverTrack> = {},
+): HandedOverTrack {
+  return {
+    track: fakeMediaStreamTrack(kind),
+    constraints: { deviceId: "lobby" },
+    muted: false,
+    ...extra,
+  };
+}
+
 function fakeLocalTrack(source: Track.Source): LocalTrack {
   const state = { isMuted: false, isUpstreamPaused: false };
   const track = {

sdk/src/media-backend/livekit/Publisher.ts

@@ -6,15 +6,23 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
+  LocalAudioTrack,
   type LocalTrackPublication,
+  LocalVideoTrack,
   ParticipantEvent,
   type Room as LivekitRoom,
   type Track,
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";
 
-import { type MediaSource, type PublishRequest } from "../../media-api";
+import {
+  type HandedOverTrack,
+  type LocalMedia,
+  type MediaSource,
+  type PublishRequest,
+} from "../../media-api";
 import { livekitSources, mediaSources } from "./LivekitMediaTrack";
+import { toLivekitProcessor } from "./videoProcessor";
 import {
   audioCaptureOptions,
   screenShareCaptureOptions,
@@ -31,6 +39,9 @@ export interface DesiredPublication {
 
 export type DesiredMedia = Map<MediaSource, DesiredPublication>;
 
+/** A lobby's tracks, moved out of its media, until each is published. */
+export type HandedOverTracks = ReturnType<LocalMedia["takeTracks"]>;
+
 /**
  * Publishes the local media on one LiveKit room, following the host's
  * requests.
@@ -46,10 +57,19 @@ export class Publisher {
   public started = false;
   private readonly room: LivekitRoom;
 
+  /** A lobby's tracks, taken over at `start`; each leaves the map as it is published. */
+  private captured: HandedOverTracks = new Map();
+
+  /**
+   * @param media - What a lobby opened ahead of the join, published in place
+   * of capturing. Taken over only at `start`, so that a publisher torn down
+   * before it publishes leaves the tracks to the next one.
+   */
   public constructor(
     room: LivekitRoom,
     private readonly desired: DesiredMedia,
     private readonly logger: Logger,
+    private readonly media?: LocalMedia,
   ) {
     this.room = room;
     room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
@@ -79,33 +99,106 @@ export class Publisher {
   public start(): void {
     if (this.started) return;
     this.started = true;
+    this.captured = this.media?.takeTracks() ?? new Map();
+    // A track muted in the lobby joins muted
+    for (const [source, { muted }] of this.captured) {
+      const wanted = this.desired.get(source);
+      if (wanted && muted) wanted.enabled = false;
+    }
     const requests = [...this.desired.values()]
-      .filter(({ enabled }) => enabled)
+      .filter(
+        ({ request, enabled }) => enabled || this.hasCaptured(request.source),
+      )
       .map(({ request }) => request);
-    const sources = new Set(requests.map((request) => request.source));
+    const toCapture = new Set(
+      requests
+        .map((request) => request.source)
+        .filter((source) => !this.hasCaptured(source)),
+    );
     // Microphone and camera in one call so that the browser asks for
     // permission once; the room defaults carry their options. LiveKit
     // resolves these once the track is published, which may block on the
     // connection; LocalTrackPublished is what tells us a track exists.
-    if (sources.has("microphone") && sources.has("camera"))
+    const captureBoth = toCapture.has("microphone") && toCapture.has("camera");
+    if (captureBoth) {
+      this.takeCaptureDefaults(requests);
       void this.room.localParticipant.enableCameraAndMicrophone();
+    }
     for (const request of requests)
       if (
         request.source === "screenShare" ||
-        !(sources.has("microphone") && sources.has("camera"))
+        this.hasCaptured(request.source) ||
+        !captureBoth
       )
         this.publish(request).catch((e) => {
           this.logger.error(`Failed to publish the ${request.source}`, e);
         });
   }
 
+  /**
+   * Makes the requests the room's capture defaults. The room took its defaults
+   * from the requests at the join, but one made since, such as a camera turned
+   * on before the start, has options of its own: a video processor dropped
+   * here would let the room see the camera before it is applied.
+   */
+  private takeCaptureDefaults(requests: PublishRequest[]): void {
+    const options = this.room.options;
+    for (const request of requests)
+      if (request.source === "microphone")
+        options.audioCaptureDefaults = {
+          ...options.audioCaptureDefaults,
+          ...audioCaptureOptions(request),
+        };
+      else if (request.source === "camera")
+        options.videoCaptureDefaults = {
+          ...options.videoCaptureDefaults,
+          ...videoCaptureOptions(request),
+        };
+  }
+
+  private hasCaptured(source: MediaSource): boolean {
+    return (
+      (source === "microphone" || source === "camera") &&
+      this.captured.has(source)
+    );
+  }
+
+  private takeCaptured(request: PublishRequest): HandedOverTrack | undefined {
+    if (request.source === "screenShare") return undefined;
+    const track = this.captured.get(request.source);
+    this.captured.delete(request.source);
+    return track;
+  }
+
+  /** A lobby's track as if this participant had captured it: LiveKit restarts and stops it as its own. */
+  private async adopt(
+    { track, constraints, muted, processor }: HandedOverTrack,
+    request: PublishRequest,
+  ): Promise<LocalTrackPublication> {
+    const local =
+      track.kind === "audio"
+        ? new LocalAudioTrack(track, constraints, false)
+        : new LocalVideoTrack(track, constraints, false);
+    if (muted) await local.mute();
+    if (processor && local instanceof LocalVideoTrack)
+      await local.setProcessor(
+        toLivekitProcessor(processor.processor, processor.processedTrack),
+      );
+    return this.room.localParticipant.publishTrack(local, {
+      source: livekitSources[request.source],
+      ...(request.source === "camera" && videoPublishOptions(request.capture)),
+    });
+  }
+
   /** Publishes one request and resolves with its publication once LiveKit has it. */
   public async publish(
     request: PublishRequest,
   ): Promise<LocalTrackPublication | undefined> {
     const participant = this.room.localParticipant;
     let publication: LocalTrackPublication | undefined;
-    if (request.source === "microphone")
+    const captured = this.takeCaptured(request);
+    if (captured) publication = await this.adopt(captured, request);
+    else if (request.source === "microphone")
       publication = await participant.setMicrophoneEnabled(
         true,
         audioCaptureOptions(request),

sdk/src/media-backend/livekit/RemotePublishTransport.test.ts

@@ -0,0 +1,172 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { MatrixError } from "matrix-js-sdk";
+import { logger } from "matrix-js-sdk/lib/logger";
+import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
+import { beforeEach, describe, expect, it, type Mocked, vi } from "vitest";
+
+import { mockRtcMembership, ownMemberMock } from "../../utils/test";
+import { type ClientGetTokenParts } from "./auth/csApi";
+import { findRemotePublishLivekitTransport } from "./RemotePublishTransport";
+
+/** A JWT whose payload states whether the issuing server lets us publish. */
+function jwtWithPublishGrant(canPublish: boolean): string {
+  return [
+    {}, // header
+    { sub: "identity", video: { room: "room", canPublish } }, // payload
+    {}, // signature
+  ]
+    .map((d) => global.btoa(JSON.stringify(d)))
+    .join(".");
+}
+
+const bobSfu = { type: "livekit", url: "wss://sfu.bob.com" };
+const carolSfu = { type: "livekit", url: "wss://sfu.carol.com" };
+const bobsTransport = { ...bobSfu, server_name: "bob.com" };
+const carolsTransport = { ...carolSfu, server_name: "carol.com" };
+
+// Bob is the oldest member, Carol joined after him
+const bob = mockRtcMembership("@bob:bob.com", "BOB", {
+  fociPreferred: [bobSfu],
+  membership: { created_ts: 100 },
+});
+const carol = mockRtcMembership("@carol:carol.com", "CAROL", {
+  fociPreferred: [carolSfu],
+  membership: { created_ts: 200 },
+});
+// Bob's other device, and Dave who publishes on Bob's SFU remotely
+const bobsLaptop = mockRtcMembership("@bob:bob.com", "LAPTOP", {
+  fociPreferred: [bobSfu],
+  membership: { created_ts: 150 },
+});
+const dave = mockRtcMembership("@dave:dave.com", "DAVE", {
+  fociPreferred: [bobsTransport],
+  membership: { created_ts: 160 },
+});
+const legacyErin = mockRtcMembership("@erin:erin.com", "ERIN", {
+  fociPreferred: [
+    { type: "livekit", livekit_service_url: "https://jwt.erin.com" },
+  ],
+  membership: { created_ts: 50 },
+});
+const ourselves = mockRtcMembership(
+  ownMemberMock.userId,
+  ownMemberMock.deviceId,
+  {
+    fociPreferred: [{ type: "livekit", url: "wss://sfu.example.org" }],
+    membership: { created_ts: 10 },
+  },
+);
+
+const csApiMissing = new MatrixError(
+  { errcode: "M_UNRECOGNIZED", error: "" },
+  404,
+);
+
+describe("findRemotePublishLivekitTransport", () => {
+  let client: Mocked<ClientGetTokenParts>;
+  const find = async (
+    memberships: CallMembership[],
+  ): ReturnType<typeof findRemotePublishLivekitTransport> =>
+    findRemotePublishLivekitTransport({
+      client,
+      memberships,
+      ownMembershipIdentity: ownMemberMock,
+      roomId: "!room:example.org",
+      slotId: "m.call#ROOM",
+      logger,
+    });
+
+  beforeEach(() => {
+    client = {
+      _unstable_getLivekitToken: vi
+        .fn()
+        .mockResolvedValue({ jwt: jwtWithPublishGrant(true) }),
+    };
+  });
+
+  it("publishes on the oldest member's SFU with its server_name", async () => {
+    await expect(find([carol, bob])).resolves.toEqual(bobsTransport);
+    expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(1);
+    expect(client._unstable_getLivekitToken).toHaveBeenCalledWith({
+      url: "wss://sfu.bob.com",
+      server_name: "bob.com",
+      room_id: "!room:example.org",
+      slot_id: "m.call#ROOM",
+      member_id: ownMemberMock.memberId,
+    });
+  });
+
+  it("asks about an SFU once however many members are on it", async () => {
+    client._unstable_getLivekitToken.mockResolvedValue({
+      jwt: jwtWithPublishGrant(false),
+    });
+
+    await expect(find([dave, bobsLaptop, bob])).resolves.toBeNull();
+    expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(1);
+  });
+
+  it("skips our own membership", async () => {
+    await expect(find([ourselves, bob])).resolves.toEqual(bobsTransport);
+    expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(1);
+  });
+
+  it("ignores members publishing through a legacy JWT service", async () => {
+    await expect(find([legacyErin, bob])).resolves.toEqual(bobsTransport);
+    expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(1);
+  });
+
+  it("moves on when a homeserver does not let us publish", async () => {
+    client._unstable_getLivekitToken.mockImplementation(
+      async ({ server_name }) =>
+        Promise.resolve({
+          jwt: jwtWithPublishGrant(server_name === "carol.com"),
+        }),
+    );
+
+    await expect(find([bob, carol])).resolves.toEqual(carolsTransport);
+    expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(2);
+  });
+
+  it("moves on when a homeserver refuses to issue a token", async () => {
+    client._unstable_getLivekitToken.mockImplementation(
+      async ({ server_name }) =>
+        server_name === "bob.com"
+          ? Promise.reject(
+              new MatrixError(
+                { errcode: "M_FORBIDDEN", error: "Not joined" },
+                403,
+              ),
+            )
+          : Promise.resolve({ jwt: jwtWithPublishGrant(true) }),
+    );
+
+    await expect(find([bob, carol])).resolves.toEqual(carolsTransport);
+  });
+
+  it("finds nothing once all candidates are exhausted", async () => {
+    client._unstable_getLivekitToken.mockResolvedValue({
+      jwt: jwtWithPublishGrant(false),
+    });
+
+    await expect(find([bob, carol])).resolves.toBeNull();
+    expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(2);
+  });
+
+  it("gives up immediately if our homeserver lacks the CS API", async () => {
+    client._unstable_getLivekitToken.mockRejectedValue(csApiMissing);
+
+    await expect(find([bob, carol])).resolves.toBeNull();
+    expect(client._unstable_getLivekitToken).toHaveBeenCalledTimes(1);
+  });
+
+  it("finds nothing where there are no other members", async () => {
+    await expect(find([ourselves])).resolves.toBeNull();
+    expect(client._unstable_getLivekitToken).not.toHaveBeenCalled();
+  });
+});

sdk/src/media-backend/livekit/RemotePublishTransport.ts

@@ -0,0 +1,106 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { type Logger } from "matrix-js-sdk/lib/logger";
+import {
+  type CallMembership,
+  isUnstableLivekitTransport,
+  type Transport,
+} from "matrix-js-sdk/lib/matrixrtc";
+import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
+
+import { serverNameOfTransport } from "../../utils/serverName";
+import { isCsApiUnavailable } from "./auth";
+import * as csApi from "./auth/csApi";
+import { decodeSFUJWTPayload } from "./auth/types";
+
+export interface RemotePublishTransportParams {
+  client: csApi.ClientGetTokenParts;
+  /** The members whose SFUs are the candidates. */
+  memberships: CallMembership[];
+  ownMembershipIdentity: CallMembershipIdentityParts;
+  roomId: string;
+  slotId: string;
+  logger: Logger;
+}
+
+/**
+ * The SFU of another member that its homeserver lets us publish on, as the
+ * transport to advertise, or null where none does. Candidates are tried
+ * oldest membership first, each SFU once, and each is checked by asking for a
+ * publisher token through our homeserver (MSC4195): a homeserver may restrict
+ * remote publishers, which shows as a token without the publish grant.
+ */
+export async function findRemotePublishLivekitTransport({
+  client,
+  memberships,
+  ownMembershipIdentity: own,
+  roomId,
+  slotId,
+  logger,
+}: RemotePublishTransportParams): Promise<Transport | null> {
+  logger.info(
+    "No local transport available, looking for another member's SFU to publish on",
+  );
+  const tried = new Set<string>();
+  const byAge = [...memberships].sort((a, b) => a.createdTs() - b.createdTs());
+  for (const membership of byAge) {
+    if (
+      membership.userId === own.userId &&
+      membership.deviceId === own.deviceId
+    )
+      continue;
+    for (const transport of membership.transports) {
+      // Publishing remotely goes through the client-server API, which a JWT
+      // service cannot stand in for, so only a `url` qualifies
+      if (!isUnstableLivekitTransport(transport) || !("url" in transport))
+        continue;
+      const { url } = transport;
+      const serverName = serverNameOfTransport(transport, membership.userId);
+      const key = JSON.stringify([url, serverName]);
+      if (tried.has(key)) continue;
+      tried.add(key);
+      try {
+        const { jwt } = await csApi.getSFUConfig({
+          client,
+          membership: own,
+          url,
+          serverName,
+          roomId,
+          slotId,
+        });
+        try {
+          // The SFU treats an absent canPublish grant as publishing allowed.
+          if (decodeSFUJWTPayload(jwt).video.canPublish !== false) {
+            logger.info(`Publishing on remote SFU ${url} of ${serverName}`);
+            return { type: "livekit", url, server_name: serverName };
+          }
+          logger.info(
+            `${serverName} does not let us publish on ${url}, skipping`,
+          );
+          continue;
+        } catch (e) {
+          logger.info(
+            `${serverName}: failed to decode SFUJWT payload: ${e}, skipping`,
+          );
+          continue;
+        }
+      } catch (e) {
+        if (isCsApiUnavailable(e)) {
+          // Without the endpoint no further candidate can work
+          logger.warn(
+            `Cannot use the MSC4195 get_token endpoint (${e}), so no remote SFU can be used`,
+          );
+          return null;
+        }
+        logger.warn(`Failed to get a token for ${url} of ${serverName}:`, e);
+        continue;
+      }
+    }
+  }
+  return null;
+}

sdk/src/media-backend/livekit/auth/index.test.ts

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
 
 import { afterEach, describe, expect, it, type Mocked, vi } from "vitest";
 import fetchMock from "fetch-mock";
-import { ConnectionError, MatrixError } from "matrix-js-sdk";
+import { MatrixError } from "matrix-js-sdk";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
 import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
@@ -144,26 +144,8 @@ describe("getSFUConfig", () => {
     expect(jwtServiceCalls()).toEqual(["/get_token"]);
   });
 
-  it("uses CS API when subscribing with url + livekit_service_url transport", async () => {
+  it("uses OpenID when subscribing with url + livekit_service_url transport", async () => {
     mockJWTService();
-    expect(
-      await getSFUConfig({
-        ...defaultParams,
-        transport: urlAndServiceUrlTransport,
-      }),
-    ).toEqual(
-      expect.objectContaining({
-        jwt: csApiJWTToken,
-        livekitAlias: "via CS API",
-      }),
-    );
-    expect(client._unstable_getLivekitToken).toHaveBeenCalled();
-    expect(jwtServiceCalls()).toEqual([]);
-  });
-
-  it("falls back to OpenID with url + livekit_service_url transport", async () => {
-    mockJWTService();
-    client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError);
     expect(
       await getSFUConfig({
         ...defaultParams,
@@ -175,25 +157,7 @@ describe("getSFUConfig", () => {
         livekitAlias: "via default endpoint",
       }),
     );
-    expect(client._unstable_getLivekitToken).toHaveBeenCalled();
-    expect(jwtServiceCalls()).toEqual(["/get_token"]);
-  });
-
-  it("falls back to OpenID when the widget host refuses the CS API", async () => {
-    mockJWTService();
-    // In widget mode a host without MSC4533 rejects the request before it
-    // reaches the homeserver, with a plain error rather than a Matrix error
-    client._unstable_getLivekitToken.mockRejectedValueOnce(
-      new Error(
-        "The rtc_livekit_get_token action is not supported by the client.",
-      ),
-    );
-    expect(
-      await getSFUConfig({
-        ...defaultParams,
-        transport: urlAndServiceUrlTransport,
-      }),
-    ).toEqual(expect.objectContaining({ jwt: defaultJWTToken }));
+    expect(client._unstable_getLivekitToken).not.toHaveBeenCalled();
     expect(jwtServiceCalls()).toEqual(["/get_token"]);
   });
 
@@ -222,7 +186,7 @@ describe("getSFUConfig", () => {
     expect(jwtServiceCalls()).toEqual([]);
   });
 
-  it("does not fall back to OpenID when publishing", async () => {
+  it("does not fall back to OpenID when publishing with url transport", async () => {
     mockJWTService();
     client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError);
     await expect(
@@ -255,33 +219,6 @@ describe("getSFUConfig", () => {
     expect(client._unstable_getLivekitToken).not.toHaveBeenCalled();
     expect(jwtServiceCalls()).toEqual(["/sfu/get"]);
   });
-
-  it("does not fall back to OpenID on connection error", async () => {
-    vi.useFakeTimers();
-    try {
-      mockJWTService();
-      const client_ = {
-        ...client,
-        _unstable_getLivekitToken: vi
-          .fn()
-          .mockRejectedValue(new ConnectionError("No connection")),
-      };
-      await Promise.all([
-        expect(
-          getSFUConfig({
-            ...defaultParams,
-            client: client_,
-            transport: urlAndServiceUrlTransport,
-          }),
-        ).rejects.toThrow(expect.any(ConnectionError)),
-        vi.runAllTimersAsync(), // Allow request to be retried
-      ]);
-      expect(client_._unstable_getLivekitToken).toHaveBeenCalled();
-      expect(jwtServiceCalls()).toEqual([]);
-    } finally {
-      vi.useRealTimers();
-    }
-  });
 });
 
 describe("delegateDelayedLeave", () => {
@@ -362,46 +299,21 @@ describe("delegateDelayedLeave", () => {
 
 describe("chooseAuthFlow", () => {
   it.each([
-    [
-      "url",
-      "publisher",
-      urlTransport,
-      { kind: "csApi", url: urlTransport.url },
-    ],
-    [
-      "url",
-      "subscriber",
-      urlTransport,
-      { kind: "csApi", url: urlTransport.url },
-    ],
+    ["url", urlTransport, { kind: "csApi", url: urlTransport.url }],
     [
       "livekit_service_url",
-      "publisher",
-      serviceUrlTransport,
-      { kind: "openID", serviceUrl },
-    ],
-    [
-      "livekit_service_url",
-      "subscriber",
       serviceUrlTransport,
       { kind: "openID", serviceUrl },
     ],
     [
       "url + livekit_service_url",
-      "publisher",
       urlAndServiceUrlTransport,
       { kind: "openID", serviceUrl },
     ],
-    [
-      "url + livekit_service_url",
-      "subscriber",
-      urlAndServiceUrlTransport,
-      { kind: "csApi", url: urlTransport.url },
-    ],
   ] as const)(
     "chooses the flow for a %s transport as %s",
-    (_shape, role, transport, flow) => {
-      expect(chooseAuthFlow(transport, role)).toEqual(flow);
+    (_shape, transport, flow) => {
+      expect(chooseAuthFlow(transport)).toEqual(flow);
     },
   );
 });

sdk/src/media-backend/livekit/auth/index.ts

@@ -48,7 +48,7 @@ function isEndpointUnsupported(e: unknown): boolean {
  * (MSC4533), which the widget API reports as a plain error. A connection
  * error is neither: the endpoint may well exist.
  */
-function isCsApiUnavailable(e: unknown): boolean {
+export function isCsApiUnavailable(e: unknown): boolean {
   if (isEndpointUnsupported(e)) return true;
   return !(e instanceof MatrixError) && !(e instanceof ConnectionError);
 }
@@ -65,7 +65,10 @@ export interface GetSFUConfigParams {
   logger: Logger;
 }
 
-/** Authenticates with a transport, through whichever flow suits it and our role. */
+/**
+ * Gets an {@link SFUConfig} appropriate for connecting to a given transport,
+ * using whichever auth flow is appropriate.
+ */
 export async function getSFUConfig({
   client,
   membership,
@@ -76,48 +79,36 @@ export async function getSFUConfig({
   role,
   logger,
 }: GetSFUConfigParams): Promise<SFUConfig> {
-  const viaOpenID = async (serviceUrl: string): Promise<SFUConfig> =>
-    await openID.getSFUConfig({
+  const flow = chooseAuthFlow(transport);
+  if (flow.kind === "openID") {
+    return await openID.getSFUConfig({
       client,
       membership,
-      serviceUrl,
+      serviceUrl: flow.serviceUrl,
       roomId,
       slotId,
       role,
       logger,
     });
-
-  const flow = chooseAuthFlow(transport, role);
-  if (flow.kind === "openID") return await viaOpenID(flow.serviceUrl);
-
-  try {
-    return await csApi.getSFUConfig({
-      client,
-      membership,
-      url: flow.url,
-      serverName,
-      roomId,
-      slotId,
-    });
-  } catch (e) {
-    // A publisher never falls back: a homeserver that advertises a `url` but
-    // does not implement MSC4195 is misconfigured, and the admin should hear
-    // about it before `livekit_service_url` is deprecated
-    const mayFallBack =
-      role === "subscriber" && "livekit_service_url" in transport;
-    if (mayFallBack && isCsApiUnavailable(e)) {
-      logger.warn(
-        `Cannot use the MSC4195 get_token endpoint (${e}), falling back to the OpenID flow with service ${transport.livekit_service_url}`,
-      );
-      return await viaOpenID(transport.livekit_service_url);
+  } else {
+    try {
+      return await csApi.getSFUConfig({
+        client,
+        membership,
+        url: flow.url,
+        serverName,
+        roomId,
+        slotId,
+      });
+    } catch (e) {
+      if (isEndpointUnsupported(e)) {
+        logger.error(
+          `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${flow.url}`,
+        );
+        throw new NoMatrix2AuthorizationService(e as Error);
+      }
+      throw e;
     }
-    if (isEndpointUnsupported(e)) {
-      logger.error(
-        `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${flow.url}`,
-      );
-      throw new NoMatrix2AuthorizationService(e as Error);
-    }
-    throw e;
   }
 }
 
@@ -149,7 +140,7 @@ export async function delegateDelayedLeave({
   delayTimeoutMs,
   logger,
 }: DelegateDelayedLeaveParams): Promise<void> {
-  const flow = chooseAuthFlow(transport, "publisher");
+  const flow = chooseAuthFlow(transport);
   if (flow.kind === "csApi") {
     try {
       await csApi.delegateDelayedLeave({
@@ -190,23 +181,18 @@ export type AuthFlow =
   | { kind: "openID"; serviceUrl: string };
 
 /**
- * Chooses between the MSC4195 client-server API and the OpenID flow against
- * the transport's JWT service.
- *
- * Only the JWT service's `/sfu/get` endpoint assigns the legacy participant
- * identity that clients predating MSC4195 expect from anyone advertising a
- * membership, so a publisher keeps using the JWT service wherever the
- * transport still offers one, even next to a `url`. A subscriber may use the
- * client-server API regardless, as nobody matches its identity.
+ * Chooses which auth flow to use for a given transport. For now this returns a
+ * single auth flow, with no provision for a fallback behavior.
  */
-export function chooseAuthFlow(
-  transport: UnstableLivekitTransport,
-  role: ConnectionRole,
-): AuthFlow {
-  if ("url" in transport) {
-    const useCsApi =
-      !("livekit_service_url" in transport) || role === "subscriber";
-    if (useCsApi) return { kind: "csApi", url: transport.url };
-  }
+export function chooseAuthFlow(transport: UnstableLivekitTransport): AuthFlow {
+  if ("url" in transport && !("livekit_service_url" in transport))
+    return { kind: "csApi", url: transport.url };
+  // All other transports use the OpenID flow, even if they contain a `url`.
+  // This is necessary since the auth flow determines the participant identity
+  // format, and we still have older clients out in the wild which would not yet
+  // know to look for a hashed participant identity in case `url` is present in
+  // someone else's transport.
+  // TODO: Once users have updated their clients, choose the CS API flow if
+  // available, keeping OpenID as a fallback.
   return { kind: "openID", serviceUrl: transport.livekit_service_url };
 }

sdk/src/media-backend/livekit/auth/types.ts

@@ -34,11 +34,13 @@ export interface SFUConfig {
 }
 
 /** The parts of the JWT payload the backend reads. */
-interface SFUJWTPayload {
+export interface SFUJWTPayload {
   /** The subject: the participant identity the token carries. */
   sub: string;
   video: {
     room: string;
+    /** Whether the token lets us publish; a homeserver may deny this to remote publishers. */
+    canPublish?: boolean;
   };
 }
 
@@ -47,8 +49,7 @@ export function extractFullConfigFromToken(sfuConfig: {
   url: string;
   jwt: string;
 }): SFUConfig {
-  const [, payloadStr] = sfuConfig.jwt.split(".");
-  const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
+  const payload = decodeSFUJWTPayload(sfuConfig.jwt);
   return {
     jwt: sfuConfig.jwt,
     url: sfuConfig.url,
@@ -56,3 +57,9 @@ export function extractFullConfigFromToken(sfuConfig: {
     livekitIdentity: payload.sub,
   };
 }
+
+/** Decodes the payload of an SFU JWT, without verifying its signature. */
+export function decodeSFUJWTPayload(jwt: string): SFUJWTPayload {
+  const [, payloadStr] = jwt.split(".");
+  return JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
+}

sdk/src/media-backend/livekit/publishOptions.ts

@@ -16,7 +16,7 @@ import {
   type PublishRequest,
   type VideoCaptureSettings,
 } from "../../media-api";
-import { convertToLivekitProcessor } from "./videoProcessor";
+import { toLivekitProcessor } from "./videoProcessor";
 
 export type PublishRequestFor<S extends PublishRequest["source"]> = Extract<
   PublishRequest,
@@ -40,7 +40,7 @@ export function videoCaptureOptions({
   if (!deviceId && !processor && !capture?.resolution) return undefined;
   return {
     ...(deviceId && { deviceId }),
-    ...(processor && { processor: convertToLivekitProcessor(processor) }),
+    ...(processor && { processor: toLivekitProcessor(processor) }),
     ...(capture?.resolution && { resolution: capture.resolution }),
   };
 }

sdk/src/media-backend/livekit/videoProcessor.test.ts

@@ -5,13 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { Track, type VideoProcessorOptions } from "livekit-client";
-import { describe, expect, it, vi } from "vitest";
+import {
+  LocalVideoTrack,
+  Track,
+  type TrackProcessor,
+  type VideoProcessorOptions,
+} from "livekit-client";
+import { beforeAll, describe, expect, it, vi } from "vitest";
 
 import { type VideoProcessor } from "../../media-api";
-import { convertToLivekitProcessor } from "./videoProcessor";
+import { fromLivekitProcessor, toLivekitProcessor } from "./videoProcessor";
 
-describe("convertToLivekitProcessor", () => {
+describe("toLivekitProcessor", () => {
   const captured = { id: "captured" } as MediaStreamTrack;
   const processed = { id: "processed" } as MediaStreamTrack;
   const restarted = { id: "restarted" } as MediaStreamTrack;
@@ -33,7 +38,7 @@ describe("convertToLivekitProcessor", () => {
 
   it("holds the processed track where LiveKit reads it", async () => {
     const processor = fakeProcessor();
-    const adapter = convertToLivekitProcessor(processor);
+    const adapter = toLivekitProcessor(processor);
     expect(adapter.name).toBe("blur");
     expect(adapter.processedTrack).toBeUndefined();
 
@@ -47,4 +52,202 @@ describe("convertToLivekitProcessor", () => {
     await adapter.destroy();
     expect(processor.destroy).toHaveBeenCalled();
   });
+
+  // A stopped track tears its processor down without waiting, while the track
+  // replacing it initialises the same one
+  it("runs a destroy and the next init on one processor in turn, across adapters", async () => {
+    const processor = fakeProcessor();
+    let finishDestroy = (): void => {};
+    vi.mocked(processor.destroy).mockImplementation(
+      async () =>
+        new Promise<void>((resolve) => {
+          finishDestroy = resolve;
+        }),
+    );
+    const previous = toLivekitProcessor(processor);
+    const next = toLivekitProcessor(processor);
+    await previous.init(options);
+    vi.mocked(processor.init).mockClear();
+
+    const destroying = previous.destroy();
+    const initialising = next.init(options);
+    await Promise.resolve();
+    expect(processor.init).not.toHaveBeenCalled();
+
+    finishDestroy();
+    await Promise.all([destroying, initialising]);
+    expect(processor.init).toHaveBeenCalledOnce();
+    expect(next.processedTrack).toBe(processed);
+  });
+});
+
+describe("a processor set on a camera that is off", () => {
+  const ended = { id: "ended", readyState: "ended" } as MediaStreamTrack;
+
+  function fakeProcessor(): VideoProcessor {
+    return {
+      name: "blur",
+      init: vi.fn().mockResolvedValue({ id: "processed" }),
+      restart: vi.fn().mockResolvedValue({ id: "restarted" }),
+      destroy: vi.fn().mockResolvedValue(undefined),
+    };
+  }
+
+  it("builds on the restart that turns the camera on, not before", async () => {
+    const processor = fakeProcessor();
+    const adapter = toLivekitProcessor(processor);
+
+    await adapter.init({ kind: Track.Kind.Video, track: ended });
+    expect(processor.init).not.toHaveBeenCalled();
+    expect(adapter.processedTrack).toBeUndefined();
+
+    const live = { id: "live", readyState: "live" } as MediaStreamTrack;
+    await adapter.restart({ kind: Track.Kind.Video, track: live });
+    expect(processor.init).toHaveBeenCalledWith(live, undefined);
+    expect(processor.restart).not.toHaveBeenCalled();
+    expect(adapter.processedTrack).toEqual({ id: "processed" });
+  });
+
+  // The processor may be the app's one pipeline, running on another track
+  it("leaves the processor alone when let go before it was built", async () => {
+    const processor = fakeProcessor();
+    const adapter = toLivekitProcessor(processor);
+    await adapter.init({ kind: Track.Kind.Video, track: ended });
+    await adapter.destroy();
+    expect(processor.destroy).not.toHaveBeenCalled();
+  });
+
+  // LiveKit's own camera track, so that the order of its restart is what is
+  // tested: a raw frame handed to the sender would reach the room
+  describe("on LiveKit's camera track", () => {
+    // jsdom has no MediaStream; LiveKit wraps every track in one
+    beforeAll(() => {
+      const original = globalThis.MediaStream;
+      globalThis.MediaStream = class {
+        public constructor(private readonly tracks: MediaStreamTrack[] = []) {}
+        public getTracks(): MediaStreamTrack[] {
+          return this.tracks;
+        }
+        public getVideoTracks(): MediaStreamTrack[] {
+          return this.tracks.filter(({ kind }) => kind === "video");
+        }
+        public getAudioTracks(): MediaStreamTrack[] {
+          return this.tracks.filter(({ kind }) => kind === "audio");
+        }
+        public addTrack(track: MediaStreamTrack): void {
+          this.tracks.push(track);
+        }
+        public removeTrack(track: MediaStreamTrack): void {
+          this.tracks.splice(this.tracks.indexOf(track), 1);
+        }
+      } as unknown as typeof MediaStream;
+      // LiveKit plays its processor element and catches a refusal
+      const play = vi
+        .spyOn(HTMLMediaElement.prototype, "play")
+        .mockResolvedValue(undefined);
+      return (): void => {
+        globalThis.MediaStream = original;
+        play.mockRestore();
+      };
+    });
+
+    function deviceTrack(id: string): MediaStreamTrack {
+      const track = {
+        kind: "video",
+        id,
+        label: "",
+        enabled: true,
+        readyState: "live",
+        getSettings: () => ({}),
+        getConstraints: () => ({}),
+        getCapabilities: () => ({}),
+        applyConstraints: vi.fn().mockResolvedValue(undefined),
+        addEventListener: vi.fn(),
+        removeEventListener: vi.fn(),
+        stop: vi.fn(() => {
+          Object.assign(track, { readyState: "ended" });
+        }),
+      };
+      return track as unknown as MediaStreamTrack;
+    }
+
+    it("hands the sender only processed frames when the camera turns on", async () => {
+      const reacquired = deviceTrack("reacquired");
+      Object.defineProperty(navigator, "mediaDevices", {
+        configurable: true,
+        value: {
+          getUserMedia: vi.fn().mockResolvedValue({
+            getTracks: () => [reacquired],
+          }),
+        },
+      });
+      const processedTrack = { id: "processed" } as MediaStreamTrack;
+      const processor = {
+        ...fakeProcessor(),
+        init: vi.fn().mockResolvedValue(processedTrack),
+      };
+      const camera = new LocalVideoTrack(deviceTrack("first"), {}, false);
+      camera.source = Track.Source.Camera;
+      const replaceTrack = vi.fn().mockResolvedValue(undefined);
+      Object.assign(camera, {
+        sender: {
+          replaceTrack,
+          getParameters: () => ({ encodings: [] }),
+          setParameters: vi.fn().mockResolvedValue(undefined),
+        },
+      });
+
+      await camera.mute();
+      expect(camera.mediaStreamTrack.readyState).toBe("ended");
+      // What the camera sent before it was turned off is not in question
+      replaceTrack.mockClear();
+      await camera.setProcessor(toLivekitProcessor(processor));
+      await camera.unmute();
+
+      expect(processor.init).toHaveBeenCalledWith(
+        reacquired,
+        expect.anything(),
+      );
+      expect(replaceTrack.mock.calls).toEqual([[processedTrack]]);
+      // LiveKit plays attached elements again on a timer
+      await new Promise((resolve) => setTimeout(resolve, 0));
+    });
+  });
+});
+
+describe("fromLivekitProcessor", () => {
+  const captured = { id: "captured" } as MediaStreamTrack;
+  const processed = { id: "processed" } as MediaStreamTrack;
+
+  function fakeLivekitProcessor(
+    processedTrack: MediaStreamTrack | undefined,
+  ): TrackProcessor<Track.Kind.Video> {
+    return {
+      name: "background-blur",
+      processedTrack,
+      init: vi.fn().mockResolvedValue(undefined),
+      restart: vi.fn().mockResolvedValue(undefined),
+      destroy: vi.fn().mockResolvedValue(undefined),
+    };
+  }
+
+  it("resolves with the track the LiveKit processor produced", async () => {
+    const livekitProcessor = fakeLivekitProcessor(processed);
+    const processor = fromLivekitProcessor(livekitProcessor);
+    expect(processor.name).toBe("background-blur");
+    await expect(processor.init(captured)).resolves.toBe(processed);
+    expect(livekitProcessor.init).toHaveBeenCalledWith({
+      kind: Track.Kind.Video,
+      track: captured,
+    });
+    await expect(processor.restart(captured)).resolves.toBe(processed);
+    await processor.destroy();
+    expect(livekitProcessor.destroy).toHaveBeenCalled();
+  });
+
+  it("rejects where the LiveKit processor produced no track", async () => {
+    await expect(
+      fromLivekitProcessor(fakeLivekitProcessor(undefined)).init(captured),
+    ).rejects.toThrow("produced no track");
+  });
 });

sdk/src/media-backend/livekit/videoProcessor.ts

@@ -5,30 +5,84 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { type Track, type TrackProcessor } from "livekit-client";
+import { Track, type TrackProcessor } from "livekit-client";
 
 import { type VideoProcessor } from "../../media-api";
+import { inTurn } from "../../utils/processorTurns";
+
+export interface LivekitProcessor extends TrackProcessor<Track.Kind.Video> {
+  /** The processor this adapts, so that setting the same one again is a no-op. */
+  source: VideoProcessor;
+}
 
 /**
- * A `VideoProcessor` as livekit-client wants it. LiveKit reads the processed
- * track off the processor after `init` and `restart` rather than taking it as
- * their result, so the adapter keeps it there.
+ * A `VideoProcessor` as a livekit-client processor. LiveKit reads the
+ * processed track off the processor after `init` and `restart` rather than
+ * taking it as their result, so the adapter keeps it there. A processor a
+ * lobby already runs on the track is adopted with its output, `running`, so
+ * that the join does not initialise it again. Every step takes its turn on
+ * the processor, which a stopped track tears down without waiting for the
+ * next one's init.
+ *
+ * Set on a camera that is off, whose track LiveKit has stopped, the build
+ * waits for the restart that turns it back on. LiveKit restarts a processor
+ * before it hands the new track to the sender, so the camera's first frame
+ * out is already processed.
  */
-export function convertToLivekitProcessor(
+export function toLivekitProcessor(
   processor: VideoProcessor,
-): TrackProcessor<Track.Kind.Video> {
-  const adapter: TrackProcessor<Track.Kind.Video> = {
+  running?: MediaStreamTrack,
+): LivekitProcessor {
+  let built = false;
+  const adapter: LivekitProcessor = {
     name: processor.name,
-    init: async (opts) => {
-      adapter.processedTrack = await processor.init(opts.track, opts.element);
-    },
-    restart: async (opts) => {
-      adapter.processedTrack = await processor.restart(
-        opts.track,
-        opts.element,
-      );
-    },
-    destroy: async () => processor.destroy(),
+    source: processor,
+    init: async (opts) =>
+      inTurn(processor, async () => {
+        if (running === undefined && opts.track.readyState === "ended") return;
+        adapter.processedTrack =
+          running ?? (await processor.init(opts.track, opts.element));
+        running = undefined;
+        built = true;
+      }),
+    restart: async (opts) =>
+      inTurn(processor, async () => {
+        adapter.processedTrack = built
+          ? await processor.restart(opts.track, opts.element)
+          : await processor.init(opts.track, opts.element);
+        built = true;
+      }),
+    destroy: async () =>
+      inTurn(processor, async () => {
+        if (built) await processor.destroy();
+      }),
   };
   return adapter;
 }
+
+/**
+ * A livekit-client processor, such as one from `@livekit/track-processors`,
+ * as a `VideoProcessor`: `init` and `restart` resolve with the track it
+ * leaves on itself.
+ */
+export function fromLivekitProcessor(
+  processor: TrackProcessor<Track.Kind.Video>,
+): VideoProcessor {
+  const processedTrack = (): MediaStreamTrack => {
+    if (processor.processedTrack === undefined)
+      throw new Error(`Processor ${processor.name} produced no track`);
+    return processor.processedTrack;
+  };
+  return {
+    name: processor.name,
+    init: async (track, element) => {
+      await processor.init({ kind: Track.Kind.Video, track, element });
+      return processedTrack();
+    },
+    restart: async (track, element) => {
+      await processor.restart({ kind: Track.Kind.Video, track, element });
+      return processedTrack();
+    },
+    destroy: async () => processor.destroy(),
+  };
+}

sdk/src/utils/array.ts

@@ -0,0 +1,16 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+/**
+ * Determine whether two arrays are equal by shallow comparison.
+ */
+export function shallowEquals<A>(first: A[], second: A[]): boolean {
+  if (first.length !== second.length) return false;
+  for (let i = 0; i < first.length; i++)
+    if (first[i] !== second[i]) return false;
+  return true;
+}

sdk/src/utils/processorTurns.ts

@@ -0,0 +1,26 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { type VideoProcessor } from "../media-api";
+
+const turns = new WeakMap<VideoProcessor, Promise<unknown>>();
+
+/**
+ * One operation at a time on a processor. A host shares one across the
+ * tracks it captures and publishes, and a second init while the first still
+ * loads its model, or a destroy while an init runs, leaves it broken: the
+ * backend's own locking is per track, and a track that is stopped tears its
+ * processor down without waiting.
+ */
+export async function inTurn<T>(
+  processor: VideoProcessor,
+  run: () => Promise<T>,
+): Promise<T> {
+  const turn = (turns.get(processor) ?? Promise.resolve()).then(run, run);
+  turns.set(processor, turn);
+  return turn;
+}

sdk/src/utils/serverName.ts

@@ -5,7 +5,23 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
+import { type Transport } from "matrix-js-sdk/lib/matrixrtc";
+
 /** The homeserver a Matrix user id belongs to: everything after the first colon. */
 export function serverNameOf(userId: string): string {
   return userId.replace(/^.*?:/, "");
 }
+
+/**
+ * The homeserver that operates a transport: the one the transport names, else
+ * the one of the member advertising it. A member publishing on another
+ * homeserver's SFU (MSC4195) names that homeserver in `server_name`.
+ */
+export function serverNameOfTransport(
+  transport: Transport,
+  userId: string,
+): string {
+  return typeof transport.server_name === "string"
+    ? transport.server_name
+    : serverNameOf(userId);
+}

src/App.tsx

@@ -35,7 +35,7 @@ import { ErrorPage, LoadingPage } from "./FullScreenView";
 import { Initializer } from "./initializer";
 import { type WidgetHelpers } from "./widget";
 import { useTheme } from "./useTheme";
-import { ProcessorProvider } from "./livekit/TrackProcessorContext";
+import { BackgroundEffectsProvider } from "./tracks/BackgroundEffectsContext";
 import { type AppViewModel } from "./state/AppViewModel";
 import { MediaDevicesContext } from "./MediaDevicesContext";
 import {
@@ -148,7 +148,7 @@ export const App: FC<Props> = ({ vm, widget }) => {
     loaded && clientReady ? (
       <ClientProvider client={widgetClient}>
         <MediaDevicesContext value={vm.mediaDevices}>
-          <ProcessorProvider>
+          <BackgroundEffectsProvider effects={vm.backgroundEffects}>
             <Sentry.ErrorBoundary
               fallback={(error) => <ErrorPage error={error} />}
             >
@@ -159,7 +159,7 @@ export const App: FC<Props> = ({ vm, widget }) => {
                 <SentryRoute path="*" element={<RoomPage />} />
               </Routes>
             </Sentry.ErrorBoundary>
-          </ProcessorProvider>
+          </BackgroundEffectsProvider>
         </MediaDevicesContext>
       </ClientProvider>
     ) : (

src/AppBar.module.css

@@ -7,6 +7,7 @@
   padding-top: env(safe-area-inset-top);
   opacity: 1;
   transition: opacity 0.15s;
+  text-align: center;
 }
 
 /* Pseudo-element for the gradient background */
@@ -52,7 +53,7 @@
     ". . ."
     "primaryButton title secondaryButton"
     ". . .";
-  place-items: center;
+  place-items: center stretch;
   column-gap: var(--cpd-space-2x);
 }
 

src/RTCConnectionStats.tsx

@@ -16,12 +16,12 @@ import classNames from "classnames";
 import { Modal } from "./Modal";
 import styles from "./RTCConnectionStats.module.css";
 import mediaViewStyles from "../src/tile/MediaView.module.css";
-import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
+import { type Transport } from "matrix-js-sdk/lib/matrixrtc";
 
 interface Props {
   audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
   video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
-  transport?: UnstableLivekitTransport;
+  transport?: Transport;
   rtcBackendIdentity?: string;
 }
 

src/button/ReactionToggleButton.test.tsx

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { act, render } from "@testing-library/react";
-import { expect, test, vi } from "vitest";
+import { expect, test } from "vitest";
 import { TooltipProvider } from "@vector-im/compound-web";
 import { userEvent } from "@testing-library/user-event";
 import { type ReactNode } from "react";
@@ -20,7 +20,6 @@ import { type MockRTCSession } from "../utils/test";
 import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
 import { initializeWidget } from "../widget";
 initializeWidget();
-vi.mock("livekit-client/e2ee-worker?worker");
 
 const localIdent = `${localRtcMember.userId}:${localRtcMember.deviceId}`;
 

src/components/BackgroundEffectGrid.module.css

@@ -0,0 +1,92 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+.grid {
+  display: grid;
+  /* A zero floor, so a long label wraps inside its tile rather than widening
+     the grid past the menu. */
+  grid-template-columns: repeat(3, minmax(0, 1fr));
+  grid-auto-rows: 1fr;
+  gap: var(--cpd-space-3x);
+  padding: var(--cpd-space-2x) var(--cpd-space-4x) 0;
+}
+
+/* Within the grid, to outweigh the menu item styles the tiles are built on. */
+.grid .tile {
+  position: relative;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: center;
+  gap: var(--cpd-space-1x);
+  min-inline-size: 0;
+  padding: var(--cpd-space-3x) var(--cpd-space-2x);
+  /* Not hidden: that makes the tile a scroll container, which focus scrolls
+     into view without its border, and so without the focus ring. */
+  overflow: clip;
+  border: var(--cpd-border-width-2) solid
+    var(--cpd-color-border-interactive-secondary);
+  border-radius: var(--cpd-space-2x);
+  background: none;
+  color: var(--cpd-color-icon-secondary);
+  font: inherit;
+  cursor: pointer;
+}
+
+/* As the menu items highlight on hover. */
+.grid .tile[data-highlighted] {
+  background: var(--cpd-color-bg-action-secondary-hovered);
+}
+
+.grid .tile[aria-checked="true"] {
+  border-color: var(--cpd-color-border-accent);
+}
+
+.grid .tile[data-disabled] {
+  cursor: default;
+  opacity: 0.55;
+}
+
+.tile[data-disabled] .picture {
+  filter: grayscale(1);
+}
+
+.swatch {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.picture {
+  position: absolute;
+  inset: 0;
+  inline-size: 100%;
+  block-size: 100%;
+  object-fit: cover;
+}
+
+/* Positioned, or the picture, positioned itself, would cover it. */
+.mark {
+  position: relative;
+  display: inline-flex;
+  color: var(--cpd-color-icon-accent-primary);
+}
+
+/* On a picture the mark needs a ground of its own to be seen. */
+.picture ~ .mark {
+  border-radius: 50%;
+  background: var(--cpd-color-bg-canvas-default);
+  box-shadow: 0 0 0 var(--cpd-border-width-2) var(--cpd-color-bg-canvas-default);
+}
+
+.label {
+  max-inline-size: 100%;
+  font: var(--cpd-font-body-sm-medium);
+  color: var(--cpd-color-text-primary);
+  text-align: center;
+  overflow-wrap: anywhere;
+}

src/components/BackgroundEffectGrid.tsx

@@ -0,0 +1,151 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { type FC, type ReactElement } from "react";
+import { useTranslation } from "react-i18next";
+import { VisuallyHidden } from "@radix-ui/react-visually-hidden";
+import { InlineSpinner, MenuItem } from "@vector-im/compound-web";
+import {
+  BlockIcon,
+  BlurIcon,
+  CheckCircleSolidIcon,
+} from "@vector-im/compound-design-tokens/assets/web/icons";
+
+import styles from "./BackgroundEffectGrid.module.css";
+
+/**
+ * One choice in the camera menu's Background effects section, as the view
+ * model offers it. The grid names it: pictures are numbered in order.
+ */
+export interface BackgroundEffectGridItem {
+  /** Its option id, as `backgroundEffect` gives the one in force. */
+  id: string;
+  kind: "none" | "blur" | "image";
+  /** For kind "image". */
+  imageUrl?: string;
+}
+
+interface Props {
+  effects: BackgroundEffectGridItem[];
+  selected: string | undefined;
+  /** Undefined where effects can't run: all but no effect are disabled. */
+  onSelect: ((id: string) => void) | undefined;
+  /** Shown on the selected tile until its effect is on screen. */
+  settling?: boolean;
+  /** Id of the text saying what the user should know before choosing. */
+  describedBy?: string;
+}
+
+/**
+ * The background effects as a grid of tiles: one radio choice, walked by the
+ * menu's arrow keys like the device rows. Sits in a menu section, which names
+ * it.
+ */
+export const BackgroundEffectGrid: FC<Props> = ({
+  effects,
+  selected,
+  onSelect,
+  settling = false,
+  describedBy,
+}) => {
+  const { t } = useTranslation();
+  const choose = (id: string): void => {
+    if (id !== selected) onSelect?.(id);
+  };
+  let pictures = 0;
+  const tiles = effects.map((effect) => {
+    const label =
+      effect.kind === "none"
+        ? t("background_effects.none")
+        : effect.kind === "blur"
+          ? t("background_effects.blur")
+          : t("background_effects.numbered", { n: ++pictures });
+    const checked = effect.id === selected;
+    const disabled = onSelect === undefined && effect.kind !== "none";
+    return (
+      <MenuItem
+        key={effect.id}
+        role="menuitemradio"
+        aria-checked={checked}
+        aria-busy={checked && settling}
+        disabled={disabled}
+        data-disabled={disabled ? "" : undefined}
+        label={null}
+        hideChevron
+        className={styles.tile}
+        // Without a handler it isn't one of the menu's items, so the keyboard
+        // passes over it.
+        onSelect={
+          disabled
+            ? null
+            : (e): void => {
+                // Kept open, so the user sees what they chose take effect.
+                e.preventDefault();
+                choose(effect.id);
+              }
+        }
+      >
+        <TileContent
+          effect={effect}
+          label={label}
+          checked={checked}
+          settling={checked && settling}
+        />
+      </MenuItem>
+    );
+  });
+
+  return (
+    // A group of its own, as the section's can't carry the description.
+    <div role="group" aria-describedby={describedBy} className={styles.grid}>
+      {tiles}
+    </div>
+  );
+};
+
+function TileContent({
+  effect,
+  label,
+  checked,
+  settling,
+}: {
+  effect: BackgroundEffectGridItem;
+  /** The tile's accessible name; drawn only where the tile has no picture. */
+  label: string;
+  checked: boolean;
+  settling: boolean;
+}): ReactElement {
+  return (
+    <>
+      <span aria-hidden className={styles.swatch}>
+        {effect.kind === "image" && (
+          <img className={styles.picture} src={effect.imageUrl} alt="" />
+        )}
+        {settling ? (
+          <span className={styles.mark}>
+            <InlineSpinner size={20} />
+          </span>
+        ) : checked ? (
+          <CheckCircleSolidIcon
+            className={styles.mark}
+            width={20}
+            height={20}
+          />
+        ) : effect.kind === "none" ? (
+          <BlockIcon width={20} height={20} />
+        ) : effect.kind === "blur" ? (
+          <BlurIcon width={20} height={20} />
+        ) : null}
+      </span>
+      {effect.kind === "image" ? (
+        <VisuallyHidden>{label}</VisuallyHidden>
+      ) : (
+        <span className={styles.label}>{label}</span>
+      )}
+    </>
+  );
+}

src/components/CallFooter.stories.tsx

@@ -22,6 +22,7 @@ import { MediaDevices } from "../state/MediaDevices";
 import { globalScope } from "../state/ObservableScope";
 import { constant } from "../state/Behavior";
 import { type LayoutMode } from "../state/LayoutSwitchViewModel";
+import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
 
 // consts for tests
 const reactionIdentifier = "@user:example.com:DEVICE";
@@ -123,8 +124,19 @@ export const Default: Story = {
     toggleAudio: fn(),
     toggleVideo: fn(),
     toggleScreenSharing: fn(),
-    toggleBlur: fn(),
-    videoBlurEnabled: true,
+    selectBackgroundEffect: fn(),
+    backgroundEffect: "blur",
+    backgroundEffects: [
+      { id: "none", kind: "none" },
+      { id: "blur", kind: "blur" },
+      ...shippedBackgrounds.map((background) => ({
+        id: `image:${background.id}`,
+        kind: "image" as const,
+        imageUrl: background.imagePath,
+      })),
+    ],
+    backgroundEffectNotice: undefined,
+    backgroundEffectSettling: false,
     hangup: fn(),
     buttonSize: "lg",
     showFooter: true,

src/components/CallFooter.tsx

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
 
 import { type FC, type JSX, type Ref, useMemo } from "react";
 import classNames from "classnames";
+import { useTranslation } from "react-i18next";
 
 import LogoMark from "../icons/LogoMark.svg?react";
 import LogoType from "../icons/LogoType.svg?react";
@@ -31,6 +32,7 @@ import { type ViewModel } from "../state/ViewModel";
 import { useBehavior } from "../useBehavior";
 import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel";
 import { LayoutSwitch } from "../room/LayoutSwitch";
+import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
 
 export interface AudioOutputSwitcher {
   targetOutput: string;
@@ -56,20 +58,29 @@ export interface FooterActions {
   toggleAudio: (() => void) | undefined;
   /** Also controls if the videoMute button is disabled */
   toggleVideo: (() => void) | undefined;
-  toggleBlur: (() => void) | undefined;
+  /** Undefined where background effects can't be chosen. */
+  selectBackgroundEffect: ((id: string) => void) | undefined;
   toggleScreenSharing: (() => void) | undefined;
   /** Also controls if the settings button is visible */
   openSettings: (() => void) | undefined;
   /** Also controls if the hangup button is visible */
   hangup: (() => void) | undefined;
 }
+
 // we do not use any ? optional properties so that the vm type is including all fields.
 export interface FooterState {
   audioEnabled: boolean;
   audioBusy: boolean;
   videoEnabled: boolean;
   videoBusy: boolean;
-  videoBlurEnabled: boolean;
+  /** The background effect in force, as its option id. */
+  backgroundEffect: string;
+  /** Every effect on offer, in the order they are shown. */
+  backgroundEffects: BackgroundEffectGridItem[];
+  /** What the user is told about effects here, if anything. */
+  backgroundEffectNotice: "unavailable" | "slow" | undefined;
+  /** Whether the first effect chosen is still being prepared. */
+  backgroundEffectSettling: boolean;
   showFooter: boolean;
 
   /* This is needed for WindowMode = "flat" */
@@ -117,12 +128,14 @@ export interface FooterProps {
   children?: JSX.Element | JSX.Element[] | false;
   vm: ViewModel<FooterSnapshot>;
 }
+
 export const CallFooter: FC<FooterProps> = ({
   className,
   ref,
   children,
   vm,
 }) => {
+  const { t } = useTranslation();
   const asOverlay = useBehavior(vm.asOverlay$);
   const showFooter = useBehavior(vm.showFooter$);
   const hideControls = useBehavior(vm.hideControls$);
@@ -151,8 +164,11 @@ export const CallFooter: FC<FooterProps> = ({
   const selectedAudioOutput = useBehavior(vm.selectedAudioOutput$);
   const selectAudioOutputOption = useBehavior(vm.selectAudioOutputOption$);
   const selectVideoButtonOption = useBehavior(vm.selectVideoButtonOption$);
-  const toggleBlur = useBehavior(vm.toggleBlur$);
-  const videoBlurEnabled = useBehavior(vm.videoBlurEnabled$);
+  const backgroundEffect = useBehavior(vm.backgroundEffect$);
+  const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
+  const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$);
+  const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$);
+  const backgroundEffects = useBehavior(vm.backgroundEffects$);
   const buttonSize = useBehavior(vm.buttonSize$);
   const showLogo = useBehavior(vm.showLogo$);
 
@@ -214,8 +230,17 @@ export const CallFooter: FC<FooterProps> = ({
         options={videoOptions}
         selectedOption={selectedVideo}
         onSelect={selectVideoButtonOption}
-        videoBlurToggleClick={toggleBlur}
-        videoBlurEnabled={videoBlurEnabled}
+        backgroundEffects={backgroundEffects}
+        selectedBackgroundEffect={backgroundEffect}
+        onSelectBackgroundEffect={selectBackgroundEffect}
+        backgroundEffectSettling={backgroundEffectSettling}
+        backgroundEffectNotice={
+          backgroundEffectNotice === "unavailable"
+            ? t("background_effects.unavailable")
+            : backgroundEffectNotice === "slow"
+              ? t("background_effects.slow")
+              : undefined
+        }
       />,
     );
   } else {

src/components/CallFooterViewModel.test.ts

@@ -5,17 +5,22 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { describe, expect, it, vi } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 import { BehaviorSubject } from "rxjs";
 
 import { testScope, mockMuteStates, mockMediaDevices } from "../utils/test";
 import { constant } from "../state/Behavior";
 import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
-import type { Alignment, Layout } from "../state/layout-types";
+import type { Layout } from "../state/layout-types";
 import type { SpotlightTileViewModel } from "../state/TileViewModel";
 import type { DeviceLabel } from "../state/MediaDevices";
-import { createCallFooterViewModel } from "./CallFooterViewModel";
+import {
+  createCallFooterViewModel,
+  createLobbyFooterViewModel,
+} from "./CallFooterViewModel";
 import { HeaderStyle } from "../UrlParams";
+import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
+import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
 
 const platformMock = vi.hoisted(() => vi.fn(() => "desktop"));
 vi.mock("../Platform", () => ({
@@ -24,10 +29,12 @@ vi.mock("../Platform", () => ({
   },
 }));
 
-// Prevent supportsBackgroundProcessors from throwing in jsdom – it is not
-// exercised by these tests (only used in `videoToggles`, not `videoOptions`).
+// The SDK's own check needs WebGL and canvas APIs jsdom does not have.
+const sdkSupportMock = vi.hoisted(() => vi.fn(() => false));
+const modernRouteMock = vi.hoisted(() => vi.fn(() => true));
 vi.mock("@livekit/track-processors", () => ({
-  supportsBackgroundProcessors: (): boolean => false,
+  supportsBackgroundProcessors: (): boolean => sdkSupportMock(),
+  supportsModernBackgroundProcessors: (): boolean => modernRouteMock(),
 }));
 
 const outputSelectionMock = vi.hoisted(() => vi.fn(() => true));
@@ -65,10 +72,6 @@ function buildMinimalCallViewModel(layout: Layout): CallViewModel {
 const gridLayout: Layout = {
   type: "grid",
   grid: [],
-  spotlightAlignment$: new BehaviorSubject<Alignment>({
-    inline: "end",
-    block: "end",
-  }),
   setVisibleTiles: (_: number) => {},
 };
 
@@ -109,6 +112,7 @@ describe("createCallFooterViewModel", () => {
         buildMinimalCallViewModel(gridLayout),
         mockMuteStates(),
         twoMicsAndOneCamMediaDevices,
+        constant(false),
         /* reactionIdentifier */ undefined,
         { showControls: true, header: HeaderStyle.Standard },
       );
@@ -151,6 +155,7 @@ describe("createCallFooterViewModel", () => {
             select: vi.fn(),
           },
         }),
+        constant(false),
         /* reactionIdentifier */ undefined,
         { showControls: true, header: HeaderStyle.Standard },
       );
@@ -169,6 +174,7 @@ describe("createCallFooterViewModel", () => {
         buildMinimalCallViewModel(layout),
         mockMuteStates(),
         twoMicsAndOneCamMediaDevices,
+        constant(false),
         /* reactionIdentifier */ undefined,
         { showControls: true, header: HeaderStyle.Standard },
       );
@@ -191,6 +197,7 @@ describe("createCallFooterViewModel", () => {
         buildMinimalCallViewModel(gridLayout),
         mockMuteStates(),
         twoMicsAndOneCamMediaDevices,
+        constant(false),
         /* reactionIdentifier */ undefined,
         { showControls: true, header: HeaderStyle.Standard },
       );
@@ -222,4 +229,127 @@ describe("createCallFooterViewModel", () => {
       ]);
     });
   });
+
+  describe("background effects", () => {
+    afterEach(() => backgroundEffectSetting.setValue("none"));
+
+    // The lobby, because in a call the whole device switcher is already
+    // withheld on a phone, whatever the verdict says.
+    function lobbyOn(
+      platform: string,
+    ): ReturnType<typeof createLobbyFooterViewModel> {
+      platformMock.mockReturnValue(platform);
+      return createLobbyFooterViewModel(
+        testScope(),
+        mockMuteStates(),
+        twoMicsAndOneCamMediaDevices,
+        constant(false),
+        /* openSettings */ undefined,
+        /* hangup */ undefined,
+        /* showLogo */ false,
+      );
+    }
+
+    it("offers them on a phone whose browser can run them", () => {
+      sdkSupportMock.mockReturnValue(true);
+      expect(lobbyOn("ios").selectBackgroundEffect$.value).toBeDefined();
+    });
+
+    it("offers them on a desktop whose browser can run them", () => {
+      sdkSupportMock.mockReturnValue(true);
+      expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeDefined();
+    });
+
+    it("offers nothing where the browser itself cannot run them", () => {
+      sdkSupportMock.mockReturnValue(false);
+      expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeUndefined();
+    });
+
+    it("puts no effect in force where the browser cannot run them", () => {
+      backgroundEffectSetting.setValue("blur");
+      sdkSupportMock.mockReturnValue(false);
+      expect(lobbyOn("desktop").backgroundEffect$.value).toBe("none");
+      sdkSupportMock.mockReturnValue(true);
+      expect(lobbyOn("desktop").backgroundEffect$.value).toBe("blur");
+    });
+
+    it("stores the effect chosen", () => {
+      sdkSupportMock.mockReturnValue(true);
+      const vm = lobbyOn("desktop");
+      vm.selectBackgroundEffect$.value?.("image:arc");
+      expect(vm.backgroundEffect$.value).toBe("image:arc");
+      // What isn't an effect on offer is stored as none.
+      vm.selectBackgroundEffect$.value?.("image:gone");
+      expect(backgroundEffectSetting.getValue()).toBe("none");
+    });
+
+    it("says they run slowly where only the slower route exists", () => {
+      sdkSupportMock.mockReturnValue(true);
+      modernRouteMock.mockReturnValue(false);
+      const slow = lobbyOn("desktop");
+      expect(slow.backgroundEffectNotice$.value).toBe("slow");
+      expect(slow.selectBackgroundEffect$.value).toBeDefined();
+      modernRouteMock.mockReturnValue(true);
+      expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBeUndefined();
+    });
+
+    it("offers every effect in order", () => {
+      sdkSupportMock.mockReturnValue(true);
+      expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([
+        { id: "none", kind: "none" },
+        { id: "blur", kind: "blur" },
+        ...shippedBackgrounds.map((background) => ({
+          id: `image:${background.id}`,
+          kind: "image",
+          imageUrl: background.imagePath,
+        })),
+      ]);
+    });
+
+    it("says they are unavailable where they cannot be chosen", () => {
+      sdkSupportMock.mockReturnValue(false);
+      expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBe(
+        "unavailable",
+      );
+    });
+
+    it("passes on the wait for the first effect", () => {
+      platformMock.mockReturnValue("desktop");
+      const settling$ = new BehaviorSubject(true);
+      const vm = createLobbyFooterViewModel(
+        testScope(),
+        mockMuteStates(),
+        twoMicsAndOneCamMediaDevices,
+        settling$,
+        /* openSettings */ undefined,
+        /* hangup */ undefined,
+        /* showLogo */ false,
+      );
+      expect(vm.backgroundEffectSettling$.value).toBe(true);
+      settling$.next(false);
+      expect(vm.backgroundEffectSettling$.value).toBe(false);
+    });
+
+    it("availability is the same before and during a call", () => {
+      for (const supported of [true, false]) {
+        sdkSupportMock.mockReturnValue(supported);
+        const lobby = lobbyOn("desktop");
+        const inCall = createCallFooterViewModel(
+          testScope(),
+          buildMinimalCallViewModel(gridLayout),
+          mockMuteStates(),
+          twoMicsAndOneCamMediaDevices,
+          constant(false),
+          /* reactionIdentifier */ undefined,
+          { showControls: true, header: HeaderStyle.Standard },
+        );
+        const offeredInLobby =
+          lobby.selectBackgroundEffect$.value !== undefined;
+        expect(inCall.selectBackgroundEffect$.value !== undefined).toBe(
+          offeredInLobby,
+        );
+        expect(offeredInLobby).toBe(supported);
+      }
+    });
+  });
 });

src/components/CallFooterViewModel.tsx

@@ -5,15 +5,30 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { combineLatest, map, type Observable, switchMap } from "rxjs";
-import { supportsBackgroundProcessors } from "@livekit/track-processors";
+import {
+  combineLatest,
+  distinctUntilChanged,
+  map,
+  type Observable,
+  switchMap,
+} from "rxjs";
 import { supportsAudioOutputSelection } from "livekit-client";
+import {
+  supportsBackgroundProcessors,
+  supportsModernBackgroundProcessors,
+} from "@livekit/track-processors";
+
+import {
+  parseEffect,
+  serializeEffect,
+  shippedBackgrounds,
+} from "../tracks/backgroundEffectCatalog";
 
 import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
 import { type MenuOptions } from "./MediaMuteAndSwitchButton";
 import { type MediaDevices } from "../state/MediaDevices";
 import {
-  backgroundBlur as backgroundBlurSettings,
+  backgroundEffect as backgroundEffectSetting,
   debugTileLayout as debugTileLayoutSetting,
 } from "../settings/settings";
 import { type Behavior, constant } from "../state/Behavior";
@@ -23,6 +38,7 @@ import { createStaticViewModel, type ViewModel } from "../state/ViewModel";
 import { HeaderStyle } from "../UrlParams";
 import { platform } from "../Platform";
 import { type FooterSnapshot } from "./CallFooter";
+import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
 
 /**
  * Shared helper: maps MuteStates into the audio/video enabled + toggle behaviors
@@ -56,13 +72,14 @@ function buildMuteBehaviors(
 
 /**
  * Shared helper: maps MediaDevices into the audio/video device-list behaviors
- * needed by FooterSnapshot (options, selection, callbacks, blur toggle).
+ * needed by FooterSnapshot (options, selection, callbacks, background effect).
  */
 function buildDeviceBehaviors(
   scope: ObservableScope,
   mediaDevices: MediaDevices,
   /** return empty arrays for  audioOptions and videoOptions*/
   disableSwitcher$: Behavior<boolean>,
+  backgroundEffectSettling$: Observable<boolean>,
 ): Pick<
   ViewModel<FooterSnapshot>,
   | "audioOptions$"
@@ -74,8 +91,11 @@ function buildDeviceBehaviors(
   | "videoOptions$"
   | "selectedVideo$"
   | "selectVideoButtonOption$"
-  | "toggleBlur$"
-  | "videoBlurEnabled$"
+  | "backgroundEffect$"
+  | "selectBackgroundEffect$"
+  | "backgroundEffects$"
+  | "backgroundEffectNotice$"
+  | "backgroundEffectSettling$"
 > {
   const options$ = (
     available$: Behavior<Map<string, MenuOptions["label"]>>,
@@ -92,6 +112,11 @@ function buildDeviceBehaviors(
       ),
     );
 
+  const supported = supportsBackgroundProcessors();
+  const slow = !supportsModernBackgroundProcessors();
+  const offered$ = disableSwitcher$.pipe(
+    map((switcherDisabled) => !switcherDisabled && supported),
+  );
   return {
     audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)),
     selectedAudio$: scope.behavior(
@@ -116,18 +141,35 @@ function buildDeviceBehaviors(
       mediaDevices.videoInput.selected$.pipe(map((s) => s?.id)),
     ),
     selectVideoButtonOption$: constant(mediaDevices.videoInput.select),
-    toggleBlur$: scope.behavior(
-      combineLatest([backgroundBlurSettings.value$, disableSwitcher$]).pipe(
-        map(([current, switcherDisabled]) => {
-          return !switcherDisabled && supportsBackgroundProcessors()
-            ? (): void => {
-                backgroundBlurSettings.setValue(!current);
-              }
-            : undefined;
-        }),
+    backgroundEffect$: scope.behavior(
+      backgroundEffectSetting.value$.pipe(
+        map((raw) => (supported ? serializeEffect(parseEffect(raw)) : "none")),
       ),
     ),
-    videoBlurEnabled$: backgroundBlurSettings.value$,
+    selectBackgroundEffect$: scope.behavior(
+      offered$.pipe(
+        map((offered) =>
+          offered
+            ? (id: string): void =>
+                backgroundEffectSetting.setValue(
+                  serializeEffect(parseEffect(id)),
+                )
+            : undefined,
+        ),
+      ),
+    ),
+    backgroundEffects$: constant(backgroundEffectChoices()),
+    backgroundEffectNotice$: scope.behavior(
+      offered$.pipe(
+        map((offered) =>
+          !offered ? "unavailable" : slow ? "slow" : undefined,
+        ),
+      ),
+    ),
+    backgroundEffectSettling$: scope.behavior(
+      backgroundEffectSettling$.pipe(distinctUntilChanged()),
+      false,
+    ),
   };
 }
 
@@ -138,6 +180,8 @@ function buildDeviceBehaviors(
  * @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc.
  * @param muteStates - Audio and video mute state + toggles.
  * @param mediaDevices - Available and selected input devices.
+ * @param backgroundEffectSettling$ - Whether the first effect chosen is still
+ *   being prepared.
  * @param reactionIdentifier - The local user's reaction identifier string, or
  *   undefined when reactions are not supported (hides the reaction button).
  * @param options - `showControls`: whether the call controls should be shown.
@@ -148,6 +192,7 @@ export function createCallFooterViewModel(
   callModel: CallViewModel,
   muteStates: MuteStates,
   mediaDevices: MediaDevices,
+  backgroundEffectSettling$: Observable<boolean>,
   reactionIdentifier: string | undefined,
   options: { showControls: boolean; header: HeaderStyle },
 ): ViewModel<FooterSnapshot> {
@@ -162,7 +207,12 @@ export function createCallFooterViewModel(
   );
   return {
     ...buildMuteBehaviors(scope, muteStates),
-    ...buildDeviceBehaviors(scope, mediaDevices, disableDeviceSwitcher$),
+    ...buildDeviceBehaviors(
+      scope,
+      mediaDevices,
+      disableDeviceSwitcher$,
+      backgroundEffectSettling$,
+    ),
     // candidat to move into the FooterViewModel
     showFooter$: callModel.showFooter$,
     hideControls$: constant(!showControls),
@@ -224,6 +274,8 @@ export function createCallFooterViewModel(
  * @param scope - ObservableScope that bounds the lifetime of derived behaviors.
  * @param muteStates - Audio and video mute state + toggles.
  * @param mediaDevices - Available and selected input devices.
+ * @param backgroundEffectSettling$ - Whether the first effect chosen is still
+ *   being prepared.
  * @param openSettings - Callback to open the settings modal, or undefined.
  * @param hangup - Callback to leave/cancel, or undefined (hides the button).
  * @param showLogo - Whether to show the Element Call logo.
@@ -232,6 +284,7 @@ export function createLobbyFooterViewModel(
   scope: ObservableScope,
   muteStates: MuteStates,
   mediaDevices: MediaDevices,
+  backgroundEffectSettling$: Observable<boolean>,
   openSettings: (() => void) | undefined,
   hangup: (() => void) | undefined,
   showLogo: boolean,
@@ -275,6 +328,23 @@ export function createLobbyFooterViewModel(
       selectVideoButtonOption: undefined,
     }),
     ...buildMuteBehaviors(scope, muteStates),
-    ...buildDeviceBehaviors(scope, mediaDevices, constant(false)),
+    ...buildDeviceBehaviors(
+      scope,
+      mediaDevices,
+      constant(false),
+      backgroundEffectSettling$,
+    ),
   };
 }
+
+function backgroundEffectChoices(): BackgroundEffectGridItem[] {
+  return [
+    { id: serializeEffect({ kind: "none" }), kind: "none" },
+    { id: serializeEffect({ kind: "blur" }), kind: "blur" },
+    ...shippedBackgrounds.map((background) => ({
+      id: serializeEffect({ kind: "shipped", id: background.id }),
+      kind: "image" as const,
+      imageUrl: background.imagePath,
+    })),
+  ];
+}

src/components/MediaMuteAndSwitchButton.module.css

@@ -36,20 +36,17 @@ Please see LICENSE in the repository root for full details.
   }
 }
 
+/* The width design sets: a long device name wraps. */
 .menu {
-  display: flex;
-  flex-direction: column;
+  inline-size: 296px;
 }
 
-/* Only the device lists scroll, bounded by the measured call area: the menu is
-   portalled out of the root, so CSS can't size it against the call. */
+/* Its share of the measured call area: the menu is portalled out of the root,
+   so CSS can't size it against the call. */
 .deviceList {
-  overflow-y: auto;
-  min-block-size: 0;
   max-block-size: var(--device-list-max-height);
-  /* Keeps a row reached by keyboard clear of the sticky heading and meter. */
-  scroll-padding-block: var(--device-list-scroll-padding-start, 0)
-    var(--device-list-scroll-padding-end, 0);
+  /* The raised headings stay raised within the list, under the menu's frame. */
+  isolation: isolate;
 }
 
 /* Compound's heading rule heads each section; the design adds space below it. */
@@ -62,23 +59,10 @@ Please see LICENSE in the repository root for full details.
   margin-block-start: var(--cpd-space-7x);
 }
 
-/* Sticky within its own section. Opaque and inset a border width so it doesn't
-   paint over the menu's outline. */
-.sectionHeading {
-  position: sticky;
-  inset-block-start: 0;
-  background: var(--cpd-color-bg-canvas-default);
-  margin-inline: var(--cpd-border-width-1);
-  margin-block-start: var(--cpd-border-width-1);
-}
-
-/* Sticky at the foot of the microphone section; opaque and inset as above. */
-.stickyMeter {
-  position: sticky;
-  inset-block-end: 0;
-  background: var(--cpd-color-bg-canvas-default);
-  margin-inline: var(--cpd-border-width-1);
-  margin-block-end: var(--cpd-border-width-1);
+/* Raised, or the positioned effect tiles scroll over the heading they sit
+   under. Compound's sticky heading is the section's aria-hidden first child. */
+.deviceList [role="group"] > [aria-hidden="true"]:first-child {
+  z-index: 1;
 }
 
 /* Radix focuses whatever the pointer is over, so the browser's ring would
@@ -92,3 +76,20 @@ Please see LICENSE in the repository root for full details.
   outline: var(--cpd-border-width-2) solid var(--cpd-color-border-focused);
   outline-offset: calc(-1 * var(--cpd-border-width-2));
 }
+
+.notice {
+  display: flex;
+  align-items: flex-start;
+  gap: var(--cpd-space-3x);
+  margin: var(--cpd-space-2x) var(--cpd-space-4x) 0;
+  padding: var(--cpd-space-3x);
+  border-radius: var(--cpd-space-2x);
+  background: var(--cpd-color-bg-subtle-secondary);
+  color: var(--cpd-color-text-primary);
+  font: var(--cpd-font-body-md-regular);
+}
+
+.notice > svg {
+  flex-shrink: 0;
+  color: var(--cpd-color-icon-secondary);
+}

src/components/MediaMuteAndSwitchButton.stories.tsx

@@ -11,6 +11,8 @@ import { useEffect, useState, type FC, type JSX, type ReactNode } from "react";
 import type { Meta, StoryObj } from "@storybook/react-vite";
 import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton";
 import styles from "./MediaMuteAndSwitchButton.module.css";
+import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
+import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
 import meterStyles from "./MicrophoneLevelMeter.module.css";
 import { MediaDevicesContext } from "../MediaDevicesContext";
 import { RootElementProvider } from "../RootElementContext";
@@ -59,14 +61,19 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => {
 };
 
 /** Supplies a call-sized root. Without one the list is bounded by the whole Storybook frame. */
-const WithACallArea: FC<{ children: ReactNode }> = ({ children }) => {
+const WithACallArea: FC<{ children: ReactNode; blockSize?: number }> = ({
+  children,
+  // A call's size: anything smaller makes a short device list scroll, and a
+  // narrower one narrows the menu.
+  blockSize = 720,
+}) => {
   const [callArea, setCallArea] = useState<HTMLElement | null>(null);
   return (
     <div
       ref={setCallArea}
       style={{
-        // A call's size: anything smaller makes a short device list scroll.
-        blockSize: 720,
+        blockSize,
+        inlineSize: 1024,
         display: "flex",
         alignItems: "flex-end",
         justifyContent: "center",
@@ -97,6 +104,10 @@ const meta = {
 export default meta;
 type Story = StoryObj<typeof meta>;
 
+// By place: Compound's section heading is aria-hidden, as its group carries
+// the name.
+const SECTION_HEADING = "[role='group'] > [aria-hidden]";
+
 export const Default: Story = {
   args: {
     iconsAndLabels: "audio",
@@ -127,8 +138,6 @@ export const AudioMute: Story = {
       { label: { type: "name", name: "Microphone 1" }, id: "1" },
       { label: { type: "name", name: "Microphone 2" }, id: "2" },
     ],
-    videoBlurEnabled: true,
-    videoBlurToggleClick: fn(),
     selectedOption: "2",
   },
   play: async ({ args, canvasElement }) => {
@@ -167,6 +176,16 @@ export const VideoMute: Story = {
   },
 };
 
+const effects: BackgroundEffectGridItem[] = [
+  { id: "none", kind: "none" },
+  { id: "blur", kind: "blur" },
+  ...shippedBackgrounds.map((background) => ({
+    id: `image:${background.id}`,
+    kind: "image" as const,
+    imageUrl: background.imagePath,
+  })),
+];
+
 export const VideoUnmute: Story = {
   args: {
     iconsAndLabels: "video",
@@ -175,9 +194,214 @@ export const VideoUnmute: Story = {
       { label: { type: "name", name: "Camera 1" }, id: "1" },
       { label: { type: "name", name: "Camera 2" }, id: "2" },
     ],
-    videoBlurEnabled: true,
-    videoBlurToggleClick: fn(),
     selectedOption: "2",
+    backgroundEffects: effects,
+    selectedBackgroundEffect: "none",
+    onSelectBackgroundEffect: fn(),
+  },
+};
+
+/** One choice among no effect, blur and the shipped images, under the cameras. */
+export const BackgroundEffects: Story = {
+  args: VideoUnmute.args,
+  play: async ({ args, canvasElement }) => {
+    const canvas = within(canvasElement);
+    await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
+    const menu = within(document.body);
+    const section = await menu.findByRole("group", {
+      name: "Background effects",
+    });
+    const tiles = within(section).getAllByRole("menuitemradio");
+    await expect(tiles.map((tile) => tile.textContent)).toEqual([
+      "None",
+      "Blur",
+      "Background 1",
+      "Background 2",
+    ]);
+    await expect(
+      within(section).getByRole("menuitemradio", { checked: true }),
+    ).toHaveTextContent("None");
+
+    const cameras = menu.getByRole("group", { name: "Camera" });
+    await expect(
+      cameras.compareDocumentPosition(section) &
+        Node.DOCUMENT_POSITION_FOLLOWING,
+    ).toBeTruthy();
+    const [first, second, third, fourth] = tiles.map((tile) =>
+      tile.getBoundingClientRect(),
+    );
+    await expect(second.top).toBe(first.top);
+    await expect(third.top).toBe(first.top);
+    await expect(fourth.top).toBeGreaterThan(first.bottom);
+    const frame = document.body
+      .querySelector("[role='menu']")!
+      .getBoundingClientRect();
+    for (const tile of [first, second, third])
+      await expect(tile.left >= frame.left && tile.right <= frame.right).toBe(
+        true,
+      );
+
+    await userEvent.click(menu.getByRole("menuitemradio", { name: "Blur" }));
+    await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith("blur");
+    await expect(menu.getByRole("menu")).toBeVisible();
+  },
+};
+
+/** Where effects can't run: shown, disabled and explained, except no effect. */
+export const BackgroundEffectsUnavailable: Story = {
+  args: {
+    ...VideoUnmute.args,
+    onSelectBackgroundEffect: undefined,
+    backgroundEffectNotice:
+      "Background effects are not supported on this platform.",
+  },
+  play: async ({ canvasElement }) => {
+    const canvas = within(canvasElement);
+    await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
+    const menu = within(document.body);
+    const section = await menu.findByRole("group", {
+      name: "Background effects",
+    });
+    // On the tiles' own group: the section's can't carry a description.
+    await expect(
+      within(section).getByRole("group"),
+    ).toHaveAccessibleDescription(
+      "Background effects are not supported on this platform.",
+    );
+    for (const tile of within(section).getAllByRole("menuitemradio"))
+      if (tile.textContent === "None") await expect(tile).toBeEnabled();
+      else await expect(tile).toBeDisabled();
+  },
+};
+
+/** Where only the slower route exists: offered and choosable, with the cost said. */
+export const BackgroundEffectsSlowInThisBrowser: Story = {
+  args: {
+    ...VideoUnmute.args,
+    backgroundEffectNotice:
+      "Background effects run slowly on this platform, which may cause your video to stutter.",
+  },
+  play: async ({ args, canvasElement }) => {
+    const canvas = within(canvasElement);
+    await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
+    const menu = within(document.body);
+    const section = await menu.findByRole("group", {
+      name: "Background effects",
+    });
+    // On the tiles' own group: the section's can't carry a description.
+    await expect(
+      within(section).getByRole("group"),
+    ).toHaveAccessibleDescription(
+      "Background effects run slowly on this platform, which may cause your video to stutter.",
+    );
+    for (const tile of within(section).getAllByRole("menuitemradio"))
+      await expect(tile).not.toHaveAttribute("aria-disabled");
+    await userEvent.click(
+      within(section).getByRole("menuitemradio", { name: "Background 1" }),
+    );
+    await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith(
+      "image:arc",
+    );
+
+    const notice = document.body.querySelector(`.${styles.notice}`)!;
+    const frame = document.body
+      .querySelector("[role='menu']")!
+      .getBoundingClientRect();
+    await expect(frame.width).toBeLessThanOrEqual(296);
+    await expect(notice.getBoundingClientRect().right).toBeLessThanOrEqual(
+      frame.right,
+    );
+  },
+};
+
+/** The first effect of a session shows its wait on its own tile. */
+export const BackgroundEffectsSettling: Story = {
+  args: {
+    ...VideoUnmute.args,
+    selectedBackgroundEffect: "blur",
+    backgroundEffectSettling: true,
+  },
+  play: async ({ canvasElement }) => {
+    const canvas = within(canvasElement);
+    await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
+    const menu = within(document.body);
+    const chosen = await menu.findByRole("menuitemradio", {
+      name: "Blur",
+      checked: true,
+    });
+    await expect(chosen).toHaveAttribute("aria-busy", "true");
+    const marks = chosen.querySelectorAll("svg");
+    await expect(marks).toHaveLength(1);
+    await expect(getComputedStyle(marks[0]).animationName).not.toBe("none");
+    const section = menu.getByRole("group", { name: "Background effects" });
+    for (const other of within(section).getAllByRole("menuitemradio", {
+      checked: false,
+    }))
+      await expect(other).toHaveAttribute("aria-busy", "false");
+  },
+};
+
+/** In a short call the effects scroll into view with the list. */
+export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = {
+  args: {
+    ...VideoUnmute.args,
+    // It sits with the effects, so it must fit the same share of the call.
+    backgroundEffectNotice:
+      "Background effects run slowly on this platform, which may cause your video to stutter.",
+  },
+  decorators: [
+    (Story): JSX.Element => (
+      <WithACallArea blockSize={300}>
+        <Story />
+      </WithACallArea>
+    ),
+  ],
+  play: async ({ canvasElement }) => {
+    const canvas = within(canvasElement);
+    await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
+    const menu = within(document.body);
+    const last = await menu.findByRole("menuitemradio", {
+      name: "Background 2",
+    });
+    const list = document.body.querySelector<HTMLElement>(
+      `.${styles.deviceList}`,
+    )!;
+    await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight);
+    await expect(last.getBoundingClientRect().top).toBeGreaterThan(
+      list.getBoundingClientRect().bottom,
+    );
+
+    // Scrolled under the section's heading, the tiles pass beneath it. By
+    // position: Compound's heading is the section's hidden first child.
+    const heading = menu.getByRole("group", { name: "Background effects" })
+      .firstElementChild as HTMLElement;
+    list.scrollTop = heading.offsetTop + 30;
+    await waitFor(async () => {
+      const box = heading.getBoundingClientRect();
+      const drawn = document.elementFromPoint(
+        box.left + box.width / 2,
+        box.bottom - 2,
+      );
+      await expect(heading.contains(drawn)).toBe(true);
+    });
+    list.scrollTop = 0;
+
+    for (let i = 0; i < 10 && document.activeElement !== last; i++)
+      await userEvent.keyboard("{ArrowDown}");
+    await expect(document.activeElement).toBe(last);
+    const scrollport = list.getBoundingClientRect();
+    const reached = last.getBoundingClientRect();
+    await expect(reached.bottom).toBeLessThanOrEqual(scrollport.bottom + 1);
+    await expect(reached.top).toBeGreaterThanOrEqual(scrollport.top - 1);
+
+    const callArea = canvasElement
+      .querySelector<HTMLElement>("[style*='block-size: 300px']")!
+      .getBoundingClientRect();
+    const frame = document.body
+      .querySelector("[role='menu']")!
+      .getBoundingClientRect();
+    await expect(frame.top).toBeGreaterThanOrEqual(callArea.top);
+    await expect(frame.bottom).toBeLessThanOrEqual(callArea.bottom);
   },
 };
 
@@ -219,9 +443,8 @@ export const SpeakerAndMicrophoneSections: Story = {
     await expect(
       document.body.querySelectorAll("[role='separator']"),
     ).toHaveLength(0);
-    const headings = document.body.querySelectorAll<HTMLElement>(
-      `.${styles.sectionHeading}`,
-    );
+    const headings =
+      document.body.querySelectorAll<HTMLElement>(SECTION_HEADING);
     await expect(headings).toHaveLength(2);
     const frame = menu.getBoundingClientRect();
     for (const heading of headings) {
@@ -256,6 +479,37 @@ export const SpeakerAndMicrophoneSections: Story = {
   },
 };
 
+/** A long name wraps rather than widening the menu, which design sets. */
+export const LongDeviceNameWraps: Story = {
+  args: {
+    ...SpeakerAndMicrophoneSections.args,
+    outputOptions: [
+      {
+        label: {
+          type: "name",
+          name: "Logitech BRIO 4K Ultra HD Pro Business Webcam Speakers (046d:085e)",
+        },
+        id: "long",
+      },
+      { label: { type: "name", name: "Headset" }, id: "spk2" },
+    ],
+    selectedOutputOption: "long",
+  },
+  play: async ({ canvasElement }) => {
+    const canvas = within(canvasElement);
+    await userEvent.click(canvas.getByRole("button", { name: "Microphone" }));
+    const body = within(document.body);
+    const long = await body.findByRole("menuitemradio", { name: /Logitech/ });
+    const short = await body.findByRole("menuitemradio", { name: "Headset" });
+
+    const menu = document.body.querySelector("[role='menu']")!;
+    await expect(Math.round(menu.getBoundingClientRect().width)).toBe(296);
+    await expect(long.getBoundingClientRect().height).toBeGreaterThan(
+      short.getBoundingClientRect().height,
+    );
+  },
+};
+
 export const OutputCannotBeChosen: Story = {
   args: {
     ...Default.args,
@@ -402,14 +656,14 @@ export const ManyDevices: Story = {
     await userEvent.click(canvas.getByRole("button", { name: "Microphone" }));
     const menu = within(document.body);
 
-    // The sticky wrapper rather than the meter: without a fake microphone (as on
+    // The meter's box rather than its level: without a fake microphone (as on
     // WebKit) the meter shows a message instead of a level.
     const list = document.body.querySelector<HTMLElement>(
       `.${styles.deviceList}`,
     )!;
     const sticky = await waitFor(() => {
       const element = document.body.querySelector<HTMLElement>(
-        `.${styles.stickyMeter}`,
+        `.${meterStyles.meter}`,
       );
       if (element === null) throw new Error("the meter has not rendered yet");
       return element;
@@ -436,10 +690,6 @@ export const ManyDevices: Story = {
       .getBoundingClientRect();
     await expect(frame.top).toBeGreaterThanOrEqual(0);
     await expect(frame.bottom).toBeLessThanOrEqual(window.innerHeight + 1);
-
-    // The meter is the menu's one opaque part, so it must stay inside the frame.
-    await expect(pinned.left).toBeGreaterThan(frame.left);
-    await expect(pinned.right).toBeLessThan(frame.right);
   },
 };
 
@@ -536,7 +786,7 @@ export const KeyboardReachesEveryDevice: Story = {
 
     const meter = await waitFor(() => {
       const element = document.body.querySelector<HTMLElement>(
-        `.${styles.stickyMeter}`,
+        `.${meterStyles.meter}`,
       );
       if (element === null) throw new Error("the meter has not rendered yet");
       return element;
@@ -545,9 +795,7 @@ export const KeyboardReachesEveryDevice: Story = {
     // Drawn over the list: the sticky headings and the meter.
     const overlays = [
       meter,
-      ...document.body.querySelectorAll<HTMLElement>(
-        `.${styles.sectionHeading}`,
-      ),
+      ...document.body.querySelectorAll<HTMLElement>(SECTION_HEADING),
     ];
     const items = within(document.body).getAllByRole("menuitemradio");
 
@@ -591,10 +839,7 @@ export const HeadingsStayWhileScrolling: Story = {
       `.${styles.deviceList}`,
     )!;
     const group = await menu.findByRole("group", { name: "Microphone" });
-    // By class: the heading is aria-hidden, and its group carries the name.
-    const heading = group.querySelector<HTMLElement>(
-      `.${styles.sectionHeading}`,
-    )!;
+    const heading = group.querySelector<HTMLElement>(":scope > [aria-hidden]")!;
 
     // Far enough that the heading is only on screen if it is stuck there.
     list.scrollTop +=
@@ -607,13 +852,6 @@ export const HeadingsStayWhileScrolling: Story = {
     await expect(heading.getBoundingClientRect().bottom).toBeGreaterThan(
       scrollport.top,
     );
-    // Clear of the menu's frame.
-    const frame = document.body
-      .querySelector("[role='menu']")!
-      .getBoundingClientRect();
-    await expect(heading.getBoundingClientRect().left).toBeGreaterThan(
-      frame.left,
-    );
   },
 };
 
@@ -628,31 +866,23 @@ function overlapping(element: Element, overlays: Element[]): number {
   }, 0);
 }
 
-/** The blur toggle gets the keyboard ring too: it is the menu's child, not the list's. */
-export const FocusRingCoversTheBlurToggle: Story = {
-  args: {
-    ...VideoUnmute.args,
-    iconsAndLabels: "video",
-    videoBlurEnabled: false,
-    videoBlurToggleClick: fn(),
-  },
+/** The effect tiles get the keyboard ring the device rows get. */
+export const FocusRingCoversTheEffectTiles: Story = {
+  args: VideoUnmute.args,
   play: async ({ canvasElement }) => {
     const canvas = within(canvasElement);
     await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
-    const toggle = await within(document.body).findByRole("menuitemcheckbox", {
-      name: /Blur background/,
+    const tile = await within(document.body).findByRole("menuitemradio", {
+      name: "Blur",
     });
 
-    // Arrow down past the cameras to the toggle.
-    for (let i = 0; i < 6 && document.activeElement !== toggle; i++)
+    for (let i = 0; i < 6 && document.activeElement !== tile; i++)
       await userEvent.keyboard("{ArrowDown}");
-    await expect(document.activeElement).toBe(toggle);
-    // The same ring the device rows get.
-    await expect(outlineWidth(toggle)).toBeGreaterThan(0);
+    await expect(document.activeElement).toBe(tile);
+    await expect(outlineWidth(tile)).toBeGreaterThan(0);
 
-    // And the pointer takes it away, with the toggle still focused.
-    await userEvent.hover(toggle);
-    await expect(document.activeElement).toBe(toggle);
-    await expect(outlineWidth(toggle)).toBe(0);
+    await userEvent.hover(tile);
+    await expect(document.activeElement).toBe(tile);
+    await expect(outlineWidth(tile)).toBe(0);
   },
 };

src/components/MediaMuteAndSwitchButton.test.tsx

@@ -9,6 +9,7 @@ import { afterEach, describe, expect, test, vi } from "vitest";
 import { axe } from "vitest-axe";
 import {
   act,
+  fireEvent,
   render,
   screen,
   within,
@@ -22,10 +23,18 @@ import {
   MediaMuteAndSwitchButton,
   type MenuOptions,
 } from "./MediaMuteAndSwitchButton";
+import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
 import { MediaDevicesContext } from "../MediaDevicesContext";
 import { type MediaDevices } from "../state/MediaDevices";
 import { restoreAudioCapture, stubAudioCapture } from "../utils/test";
 
+const effects: BackgroundEffectGridItem[] = [
+  { id: "none", kind: "none" },
+  { id: "blur", kind: "blur" },
+  { id: "image:arc", kind: "image", imageUrl: "" },
+  { id: "image:glow", kind: "image", imageUrl: "" },
+];
+
 interface RenderOptions {
   requestDeviceNames: () => void;
 }
@@ -305,30 +314,88 @@ describe("MediaMuteAndSwitchButton", () => {
     expect(mic2ItemAfter.querySelector(".rotate")).toBeNull();
   });
 
-  test("renders menu with toggle control and calls toggle callback", async () => {
+  test("background effects are one keyboard-operable labelled choice", async () => {
     const user = userEvent.setup();
-    const onSelect = vi.fn();
-    const onVideoBlurToggle = vi.fn();
+    const onSelectBackgroundEffect = vi.fn();
     const { getByRole } = renderComponent(
       <MediaMuteAndSwitchButton
         iconsAndLabels="video"
         enabled={true}
-        videoBlurToggleClick={onVideoBlurToggle}
-        onSelect={onSelect}
+        options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
+        selectedOption="cam1"
+        onSelect={vi.fn()}
+        backgroundEffects={effects}
+        selectedBackgroundEffect="none"
+        onSelectBackgroundEffect={onSelectBackgroundEffect}
       />,
     );
 
     await user.click(getByRole("button", { name: "Camera" }));
-
-    const toggle = screen.getByRole("menuitemcheckbox", {
-      name: "Blur background",
+    const section = screen.getByRole("group", { name: "Background effects" });
+    within(section).getByRole("menuitemradio", { name: "None", checked: true });
+    within(section).getByRole("menuitemradio", {
+      name: "Background 1",
+      checked: false,
     });
-    expect(toggle).toBeInTheDocument();
-    expect(toggle).toHaveAttribute("aria-checked", "false");
 
-    await user.click(toggle);
+    const blur = within(section).getByRole("menuitemradio", { name: "Blur" });
+    for (let i = 0; i < 6 && document.activeElement !== blur; i++)
+      await user.keyboard("{ArrowDown}");
+    expect(document.activeElement).toBe(blur);
+    await user.keyboard("{Enter}");
+    expect(onSelectBackgroundEffect).toHaveBeenCalledWith("blur");
+    expect(screen.getByRole("menu")).toBeInTheDocument();
+  });
 
-    expect(onVideoBlurToggle).toHaveBeenCalled();
+  test("does not choose the effect already in force again", async () => {
+    const user = userEvent.setup();
+    const onSelectBackgroundEffect = vi.fn();
+    const { getByRole } = renderComponent(
+      <MediaMuteAndSwitchButton
+        iconsAndLabels="video"
+        enabled={true}
+        options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
+        selectedOption="cam1"
+        backgroundEffects={effects}
+        selectedBackgroundEffect="blur"
+        onSelectBackgroundEffect={onSelectBackgroundEffect}
+      />,
+    );
+
+    await user.click(getByRole("button", { name: "Camera" }));
+    await user.click(screen.getByRole("menuitemradio", { name: "Blur" }));
+    expect(onSelectBackgroundEffect).not.toHaveBeenCalled();
+  });
+
+  test("offers the background effects in a phone's drawer", async () => {
+    const userAgent = vi
+      .spyOn(navigator, "userAgent", "get")
+      .mockReturnValue("Mozilla/5.0 (Linux; Android 14)");
+    try {
+      const user = userEvent.setup();
+      const onSelectBackgroundEffect = vi.fn();
+      const { getByRole } = renderComponent(
+        <MediaMuteAndSwitchButton
+          iconsAndLabels="video"
+          enabled={true}
+          options={[{ label: { type: "name", name: "Camera 1" }, id: "cam1" }]}
+          selectedOption="cam1"
+          backgroundEffects={effects}
+          selectedBackgroundEffect="none"
+          onSelectBackgroundEffect={onSelectBackgroundEffect}
+        />,
+      );
+
+      await user.click(getByRole("button", { name: "Camera" }));
+      const section = screen.getByRole("group", { name: "Background effects" });
+      // A click alone: the drawer's drag handling reads layout jsdom lacks.
+      fireEvent.click(
+        within(section).getByRole("menuitemradio", { name: "Background 2" }),
+      );
+      expect(onSelectBackgroundEffect).toHaveBeenCalledWith("image:glow");
+    } finally {
+      userAgent.mockRestore();
+    }
   });
 
   test("marks the selected menu item as checked", async () => {
@@ -552,7 +619,9 @@ describe("MediaMuteAndSwitchButton", () => {
         ]}
         selectedOption="cam1"
         onSelect={vi.fn()}
-        videoBlurToggleClick={vi.fn()}
+        backgroundEffects={effects}
+        selectedBackgroundEffect="none"
+        onSelectBackgroundEffect={vi.fn()}
       />,
     );
 
@@ -560,9 +629,10 @@ describe("MediaMuteAndSwitchButton", () => {
 
     screen.getByRole("menuitemradio", { name: "Camera 1", checked: true });
     screen.getByRole("menuitemradio", { name: "Camera 2", checked: false });
-    expect(
-      screen.getByRole("menuitemcheckbox", { name: "Blur background" }),
-    ).toBeInTheDocument();
+    within(screen.getByRole("group", { name: "Background effects" })).getByRole(
+      "menuitemradio",
+      { name: "Blur" },
+    );
   });
 
   test("marks focus as keyboard-driven only when the keyboard moved it", async () => {

src/components/MediaMuteAndSwitchButton.tsx

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
 
 import {
   useCallback,
+  useId,
   useState,
   type CSSProperties,
   type FC,
@@ -17,13 +18,14 @@ import {
   Button,
   Menu,
   MenuItem,
-  MenuTitle,
+  MenuScrollArea,
+  MenuScrollSection,
   RadioInput,
-  ToggleMenuItem,
 } from "@vector-im/compound-web";
 import {
   ChevronUpIcon,
   ChevronDownIcon,
+  InfoIcon,
   SpinnerIcon,
 } from "@vector-im/compound-design-tokens/assets/web/icons";
 import classNames from "classnames";
@@ -40,6 +42,10 @@ import { useMediaDevices } from "../MediaDevicesContext";
 import { useRootElement } from "../RootElementContext";
 import { observeElementSize$ } from "../utils/elementSize";
 import { LiveMicrophoneLevelMeter } from "./MicrophoneLevelMeter";
+import {
+  BackgroundEffectGrid,
+  type BackgroundEffectGridItem,
+} from "./BackgroundEffectGrid";
 
 export interface MenuOptions {
   label: DeviceLabel | AudioOutputDeviceLabel;
@@ -64,8 +70,15 @@ export interface MediaMuteAndSwitchButtonProps {
   selectedOutputOption?: string;
   /** Picks an output device. Undefined disables the speaker section. */
   onSelectOutput?: (id: string) => void;
-  videoBlurToggleClick?: () => void;
-  videoBlurEnabled?: boolean;
+  /** Camera menu only. None or an empty list leaves the section out. */
+  backgroundEffects?: BackgroundEffectGridItem[];
+  selectedBackgroundEffect?: string;
+  /** Undefined where effects can't run: all but no effect are disabled. */
+  onSelectBackgroundEffect?: (id: string) => void;
+  /** What the user should know before choosing an effect, if anything. */
+  backgroundEffectNotice?: string;
+  /** Whether the first effect chosen is still being prepared. */
+  backgroundEffectSettling?: boolean;
   /**
    * For any toggle and option this method will be called.
    * So toggles need to be implemented by listening here and setting the right toggle item to `enabled`
@@ -73,8 +86,6 @@ export interface MediaMuteAndSwitchButtonProps {
   onSelect?: (id: string) => void;
 }
 
-const BLUR_ID = "blur";
-
 /** Id of the placeholder "Default" row, shown when the platform lists no outputs. */
 const DEFAULT_OUTPUT_ID = "default";
 
@@ -84,6 +95,9 @@ const LIST_SHARE_OF_CALL = 0.6;
 /** Smallest device list height in px, so a short call still shows more than one device. */
 const MIN_LIST_HEIGHT = 160;
 
+/** Space kept between the menu and the call area's sides. */
+const MENU_MARGIN = 16;
+
 export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
   enabled,
   busy,
@@ -94,8 +108,11 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
   outputOptions,
   selectedOutputOption,
   onSelectOutput,
-  videoBlurEnabled,
-  videoBlurToggleClick,
+  backgroundEffects,
+  selectedBackgroundEffect,
+  onSelectBackgroundEffect,
+  backgroundEffectNotice,
+  backgroundEffectSettling,
   onSelect,
 }) => {
   // Requested but not yet selected. Keyed by kind too, since Chrome uses
@@ -113,6 +130,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
   const isBusy = busy ?? false;
   const { t } = useTranslation();
   const devices = useMediaDevices();
+  const noticeId = useId();
 
   /**
    * Records on the menu whether the keyboard or the pointer moved focus, for
@@ -157,18 +175,18 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
     return (): void => subscription.unsubscribe();
   }, [menuOpen, rootElement]);
 
-  // Kept clear at the list's foot, so a row reached by keyboard isn't under
-  // the meter.
-  const [meterHeight, meter] = useMeasuredHeight();
-
-  // Likewise at its head, for the sticky headings.
-  const [headingHeight, heading] = useMeasuredHeight();
-
   useEffect(() => {
     if (menuOpen) devices.requestDeviceNames(); // No-op after the first call
   }, [menuOpen, devices]);
 
-  const MuteButton = iconsAndLabels === "audio" ? MicButton : VideoButton;
+  const audioMeter = iconsAndLabels === "audio" && (
+    <LiveMicrophoneLevelMeter
+      deviceId={selectedOption}
+      // Only while open, so the microphone isn't held all call.
+      active={menuOpen}
+    />
+  );
+  const MuteButton = audioMeter ? MicButton : VideoButton;
   const button = (
     <MuteButton
       enabled={enabled ?? false}
@@ -179,22 +197,12 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
         e.stopPropagation();
       }}
       disabled={isBusy || onMuteClick === undefined}
-      data-testid={
-        iconsAndLabels === "audio" ? "incall_mute" : "incall_videomute"
-      }
+      data-testid={audioMeter ? "incall_mute" : "incall_videomute"}
     />
   );
 
-  const toggles =
-    iconsAndLabels === "video" && videoBlurToggleClick !== undefined
-      ? [
-          {
-            label: t("action.blur_background"),
-            enabled: videoBlurEnabled ?? false,
-            id: BLUR_ID,
-          },
-        ]
-      : [];
+  const showEffects =
+    iconsAndLabels === "video" && (backgroundEffects?.length ?? 0) > 0;
 
   let optionsButtonLabel: string;
   let menuTitle: string;
@@ -326,6 +334,8 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
         open={menuOpen}
         onOpenChange={onOpenChange}
         side="top"
+        collisionBoundary={rootElement}
+        collisionPadding={MENU_MARGIN}
         trigger={
           <Button
             iconOnly
@@ -340,101 +350,62 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
           />
         }
       >
-        <div
+        <MenuScrollArea
           ref={trackFocusSource}
-          // Keeps the items the menu's own children for assistive tech.
-          role="none"
           className={styles.deviceList}
           data-testid="device-list"
           style={
             {
               "--device-list-max-height":
                 listMaxHeight === undefined ? undefined : `${listMaxHeight}px`,
-              "--device-list-scroll-padding-end":
-                meterHeight === undefined ? undefined : `${meterHeight}px`,
-              "--device-list-scroll-padding-start":
-                headingHeight === undefined ? undefined : `${headingHeight}px`,
             } as CSSProperties
           }
         >
-          {iconsAndLabels === "audio" && speakerOptions && (
-            <>
-              {/* A menu may only contain items, separators and groups, so each
-                  heading is a hidden part of a named group. */}
-              <div role="group" aria-label={t("settings.devices.speaker")}>
-                <div aria-hidden className={styles.sectionHeading}>
-                  <MenuTitle title={t("settings.devices.speaker")} />
-                </div>
-                {deviceItems(
-                  "output",
-                  speakerOptions,
-                  selectedSpeaker,
-                  onSelectOutput,
-                  (n) => t("settings.devices.speaker_numbered", { n }),
-                )}
-              </div>
-            </>
-          )}
-          <div role="group" aria-label={optionsButtonLabel}>
-            <div ref={heading} aria-hidden className={styles.sectionHeading}>
-              <MenuTitle title={optionsButtonLabel} />
-            </div>
-            {/* Apart from the heading, so the sticky meter can't ride over it. */}
-            <div role="none">
+          {audioMeter && speakerOptions && (
+            <MenuScrollSection title={t("settings.devices.speaker")}>
               {deviceItems(
-                "input",
-                options,
-                selectedOption,
-                onSelect,
-                numberedLabel,
+                "output",
+                speakerOptions,
+                selectedSpeaker,
+                onSelectOutput,
+                (n) => t("settings.devices.speaker_numbered", { n }),
               )}
-              {iconsAndLabels === "audio" && (
-                <LiveMicrophoneLevelMeter
-                  ref={meter}
-                  deviceId={selectedOption}
-                  // Only while open, so the microphone isn't held all call.
-                  active={menuOpen}
-                  className={styles.stickyMeter}
-                />
+            </MenuScrollSection>
+          )}
+          <MenuScrollSection
+            title={optionsButtonLabel}
+            stickyBottomItem={audioMeter}
+          >
+            {deviceItems(
+              "input",
+              options,
+              selectedOption,
+              onSelect,
+              numberedLabel,
+            )}
+          </MenuScrollSection>
+          {showEffects && (
+            <MenuScrollSection title={t("settings.background_effects_header")}>
+              <BackgroundEffectGrid
+                effects={backgroundEffects ?? []}
+                selected={selectedBackgroundEffect}
+                onSelect={onSelectBackgroundEffect}
+                settling={backgroundEffectSettling}
+                describedBy={
+                  backgroundEffectNotice === undefined ? undefined : noticeId
+                }
+              />
+              {/* In the list, so the menu grows no taller for it. */}
+              {backgroundEffectNotice !== undefined && (
+                <div id={noticeId} role="none" className={styles.notice}>
+                  <InfoIcon width={20} height={20} />
+                  <span>{backgroundEffectNotice}</span>
+                </div>
               )}
-            </div>
-          </div>
-        </div>
-        {toggles.length > 0 && <hr />}
-        {toggles.map((toggle) => (
-          <ToggleMenuItem
-            label={toggle.label}
-            onSelect={(e) => {
-              videoBlurToggleClick?.();
-              e.preventDefault();
-            }}
-            checked={toggle.enabled}
-            key={toggle.id}
-          />
-        ))}
+            </MenuScrollSection>
+          )}
+        </MenuScrollArea>
       </Menu>
     </div>
   );
 };
-
-/** Follows an element's height. */
-function useMeasuredHeight(): [
-  number | undefined,
-  (element: HTMLElement | null) => (() => void) | undefined,
-] {
-  const [height, setHeight] = useState<number>();
-  const ref = useCallback(
-    (element: HTMLElement | null): (() => void) | undefined => {
-      if (element === null) return;
-      const subscription = observeElementSize$(element)
-        .pipe(
-          map((size) => size.height),
-          distinctUntilChanged(),
-        )
-        .subscribe(setHeight);
-      return (): void => subscription.unsubscribe();
-    },
-    [],
-  );
-  return [height, ref];
-}

src/components/MicrophoneLevelMeter.module.css

@@ -16,10 +16,10 @@ Please see LICENSE in the repository root for full details.
   align-items: center;
   gap: var(--cpd-space-3x);
   padding-block: var(--cpd-space-2x);
-  /* Centres the icon on the radio column: less the row's inset border and half
-     the icon's extra width. Checked in the MeterAlignsWithTheDeviceRows story. */
+  /* Centres the icon on the radio column: less half the icon's extra width.
+     Checked in the MeterAlignsWithTheDeviceRows story. */
   padding-inline-start: calc(
-    var(--cpd-space-4x) - var(--cpd-border-width-1) -
+    var(--cpd-space-4x) -
       (var(--meter-icon-size) - var(--device-control-size)) / 2
   );
   padding-inline-end: calc(var(--cpd-space-4x) * 2 + var(--cpd-space-2x));

src/components/MicrophoneLevelMeter.tsx

@@ -5,17 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import {
-  useCallback,
-  useLayoutEffect,
-  useRef,
-  useState,
-  type FC,
-  type Ref,
-} from "react";
+import { useCallback, useLayoutEffect, useRef, useState, type FC } from "react";
 import { Text } from "@vector-im/compound-web";
 import { MicOnIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
-import classNames from "classnames";
 import { useTranslation } from "react-i18next";
 
 import { distinctUntilChanged, map } from "rxjs";
@@ -30,8 +22,6 @@ export interface LiveMicrophoneLevelMeterProps {
   deviceId: string | undefined;
   /** Whether to hold a capture at all. */
   active: boolean;
-  className?: string;
-  ref?: Ref<HTMLDivElement>;
 }
 
 /**
@@ -41,24 +31,18 @@ export interface LiveMicrophoneLevelMeterProps {
 export const LiveMicrophoneLevelMeter: FC<LiveMicrophoneLevelMeterProps> = ({
   deviceId,
   active,
-  className,
-  ref,
 }) => {
   const state = useMicrophoneLevel(deviceId, active);
-  return <MicrophoneLevelMeter state={state} className={className} ref={ref} />;
+  return <MicrophoneLevelMeter state={state} />;
 };
 
 export interface MicrophoneLevelMeterProps {
   state: MicrophoneState;
-  className?: string;
-  ref?: Ref<HTMLDivElement>;
 }
 
 /** The input level of a microphone, or why there is none. */
 export const MicrophoneLevelMeter: FC<MicrophoneLevelMeterProps> = ({
   state,
-  className,
-  ref,
 }) => {
   const { t } = useTranslation();
   // Bars keep one size, so their count follows the width. Starts full so the
@@ -104,7 +88,7 @@ export const MicrophoneLevelMeter: FC<MicrophoneLevelMeterProps> = ({
   }, [state, barCount, t]);
 
   return (
-    <div ref={ref} className={classNames(styles.meter, className)}>
+    <div className={styles.meter}>
       <MicOnIcon className={styles.icon} aria-hidden />
       {state.type !== "level" ? (
         <Text size="sm" className={styles.message}>

src/components/__snapshots__/MediaMuteAndSwitchButton.test.tsx.snap

@@ -9,7 +9,7 @@ exports[`MediaMuteAndSwitchButton > renders 1`] = `
       aria-busy="false"
       aria-checked="false"
       aria-disabled="true"
-      aria-labelledby="_r_0_"
+      aria-labelledby="_r_1_"
       class="_button_1nw83_8 _has-icon_1nw83_60 _icon-only_1nw83_53"
       data-kind="primary"
       data-size="lg"
@@ -39,7 +39,7 @@ exports[`MediaMuteAndSwitchButton > renders 1`] = `
       data-kind="tertiary"
       data-size="lg"
       data-state="closed"
-      id="radix-_r_5_"
+      id="radix-_r_6_"
       role="button"
       tabindex="0"
       type="button"

src/config/Config.test.ts

@@ -95,6 +95,19 @@ describe("Config.initWith", () => {
     expect(Config.get().ssla).toBe("https://example.invalid/ssla");
   });
 
+  it("falls back to the defaults when fetching config.json rejects", async () => {
+    // A rejected promise is what a webview gives for a missing local file.
+    vi.spyOn(globalThis, "fetch").mockRejectedValue(
+      new TypeError("Load failed"),
+    );
+    vi.spyOn(logger, "info").mockImplementation(() => {});
+    Reflect.set(Config, "internalInstance", undefined);
+
+    await expect(Config.init()).resolves.toBeUndefined();
+
+    expect(Config.get().media_quality).toEqual(DEFAULT_CONFIG.media_quality);
+  });
+
   it("replaces a config initialized earlier", () => {
     Config.initWith({ ssla: "https://first.invalid/ssla" });
     Config.initWith({ ssla: "https://second.invalid/ssla" });

src/config/Config.ts

@@ -140,13 +140,20 @@ export function validateConfig(config: ConfigOptions): ConfigOptions {
 }
 
 async function downloadConfig(fetchTarget: string): Promise<ConfigOptions> {
-  const response = await fetch(fetchTarget);
+  let response: Response;
+  try {
+    response = await fetch(fetchTarget);
+  } catch (e) {
+    // The embedded package ships no config.json, and some webviews (Element X
+    // iOS) reject the fetch of a missing local file rather than answering 404.
+    logger.info(`No config at ${fetchTarget}, using defaults`, e);
+    // resolveConfig merges in the defaults.
+    return {};
+  }
 
   if (isFailure(response)) {
-    // Lack of a config isn't an error, we should just use the defaults.
-    // Also treat a blank config as no config, assuming the status code is 0, because we don't get 404s from file:
-    // URIs so this is the only way we can not fail if the file doesn't exist when loading from a file:// URI.
-    return DEFAULT_CONFIG;
+    // resolveConfig merges in the defaults.
+    return {};
   }
 
   return response.json();

src/config/ConfigOptions.ts

@@ -6,23 +6,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-/**
- * The MatrixRTC mode determines how Element Call interacts with the
- * MatrixRTC backend and other participants. Selectable via the Developer
- * Settings, or pinned for a deployment via `matrix_rtc_mode` in config.json.
- */
-export enum MatrixRTCMode {
-  /** Multi-SFU transport, legacy JWT endpoint, state events. */
-  Compatibility = "compatibility",
-  /**
-   * Multi-SFU transport with:
-   *  - sticky events
-   *  - hashed RTC backend identity
-   *  - the new endpoint for the jwt token on the local membership (remote memberships will always try the new jwt endpoint first -> then the legacy one)
-   *  - use the hashed identity for the local membership
-   */
-  Matrix_2_0 = "matrix_2_0",
-}
+import { MatrixRTCMode } from "@element-hq/matrixrtc-sdk";
+
+export { MatrixRTCMode };
 
 export interface DelayedLeaveTimings {
   /**

src/e2ee/e2eeType.ts

@@ -1,12 +1,8 @@
 /*
-Copyright 2023, 2024 New Vector Ltd.
+Copyright 2026 Element Creations Ltd.
 
 SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-export enum E2eeType {
-  NONE = 0,
-  PER_PARTICIPANT = 1,
-  SHARED_KEY = 2,
-}
+export { E2eeType } from "@element-hq/matrixrtc-sdk";

src/e2ee/matrixKeyProvider.test.ts

@@ -1,72 +0,0 @@
-/*
-Copyright 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import { describe, expect, test, vi } from "vitest";
-import {
-  type MatrixRTCSession,
-  MatrixRTCSessionEvent,
-} from "matrix-js-sdk/lib/matrixrtc";
-import { KeyProviderEvent } from "livekit-client";
-
-import { MatrixKeyProvider } from "./matrixKeyProvider";
-
-function mockRTCSession(): MatrixRTCSession {
-  return {
-    on: vi.fn(),
-    off: vi.fn(),
-    reemitEncryptionKeys: vi.fn(),
-  } as unknown as MatrixRTCSession;
-}
-
-describe("matrixKeyProvider", () => {
-  test("initializes", () => {
-    const keyProvider = new MatrixKeyProvider();
-    expect(keyProvider).toBeTruthy();
-  });
-
-  test("listens for key requests and emits existing keys", () => {
-    const keyProvider = new MatrixKeyProvider();
-
-    const session = mockRTCSession();
-
-    keyProvider.setRTCSession(session);
-
-    expect(session.on).toHaveBeenCalledWith(
-      MatrixRTCSessionEvent.EncryptionKeyChanged,
-      expect.any(Function),
-    );
-    expect(session.off).not.toHaveBeenCalled();
-  });
-
-  test("stops listening when session changes", () => {
-    const keyProvider = new MatrixKeyProvider();
-
-    const session1 = mockRTCSession();
-    const session2 = mockRTCSession();
-
-    keyProvider.setRTCSession(session1);
-    expect(session1.off).not.toHaveBeenCalled();
-
-    keyProvider.setRTCSession(session2);
-    expect(session1.off).toHaveBeenCalledWith(
-      MatrixRTCSessionEvent.EncryptionKeyChanged,
-      expect.any(Function),
-    );
-  });
-
-  test("emits existing keys", () => {
-    const keyProvider = new MatrixKeyProvider();
-    const setKeyListener = vi.fn();
-    keyProvider.on(KeyProviderEvent.SetKey, setKeyListener);
-
-    const session = mockRTCSession();
-
-    keyProvider.setRTCSession(session);
-
-    expect(session.reemitEncryptionKeys).toHaveBeenCalled();
-  });
-});

src/e2ee/matrixKeyProvider.ts

@@ -1,75 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import { BaseKeyProvider } from "livekit-client";
-import {
-  type MatrixRTCSession,
-  MatrixRTCSessionEvent,
-} from "matrix-js-sdk/lib/matrixrtc";
-import { logger as rootLogger, type Logger } from "matrix-js-sdk/lib/logger";
-import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
-
-export class MatrixKeyProvider extends BaseKeyProvider {
-  private rtcSession?: MatrixRTCSession;
-  private logger: Logger;
-  public constructor() {
-    super({ ratchetWindowSize: 10, keyringSize: 256 });
-    this.logger = rootLogger.getChild("[MatrixKeyProvider]");
-  }
-
-  public setRTCSession(rtcSession: MatrixRTCSession): void {
-    if (this.rtcSession) {
-      this.rtcSession.off(
-        MatrixRTCSessionEvent.EncryptionKeyChanged,
-        this.onEncryptionKeyChanged,
-      );
-    }
-
-    this.rtcSession = rtcSession;
-
-    this.rtcSession.on(
-      MatrixRTCSessionEvent.EncryptionKeyChanged,
-      this.onEncryptionKeyChanged,
-    );
-
-    // The new session could be aware of keys of which the old session wasn't,
-    // so emit key changed events
-    this.rtcSession.reemitEncryptionKeys();
-  }
-
-  private onEncryptionKeyChanged = (
-    encryptionKey: Uint8Array<ArrayBuffer>,
-    encryptionKeyIndex: number,
-    membershipParts: CallMembershipIdentityParts,
-    rtcBackendIdentity: string,
-  ): void => {
-    crypto.subtle
-      .importKey("raw", encryptionKey, "HKDF", false, [
-        "deriveBits",
-        "deriveKey",
-      ])
-      .then(
-        (keyMaterial) => {
-          this.onSetEncryptionKey(
-            keyMaterial,
-            rtcBackendIdentity,
-            encryptionKeyIndex,
-          );
-
-          this.logger.debug(
-            `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}:${membershipParts.deviceId}) encryptionKeyIndex=${encryptionKeyIndex}`,
-          );
-        },
-        (e) => {
-          this.logger.error(
-            `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipParts.userId}:${membershipParts.deviceId} encryptionKeyIndex=${encryptionKeyIndex}`,
-            e,
-          );
-        },
-      );
-  };
-}

src/e2ee/sharedKeyManagement.test.ts

@@ -5,11 +5,24 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { afterEach, describe, expect, test } from "vitest";
+import { afterEach, describe, expect, test, vi } from "vitest";
+import { act, renderHook } from "@testing-library/react";
+import { E2eeType } from "@element-hq/matrixrtc-sdk";
 
-import { getKeyForRoom, saveKeyForRoom } from "./sharedKeyManagement";
+import {
+  getKeyForRoom,
+  saveKeyForRoom,
+  useRoomEncryptionSystem,
+} from "./sharedKeyManagement";
+
+vi.mock("../ClientContext", () => {
+  const room = { hasEncryptionStateEvent: (): boolean => false };
+  const client = { getRoom: (): typeof room => room };
+  return { useClient: (): { client: typeof client } => ({ client }) };
+});
 
 const roomId = "!room:example.org";
+const otherRoomId = "!other:example.org";
 
 describe("getKeyForRoom", () => {
   afterEach(() => {
@@ -33,4 +46,58 @@ describe("getKeyForRoom", () => {
   test("has no key to offer for a room it has never seen", () => {
     expect(getKeyForRoom(roomId)).toBeNull();
   });
+
+  test("prefers the stored key when the parameters are for another room", () => {
+    saveKeyForRoom(roomId, "stored");
+    window.location.hash = `#?roomId=${encodeURIComponent(otherRoomId)}&password=other-room`;
+
+    expect(getKeyForRoom(roomId)).toBe("stored");
+  });
+
+  test("ignores the password in the parameters for another room", () => {
+    window.location.hash = `#?roomId=${encodeURIComponent(otherRoomId)}&password=other-room`;
+
+    expect(getKeyForRoom(roomId)).toBeNull();
+  });
+});
+
+describe("useRoomEncryptionSystem", () => {
+  afterEach(() => {
+    window.location.hash = "#";
+    localStorage.clear();
+  });
+
+  test("picks up a key saved for the room after it rendered", () => {
+    const { result } = renderHook(() => useRoomEncryptionSystem(roomId));
+    expect(result.current).toEqual({ kind: E2eeType.NONE });
+
+    act(() => saveKeyForRoom(roomId, "stored"));
+
+    expect(result.current).toEqual({
+      kind: E2eeType.SHARED_KEY,
+      secret: "stored",
+    });
+  });
+
+  test("stores a key from the parameters under the room's key only", () => {
+    window.location.hash = `#?roomId=${encodeURIComponent(roomId)}&password=from-the-link`;
+
+    renderHook(() => useRoomEncryptionSystem(roomId));
+
+    expect(Object.keys(localStorage)).toEqual([`room-shared-key-${roomId}`]);
+    expect(getKeyForRoom(roomId)).toBe("from-the-link");
+  });
+
+  test("keeps the stored key when the parameters are for another room", () => {
+    saveKeyForRoom(roomId, "stored");
+    window.location.hash = `#?roomId=${encodeURIComponent(otherRoomId)}&password=other-room`;
+
+    const { result } = renderHook(() => useRoomEncryptionSystem(roomId));
+
+    expect(result.current).toEqual({
+      kind: E2eeType.SHARED_KEY,
+      secret: "stored",
+    });
+    expect(localStorage.getItem(`room-shared-key-${roomId}`)).toBe("stored");
+  });
 });

src/e2ee/sharedKeyManagement.ts

@@ -13,7 +13,7 @@ import {
   useLocalStorage,
 } from "../useLocalStorage";
 import { getUrlParams, useUrlParams } from "../UrlParams";
-import { E2eeType } from "./e2eeType";
+import { E2eeType, type EncryptionSystem } from "@element-hq/matrixrtc-sdk";
 import { useClient } from "../ClientContext";
 
 /**
@@ -59,22 +59,27 @@ const useRoomSharedKey = (
 
 /**
  * The shared key for a room, preferring one supplied in the parameters Element
- * Call was started with over whatever is in local storage.
+ * Call was started with over whatever is in local storage. If the parameters
+ * are for another room, only the stored key is used, because the password in
+ * the parameters belongs to that other room.
  */
 function keyForRoom(
   roomId: string,
   paramsRoomId: string | null,
   password: string | null,
 ): string | null {
-  if (roomId !== paramsRoomId)
+  const storedKey = localStorage.getItem(
+    getRoomSharedKeyLocalStorageKey(roomId),
+  );
+  if (roomId !== paramsRoomId) {
     logger.warn(
       "requested key for a roomId which is not the current call room id (from the URL)",
       roomId,
       paramsRoomId,
     );
-  return (
-    password ?? localStorage.getItem(getRoomSharedKeyLocalStorageKey(roomId))
-  );
+    return storedKey;
+  }
+  return password ?? storedKey;
 }
 
 export function getKeyForRoom(roomId: string): string | null {
@@ -82,26 +87,19 @@ export function getKeyForRoom(roomId: string): string | null {
   return keyForRoom(roomId, paramsRoomId, password);
 }
 
-export type Unencrypted = { kind: E2eeType.NONE };
-export type SharedSecret = { kind: E2eeType.SHARED_KEY; secret: string };
-export type PerParticipantE2EE = { kind: E2eeType.PER_PARTICIPANT };
-export type EncryptionSystem = Unencrypted | SharedSecret | PerParticipantE2EE;
+export { type EncryptionSystem } from "@element-hq/matrixrtc-sdk";
 
 export function useRoomEncryptionSystem(roomId: string): EncryptionSystem {
   const { client } = useClient();
   const { roomId: paramsRoomId, password } = useUrlParams();
 
   const [storedPassword] = useRoomSharedKey(
-    // TODO: this passes an already-prefixed key where a room ID is expected, so
-    // the local storage key ends up prefixed twice and never matches what
-    // saveKeyForRoom writes. Preserved as-is here to keep this commit a pure
-    // refactor; the reactive read is effectively dead until it is fixed.
-    getRoomSharedKeyLocalStorageKey(roomId),
+    roomId,
     keyForRoom(roomId, paramsRoomId, password) ?? undefined,
   );
 
   const room = client?.getRoom(roomId);
-  const e2eeSystem = <EncryptionSystem>useMemo(() => {
+  const e2eeSystem = useMemo<EncryptionSystem>(() => {
     if (!room) return { kind: E2eeType.NONE };
     if (storedPassword)
       return {

src/grid/CallLayout.test.tsx

@@ -5,28 +5,25 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { act, render, screen } from "@testing-library/react";
-import { BehaviorSubject } from "rxjs";
-import { describe, expect, it, vi } from "vitest";
+import { render } from "@testing-library/react";
+import { expect, test, vi } from "vitest";
 
-import { arrangeTiles, type Bounds } from "./CallLayout";
+import { type CallLayout } from "./CallLayout";
 import { makeGridLayout } from "./GridLayout";
 import { makeSpotlightLandscapeLayout } from "./SpotlightLandscapeLayout";
 import { makeSpotlightPortraitLayout } from "./SpotlightPortraitLayout";
-import { type DragCallback, type LayoutProps } from "./Grid";
+import { type LayoutProps } from "./Grid";
 import { constant } from "../state/Behavior";
-import {
-  type Alignment,
-  type GridLayout,
-  type SpotlightLandscapeLayout,
-  type SpotlightPortraitLayout,
-} from "../state/layout-types";
+import { type Alignment } from "../state/layout-types";
 import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
 import {
   GridTileViewModel,
   SpotlightTileViewModel,
   type TileViewModel,
 } from "../state/TileViewModel";
+import { makeOneOnOneMobileLayout } from "./OneOnOneMobileLayout";
+import { makeSpotlightExpandedLayout } from "./SpotlightExpandedLayout";
+import { makeOneOnOneDesktopLayout } from "./OneOnOneDesktopLayout";
 
 // Grid supplies these hooks to the layouts it renders; on their own the
 // layouts have no grid to report to
@@ -36,152 +33,105 @@ vi.mock("./Grid", async (importOriginal) => ({
   useVisibleTiles: (): void => {},
 }));
 
-describe("grid layout", () => {
-  it("sizes the tiles from the minimum bounds", () => {
-    const minBounds$ = new BehaviorSubject<Bounds>({ width: 800, height: 600 });
-    const { scrolling: Scrolling } = makeGridLayout({ minBounds$ });
-    render(<Scrolling model={gridModel()} Slot={Slot} />);
-    const layer = screen.getByTestId("slot-0").parentElement!;
-    expect(layer.style.width).toBe("800px");
-    expect(layer.style.getPropertyValue("--width")).toBe(
-      `${Math.floor(arrangeTiles(800, 600, 3).tileWidth)}px`,
-    );
+const minBounds$ = constant({ width: 800, height: 800 });
 
-    act(() => minBounds$.next({ width: 400, height: 300 }));
-    expect(layer.style.width).toBe("400px");
-    expect(layer.style.getPropertyValue("--width")).toBe(
-      `${Math.floor(arrangeTiles(400, 300, 3).tileWidth)}px`,
-    );
-  });
-
-  it("moves the spotlight only when a drag reaches another corner", () => {
-    const model = gridModel();
-    const { fixed: Fixed } = makeGridLayout({ minBounds$: bounds });
-    render(<Fixed model={model} Slot={Slot} />);
-    const alignments: Alignment[] = [];
-    model.spotlightAlignment$.subscribe((a) => alignments.push(a));
-    const drag = drags.get("spotlight")!;
-
-    // Still in the bottom right corner
-    act(() => drag({ x: 0, y: 0, xRatio: 0.9, yRatio: 0.9 }));
-    // Over to the bottom left
-    act(() => drag({ x: 0, y: 0, xRatio: 0.1, yRatio: 0.9 }));
-    // And a little further, still bottom left
-    act(() => drag({ x: 0, y: 0, xRatio: 0.2, yRatio: 0.8 }));
-    // Up to the top left
-    act(() => drag({ x: 0, y: 0, xRatio: 0.2, yRatio: 0.1 }));
-
-    expect(alignments).toEqual([
-      { inline: "end", block: "end" },
-      { inline: "start", block: "end" },
-      { inline: "start", block: "start" },
-    ]);
-    const slot = screen.getByTestId("slot-spotlight");
-    expect(slot.getAttribute("data-inline-alignment")).toBe("start");
-    expect(slot.getAttribute("data-block-alignment")).toBe("start");
-  });
-});
-
-describe("spotlight landscape layout", () => {
-  it("puts the spotlight in the fixed layer and the grid in the scrolling one", () => {
-    const { fixed: Fixed, scrolling: Scrolling } = makeSpotlightLandscapeLayout(
-      { minBounds$: bounds },
-    );
-    const model = landscapeModel();
-    const { unmount } = render(<Fixed model={model} Slot={Slot} />);
-    expect(screen.getByTestId("slot-spotlight")).toBeInTheDocument();
-    unmount();
-
-    render(<Scrolling model={model} Slot={Slot} />);
-    expect(slotIds()).toEqual(model.grid.map((tile) => tile.id));
-  });
-});
-
-describe("spotlight portrait layout", () => {
-  it("sizes the grid tiles from the minimum width", () => {
-    const minBounds$ = new BehaviorSubject<Bounds>({ width: 300, height: 700 });
-    const { scrolling: Scrolling } = makeSpotlightPortraitLayout({
-      minBounds$,
-    });
-    const model = portraitModel();
-    render(<Scrolling model={model} Slot={Slot} />);
-    expect(slotIds()).toEqual(model.grid.map((tile) => tile.id));
-    const layer = screen.getByTestId(`slot-${model.grid[0].id}`).parentElement!
-      .parentElement!;
-    expect(layer.style.getPropertyValue("--grid-tile-width")).toBe(
-      `${Math.floor(arrangeTiles(300, 300, 3).tileWidth)}px`,
-    );
-
-    act(() => minBounds$.next({ width: 500, height: 700 }));
-    expect(layer.style.getPropertyValue("--grid-tile-width")).toBe(
-      `${Math.floor(arrangeTiles(500, 500, 3).tileWidth)}px`,
-    );
-  });
-});
-
-const bounds = constant<Bounds>({ width: 800, height: 600 });
-
-// The layouts only pass tile view models through to their slots
-const media = {} as UserMediaViewModel;
-const spotlight = new SpotlightTileViewModel(
-  constant([]),
-  constant(false),
-  constant("solid"),
+const spotlight = {
+  vm: new SpotlightTileViewModel(
+    constant([]),
+    constant(false),
+    constant("solid"),
+  ),
+  alignment$: constant<Alignment>({ inline: "end", block: "end" }),
+  onDrag: () => {},
+};
+const grid = Array.from(
+  { length: 3 },
+  () => new GridTileViewModel(constant({} as UserMediaViewModel)),
 );
-const tiles = (): GridTileViewModel[] =>
-  Array.from({ length: 3 }, () => new GridTileViewModel(constant(media)));
+const pip = {
+  alignment$: constant<Alignment>({ inline: "end", block: "end" }),
+  onDrag: () => {},
+  vm: { id: "pip" } as GridTileViewModel,
+  size$: constant("sm" as const),
+};
 
-function gridModel(): GridLayout {
-  return {
-    type: "grid",
-    spotlight,
-    grid: tiles(),
-    spotlightAlignment$: new BehaviorSubject<Alignment>({
-      inline: "end",
-      block: "end",
-    }),
-    setVisibleTiles: () => {},
-  };
-}
-
-function landscapeModel(): SpotlightLandscapeLayout {
-  return {
-    type: "spotlight-landscape",
-    spotlight,
-    grid: tiles(),
-    setVisibleTiles: () => {},
-  };
-}
-
-function portraitModel(): SpotlightPortraitLayout {
-  return {
-    type: "spotlight-portrait",
-    spotlight,
-    grid: tiles(),
-    setVisibleTiles: () => {},
-  };
-}
-
-// Slots record their drag callback so a test can drag them, and show the id
-// of the tile they hold
-const drags = new Map<string, DragCallback | undefined>();
 const Slot: LayoutProps<unknown, TileViewModel, HTMLDivElement>["Slot"] = ({
   id,
   model,
   onDrag,
+  children,
   ...props
-}) => {
-  drags.set(id, onDrag);
-  return (
-    <div data-testid={`slot-${id}`} {...props}>
-      {model instanceof GridTileViewModel ? model.id : "spotlight"}
-    </div>
-  );
-};
+}) => (
+  <div data-testid={`slot-${id}`} {...props}>
+    {children}
+  </div>
+);
 
-function slotIds(): string[] {
-  return screen
-    .getAllByTestId(/^slot-/)
-    .map((slot) => slot.textContent ?? "")
-    .filter((id) => id !== "spotlight");
+function snapshotLayout<Model>(layout: CallLayout<Model>, model: Model): void {
+  const {
+    foreground,
+    scrolling: Scrolling,
+    fixed: Fixed,
+  } = layout({ minBounds$ });
+  const scrollingLayer = <Scrolling model={model} Slot={Slot} />;
+  const fixedLayer = <Fixed model={model} Slot={Slot} />;
+  const { container } = render(
+    foreground === "fixed" ? (
+      <>
+        {scrollingLayer}
+        {fixedLayer}
+      </>
+    ) : (
+      <>
+        {fixedLayer}
+        {scrollingLayer}
+      </>
+    ),
+  );
+  expect(container).toMatchSnapshot();
 }
+
+test("grid layout", () =>
+  snapshotLayout(makeGridLayout, {
+    type: "grid",
+    spotlight,
+    grid,
+    setVisibleTiles: () => {},
+  }));
+
+test("spotlight landscape layout", () =>
+  snapshotLayout(makeSpotlightLandscapeLayout, {
+    type: "spotlight-landscape",
+    spotlight,
+    grid,
+    setVisibleTiles: () => {},
+  }));
+
+test("spotlight portrait layout", () =>
+  snapshotLayout(makeSpotlightPortraitLayout, {
+    type: "spotlight-portrait",
+    spotlight,
+    grid,
+    setVisibleTiles: () => {},
+  }));
+
+test("spotlight expanded layout", () =>
+  snapshotLayout(makeSpotlightExpandedLayout, {
+    type: "spotlight-expanded",
+    spotlight: spotlight.vm,
+    pip,
+  }));
+
+test("one-on-one desktop layout", () =>
+  snapshotLayout(makeOneOnOneDesktopLayout, {
+    type: "one-on-one-desktop",
+    spotlight: { id: "spotlight" } as GridTileViewModel,
+    pip,
+  }));
+
+test("one-on-one mobile layout", () =>
+  snapshotLayout(makeOneOnOneMobileLayout, {
+    type: "one-on-one-mobile",
+    spotlight: spotlight.vm,
+    pip,
+  }));