Run the media backend inside a combination of backends

The participation now builds its default backend by combining every backend
the SDK has, publishing on the type of the transport the host names (LiveKit
without one) and handing each remote member to the backend of its transport.
LiveKit is the only member of the combination for now.

The LiveKit backend can run without a local transport, so it subscribes only
while another backend publishes, and probes the homeserver for delayed leave
delegation only when it first publishes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

sdk/SdkArchitecture.md

@@ -66,9 +66,13 @@ media backend: whatever carries the media, behind the `MediaBackend` interface i
 `sdk/src/media-backend/api.ts`. The client is the join of the two: a member exists
 once its membership does, and gets its media once the backend has something
 for that membership on its transport; `state$` is the membership state combined
-with the local media connection. LiveKit is the one backend today; a
-cascading SFU or full mesh is another folder under `sdk/src/media-backend/`, handed
-in through `options.backend`, and nothing in the left box changes.
+with the local media connection. LiveKit (`livekit/`) is the one backend today,
+and the client runs it inside a combination (`combineBackends.ts`). It publishes
+on the backend of the type of `options.transport`, LiveKit without one, and
+hands every remote member to the backend of that member's transport. A backend
+nobody is on connects nowhere. A cascading SFU or full mesh is another folder
+under `sdk/src/media-backend/`, added to the combination, and nothing in the
+left box changes; `options.backend` replaces the combination altogether.
 
 ### How Element Call uses it
 
@@ -149,7 +153,7 @@ the sections below still describe `MatrixRTCClient` until that lands.
 | publish request             | a `PublishRequest`: a source, where to capture it from and how to encode it. Given at the join in `publish`, or later to `publish` on the participation                                                                                                |                                                                                                             |
 | attach, detach              | handing a `<video>` or `<audio>` element to a track and taking it back. The SDK owns the stream and the observers on the element                                                                                                                       | LiveKit `Track.attach`                                                                                      |
 | transport                   | an `RTCTransport`: where media is exchanged, a LiveKit service url today. Carries the raw js-sdk `Transport` from the membership and, in `resolved$`, what the backend fetched to connect to it                                                        | MSC4143 "focus"                                                                                             |
-| media backend               | a `MediaBackend`: what carries the media, behind `sdk/src/media-backend/api.ts`. One per participation, chosen with `backend` in the join options; LiveKit under `sdk/src/media-backend/livekit/` is the one that exists                               | "SFU", a LiveKit room, a peer connection                                                                    |
+| media backend               | a `MediaBackend`: what carries the media, behind `sdk/src/media-backend/api.ts`. One per participation: the SDK's backends combined, publishing on the type of `transport` in the join options, unless `backend` replaces them                         | "SFU", a LiveKit room, a peer connection                                                                    |
 
 Names use `RTC` in capitals, as the js-sdk does: `RTCSlot`, `RTCParticipation`,
 `RTCMember`, `matrixRTCMode`.

sdk/src/RTCParticipation.ts

@@ -50,6 +50,7 @@ import {
   type MediaBackendContext,
   type MediaKey,
 } from "./media-backend/api";
+import { combineBackends } from "./media-backend/combineBackends";
 import { createLivekitBackend } from "./media-backend/livekit/LivekitBackend";
 import { findRemotePublishLivekitTransport } from "./media-backend/livekit/RemotePublishTransport";
 import { createHomeserverConnected$ } from "./matrixrtc/HomeserverConnected";
@@ -310,15 +311,23 @@ export function createRTCParticipation(
   };
 }
 
-/** The host's backend, or LiveKit configured from the participation options. */
+/**
+ * The host's backend, or every backend the SDK has, combined: we publish on
+ * the type of the transport the host named (LiveKit without one) and receive
+ * from members on any of them.
+ */
 function createBackend(
   scope: ObservableScope,
   client: MatrixClient,
-  { backend, mediaQuality }: RTCParticipationOptions,
+  { backend, mediaQuality, transport }: RTCParticipationOptions,
   context: MediaBackendContext,
 ): MediaBackend {
   if (backend) return backend(scope, context);
-  return createLivekitBackend(scope, context, { client, mediaQuality });
+  return combineBackends(
+    scope,
+    [createLivekitBackend(scope, context, { client, mediaQuality })],
+    transport?.type ?? "livekit",
+  );
 }
 
 /** The slot's per-participant keys: the ones it already holds on subscribe, then each new one. */

sdk/src/media-backend/combineBackends.test.ts

@@ -0,0 +1,91 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { describe, expect, it, vi } from "vitest";
+import { BehaviorSubject, NEVER } from "rxjs";
+
+import { type RTCMember, type TransportMetadata } from "../api";
+import { type MemberMedia } from "../media-api";
+import { constant } from "../reactive/Behavior";
+import { ObservableScope } from "../reactive/ObservableScope";
+import { type MediaBackend } from "./api";
+import { combineBackends } from "./combineBackends";
+
+function fakeBackend(type: string): MediaBackend & { ended: string[] } {
+  const ended: string[] = [];
+  return {
+    ended,
+    transportType: type,
+    prepareLocalTransport: vi.fn(async () =>
+      Promise.resolve({ canDelegateDelayedLeave: false }),
+    ),
+    delegateDelayedLeave: vi.fn(),
+    local: {} as MediaBackend["local"],
+    mediaFor: (scope, member): MemberMedia => {
+      scope.onEnd(() => ended.push(member.memberId));
+      return {
+        tracks$: constant([{ id: `${type}:${member.memberId}` }] as never),
+        encryptionError$: NEVER,
+      };
+    },
+    connections$: constant([
+      { transport: { type }, state: "connected" as never, resolved: undefined },
+    ]),
+    setAudioOutputDeviceId: vi.fn(),
+    sendData: vi.fn(),
+    data$: NEVER,
+  };
+}
+
+function transport(type: string): TransportMetadata {
+  return { type, id: type, raw: { type }, resolved$: constant(undefined) };
+}
+
+describe("combineBackends", () => {
+  it("routes each member to the backend of its transport type", () => {
+    const scope = new ObservableScope();
+    const livekit = fakeBackend("livekit");
+    const other = fakeBackend("other");
+    const combined = combineBackends(scope, [livekit, other], "livekit");
+    expect(combined.transportType).toBe("livekit");
+    expect(combined.connections$.value).toHaveLength(2);
+
+    const transport$ = new BehaviorSubject<TransportMetadata | undefined>(
+      transport("other"),
+    );
+    const memberScope = new ObservableScope();
+    const media = combined.mediaFor(memberScope, {
+      memberId: "m1",
+      transport$,
+    } as unknown as RTCMember);
+    expect(media.tracks$.value).toEqual([{ id: "other:m1" }]);
+
+    transport$.next(transport("livekit"));
+    expect(other.ended).toEqual(["m1"]);
+    expect(media.tracks$.value).toEqual([{ id: "livekit:m1" }]);
+
+    transport$.next(transport("unknown"));
+    expect(media.tracks$.value).toBeNull();
+    // The publishing backend keeps every member for as long as it is one
+    expect(livekit.ended).toEqual([]);
+    memberScope.end();
+    expect(livekit.ended).toEqual(["m1"]);
+    scope.end();
+  });
+
+  it("publishes on the backend of the publish type", () => {
+    const scope = new ObservableScope();
+    const livekit = fakeBackend("livekit");
+    const other = fakeBackend("other");
+    const combined = combineBackends(scope, [livekit, other], "other");
+    expect(combined.transportType).toBe("other");
+    expect(() => combineBackends(scope, [livekit, other], "unknown")).toThrow(
+      "No media backend for transport unknown",
+    );
+    scope.end();
+  });
+});

sdk/src/media-backend/combineBackends.ts

@@ -0,0 +1,80 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { combineLatest, map, merge, NEVER, of, switchMap } from "rxjs";
+
+import { type RTCMember } from "../api";
+import { MatrixRTCError } from "../errors";
+import { type MemberMedia } from "../media-api";
+import { type ObservableScope } from "../reactive/ObservableScope";
+import { mapScoped } from "../utils/mapScoped";
+import { type MediaBackend } from "./api";
+
+/**
+ * Several backends as one: we publish on the backend of `publishType`, and
+ * receive from every member on a transport any of the backends serves. A
+ * member is handed to the backend of its current transport type. A backend
+ * nobody is on costs nothing: it connects only for the members handed to it.
+ */
+export function combineBackends(
+  scope: ObservableScope,
+  all: MediaBackend[],
+  publishType: string,
+): MediaBackend {
+  const byType = new Map(all.map((b) => [b.transportType, b]));
+  const publish = byType.get(publishType);
+  if (publish === undefined)
+    throw new MatrixRTCError(`No media backend for transport ${publishType}`);
+
+  const mediaFor = (
+    memberScope: ObservableScope,
+    member: RTCMember,
+  ): MemberMedia => {
+    // The publishing backend sees every member, whatever its transport, so
+    // that it can tell members from strangers when a backend is asked for our
+    // media by members on other transports. It connects nowhere for them
+    const publishMedia = publish.mediaFor(memberScope, member);
+    const type$ = memberScope.behavior(
+      member.transport$.pipe(map((transport) => transport?.type)),
+    );
+    const media$ = mapScoped(memberScope, type$, (s, type) =>
+      type === publish.transportType
+        ? publishMedia
+        : byType.get(type)?.mediaFor(s, member),
+    );
+    return {
+      tracks$: memberScope.behavior(
+        media$.pipe(switchMap((media) => media?.tracks$ ?? of(null))),
+      ),
+      encryptionError$: media$.pipe(
+        switchMap((media) => media?.encryptionError$ ?? NEVER),
+      ),
+    };
+  };
+
+  return {
+    transportType: publish.transportType,
+    prepareLocalTransport: async (transport) =>
+      publish.prepareLocalTransport(transport),
+    delegateDelayedLeave: async (delayId) =>
+      publish.delegateDelayedLeave(delayId),
+    local: publish.local,
+    mediaFor,
+    connections$: scope.behavior(
+      combineLatest(all.map((b) => b.connections$)).pipe(
+        map((lists) => lists.flat()),
+      ),
+    ),
+    setAudioOutputDeviceId: async (deviceId) => {
+      await Promise.all(
+        all.map(async (b) => b.setAudioOutputDeviceId(deviceId)),
+      );
+    },
+    sendData: async (topic, text) => publish.sendData(topic, text),
+    data$: merge(...all.map((b) => b.data$)),
+  };
+}

sdk/src/media-backend/livekit/ConnectionMap.ts

@@ -101,8 +101,11 @@ export class ConnectionMap {
 interface Props {
   scope: ObservableScope;
   connectionFactory: ConnectionFactory;
-  /** The transport we publish on; the one connection with the publisher role. */
-  localTransport: TransportLocator;
+  /**
+   * The transport we publish on; the one connection with the publisher role.
+   * Null for a backend that only subscribes, while another backend publishes.
+   */
+  localTransport: TransportLocator | null;
   /** Every transport some member is on, deduplicated against the local one. */
   remoteTransports$: Behavior<Epoch<TransportLocator[]>>;
   logger: Logger;
@@ -128,7 +131,11 @@ export function createConnectionMap$({
     scope.behavior(
       remoteTransports$.pipe(
         mapEpoch((remoteTransports) =>
-          removeDuplicateTransports([localTransport, ...remoteTransports]),
+          removeDuplicateTransports(
+            localTransport
+              ? [localTransport, ...remoteTransports]
+              : remoteTransports,
+          ),
         ),
         tap((transports) =>
           logger.debug("localAndRemoteTransports$ = ", transports),
@@ -136,7 +143,7 @@ export function createConnectionMap$({
       ),
     );
 
-  const localTransportKey = keyFor(localTransport);
+  const localTransportKey = localTransport && keyFor(localTransport);
 
   const connections$ = scope.behavior(
     localAndRemoteTransports$.pipe(

sdk/src/media-backend/livekit/LivekitBackend.ts

@@ -115,9 +115,14 @@ export function createLivekitBackend(
   );
 
   // A new map, in a scope of its own, whenever the local transport changes:
-  // the publisher role sits on one connection, and which one is fixed per map
+  // the publisher role sits on one connection, and which one is fixed per map.
+  // Without a local transport the map only subscribes, for when another
+  // backend publishes; boxed so that `mapScoped` builds a map for null too
+  const boxedLocalTransport$ = scope.behavior(
+    localTransport$.pipe(map((localTransport) => ({ localTransport }))),
+  );
   const connectionMap$ = scope.behavior<Epoch<ConnectionMap>>(
-    mapScoped(scope, localTransport$, (mapScope, localTransport) =>
+    mapScoped(scope, boxedLocalTransport$, (mapScope, { localTransport }) =>
       createConnectionMap$({
         scope: mapScope,
         connectionFactory,
@@ -139,12 +144,15 @@ export function createLivekitBackend(
     logger,
   });
 
-  // Probed once: the answer cannot change within a session
-  const homeserverSupportsDelegation = supportsDelegation(
-    `${client.baseUrl}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`,
-    "homeserver",
-    logger,
-  );
+  // Probed once, when first needed: the answer cannot change within a
+  // session, and a backend that only subscribes never needs it
+  let homeserverProbe: Promise<boolean> | undefined;
+  const homeserverSupportsDelegation = async (): Promise<boolean> =>
+    (homeserverProbe ??= supportsDelegation(
+      `${client.baseUrl}/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave`,
+      "homeserver",
+      logger,
+    ));
 
   const ownServerName = serverNameOf(ownMembershipIdentity.userId);
 
@@ -176,7 +184,7 @@ export function createLivekitBackend(
               `transport ${flow.serviceUrl}`,
               logger,
             )
-          : homeserverSupportsDelegation;
+          : homeserverSupportsDelegation();
     }
     localTransport$.next(locator);
     // The membership is only sent once the publisher connection holds its