flatten media api (no MemberMedia wrapper object

sdk/README.md

@@ -39,7 +39,7 @@ const rtcClient = createMatrixRTCClient(
 rtcClient.join();
 
 rtcClient.remoteMembers$.subscribe((members) => {
-  // each member has displayName$, media$ and more; a media track is rendered
+  // each member has displayName$, tracks$ and more; a media track is rendered
   // by handing it a <video> or <audio> element: track.attach(element)
 });
 

sdk/SdkArchitecture.md

@@ -29,8 +29,7 @@ therefore `Element Call → sdk`, never the reverse. The migration from today's
 │     keyRotationSuppressed$ connectedTransports$                              │
 │                                                                              │
 │   per member    RTCMember   { id userId deviceId membership$ displayName$    │
-│                               avatarUrl$ transport$ media$ }                 │
+│                               avatarUrl$ transport$ tracks$ encryptionError$ }│
-│   per media     MemberMedia { tracks$ encryptionError$ }                     │
 │   per track     MediaTrack  { source kind muted$ encrypted$ stats$ attach() }│
 │                                                                              │
 │  ┌───────────────────────────┐  MediaBackend   ┌───────────────────────────┐ │
@@ -80,7 +79,7 @@ in through `options.backend`, and nothing in the left box changes.
 │ CallViewModel                                                  │
 │   layout, ringing and notifications, auto-leave, sounds,       │
 │   reactions, hand raise, settings, audio routing, host bridge  │
-│    ▲ status$, localMember$, remoteMembers$, MemberMedia        │
+│    ▲ status$, localMember$, remoteMembers$, tracks$            │
 │ MatrixRTCClient                   @element-hq/matrixrtc-sdk    │
 └────────────────────────────────────────────────────────────────┘
      │ MatrixClient + Room                        │ livekit-client
@@ -91,7 +90,7 @@ in through `options.backend`, and nothing in the left box changes.
 All three modes already hold a `MatrixClient` and a `Room` by the time a call
 starts: standalone after `useLoadGroupCall`, the component from the host's `client`
 prop, the widget from `createRoomWidgetClient`. The view model passes them on and
-builds its media view models from `RTCMember` and `MemberMedia` instead of from
+builds its media view models from `RTCMember` and its `tracks$` instead of from
 LiveKit participants.
 
 ### Joining, in order
@@ -112,7 +111,7 @@ LiveKit participants.
   │             │ create tracks, resume upstream ──────────────────────────────▶│
   │             │ media key to every other member (to-device) ──▶│               │
   │             │◀── other memberships (sync) ──────────│◀── participants ────────│
-  │◀─ status$ "connected", localMember$, remoteMembers$ with media$ ──────────────│
+  │◀─ status$ "connected", localMember$, remoteMembers$ with tracks$ ─────────────│
 ```
 
 The session discovers the transport from the homeserver and the backend prepares
@@ -132,7 +131,7 @@ it, and `leave()` also sends the leave.
 | client            | a `MatrixRTCClient`: one room, one slot, one local member                                                                                                                                             | `MatrixClient` is always written out                              |
 | membership        | one device's MatrixRTC state event, seen by hosts as `RTCMembership`, the js-sdk `CallMembership` behind it                                                                                           |                                                                   |
 | member            | an `RTCMember`: a membership plus what the SDK derives from it (display name, transport, media)                                                                                                       | Element Call's "matrixLivekitMember"                              |
-| media             | a `MemberMedia`: what a member sends, once it has arrived on its transport. `null` while waiting for media                                                                                            | Element Call's `MediaViewModel` is a view of it                   |
+| media             | what a member sends, once it has arrived on its transport: `tracks$` on the member, `null` while waiting for media, and `encryptionError$` beside it. The backend supplies the same two fields        | Element Call's `MediaViewModel` is a view of it                   |
 | track             | a `MediaTrack`: one publication of a member, audio or video, with `source`, `kind`, `id` and the behaviors on it. `LocalAudioMediaTrack` and `LocalVideoMediaTrack` add the controls over our own     | LiveKit `TrackPublication` and `Track`, which never leave the SDK |
 | source            | a `MediaSource`: what a track is captured from, `microphone`, `camera`, `screenShare`, `screenShareAudio` or `unknown`. Picked out of `tracks$` with `trackBySource$`                                 | LiveKit `Track.Source`                                            |
 | publish request   | a `PublishRequest`: a source, where to capture it from and how to encode it. Given at the join in `options.publish`, or later to `publish` on the local member                                        |                                                                   |
@@ -396,12 +395,16 @@ export interface RTCMember {
   /** Which transport this member is on; undefined when the membership names none. */
   transport$: Behavior<TransportMetadata | undefined>;
   /**
-   * Null while the member has a transport but no participant has shown up on
+   * The member's tracks, in publication order, once it has shown up on its
-   * it yet ("waiting for media"). Hand raise and reactions are room events
+   * transport; null until then ("waiting for media"). An entry stays the same
-   * keyed by member and stay with the host for now; they are the obvious next
+   * object for as long as the same publication is behind it. Which track is
-   * fields here.
+   * which is in its `source`; `trackBySource$` picks one out. Hand raise and
+   * reactions are room events keyed by member and stay with the host for now;
+   * they are the obvious next fields here.
    */
-  media$: Behavior<MemberMedia | null>;
+  tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[] | null>;
+  /** Emits when the SFU reports a key problem for this member. */
+  encryptionError$: Observable<EncryptionError>;
 }
 
 export interface RemoteRTCMember extends RTCMember {
@@ -410,7 +413,7 @@ export interface RemoteRTCMember extends RTCMember {
 
 export interface LocalRTCMember extends RTCMember {
   local: true;
-  media$: Behavior<LocalMemberMedia | null>;
+  tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>;
   /**
    * Publishes a source and resolves with its track once it is in `tracks$`.
    * Before the transport is connected the request is remembered and applied
@@ -505,6 +508,9 @@ and full mesh has one peer connection per member and no token service. So a
 backend is handed the whole session's view and gives back media:
 
 ```ts
+/** The fields of a member the backend supplies, as the member exposes them. */
+type MemberMediaFields = Pick<RTCMember, "tracks$" | "encryptionError$">;
+
 export interface MediaBackend {
   readonly transportType: string; // "livekit"
   /** Authenticate with the transport the client found; the probe for delegation runs here. */
@@ -513,12 +519,12 @@ export interface MediaBackend {
   ): Promise<{ canDelegateDelayedLeave: boolean }>;
   /** Hand over the delayed leave, each time the membership manager has a new delay id. */
   delegateDelayedLeave(delayId: string): Promise<void>;
-  readonly local: LocalMediaBackend; // connectionState$ media$ setPublishing publishError$ publish unpublish
+  readonly local: LocalMediaBackend; // tracks$ encryptionError$ connectionState$ setPublishing publishError$ publish unpublish
   /** A remote member's media; also how the backend learns which transports to connect to. */
   mediaFor$(
     scope: ObservableScope,
     membership$: Behavior<RTCMembership>,
-  ): Behavior<MemberMedia | null>;
+  ): MemberMediaFields;
   readonly connections$: Behavior<BackendConnection[]>; // transport, state, resolved; diagnostics
   sendData(topic: string, text: string): Promise<void>;
   readonly data$: Observable<DataPacket>;
@@ -543,13 +549,16 @@ without one, LiveKit is built from the LiveKit-flavoured client options. The
 reasoning behind each of these choices is in
 [`MediaBackendPlan.md`](./MediaBackendPlan.md).
 
-## Media: `MemberMedia`
+## Media: `tracks$` and `MediaTrack`
 
 Every member has media. Today, in Element Call, that media _is_ a `livekit-client`
 participant, handed straight through to the view models. In the SDK the participant
-is wrapped in a `MemberMedia`, so that nothing outside the SDK imports
+becomes the member's `tracks$`, a list of `MediaTrack`s, so that nothing outside
-`Participant`, `TrackPublication`, `Track` or `TrackReference`. The adapter in
+the SDK imports `Participant`, `TrackPublication`, `Track` or `TrackReference`.
-`sdk/src/media-backend/livekit/` is the only place that reads a participant.
+The adapter in `sdk/src/media-backend/livekit/` is the only place that reads a
+participant. The backend supplies the same two fields through `mediaFor$`, so
+there is one shape from the LiveKit participant to the host and nobody has to
+subscribe through one Behavior to reach another.
 
 ```ts
 export type MediaSource =
@@ -632,34 +641,12 @@ export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack {
 
 export type EncryptionError = "MissingKey" | "InvalidKey";
 
-/**
+/** The member's first track of a source; undefined while there is none, or no media yet. */
- * The media of one member, backed by a LiveKit participant inside the SDK.
- * There is no identity field: the LiveKit identity is the member's `id`, and
- * the SDK does the matching before a MemberMedia exists.
- */
-export interface MemberMedia {
-  local: boolean;
-  /**
-   * One entry per published track, in publication order. An entry stays the
-   * same object for as long as the same publication is behind it. Which
-   * track is which is in its `source`; `trackBySource$` picks one out.
-   */
-  tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[]>;
-  /** Emits when the SFU reports a key problem for this member. */
-  encryptionError$: Observable<EncryptionError>;
-}
-
-/** The member's first track of a source; undefined while there is none. */
 export function trackBySource$<S extends MediaSource>(
   scope: ObservableScope,
-  tracks$: Behavior<MediaTrack[]>,
+  tracks$: Behavior<MediaTrack[] | null>,
   source: S,
 ): Behavior<TrackOfSource<S> | undefined>;
-
-export interface LocalMemberMedia extends MemberMedia {
-  local: true;
-  tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]>;
-}
 ```
 
 ### Rendering: `attach` and `detach`
@@ -708,7 +695,7 @@ Layout:
 sdk/
   index.ts            the entry point: re-exports the API and the primitives
   src/api.ts          the public types above: client, members, transports
-  src/media-api.ts    the public media types: publish requests, tracks, MemberMedia
+  src/media-api.ts    the public media types: publish requests, tracks
   src/errors.ts       MatrixRTCError and its codes
   src/config.ts       MatrixRTCMode, session timings, media quality
   src/encryption.ts   E2eeType, EncryptionSystem
@@ -717,7 +704,7 @@ sdk/
   src/matrixrtc/      the MatrixRTC side: the local member, memberships, member
                       metadata, transports, discovery, status, the js-sdk join
   src/media-backend/  the MediaBackend interface in api.ts; livekit/ is the one
-                      backend: connections, publisher, JWT, keys, MemberMedia, MediaTrack
+                      backend: connections, publisher, JWT, keys, the tracks
   src/utils/          LazyBehavior, mapScoped, network retry, display names, test helpers
   dev/                the harness (below)
   SdkArchitecture.md  this document
@@ -743,7 +730,7 @@ homeserver rate-limits); join the room and wait for the sync to hold it; create
 client with microphone and camera enabled; show one tile per member with the display
 name, a `<video>` attached from the member's `"camera"` track and, for remote
 members, an `<audio>` from the `"microphone"` track, both picked out of
-`media$.tracks$` with `trackBySource$`; toggle the microphone and camera; leave.
+`tracks$` with `trackBySource$`; toggle the microphone and camera; leave.
 
 Every media element carries the track's state as `data-*` attributes (`muted`,
 `encrypted`, `active`, `frameWidth`, `frames`), which is what the tests read to

sdk/dev/main.ts

@@ -121,9 +121,7 @@ async function setPublished(
 ): Promise<boolean> {
   const member = rtcClient.localMember$.value;
   if (member === null) return !enabled;
-  const track = member.media$.value?.tracks$.value.find(
+  const track = member.tracks$.value?.find((t) => t.source === source);
-    (t) => t.source === source,
-  );
   if (track && source === "screenShare") {
     if (!enabled) await member.unpublish(track.id);
     return enabled;
@@ -203,19 +201,16 @@ function memberTile(scope: ObservableScope, member: RTCMember): HTMLElement {
     name.textContent = n;
   });
 
-  const tracks$ = scope.behavior(
-    member.media$.pipe(switchMap((media) => media?.tracks$ ?? of([]))),
-  );
   render(
     scope,
-    trackBySource$(scope, tracks$, "camera"),
+    trackBySource$(scope, member.tracks$, "camera"),
     tile.appendChild(document.createElement("video")),
   );
   // Our own microphone would only echo
   if (!member.local)
     render(
       scope,
-      trackBySource$(scope, tracks$, "microphone"),
+      trackBySource$(scope, member.tracks$, "microphone"),
       tile.appendChild(document.createElement("audio")),
     );
   return tile;

sdk/src/api.ts

@@ -20,11 +20,12 @@ import { type Observable } from "rxjs";
 
 import { type Behavior } from "./reactive/Behavior";
 import {
+  type AudioMediaTrack,
+  type EncryptionError,
   type LocalAudioMediaTrack,
-  type LocalMemberMedia,
   type LocalVideoMediaTrack,
-  type MemberMedia,
   type PublishRequest,
+  type VideoMediaTrack,
 } from "./media-api";
 import { type MediaBackendFactory } from "./media-backend/api";
 import { type EncryptionSystem } from "./encryption";
@@ -219,10 +220,14 @@ export interface RTCMember {
   /** Which transport this member is on; undefined when the membership has none. */
   transport$: Behavior<TransportMetadata | undefined>;
   /**
-   * Null while the member has a transport but no media has arrived on it yet
+   * The member's tracks, in publication order, once it has shown up on its
-   * ("waiting for media").
+   * transport; null until then ("waiting for media"). An entry stays the same
+   * object for as long as the same publication is behind it. Which track is
+   * which is in its `source`; `trackBySource$` picks one out.
    */
-  media$: Behavior<MemberMedia | null>;
+  tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[] | null>;
+  /** Emits when the SFU reports a key problem for this member. */
+  encryptionError$: Observable<EncryptionError>;
 }
 
 export interface RemoteRTCMember extends RTCMember {
@@ -231,7 +236,7 @@ export interface RemoteRTCMember extends RTCMember {
 
 export interface LocalRTCMember extends RTCMember {
   local: true;
-  media$: Behavior<LocalMemberMedia | null>;
+  tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>;
   /**
    * Publishes a source and resolves with its track once it is in `tracks$`.
    * Before the transport is connected the request is remembered and applied

sdk/src/matrixrtc/LocalMember.test.ts

@@ -7,7 +7,13 @@ Please see LICENSE in the repository root for full details.
 
 import { logger } from "matrix-js-sdk/lib/logger";
 import { Status, type Transport } from "matrix-js-sdk/lib/matrixrtc";
-import { BehaviorSubject, type Observable, Subject, throwError } from "rxjs";
+import {
+  BehaviorSubject,
+  EMPTY,
+  type Observable,
+  Subject,
+  throwError,
+} from "rxjs";
 import { describe, expect, it, vi } from "vitest";
 
 import { defaultSessionTimings } from "../config";
@@ -192,7 +198,8 @@ function fakeLocalMediaBackend(): FakeLocalMediaBackend {
     connectionState$: new BehaviorSubject<MediaConnectionState | Error>(
       MediaConnectionState.Initialized,
     ),
-    media$: new BehaviorSubject(null),
+    tracks$: new BehaviorSubject(null),
+    encryptionError$: EMPTY,
     setPublishing: vi.fn<LocalMediaBackend["setPublishing"]>(),
     publishError$: new BehaviorSubject<Error | null>(null),
     publish: vi.fn<LocalMediaBackend["publish"]>(),

sdk/src/matrixrtc/Members.ts

@@ -16,8 +16,10 @@ import {
   type RemoteRTCMember,
   type RTCMember,
 } from "../api";
-import { type MemberMedia } from "../media-api";
+import {
-import { type LocalMediaBackend } from "../media-backend/api";
+  type LocalMediaBackend,
+  type MemberMediaFields,
+} from "../media-backend/api";
 import { type TransportRegistry } from "./Transports";
 
 /** What every member is built from, besides its own membership and media. */
@@ -41,26 +43,31 @@ export function membershipKeys(
 export function createRemoteRTCMember(
   scope: ObservableScope,
   membership$: Behavior<CallMembership>,
-  media$: Behavior<MemberMedia | null>,
+  media: MemberMediaFields,
   context: MemberContext,
 ): RemoteRTCMember {
   return {
     ...createRTCMember(scope, membership$, context),
     local: false,
-    media$,
+    tracks$: media.tracks$,
+    encryptionError$: media.encryptionError$,
   };
 }
 
 export function createLocalRTCMember(
   scope: ObservableScope,
   membership$: Behavior<CallMembership>,
-  local: Pick<LocalMediaBackend, "media$" | "publish" | "unpublish">,
+  local: Pick<
+    LocalMediaBackend,
+    "tracks$" | "encryptionError$" | "publish" | "unpublish"
+  >,
   context: MemberContext,
 ): LocalRTCMember {
   return {
     ...createRTCMember(scope, membership$, context),
     local: true,
-    media$: local.media$,
+    tracks$: local.tracks$,
+    encryptionError$: local.encryptionError$,
     publish: async (request) => local.publish(request),
     unpublish: async (id) => local.unpublish(id),
   };
@@ -70,7 +77,7 @@ function createRTCMember(
   scope: ObservableScope,
   membership$: Behavior<CallMembership>,
   { metadata, transports }: MemberContext,
-): Omit<RTCMember, "local" | "media$"> {
+): Omit<RTCMember, "local" | "tracks$" | "encryptionError$"> {
   const { userId, deviceId, rtcBackendIdentity } = membership$.value;
   return {
     id: rtcBackendIdentity,

sdk/src/media-api.ts

@@ -16,8 +16,6 @@ import {
   type TrackProcessor,
   type VideoCodec,
 } from "livekit-client";
-import { type Observable } from "rxjs";
-
 import { type Behavior } from "./reactive/Behavior";
 
 export interface AudioCaptureSettings {
@@ -137,25 +135,3 @@ export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack {
 }
 
 export type EncryptionError = "MissingKey" | "InvalidKey";
-
-/**
- * The media of one member, backed by a LiveKit participant inside the SDK.
- * There is no identity field: the LiveKit identity is the member's `id`.
- */
-export interface MemberMedia {
-  local: boolean;
-  /**
-   * One entry per published track, in publication order. An entry stays the
-   * same object for as long as the same publication is behind it. Which
-   * track is which is in its `source`; `trackBySource$` picks one out.
-   */
-  tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[]>;
-
-  /** Emits when the SFU reports a key problem for this member. */
-  encryptionError$: Observable<EncryptionError>;
-}
-
-export interface LocalMemberMedia extends MemberMedia {
-  local: true;
-  tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]>;
-}

sdk/src/media-backend/api.ts

@@ -17,12 +17,15 @@ import { type Transport } from "matrix-js-sdk/lib/matrixrtc";
 import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
 import { type Observable } from "rxjs";
 
-import { type ResolvedTransport, type RTCMembership } from "../api";
+import {
+  type LocalRTCMember,
+  type ResolvedTransport,
+  type RTCMember,
+  type RTCMembership,
+} from "../api";
 import {
   type LocalAudioMediaTrack,
-  type LocalMemberMedia,
   type LocalVideoMediaTrack,
-  type MemberMedia,
   type PublishRequest,
 } from "../media-api";
 import { type SessionTimings } from "../config";
@@ -50,6 +53,13 @@ export interface MediaKey {
   key: Uint8Array<ArrayBuffer>;
 }
 
+/** The fields of a member the backend supplies, as the member exposes them. */
+export type MemberMediaFields = Pick<RTCMember, "tracks$" | "encryptionError$">;
+export type LocalMemberMediaFields = Pick<
+  LocalRTCMember,
+  "tracks$" | "encryptionError$"
+>;
+
 export interface MediaBackendContext {
   roomId: string;
   ownMembershipIdentity: CallMembershipIdentityParts;
@@ -132,16 +142,17 @@ export interface MediaBackend {
   readonly local: LocalMediaBackend;
 
   /**
-   * The media of one remote member, null while nothing has arrived for it on
+   * The media of one remote member: its tracks, null while nothing has
-   * its transport. Built in `scope`, which the caller ends with the member.
+   * arrived for it on its transport, and its key errors. Built in `scope`,
-   * This is also how the backend learns which remote transports exist: it
+   * which the caller ends with the member. This is also how the backend
-   * follows the membership's transport for as long as `scope` lives, and
+   * learns which remote transports exist: it follows the membership's
-   * connects to every transport some member is on.
+   * transport for as long as `scope` lives, and connects to every transport
+   * some member is on.
    */
   mediaFor$(
     scope: ObservableScope,
     membership$: Behavior<RTCMembership>,
-  ): Behavior<MemberMedia | null>;
+  ): MemberMediaFields;
 
   /** Every connection the backend holds. For debugging and devtool purposes.*/
   readonly connections$: Behavior<BackendConnection[]>;
@@ -158,16 +169,14 @@ export interface MediaBackend {
   readonly data$: Observable<DataPacket>;
 }
 
-export interface LocalMediaBackend {
+/** `tracks$` is null until the connection carries a local participant. */
+export interface LocalMediaBackend extends LocalMemberMediaFields {
   /**
    * State of the connection the local member publishes on. `Initialized`
    * while there is none; an `Error` is the connection's failure.
    */
   readonly connectionState$: Behavior<MediaConnectionState | Error>;
 
-  /** The local member's own media; null until the connection carries a local participant. */
-  readonly media$: Behavior<LocalMemberMedia | null>;
-
   /**
    * Whether the local tracks reach the transport. The session sets it to
    * "joined and the homeserver is reachable". Remembered across connections:
@@ -179,8 +188,8 @@ export interface LocalMediaBackend {
   readonly publishError$: Behavior<Error | null>;
 
   /**
-   * Publishes a source and resolves with its track once it is in `media$`'s
+   * Publishes a source and resolves with its track once it is in `tracks$`.
-   * `tracks$`. Remembered across connections, so a request before the
+   * Remembered across connections, so a request before the
    * transport is up is applied once it is, and republished after a
    * reconnection. Rejects where the device could not be used.
    */

sdk/src/media-backend/livekit/LivekitBackend.ts

@@ -24,7 +24,7 @@ import {
 } from "rxjs";
 
 import { type ResolvedTransport, type RTCMembership } from "../../api";
-import { type MemberMedia } from "../../media-api";
+
 import { type MediaQuality } from "../../config";
 import { MatrixRTCError } from "../../errors";
 import { type Behavior } from "../../reactive/Behavior";
@@ -33,11 +33,11 @@ import {
   type ObservableScope,
   trackEpoch,
 } from "../../reactive/ObservableScope";
-import { mapScoped } from "../../utils/mapScoped";
 import {
   type BackendConnection,
   type MediaBackend,
   type MediaBackendContext,
+  type MemberMediaFields,
   type TransportCapabilities,
 } from "../api";
 import { type Connection } from "./Connection";
@@ -150,7 +150,7 @@ export function createLivekitBackend(
   const mediaFor$ = (
     memberScope: ObservableScope,
     membership$: Behavior<RTCMembership>,
-  ): Behavior<MemberMedia | null> => {
+  ): MemberMediaFields => {
     const identity = membership$.value.rtcBackendIdentity;
     const transport$ = memberScope.behavior(
       membership$.pipe(
@@ -190,16 +190,7 @@ export function createLivekitBackend(
         `${identity}: LiveKit participant ${source ? `matched (${source.participant.sid})` : "missing"}`,
       );
     });
-    return memberScope.behavior(
+    return createLivekitMemberMedia(memberScope, source$, encryptionSystem);
-      mapScoped(memberScope, source$, (mediaScope, { participant, room }) =>
-        createLivekitMemberMedia(
-          mediaScope,
-          participant,
-          room,
-          encryptionSystem,
-        ),
-      ).pipe(map((media) => media ?? null)),
-    );
   };
 
   const connections$ = scope.behavior<BackendConnection[]>(

sdk/src/media-backend/livekit/LivekitMemberMedia.test.ts

@@ -16,29 +16,32 @@ import {
 import { describe, expect, it } from "vitest";
 
 import { E2eeType } from "../../encryption";
+import { constant } from "../../reactive/Behavior";
 import { testScope } from "../../utils/test";
 import { createLivekitMemberMedia } from "./LivekitMemberMedia";
 
 describe("createLivekitMemberMedia", () => {
   it("lists one stable track per publication, in publication order", () => {
     const { participant, publications, room, emitter } = fakes();
-    const media = createLivekitMemberMedia(testScope(), participant, room, {
+    const media = createLivekitMemberMedia(
-      kind: E2eeType.NONE,
+      testScope(),
-    });
+      constant({ participant, room }),
+      { kind: E2eeType.NONE },
+    );
     expect(media.tracks$.value).toEqual([]);
 
     const microphone = publish(publications, "TR_1", Track.Source.Microphone);
     const camera = publish(publications, "TR_2", Track.Source.Camera);
     emitter.emit(ParticipantEvent.TrackPublished, camera);
-    const [first, second] = media.tracks$.value;
+    const [first, second] = media.tracks$.value!;
     expect(first.source).toBe("microphone");
     expect(second.source).toBe("camera");
 
     // An unrelated media event keeps the same objects
     microphone.isMuted = true;
     emitter.emit(ParticipantEvent.TrackMuted, microphone);
-    expect(media.tracks$.value[0]).toBe(first);
+    expect(media.tracks$.value![0]).toBe(first);
-    expect(media.tracks$.value[1]).toBe(second);
+    expect(media.tracks$.value![1]).toBe(second);
 
     publications.delete("TR_1");
     emitter.emit(ParticipantEvent.TrackUnpublished, microphone);
@@ -47,14 +50,16 @@ describe("createLivekitMemberMedia", () => {
 
   it("includes a track published without a source", () => {
     const { participant, publications, room, emitter } = fakes();
-    const media = createLivekitMemberMedia(testScope(), participant, room, {
+    const media = createLivekitMemberMedia(
-      kind: E2eeType.NONE,
+      testScope(),
-    });
+      constant({ participant, room }),
+      { kind: E2eeType.NONE },
+    );
     emitter.emit(
       ParticipantEvent.TrackPublished,
       publish(publications, "TR_1", Track.Source.Unknown),
     );
-    expect(media.tracks$.value.map((t) => t.source)).toEqual(["unknown"]);
+    expect(media.tracks$.value!.map((t) => t.source)).toEqual(["unknown"]);
   });
 });
 

sdk/src/media-backend/livekit/LivekitMemberMedia.ts

@@ -16,59 +16,58 @@ import {
   RoomEvent,
   type TrackPublication,
 } from "livekit-client";
-import { filter, map, type Observable } from "rxjs";
+import { EMPTY, filter, map, type Observable, of, switchMap } from "rxjs";
 
 import { type ObservableScope } from "../../reactive/ObservableScope";
 import { generateItems } from "../../reactive/observable";
-import { E2eeType } from "../../encryption";
+import { E2eeType, type EncryptionSystem } from "../../encryption";
-import { type EncryptionSystem } from "../../encryption";
 import { type Behavior } from "../../reactive/Behavior";
-import {
+import { mapScoped } from "../../utils/mapScoped";
-  type EncryptionError,
+import { type EncryptionError, type MediaSource } from "../../media-api";
-  type LocalMemberMedia,
+import { type LocalMemberMediaFields, type MemberMediaFields } from "../api";
-  type MediaSource,
-  type MemberMedia,
-} from "../../media-api";
 import {
   createLivekitMediaTrack,
   createLocalLivekitMediaTrack,
 } from "./LivekitMediaTrack";
 
+/** Where a member's media comes from: its participant on a room, null while it has none. */
+export type ParticipantSource<P extends Participant = Participant> = {
+  participant: P;
+  room: LivekitRoom;
+} | null;
+
 /**
- * A participant as `MemberMedia`. The only place, with `LivekitMediaTrack`,
+ * A member's media from its participant. The only place, with
- * that reads a LiveKit participant; everything above sees tracks and
+ * `LivekitMediaTrack`, that reads a LiveKit participant; everything above
- * behaviors.
+ * sees tracks and behaviors.
  */
 export function createLivekitMemberMedia(
   scope: ObservableScope,
-  participant: Participant,
+  source$: Behavior<ParticipantSource>,
-  room: LivekitRoom,
   encryptionSystem: EncryptionSystem,
-): MemberMedia {
+): MemberMediaFields {
   return {
-    local: false,
+    tracks$: memberTracks$(
-    tracks$: memberTracks$(scope, participant, (trackScope, publication) =>
+      scope,
+      source$,
+      (trackScope, participant, publication, room) =>
         createLivekitMediaTrack(trackScope, participant, publication, room),
     ),
-    encryptionError$: encryptionErrors$(
+    encryptionError$: encryptionErrors$(source$, encryptionSystem),
-      scope,
-      participant,
-      room,
-      encryptionSystem,
-    ),
   };
 }
 
 export function createLocalLivekitMemberMedia(
   scope: ObservableScope,
-  participant: LocalParticipant,
+  source$: Behavior<ParticipantSource<LocalParticipant>>,
-  room: LivekitRoom,
   encryptionSystem: EncryptionSystem,
   setEnabled: (source: MediaSource, enabled: boolean) => Promise<boolean>,
-): LocalMemberMedia {
+): LocalMemberMediaFields {
   return {
-    local: true,
+    tracks$: memberTracks$(
-    tracks$: memberTracks$(scope, participant, (trackScope, publication) =>
+      scope,
+      source$,
+      (trackScope, participant, publication, room) =>
         createLocalLivekitMediaTrack(
           trackScope,
           participant,
@@ -77,22 +76,27 @@ export function createLocalLivekitMemberMedia(
           setEnabled,
         ),
     ),
-    encryptionError$: encryptionErrors$(
+    encryptionError$: encryptionErrors$(source$, encryptionSystem),
-      scope,
-      participant,
-      room,
-      encryptionSystem,
-    ),
   };
 }
 
-/** One track per publication, each living as long as its publication is in the participant's map. */
+/**
-function memberTracks$<T>(
+ * One track per publication, each living as long as its publication is in
+ * the participant's map; null while there is no participant.
+ */
+function memberTracks$<P extends Participant, T>(
   scope: ObservableScope,
-  participant: Participant,
+  source$: Behavior<ParticipantSource<P>>,
-  factory: (scope: ObservableScope, publication: TrackPublication) => T,
+  factory: (
-): Behavior<T[]> {
+    scope: ObservableScope,
+    participant: P,
+    publication: TrackPublication,
+    room: LivekitRoom,
+  ) => T,
+): Behavior<T[] | null> {
   return scope.behavior(
+    mapScoped(scope, source$, (participantScope, { participant, room }) =>
+      participantScope.behavior(
         observeParticipantMedia(participant).pipe(
           generateItems(
             `${participant.identity} tracks$`,
@@ -100,27 +104,32 @@ function memberTracks$<T>(
               for (const publication of participant.trackPublications.values())
                 yield { keys: [publication.trackSid], data: publication };
             },
-        (trackScope, publication$) => factory(trackScope, publication$.value),
+            (trackScope, publication$) =>
+              factory(trackScope, participant, publication$.value, room),
           ),
         ),
+      ),
+    ).pipe(switchMap((tracks$) => tracks$ ?? of(null))),
   );
 }
 
 function encryptionErrors$(
-  scope: ObservableScope,
+  source$: Behavior<ParticipantSource>,
-  participant: Participant,
-  room: LivekitRoom,
   encryptionSystem: EncryptionSystem,
 ): Observable<EncryptionError> {
-  return roomEventSelector(room, RoomEvent.EncryptionError).pipe(
+  return source$.pipe(
+    switchMap((source) =>
+      source === null
+        ? EMPTY
+        : roomEventSelector(source.room, RoomEvent.EncryptionError).pipe(
             map(([error]) => error?.message ?? ""),
-    // The participant the error is about does not survive the trip from the
+            // The participant the error is about does not survive the trip
-    // worker, so the identity is matched in the message. A shared key is the
+            // from the worker, so the identity is matched in the message. A
-    // same for everyone, so its errors are too.
+            // shared key is the same for everyone, so its errors are too.
             filter(
               (message) =>
                 encryptionSystem.kind === E2eeType.SHARED_KEY ||
-        message.includes(participant.identity),
+                message.includes(source.participant.identity),
             ),
             map((message): EncryptionError | undefined =>
               message.includes("MissingKey")
@@ -130,6 +139,7 @@ function encryptionErrors$(
                   : undefined,
             ),
             filter((error) => error !== undefined),
-    scope.bind(),
+          ),
+    ),
   );
 }

sdk/src/media-backend/livekit/LocalMedia.ts

@@ -28,7 +28,6 @@ import {
 
 import {
   type LocalAudioMediaTrack,
-  type LocalMemberMedia,
   type LocalVideoMediaTrack,
   type MediaSource,
   type PublishRequest,
@@ -37,7 +36,6 @@ import { type EncryptionSystem } from "../../encryption";
 import { FailToStartLivekitConnection, toMatrixRTCError } from "../../errors";
 import { type Behavior } from "../../reactive/Behavior";
 import { type ObservableScope } from "../../reactive/ObservableScope";
-import { mapScoped } from "../../utils/mapScoped";
 import { type LocalMediaBackend, MediaConnectionState } from "../api";
 import { type Connection } from "./Connection";
 import {
@@ -169,19 +167,11 @@ export function createLivekitLocalMedia({
     return result;
   };
 
-  const media$ = scope.behavior<LocalMemberMedia | null>(
+  const { tracks$, encryptionError$ } = createLocalLivekitMemberMedia(
-    mapScoped(
     scope,
     participantAndRoom$(scope, participant$, connection$),
-      (mediaScope, { participant, room }) =>
-        createLocalLivekitMemberMedia(
-          mediaScope,
-          participant,
-          room,
     encryptionSystem,
     setEnabled,
-        ),
-    ).pipe(map((media) => media ?? null)),
   );
 
   const publish = async (
@@ -194,7 +184,7 @@ export function createLivekitLocalMedia({
       const publication =
         (publisher?.started ? await publisher.publish(request) : undefined) ??
         (await firstValueFrom(publication$(participant$, request.source)));
-      return await firstValueFrom(trackWithId$(media$, publication.trackSid));
+      return await firstValueFrom(trackWithId$(tracks$, publication.trackSid));
     } catch (e) {
       desired.delete(request.source);
       throw toMatrixRTCError(e);
@@ -202,7 +192,7 @@ export function createLivekitLocalMedia({
   };
 
   const unpublish = async (id: string): Promise<void> => {
-    const track = media$.value?.tracks$.value.find((t) => t.id === id);
+    const track = tracks$.value?.find((t) => t.id === id);
     if (track === undefined) return;
     // The screen share audio goes with its video
     const source =
@@ -219,7 +209,8 @@ export function createLivekitLocalMedia({
   return {
     connection$,
     connectionState$,
-    media$,
+    tracks$,
+    encryptionError$,
     setPublishing: (publish) => publishing$.next(publish),
     publishError$,
     publish,
@@ -228,7 +219,7 @@ export function createLivekitLocalMedia({
   };
 }
 
-/** The pair a `MemberMedia` is built on, changing only when one of the two does. */
+/** The pair the local tracks are built on, changing only when one of the two does. */
 function participantAndRoom$(
   scope: ObservableScope,
   participant$: Behavior<LocalParticipant | null>,
@@ -269,12 +260,11 @@ function publication$(
 }
 
 function trackWithId$(
-  media$: Behavior<LocalMemberMedia | null>,
+  tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>,
   id: string,
 ): Observable<LocalAudioMediaTrack | LocalVideoMediaTrack> {
-  return media$.pipe(
+  return tracks$.pipe(
-    switchMap((media) => media?.tracks$ ?? of([])),
+    map((tracks) => tracks?.find((track) => track.id === id)),
-    map((tracks) => tracks.find((track) => track.id === id)),
     filter((track) => track !== undefined),
   );
 }

sdk/src/utils/tracks.ts

@@ -24,17 +24,17 @@ export type TrackOfSource<S extends MediaSource> = S extends
     ? VideoMediaTrack
     : AudioMediaTrack | VideoMediaTrack;
 
-/** The member's first track of a source; undefined while there is none. */
+/** The member's first track of a source; undefined while there is none, or no media yet. */
 export function trackBySource$<S extends MediaSource>(
   scope: ObservableScope,
-  tracks$: Behavior<MediaTrack[]>,
+  tracks$: Behavior<MediaTrack[] | null>,
   source: S,
 ): Behavior<TrackOfSource<S> | undefined> {
   return scope.behavior(
     tracks$.pipe(
       map(
         (tracks) =>
-          tracks.find((t) => t.source === source) as
+          tracks?.find((t) => t.source === source) as
             | TrackOfSource<S>
             | undefined,
       ),