[SDK] split the clint object into -> slot and participation

sdk/src/media-backend/api.ts

@@ -17,15 +17,13 @@ import { type Transport } from "matrix-js-sdk/lib/matrixrtc";
 import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
 import { type Observable } from "rxjs";
 
-import {
-  type LocalRTCMember,
-  type ResolvedTransport,
-  type RTCMember,
-  type RTCMembership,
-} from "../api";
+import { type ResolvedTransport, type RTCMember } from "../api";
 import {
   type LocalAudioMediaTrack,
+  type LocalMemberMedia,
   type LocalVideoMediaTrack,
+  type MediaSource,
+  type MemberMedia,
   type PublishRequest,
 } from "../media-api";
 import { type SessionTimings } from "../config";
@@ -53,13 +51,6 @@ export interface MediaKey {
   key: Uint8Array<ArrayBuffer>;
 }
 
-/** The fields of a member the backend supplies, as the member exposes them. */
-export type MemberMediaFields = Pick<RTCMember, "tracks$" | "encryptionError$">;
-export type LocalMemberMediaFields = Pick<
-  LocalRTCMember,
-  "tracks$" | "encryptionError$"
->;
-
 export interface MediaBackendContext {
   roomId: string;
   ownMembershipIdentity: CallMembershipIdentityParts;
@@ -144,15 +135,12 @@ export interface MediaBackend {
   /**
    * The media of one remote member: its tracks, null while nothing has
    * arrived for it on its transport, and its key errors. Built in `scope`,
-   * which the caller ends with the member. This is also how the backend
-   * learns which remote transports exist: it follows the membership's
-   * transport for as long as `scope` lives, and connects to every transport
-   * some member is on.
+   * which the caller ends with the member or the participation, whichever
+   * goes first. This is also how the backend learns which remote transports
+   * exist: it follows the member's `transport$` for as long as `scope` lives,
+   * and connects to every transport some member is on.
    */
-  mediaFor$(
-    scope: ObservableScope,
-    membership$: Behavior<RTCMembership>,
-  ): MemberMediaFields;
+  mediaFor(scope: ObservableScope, member: RTCMember): MemberMedia;
 
   /** Every connection the backend holds. For debugging and devtool purposes.*/
   readonly connections$: Behavior<BackendConnection[]>;
@@ -175,7 +163,7 @@ export interface MediaBackend {
 }
 
 /** `tracks$` is null until the connection carries a local participant. */
-export interface LocalMediaBackend extends LocalMemberMediaFields {
+export interface LocalMediaBackend extends LocalMemberMedia {
   /**
    * State of the connection the local member publishes on. `Initialized`
    * while there is none; an `Error` is the connection's failure.
@@ -201,6 +189,6 @@ export interface LocalMediaBackend extends LocalMemberMediaFields {
   publish(
     request: PublishRequest,
   ): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack>;
-  /** Removes one of our tracks by id; a screen share takes its audio with it. */
-  unpublish(id: string): Promise<void>;
+  /** Removes one of our publications; a screen share takes its audio with it. */
+  unpublish(source: MediaSource): Promise<void>;
 }

sdk/src/media-backend/livekit/LivekitBackend.ts

@@ -23,11 +23,12 @@ import {
   switchMap,
 } from "rxjs";
 
-import { type ResolvedTransport, type RTCMembership } from "../../api";
+import { type ResolvedTransport, type RTCMember } from "../../api";
 
 import { type MediaQuality } from "../../config";
 import { MatrixRTCError } from "../../errors";
 import { type Behavior } from "../../reactive/Behavior";
+import { type MemberMedia } from "../../media-api";
 import {
   type Epoch,
   type ObservableScope,
@@ -37,7 +38,6 @@ import {
   type BackendConnection,
   type MediaBackend,
   type MediaBackendContext,
-  type MemberMediaFields,
   type TransportCapabilities,
 } from "../api";
 import { type Connection } from "./Connection";
@@ -147,14 +147,14 @@ export function createLivekitBackend(
     );
   };
 
-  const mediaFor$ = (
+  const mediaFor = (
     memberScope: ObservableScope,
-    membership$: Behavior<RTCMembership>,
-  ): MemberMediaFields => {
-    const identity = membership$.value.rtcBackendIdentity;
+    member: RTCMember,
+  ): MemberMedia => {
+    const identity = member.rtcBackendIdentity;
     const transport$ = memberScope.behavior(
-      membership$.pipe(
-        map((membership) => membership.getTransport()),
+      member.transport$.pipe(
+        map((transport) => transport?.raw),
         map((transport) =>
           isLivekitTransport(transport) ? transport : undefined,
         ),
@@ -224,7 +224,7 @@ export function createLivekitBackend(
     prepareLocalTransport,
     delegateDelayedLeave,
     local: localMedia_,
-    mediaFor$,
+    mediaFor,
     connections$,
     setAudioOutputDeviceId: localMedia_.setAudioOutputDeviceId,
     sendData,

sdk/src/media-backend/livekit/LivekitMemberMedia.ts

@@ -23,8 +23,12 @@ import { generateItems } from "../../reactive/observable";
 import { E2eeType, type EncryptionSystem } from "../../encryption";
 import { type Behavior } from "../../reactive/Behavior";
 import { mapScoped } from "../../utils/mapScoped";
-import { type EncryptionError, type MediaSource } from "../../media-api";
-import { type LocalMemberMediaFields, type MemberMediaFields } from "../api";
+import {
+  type EncryptionError,
+  type LocalMemberMedia,
+  type MediaSource,
+  type MemberMedia,
+} from "../../media-api";
 import {
   createLivekitMediaTrack,
   createLocalLivekitMediaTrack,
@@ -45,7 +49,7 @@ export function createLivekitMemberMedia(
   scope: ObservableScope,
   source$: Behavior<ParticipantSource>,
   encryptionSystem: EncryptionSystem,
-): MemberMediaFields {
+): MemberMedia {
   return {
     tracks$: memberTracks$(
       scope,
@@ -62,7 +66,7 @@ export function createLocalLivekitMemberMedia(
   source$: Behavior<ParticipantSource<LocalParticipant>>,
   encryptionSystem: EncryptionSystem,
   setEnabled: (source: MediaSource, enabled: boolean) => Promise<boolean>,
-): LocalMemberMediaFields {
+): LocalMemberMedia {
   return {
     tracks$: memberTracks$(
       scope,

sdk/src/media-backend/livekit/LocalMedia.ts

@@ -191,14 +191,11 @@ export function createLivekitLocalMedia({
     }
   };
 
-  const unpublish = async (id: string): Promise<void> => {
-    const track = tracks$.value?.find((t) => t.id === id);
-    if (track === undefined) return;
-    // The screen share audio goes with its video
-    const source =
-      track.source === "screenShareAudio" ? "screenShare" : track.source;
-    desired.delete(source);
-    await publisher$.value?.unpublish(source);
+  const unpublish = async (source: MediaSource): Promise<void> => {
+    const published = publishedSource(source);
+    if (published === undefined) return;
+    desired.delete(published);
+    await publisher$.value?.unpublish(published);
   };
 
   const setAudioOutputDeviceId = async (deviceId: string): Promise<void> => {
@@ -268,3 +265,17 @@ function trackWithId$(
     filter((track) => track !== undefined),
   );
 }
+
+/** The publication a source belongs to: the screen share audio goes with its video. */
+function publishedSource(
+  source: MediaSource,
+): PublishRequest["source"] | undefined {
+  switch (source) {
+    case "screenShareAudio":
+      return "screenShare";
+    case "unknown":
+      return undefined;
+    default:
+      return source;
+  }
+}