[SDK] simplify public api (no lk processor, no encryption$ as obs)

sdk/SdkArchitecture.md

@@ -30,7 +30,7 @@ therefore `Element Call → sdk`, never the reverse. The migration from today's
 │                                                                              │
 │   per member    RTCMember   { id userId deviceId membership$ displayName$    │
 │                               avatarUrl$ transport$ tracks$ encryptionError$ }│
-│   per track     MediaTrack  { source kind muted$ encrypted$ stats$ attach() }│
+│   per track     MediaTrack  { source kind muted$ encrypted stats$ attach() } │
 │                                                                              │
 │  ┌───────────────────────────┐  MediaBackend   ┌───────────────────────────┐ │
 │  │ session: MatrixRTC        │ media-backend/  │ media backend             │ │
@@ -232,7 +232,7 @@ export type PublishRequest =
       source: "camera";
       deviceId?: string;
       /** Background blur and the like. */
-      processor?: TrackProcessor<Track.Kind.Video>;
+      processor?: VideoProcessor;
       capture?: VideoCaptureSettings;
     }
   | {
@@ -596,8 +596,8 @@ export interface MediaTrack {
   /** Stable for the life of the track. */
   id: string;
   muted$: Behavior<boolean>;
-  /** False when the SFU reports the track as unencrypted. */
+  /** False when the SFU reports the track as unencrypted. Fixed for the life of the track. */
-  encrypted$: Behavior<boolean>;
+  encrypted: boolean;
   /**
    * Inbound statistics for a remote track, outbound for a local one. Polled
    * once a second, but only while someone is subscribed, so a statistics
@@ -651,9 +651,7 @@ export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack {
   /** Restarts the camera facing the other way; resolves with the device now in use. */
   switchFacingMode(): Promise<string | undefined>;
   /** Background blur and the like; undefined removes the processor. */
-  setProcessor(
+  setProcessor(processor: VideoProcessor | undefined): Promise<void>;
-    processor: TrackProcessor<Track.Kind.Video> | undefined,
-  ): Promise<void>;
 }
 
 export type EncryptionError = "MissingKey" | "InvalidKey";

sdk/dev/main.ts

@@ -263,7 +263,7 @@ function render(
   );
   label(
     "encrypted",
-    of$((t) => t.encrypted$),
+    of$((t) => of(t.encrypted)),
   );
   label(
     "active",

sdk/src/media-api.ts

@@ -11,11 +11,6 @@ Please see LICENSE in the repository root for full details.
  * `api.ts`.
  */
 
-import {
-  type Track,
-  type TrackProcessor,
-  type VideoCodec,
-} from "livekit-client";
 import { type Observable } from "rxjs";
 
 import { type Behavior } from "./reactive/Behavior";
@@ -26,6 +21,8 @@ export interface AudioCaptureSettings {
   autoGainControl?: boolean;
 }
 
+export type VideoCodec = "vp8" | "h264" | "vp9" | "av1" | "h265";
+
 export interface VideoCaptureSettings {
   resolution?: { width: number; height: number; frameRate?: number };
   maxBitrate?: number;
@@ -33,6 +30,26 @@ export interface VideoCaptureSettings {
   codec?: VideoCodec;
 }
 
+/** What a processor is given to work on: the captured track and, where a view is attached, its element. */
+export interface VideoProcessorInit {
+  track: MediaStreamTrack;
+  element?: HTMLMediaElement;
+}
+
+/**
+ * Transforms a camera track before it is published: background blur and the
+ * like. `processedTrack` is what gets published once `init` has resolved;
+ * `restart` is called with the new track where the camera changes. The shape
+ * is that of a LiveKit track processor, so one of those can be passed as is.
+ */
+export interface VideoProcessor {
+  name: string;
+  processedTrack?: MediaStreamTrack;
+  init(opts: VideoProcessorInit): Promise<void>;
+  restart(opts: VideoProcessorInit): Promise<void>;
+  destroy(): Promise<void>;
+}
+
 /**
  * One thing to publish: a source, where to capture it from and how to encode
  * it. Where the device and the settings are left out, the browser's and
@@ -43,13 +60,8 @@ export type PublishRequest =
   | {
       source: "camera";
       deviceId?: string;
-      /** Background blur and the like. */
+      /** Background blur and the like, applied before the first frame is published. */
-      // REVIEW: this is ugly makes us need to import lk
+      processor?: VideoProcessor;
-      // We need an interface:
-      //  - super high level: blur + background img
-      //  - lower level: rtc stream insertable stream (copy lk TrackProcessor concept)
-      //  - florian: should be independent of webCodec or WebRtc
-      processor?: TrackProcessor<Track.Kind.Video>;
       capture?: VideoCaptureSettings;
     }
   | {
@@ -79,9 +91,11 @@ export interface MediaTrack {
   /** Stable for the life of the track. */
   id: string;
   muted$: Behavior<boolean>;
-  /** False when the SFU reports the track as unencrypted. */
+  /**
-  // REVIEW: does it need to be a behavior
+   * False when the SFU reports the track as unencrypted. Fixed for the life
-  encrypted$: Behavior<boolean>;
+   * of the track: a publisher that changes its encryption publishes anew.
+   */
+  encrypted: boolean;
   /** Polled while subscribed. */
   // REVIEW: we want the perf gain of not having it always run.
   //
@@ -143,9 +157,7 @@ export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack {
    */
   switchFacingMode(): Promise<string | undefined>;
   /** Background blur and the like; undefined removes the processor. */
-  setProcessor(
+  setProcessor(processor: VideoProcessor | undefined): Promise<void>;
-    processor: TrackProcessor<Track.Kind.Video> | undefined,
-  ): Promise<void>;
 }
 
 export type EncryptionError = "MissingKey" | "InvalidKey";

sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts

@@ -33,21 +33,20 @@ describe("createLivekitMediaTrack", () => {
   afterEach(() => scope.end());
 
   it("describes the publication", () => {
-    const { participant, publication, room } = fakes();
+    const { participant, publication } = fakes();
     const track = createLivekitMediaTrack(
       scope,
       participant,
       publication as unknown as TrackPublication,
-      room,
     );
     expect(track.kind).toBe("video");
     expect(track.source).toBe("camera");
     expect(track.id).toBe("TR_1");
-    expect(track.encrypted$.value).toBe(true);
+    expect(track.encrypted).toBe(true);
   });
 
   it("attaches elements to the track, even one that arrives later", () => {
-    const { participant, publication, room, emitter } = fakes({
+    const { participant, publication, emitter } = fakes({
       track: undefined,
     });
     const element = document.createElement("video");
@@ -55,7 +54,6 @@ describe("createLivekitMediaTrack", () => {
       scope,
       participant,
       publication as unknown as TrackPublication,
-      room,
     );
     track.attach(element);
     track.attach(element);
@@ -71,14 +69,13 @@ describe("createLivekitMediaTrack", () => {
   });
 
   it("detaches everything when its scope ends", () => {
-    const { participant, publication, room } = fakes();
+    const { participant, publication } = fakes();
     const element = document.createElement("video");
     const scope = new ObservableScope();
     const track = createLivekitMediaTrack(
       scope,
       participant,
       publication as unknown as TrackPublication,
-      room,
     );
     track.attach(element);
     scope.end();
@@ -86,12 +83,11 @@ describe("createLivekitMediaTrack", () => {
   });
 
   it("follows the mute state", () => {
-    const { participant, publication, room, emitter } = fakes();
+    const { participant, publication, emitter } = fakes();
     const track = createLivekitMediaTrack(
       scope,
       participant,
       publication as unknown as TrackPublication,
-      room,
     );
     expect(track.muted$.value).toBe(false);
     publication.isMuted = true;
@@ -100,28 +96,26 @@ describe("createLivekitMediaTrack", () => {
   });
 
   it("scales a remote member's volume on the participant", () => {
-    const { participant, publication, room } = fakes({
+    const { participant, publication } = fakes({
       kind: Track.Kind.Audio,
     });
     const track = createLivekitMediaTrack(
       scope,
       participant,
       publication as unknown as TrackPublication,
-      room,
     ) as AudioMediaTrack;
     track.setVolume(0.5);
     expect(publication.track!.setVolume).toHaveBeenCalledWith(0.5);
   });
 
   it("is active while the member speaks and the track is not muted", () => {
-    const { participant, publication, room, emitter } = fakes({
+    const { participant, publication, emitter } = fakes({
       kind: Track.Kind.Audio,
     });
     const track = createLivekitMediaTrack(
       scope,
       participant,
       publication as unknown as TrackPublication,
-      room,
     ) as AudioMediaTrack;
     expect(track.isActive$.value).toBe(false);
     participant.isSpeaking = true;

sdk/src/media-backend/livekit/LivekitMediaTrack.ts

@@ -8,7 +8,6 @@ Please see LICENSE in the repository root for full details.
 import {
   observeParticipantEvents,
   observeParticipantMedia,
-  roomEventSelector,
 } from "@livekit/components-core";
 import {
   facingModeFromLocalTrack,
@@ -20,7 +19,6 @@ import {
   RemoteAudioTrack,
   RemoteTrack,
   type Room as LivekitRoom,
-  RoomEvent,
   Track,
   TrackEvent,
   type TrackPublication,
@@ -81,7 +79,6 @@ export function createLivekitMediaTrack(
   scope: ObservableScope,
   participant: Participant,
   publication: TrackPublication,
-  room: LivekitRoom,
 ): AudioMediaTrack | VideoMediaTrack {
   const mediaChanged$ = observeParticipantMedia(participant);
   const track$ = publicationTrack$(scope, participant, publication);
@@ -118,13 +115,7 @@ export function createLivekitMediaTrack(
     kind: publication.kind === Track.Kind.Audio ? "audio" : "video",
     id: publication.trackSid,
     muted$,
-    encrypted$: scope.behavior(
+    encrypted: publication.isEncrypted,
-      merge(
-        mediaChanged$,
-        roomEventSelector(room, RoomEvent.ParticipantEncryptionStatusChanged),
-      ).pipe(map(() => publication.isEncrypted)),
-      publication.isEncrypted,
-    ),
     stats$: new LazyBehavior<MediaStreamStats>(
       refreshStats$.pipe(
         switchMap(async () => rtpStreamStats(publication, participant.isLocal)),
@@ -189,7 +180,7 @@ export function createLocalLivekitMediaTrack(
   room: LivekitRoom,
   setEnabled: (source: MediaSource, enabled: boolean) => Promise<boolean>,
 ): LocalAudioMediaTrack | LocalVideoMediaTrack {
-  const base = createLivekitMediaTrack(scope, participant, publication, room);
+  const base = createLivekitMediaTrack(scope, participant, publication);
   const local = {
     setEnabled: async (enabled: boolean) => setEnabled(base.source, enabled),
     setDevice: async (deviceId: string): Promise<void> => {

sdk/src/media-backend/livekit/LivekitMemberMedia.ts

@@ -54,8 +54,8 @@ export function createLivekitMemberMedia(
     tracks$: memberTracks$(
       scope,
       source$,
-      (trackScope, participant, publication, room) =>
+      (trackScope, participant, publication) =>
-        createLivekitMediaTrack(trackScope, participant, publication, room),
+        createLivekitMediaTrack(trackScope, participant, publication),
     ),
     encryptionError$: encryptionErrors$(source$, encryptionSystem),
   };