From aa897ae9bce15c7e2dc70f65b86fa7b8197e8efc Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Thu, 8 Oct 2026 14:18:26 +0200 Subject: [PATCH] [SDK] simplify public api (no lk processor, no encryption$ as obs) --- sdk/SdkArchitecture.md | 12 ++--- sdk/dev/main.ts | 2 +- sdk/src/media-api.ts | 48 ++++++++++++------- .../livekit/LivekitMediaTrack.test.ts | 20 +++----- .../livekit/LivekitMediaTrack.ts | 13 +---- .../livekit/LivekitMemberMedia.ts | 4 +- 6 files changed, 47 insertions(+), 52 deletions(-) diff --git a/sdk/SdkArchitecture.md b/sdk/SdkArchitecture.md index ceb44ee5b..8ad211853 100644 --- a/sdk/SdkArchitecture.md +++ b/sdk/SdkArchitecture.md @@ -30,7 +30,7 @@ therefore `Element Call → sdk`, never the reverse. The migration from today's │ │ │ per member RTCMember { id userId deviceId membership$ displayName$ │ │ avatarUrl$ transport$ tracks$ encryptionError$ }│ -│ per track MediaTrack { source kind muted$ encrypted$ stats$ attach() }│ +│ per track MediaTrack { source kind muted$ encrypted stats$ attach() } │ │ │ │ ┌───────────────────────────┐ MediaBackend ┌───────────────────────────┐ │ │ │ session: MatrixRTC │ media-backend/ │ media backend │ │ @@ -232,7 +232,7 @@ export type PublishRequest = source: "camera"; deviceId?: string; /** Background blur and the like. */ - processor?: TrackProcessor; + processor?: VideoProcessor; capture?: VideoCaptureSettings; } | { @@ -596,8 +596,8 @@ export interface MediaTrack { /** Stable for the life of the track. */ id: string; muted$: Behavior; - /** False when the SFU reports the track as unencrypted. */ - encrypted$: Behavior; + /** False when the SFU reports the track as unencrypted. Fixed for the life of the track. */ + encrypted: boolean; /** * Inbound statistics for a remote track, outbound for a local one. Polled * once a second, but only while someone is subscribed, so a statistics @@ -651,9 +651,7 @@ export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack { /** Restarts the camera facing the other way; resolves with the device now in use. */ switchFacingMode(): Promise; /** Background blur and the like; undefined removes the processor. */ - setProcessor( - processor: TrackProcessor | undefined, - ): Promise; + setProcessor(processor: VideoProcessor | undefined): Promise; } export type EncryptionError = "MissingKey" | "InvalidKey"; diff --git a/sdk/dev/main.ts b/sdk/dev/main.ts index 0753cd7fc..37db72bf0 100644 --- a/sdk/dev/main.ts +++ b/sdk/dev/main.ts @@ -263,7 +263,7 @@ function render( ); label( "encrypted", - of$((t) => t.encrypted$), + of$((t) => of(t.encrypted)), ); label( "active", diff --git a/sdk/src/media-api.ts b/sdk/src/media-api.ts index 4ed5a1d0e..b01fa5bfa 100644 --- a/sdk/src/media-api.ts +++ b/sdk/src/media-api.ts @@ -11,11 +11,6 @@ Please see LICENSE in the repository root for full details. * `api.ts`. */ -import { - type Track, - type TrackProcessor, - type VideoCodec, -} from "livekit-client"; import { type Observable } from "rxjs"; import { type Behavior } from "./reactive/Behavior"; @@ -26,6 +21,8 @@ export interface AudioCaptureSettings { autoGainControl?: boolean; } +export type VideoCodec = "vp8" | "h264" | "vp9" | "av1" | "h265"; + export interface VideoCaptureSettings { resolution?: { width: number; height: number; frameRate?: number }; maxBitrate?: number; @@ -33,6 +30,26 @@ export interface VideoCaptureSettings { codec?: VideoCodec; } +/** What a processor is given to work on: the captured track and, where a view is attached, its element. */ +export interface VideoProcessorInit { + track: MediaStreamTrack; + element?: HTMLMediaElement; +} + +/** + * Transforms a camera track before it is published: background blur and the + * like. `processedTrack` is what gets published once `init` has resolved; + * `restart` is called with the new track where the camera changes. The shape + * is that of a LiveKit track processor, so one of those can be passed as is. + */ +export interface VideoProcessor { + name: string; + processedTrack?: MediaStreamTrack; + init(opts: VideoProcessorInit): Promise; + restart(opts: VideoProcessorInit): Promise; + destroy(): Promise; +} + /** * One thing to publish: a source, where to capture it from and how to encode * it. Where the device and the settings are left out, the browser's and @@ -43,13 +60,8 @@ export type PublishRequest = | { source: "camera"; deviceId?: string; - /** Background blur and the like. */ - // REVIEW: this is ugly makes us need to import lk - // We need an interface: - // - super high level: blur + background img - // - lower level: rtc stream insertable stream (copy lk TrackProcessor concept) - // - florian: should be independent of webCodec or WebRtc - processor?: TrackProcessor; + /** Background blur and the like, applied before the first frame is published. */ + processor?: VideoProcessor; capture?: VideoCaptureSettings; } | { @@ -79,9 +91,11 @@ export interface MediaTrack { /** Stable for the life of the track. */ id: string; muted$: Behavior; - /** False when the SFU reports the track as unencrypted. */ - // REVIEW: does it need to be a behavior - encrypted$: Behavior; + /** + * False when the SFU reports the track as unencrypted. Fixed for the life + * of the track: a publisher that changes its encryption publishes anew. + */ + encrypted: boolean; /** Polled while subscribed. */ // REVIEW: we want the perf gain of not having it always run. // @@ -143,9 +157,7 @@ export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack { */ switchFacingMode(): Promise; /** Background blur and the like; undefined removes the processor. */ - setProcessor( - processor: TrackProcessor | undefined, - ): Promise; + setProcessor(processor: VideoProcessor | undefined): Promise; } export type EncryptionError = "MissingKey" | "InvalidKey"; diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts index 1992c383b..4cd66bed1 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts @@ -33,21 +33,20 @@ describe("createLivekitMediaTrack", () => { afterEach(() => scope.end()); it("describes the publication", () => { - const { participant, publication, room } = fakes(); + const { participant, publication } = fakes(); const track = createLivekitMediaTrack( scope, participant, publication as unknown as TrackPublication, - room, ); expect(track.kind).toBe("video"); expect(track.source).toBe("camera"); expect(track.id).toBe("TR_1"); - expect(track.encrypted$.value).toBe(true); + expect(track.encrypted).toBe(true); }); it("attaches elements to the track, even one that arrives later", () => { - const { participant, publication, room, emitter } = fakes({ + const { participant, publication, emitter } = fakes({ track: undefined, }); const element = document.createElement("video"); @@ -55,7 +54,6 @@ describe("createLivekitMediaTrack", () => { scope, participant, publication as unknown as TrackPublication, - room, ); track.attach(element); track.attach(element); @@ -71,14 +69,13 @@ describe("createLivekitMediaTrack", () => { }); it("detaches everything when its scope ends", () => { - const { participant, publication, room } = fakes(); + const { participant, publication } = fakes(); const element = document.createElement("video"); const scope = new ObservableScope(); const track = createLivekitMediaTrack( scope, participant, publication as unknown as TrackPublication, - room, ); track.attach(element); scope.end(); @@ -86,12 +83,11 @@ describe("createLivekitMediaTrack", () => { }); it("follows the mute state", () => { - const { participant, publication, room, emitter } = fakes(); + const { participant, publication, emitter } = fakes(); const track = createLivekitMediaTrack( scope, participant, publication as unknown as TrackPublication, - room, ); expect(track.muted$.value).toBe(false); publication.isMuted = true; @@ -100,28 +96,26 @@ describe("createLivekitMediaTrack", () => { }); it("scales a remote member's volume on the participant", () => { - const { participant, publication, room } = fakes({ + const { participant, publication } = fakes({ kind: Track.Kind.Audio, }); const track = createLivekitMediaTrack( scope, participant, publication as unknown as TrackPublication, - room, ) as AudioMediaTrack; track.setVolume(0.5); expect(publication.track!.setVolume).toHaveBeenCalledWith(0.5); }); it("is active while the member speaks and the track is not muted", () => { - const { participant, publication, room, emitter } = fakes({ + const { participant, publication, emitter } = fakes({ kind: Track.Kind.Audio, }); const track = createLivekitMediaTrack( scope, participant, publication as unknown as TrackPublication, - room, ) as AudioMediaTrack; expect(track.isActive$.value).toBe(false); participant.isSpeaking = true; diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts index 93e3fa6b9..1a0bee65c 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts @@ -8,7 +8,6 @@ Please see LICENSE in the repository root for full details. import { observeParticipantEvents, observeParticipantMedia, - roomEventSelector, } from "@livekit/components-core"; import { facingModeFromLocalTrack, @@ -20,7 +19,6 @@ import { RemoteAudioTrack, RemoteTrack, type Room as LivekitRoom, - RoomEvent, Track, TrackEvent, type TrackPublication, @@ -81,7 +79,6 @@ export function createLivekitMediaTrack( scope: ObservableScope, participant: Participant, publication: TrackPublication, - room: LivekitRoom, ): AudioMediaTrack | VideoMediaTrack { const mediaChanged$ = observeParticipantMedia(participant); const track$ = publicationTrack$(scope, participant, publication); @@ -118,13 +115,7 @@ export function createLivekitMediaTrack( kind: publication.kind === Track.Kind.Audio ? "audio" : "video", id: publication.trackSid, muted$, - encrypted$: scope.behavior( - merge( - mediaChanged$, - roomEventSelector(room, RoomEvent.ParticipantEncryptionStatusChanged), - ).pipe(map(() => publication.isEncrypted)), - publication.isEncrypted, - ), + encrypted: publication.isEncrypted, stats$: new LazyBehavior( refreshStats$.pipe( switchMap(async () => rtpStreamStats(publication, participant.isLocal)), @@ -189,7 +180,7 @@ export function createLocalLivekitMediaTrack( room: LivekitRoom, setEnabled: (source: MediaSource, enabled: boolean) => Promise, ): LocalAudioMediaTrack | LocalVideoMediaTrack { - const base = createLivekitMediaTrack(scope, participant, publication, room); + const base = createLivekitMediaTrack(scope, participant, publication); const local = { setEnabled: async (enabled: boolean) => setEnabled(base.source, enabled), setDevice: async (deviceId: string): Promise => { diff --git a/sdk/src/media-backend/livekit/LivekitMemberMedia.ts b/sdk/src/media-backend/livekit/LivekitMemberMedia.ts index f4e19eb72..32b99b8c3 100644 --- a/sdk/src/media-backend/livekit/LivekitMemberMedia.ts +++ b/sdk/src/media-backend/livekit/LivekitMemberMedia.ts @@ -54,8 +54,8 @@ export function createLivekitMemberMedia( tracks$: memberTracks$( scope, source$, - (trackScope, participant, publication, room) => - createLivekitMediaTrack(trackScope, participant, publication, room), + (trackScope, participant, publication) => + createLivekitMediaTrack(trackScope, participant, publication), ), encryptionError$: encryptionErrors$(source$, encryptionSystem), };