diff --git a/sdk/index.ts b/sdk/index.ts index 31719fe84..e6e417cf1 100644 --- a/sdk/index.ts +++ b/sdk/index.ts @@ -61,6 +61,8 @@ export { createLivekitBackend, type LivekitBackendOptions, } from "./src/media-backend/livekit/LivekitBackend"; +// For a host that creates LiveKit tracks itself, such as a preview before joining +export { convertToLivekitProcessor } from "./src/media-backend/livekit/videoProcessor"; // For a developer panel to check a transport url before using it export { getSFUConfigWithOpenID as authenticateWithTransport, diff --git a/sdk/src/media-api.ts b/sdk/src/media-api.ts index b01fa5bfa..5e341914e 100644 --- a/sdk/src/media-api.ts +++ b/sdk/src/media-api.ts @@ -30,23 +30,21 @@ export interface VideoCaptureSettings { codec?: VideoCodec; } -/** What a processor is given to work on: the captured track and, where a view is attached, its element. */ -export interface VideoProcessorInit { - track: MediaStreamTrack; - element?: HTMLMediaElement; -} - /** * Transforms a camera track before it is published: background blur and the - * like. `processedTrack` is what gets published once `init` has resolved; - * `restart` is called with the new track where the camera changes. The shape - * is that of a LiveKit track processor, so one of those can be passed as is. + * like. `init` resolves with the track to publish in place of the captured + * one; `restart` does the same for a new capture, where the camera changes. */ export interface VideoProcessor { name: string; - processedTrack?: MediaStreamTrack; - init(opts: VideoProcessorInit): Promise; - restart(opts: VideoProcessorInit): Promise; + init( + track: MediaStreamTrack, + element?: HTMLMediaElement, + ): Promise; + restart( + track: MediaStreamTrack, + element?: HTMLMediaElement, + ): Promise; destroy(): Promise; } diff --git a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts index 1a0bee65c..a0111eda7 100644 --- a/sdk/src/media-backend/livekit/LivekitMediaTrack.ts +++ b/sdk/src/media-backend/livekit/LivekitMediaTrack.ts @@ -49,6 +49,7 @@ import { } from "../../media-api"; import { MatrixRTCError } from "../../errors"; import { LazyBehavior } from "../../utils/LazyBehavior"; +import { convertToLivekitProcessor } from "./videoProcessor"; export const mediaSources: Record = { [Track.Source.Microphone]: "microphone", @@ -219,7 +220,8 @@ export function createLocalLivekitMediaTrack( const track = videoTrack(); // A processor cannot be built on a track that has already ended if (!track || track.mediaStreamTrack.readyState === "ended") return; - if (processor) await track.setProcessor(processor); + if (processor) + await track.setProcessor(convertToLivekitProcessor(processor)); else if (track.getProcessor()) await track.stopProcessor(); }, }; diff --git a/sdk/src/media-backend/livekit/publishOptions.ts b/sdk/src/media-backend/livekit/publishOptions.ts index ccf1595fa..f030c3fd4 100644 --- a/sdk/src/media-backend/livekit/publishOptions.ts +++ b/sdk/src/media-backend/livekit/publishOptions.ts @@ -16,6 +16,7 @@ import { type PublishRequest, type VideoCaptureSettings, } from "../../media-api"; +import { convertToLivekitProcessor } from "./videoProcessor"; export type PublishRequestFor = Extract< PublishRequest, @@ -39,7 +40,7 @@ export function videoCaptureOptions({ if (!deviceId && !processor && !capture?.resolution) return undefined; return { ...(deviceId && { deviceId }), - ...(processor && { processor }), + ...(processor && { processor: convertToLivekitProcessor(processor) }), ...(capture?.resolution && { resolution: capture.resolution }), }; } diff --git a/sdk/src/media-backend/livekit/videoProcessor.test.ts b/sdk/src/media-backend/livekit/videoProcessor.test.ts new file mode 100644 index 000000000..468379fe0 --- /dev/null +++ b/sdk/src/media-backend/livekit/videoProcessor.test.ts @@ -0,0 +1,50 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { Track, type VideoProcessorOptions } from "livekit-client"; +import { describe, expect, it, vi } from "vitest"; + +import { type VideoProcessor } from "../../media-api"; +import { convertToLivekitProcessor } from "./videoProcessor"; + +describe("convertToLivekitProcessor", () => { + const captured = { id: "captured" } as MediaStreamTrack; + const processed = { id: "processed" } as MediaStreamTrack; + const restarted = { id: "restarted" } as MediaStreamTrack; + const element = document.createElement("video"); + const options: VideoProcessorOptions = { + kind: Track.Kind.Video, + track: captured, + element, + }; + + function fakeProcessor(): VideoProcessor { + return { + name: "blur", + init: vi.fn().mockResolvedValue(processed), + restart: vi.fn().mockResolvedValue(restarted), + destroy: vi.fn().mockResolvedValue(undefined), + }; + } + + it("holds the processed track where LiveKit reads it", async () => { + const processor = fakeProcessor(); + const adapter = convertToLivekitProcessor(processor); + expect(adapter.name).toBe("blur"); + expect(adapter.processedTrack).toBeUndefined(); + + await adapter.init(options); + expect(processor.init).toHaveBeenCalledWith(captured, element); + expect(adapter.processedTrack).toBe(processed); + + await adapter.restart(options); + expect(adapter.processedTrack).toBe(restarted); + + await adapter.destroy(); + expect(processor.destroy).toHaveBeenCalled(); + }); +}); diff --git a/sdk/src/media-backend/livekit/videoProcessor.ts b/sdk/src/media-backend/livekit/videoProcessor.ts new file mode 100644 index 000000000..546ea2113 --- /dev/null +++ b/sdk/src/media-backend/livekit/videoProcessor.ts @@ -0,0 +1,34 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type Track, type TrackProcessor } from "livekit-client"; + +import { type VideoProcessor } from "../../media-api"; + +/** + * A `VideoProcessor` as livekit-client wants it. LiveKit reads the processed + * track off the processor after `init` and `restart` rather than taking it as + * their result, so the adapter keeps it there. + */ +export function convertToLivekitProcessor( + processor: VideoProcessor, +): TrackProcessor { + const adapter: TrackProcessor = { + name: processor.name, + init: async (opts) => { + adapter.processedTrack = await processor.init(opts.track, opts.element); + }, + restart: async (opts) => { + adapter.processedTrack = await processor.restart( + opts.track, + opts.element, + ); + }, + destroy: async () => processor.destroy(), + }; + return adapter; +}