From 79c1f824489fdb666ce1fb886b936c96e12590cf Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Thu, 8 Oct 2026 16:10:04 +0200 Subject: [PATCH] use abstracted livekti processor --- src/room/LobbyView.tsx | 4 +- src/tracks/TrackProcessorContext.test.ts | 56 +++++++++++++++++++++--- src/tracks/TrackProcessorContext.tsx | 47 +++++++++++++++++--- 3 files changed, 94 insertions(+), 13 deletions(-) diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index 13ba55b74..660e06953 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -24,6 +24,7 @@ import { type LocalVideoTrack, Track, } from "livekit-client"; +import { convertToLivekitProcessor } from "@element-hq/matrixrtc-sdk"; import inCallStyles from "./InCallView.module.css"; import styles from "./LobbyView.module.css"; @@ -150,7 +151,8 @@ export const LobbyView: FC = ({ audio: Object.assign({}, initialAudioOptions), video: videoEnabled && { deviceId: videoInputId, - processor: initialProcessor, + processor: + initialProcessor && convertToLivekitProcessor(initialProcessor), }, }), [initialAudioOptions, videoEnabled, videoInputId, initialProcessor], diff --git a/src/tracks/TrackProcessorContext.test.ts b/src/tracks/TrackProcessorContext.test.ts index 5b977bd4d..5206a5837 100644 --- a/src/tracks/TrackProcessorContext.test.ts +++ b/src/tracks/TrackProcessorContext.test.ts @@ -11,16 +11,21 @@ import { type BackgroundOptions, type ProcessorWrapper, } from "@livekit/track-processors"; +import { type VideoProcessor } from "@element-hq/matrixrtc-sdk"; -import { applyProcessor, trackProcessorSync } from "./TrackProcessorContext"; +import { + applyProcessor, + fromLivekitProcessor, + trackProcessorSync, +} from "./TrackProcessorContext"; import { constant } from "../state/Behavior"; import { flushPromises, testScope } from "../utils/test"; -const processor = {} as ProcessorWrapper; +const processor = { name: "blur" } as VideoProcessor; function mockTrack( readyState: MediaStreamTrackState, - current?: ProcessorWrapper, + current?: VideoProcessor, ): LocalVideoTrack { return { mediaStreamTrack: { readyState }, @@ -34,7 +39,9 @@ describe("applyProcessor", () => { it("attaches the processor to a live track", () => { const track = mockTrack("live"); applyProcessor(track, processor); - expect(track.setProcessor).toHaveBeenCalledWith(processor); + expect(track.setProcessor).toHaveBeenCalledWith( + expect.objectContaining({ name: "blur" }), + ); }); it("does not attach the processor to an ended track", () => { @@ -82,6 +89,45 @@ describe("trackProcessorSync", () => { constant(track), constant({ supported: true, processor }), ); - expect(track.setProcessor).toHaveBeenCalledWith(processor); + expect(track.setProcessor).toHaveBeenCalledWith( + expect.objectContaining({ name: "blur" }), + ); + }); +}); + +describe("fromLivekitProcessor", () => { + const captured = { id: "captured" } as MediaStreamTrack; + const processed = { id: "processed" } as MediaStreamTrack; + + function mockWrapper( + processedTrack: MediaStreamTrack | undefined, + ): ProcessorWrapper { + return { + name: "background-blur", + processedTrack, + init: vi.fn().mockResolvedValue(undefined), + restart: vi.fn().mockResolvedValue(undefined), + destroy: vi.fn().mockResolvedValue(undefined), + } as unknown as ProcessorWrapper; + } + + it("resolves with the track the wrapper produced", async () => { + const wrapper = mockWrapper(processed); + const sdkProcessor = fromLivekitProcessor(wrapper); + expect(sdkProcessor.name).toBe("background-blur"); + await expect(sdkProcessor.init(captured)).resolves.toBe(processed); + expect(wrapper.init).toHaveBeenCalledWith({ + kind: "video", + track: captured, + }); + await expect(sdkProcessor.restart(captured)).resolves.toBe(processed); + await sdkProcessor.destroy(); + expect(wrapper.destroy).toHaveBeenCalled(); + }); + + it("rejects where the wrapper produced no track", async () => { + await expect( + fromLivekitProcessor(mockWrapper(undefined)).init(captured), + ).rejects.toThrow("produced no track"); }); }); diff --git a/src/tracks/TrackProcessorContext.tsx b/src/tracks/TrackProcessorContext.tsx index 9b930949b..785928b62 100644 --- a/src/tracks/TrackProcessorContext.tsx +++ b/src/tracks/TrackProcessorContext.tsx @@ -18,9 +18,13 @@ import { useEffect, useMemo, } from "react"; -import { type LocalVideoTrack } from "livekit-client"; +import { type LocalVideoTrack, Track } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { combineLatest } from "rxjs"; +import { + convertToLivekitProcessor, + type VideoProcessor, +} from "@element-hq/matrixrtc-sdk"; import { backgroundBlur as backgroundBlurSettings, @@ -38,9 +42,36 @@ import { useValueBehavior } from "../useValueBehavior"; export type ProcessorState = { supported: boolean | undefined; - processor: undefined | ProcessorWrapper; + processor: undefined | VideoProcessor; }; +/** + * This allows to use a LiveKit processor for the matrixRTCSDK (VideoProcessor). + * They have slightly different apis so we need to convert them with this + * function before we can use them in the matrixRTCSDK. + */ +export function fromLivekitProcessor( + wrapper: ProcessorWrapper, +): VideoProcessor { + const processedTrack = (): MediaStreamTrack => { + if (wrapper.processedTrack === undefined) + throw new Error(`Processor ${wrapper.name} produced no track`); + return wrapper.processedTrack; + }; + return { + name: wrapper.name, + init: async (track: MediaStreamTrack, element?: HTMLMediaElement) => { + await wrapper.init({ kind: Track.Kind.Video, track, element }); + return processedTrack(); + }, + restart: async (track: MediaStreamTrack, element?: HTMLMediaElement) => { + await wrapper.restart({ kind: Track.Kind.Video, track, element }); + return processedTrack(); + }, + destroy: async () => wrapper.destroy(), + }; +} + const ProcessorContext = createContext(undefined); export function useTrackProcessor(): ProcessorState { @@ -62,7 +93,7 @@ export function useTrackProcessorState$(): Behavior { */ export function applyProcessor( videoTrack: LocalVideoTrack, - processor: ProcessorWrapper | undefined, + processor: VideoProcessor | undefined, ): void { if (processor && !videoTrack.getProcessor()) { // A MediaStreamTrackProcessor cannot be constructed on an ended track @@ -73,7 +104,7 @@ export function applyProcessor( logger.debug("Not attaching video processor to an ended track"); return; } - videoTrack.setProcessor(processor).catch((e) => { + videoTrack.setProcessor(convertToLivekitProcessor(processor)).catch((e) => { logger.warn("Failed to attach video processor", e); }); } @@ -125,9 +156,11 @@ export const ProcessorProvider: FC = ({ children }) => { const supported = useMemo(() => supportsBackgroundProcessors(), []); const blur = useMemo( () => - new ProcessorWrapper( - new BlurBackgroundTransformer({ blurRadius: 15 }), - "background-blur", + fromLivekitProcessor( + new ProcessorWrapper( + new BlurBackgroundTransformer({ blurRadius: 15 }), + "background-blur", + ), ), [], );