use abstracted livekti processor

src/room/LobbyView.tsx

@@ -24,6 +24,7 @@ import {
   type LocalVideoTrack,
   Track,
 } from "livekit-client";
+import { convertToLivekitProcessor } from "@element-hq/matrixrtc-sdk";
 
 import inCallStyles from "./InCallView.module.css";
 import styles from "./LobbyView.module.css";
@@ -150,7 +151,8 @@ export const LobbyView: FC<Props> = ({
       audio: Object.assign({}, initialAudioOptions),
       video: videoEnabled && {
         deviceId: videoInputId,
-        processor: initialProcessor,
+        processor:
+          initialProcessor && convertToLivekitProcessor(initialProcessor),
       },
     }),
     [initialAudioOptions, videoEnabled, videoInputId, initialProcessor],

src/tracks/TrackProcessorContext.test.ts

@@ -11,16 +11,21 @@ import {
   type BackgroundOptions,
   type ProcessorWrapper,
 } from "@livekit/track-processors";
+import { type VideoProcessor } from "@element-hq/matrixrtc-sdk";
 
-import { applyProcessor, trackProcessorSync } from "./TrackProcessorContext";
+import {
+  applyProcessor,
+  fromLivekitProcessor,
+  trackProcessorSync,
+} from "./TrackProcessorContext";
 import { constant } from "../state/Behavior";
 import { flushPromises, testScope } from "../utils/test";
 
-const processor = {} as ProcessorWrapper<BackgroundOptions>;
+const processor = { name: "blur" } as VideoProcessor;
 
 function mockTrack(
   readyState: MediaStreamTrackState,
-  current?: ProcessorWrapper<BackgroundOptions>,
+  current?: VideoProcessor,
 ): LocalVideoTrack {
   return {
     mediaStreamTrack: { readyState },
@@ -34,7 +39,9 @@ describe("applyProcessor", () => {
   it("attaches the processor to a live track", () => {
     const track = mockTrack("live");
     applyProcessor(track, processor);
-    expect(track.setProcessor).toHaveBeenCalledWith(processor);
+    expect(track.setProcessor).toHaveBeenCalledWith(
+      expect.objectContaining({ name: "blur" }),
+    );
   });
 
   it("does not attach the processor to an ended track", () => {
@@ -82,6 +89,45 @@ describe("trackProcessorSync", () => {
       constant(track),
       constant({ supported: true, processor }),
     );
-    expect(track.setProcessor).toHaveBeenCalledWith(processor);
+    expect(track.setProcessor).toHaveBeenCalledWith(
+      expect.objectContaining({ name: "blur" }),
+    );
+  });
+});
+
+describe("fromLivekitProcessor", () => {
+  const captured = { id: "captured" } as MediaStreamTrack;
+  const processed = { id: "processed" } as MediaStreamTrack;
+
+  function mockWrapper(
+    processedTrack: MediaStreamTrack | undefined,
+  ): ProcessorWrapper<BackgroundOptions> {
+    return {
+      name: "background-blur",
+      processedTrack,
+      init: vi.fn().mockResolvedValue(undefined),
+      restart: vi.fn().mockResolvedValue(undefined),
+      destroy: vi.fn().mockResolvedValue(undefined),
+    } as unknown as ProcessorWrapper<BackgroundOptions>;
+  }
+
+  it("resolves with the track the wrapper produced", async () => {
+    const wrapper = mockWrapper(processed);
+    const sdkProcessor = fromLivekitProcessor(wrapper);
+    expect(sdkProcessor.name).toBe("background-blur");
+    await expect(sdkProcessor.init(captured)).resolves.toBe(processed);
+    expect(wrapper.init).toHaveBeenCalledWith({
+      kind: "video",
+      track: captured,
+    });
+    await expect(sdkProcessor.restart(captured)).resolves.toBe(processed);
+    await sdkProcessor.destroy();
+    expect(wrapper.destroy).toHaveBeenCalled();
+  });
+
+  it("rejects where the wrapper produced no track", async () => {
+    await expect(
+      fromLivekitProcessor(mockWrapper(undefined)).init(captured),
+    ).rejects.toThrow("produced no track");
   });
 });

src/tracks/TrackProcessorContext.tsx

@@ -18,9 +18,13 @@ import {
   useEffect,
   useMemo,
 } from "react";
-import { type LocalVideoTrack } from "livekit-client";
+import { type LocalVideoTrack, Track } from "livekit-client";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { combineLatest } from "rxjs";
+import {
+  convertToLivekitProcessor,
+  type VideoProcessor,
+} from "@element-hq/matrixrtc-sdk";
 
 import {
   backgroundBlur as backgroundBlurSettings,
@@ -38,9 +42,36 @@ import { useValueBehavior } from "../useValueBehavior";
 
 export type ProcessorState = {
   supported: boolean | undefined;
-  processor: undefined | ProcessorWrapper<BackgroundOptions>;
+  processor: undefined | VideoProcessor;
 };
 
+/**
+ * This allows to use a LiveKit processor for the matrixRTCSDK (VideoProcessor).
+ * They have slightly different apis so we need to convert them with this
+ * function before we can use them in the matrixRTCSDK.
+ */
+export function fromLivekitProcessor(
+  wrapper: ProcessorWrapper<BackgroundOptions>,
+): VideoProcessor {
+  const processedTrack = (): MediaStreamTrack => {
+    if (wrapper.processedTrack === undefined)
+      throw new Error(`Processor ${wrapper.name} produced no track`);
+    return wrapper.processedTrack;
+  };
+  return {
+    name: wrapper.name,
+    init: async (track: MediaStreamTrack, element?: HTMLMediaElement) => {
+      await wrapper.init({ kind: Track.Kind.Video, track, element });
+      return processedTrack();
+    },
+    restart: async (track: MediaStreamTrack, element?: HTMLMediaElement) => {
+      await wrapper.restart({ kind: Track.Kind.Video, track, element });
+      return processedTrack();
+    },
+    destroy: async () => wrapper.destroy(),
+  };
+}
+
 const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
 
 export function useTrackProcessor(): ProcessorState {
@@ -62,7 +93,7 @@ export function useTrackProcessorState$(): Behavior<ProcessorState> {
  */
 export function applyProcessor(
   videoTrack: LocalVideoTrack,
-  processor: ProcessorWrapper<BackgroundOptions> | undefined,
+  processor: VideoProcessor | undefined,
 ): void {
   if (processor && !videoTrack.getProcessor()) {
     // A MediaStreamTrackProcessor cannot be constructed on an ended track
@@ -73,7 +104,7 @@ export function applyProcessor(
       logger.debug("Not attaching video processor to an ended track");
       return;
     }
-    videoTrack.setProcessor(processor).catch((e) => {
+    videoTrack.setProcessor(convertToLivekitProcessor(processor)).catch((e) => {
       logger.warn("Failed to attach video processor", e);
     });
   }
@@ -125,10 +156,12 @@ export const ProcessorProvider: FC<Props> = ({ children }) => {
   const supported = useMemo(() => supportsBackgroundProcessors(), []);
   const blur = useMemo(
     () =>
+      fromLivekitProcessor(
         new ProcessorWrapper(
           new BlurBackgroundTransformer({ blurRadius: 15 }),
           "background-blur",
         ),
+      ),
     [],
   );