refactor

- still should not work without a fixing upstream LK:
https://github.com/livekit/components-js/pull/1042
https://github.com/livekit/components-js/pull/1043

src/livekit/useLiveKit.ts

@@ -9,9 +9,8 @@ import {
   ConnectionState,
   E2EEOptions,
   ExternalE2EEKeyProvider,
-  LocalTrackPublication,
+  LocalVideoTrack,
   Room,
-  RoomEvent,
   RoomOptions,
   Track,
 } from "livekit-client";
@@ -19,7 +18,6 @@ import { useEffect, useMemo, useRef } from "react";
 import E2EEWorker from "livekit-client/e2ee-worker?worker";
 import { logger } from "matrix-js-sdk/src/logger";
 import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
-import { BackgroundBlur as backgroundBlur } from "@livekit/track-processors";
 
 import { defaultLiveKitOptions } from "./options";
 import { SFUConfig } from "./openIDSFU";
@@ -29,7 +27,6 @@ import {
   MediaDevices,
   useMediaDevices,
 } from "./MediaDevicesContext";
-import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
 import {
   ECConnectionState,
   useECConnectionState,
@@ -37,7 +34,11 @@ import {
 import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
 import { E2eeType } from "../e2ee/e2eeType";
 import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
-import { useSetting } from "../settings/settings";
+import {
+  useTrackProcessor,
+  useTrackProcessorSync,
+} from "./TrackProcessorContext";
+import { useInitial } from "../useInitial";
 
 interface UseLivekitResult {
   livekitRoom?: Room;
@@ -83,22 +84,16 @@ export function useLiveKit(
   const initialMuteStates = useRef<MuteStates>(muteStates);
   const devices = useMediaDevices();
   const initialDevices = useRef<MediaDevices>(devices);
-  const blur = useMemo(() => {
-    let b = undefined;
-    try {
-      b = backgroundBlur(15, { delegate: "GPU" });
-    } catch (e) {
-      logger.error("disable background blur", e);
-    }
-    return b;
-  }, []);
+
+  const { processor } = useTrackProcessor() || {};
+  const initialProcessor = useInitial(() => processor);
   const roomOptions = useMemo(
     (): RoomOptions => ({
       ...defaultLiveKitOptions,
       videoCaptureDefaults: {
         ...defaultLiveKitOptions.videoCaptureDefaults,
         deviceId: initialDevices.current.videoInput.selectedId,
-        processor: blur,
+        processor: initialProcessor,
       },
       audioCaptureDefaults: {
         ...defaultLiveKitOptions.audioCaptureDefaults,
@@ -109,7 +104,7 @@ export function useLiveKit(
       },
       e2ee: e2eeOptions,
     }),
-    [blur, e2eeOptions],
+    [e2eeOptions, initialProcessor],
   );
 
   // Store if audio/video are currently updating. If to prohibit unnecessary calls
@@ -134,6 +129,15 @@ export function useLiveKit(
     return r;
   }, [roomOptions, e2eeSystem]);
 
+  const videoTrack = useMemo(
+    () =>
+      Array.from(room.localParticipant.videoTrackPublications.values()).find(
+        (v) => v.source === Track.Source.Camera,
+      )?.track as LocalVideoTrack | null,
+    [room.localParticipant.videoTrackPublications],
+  );
+  useTrackProcessorSync(videoTrack);
+
   const connectionState = useECConnectionState(
     {
       deviceId: initialDevices.current.audioInput.selectedId,
@@ -143,58 +147,6 @@ export function useLiveKit(
     sfuConfig,
   );
 
-  const [showBackgroundBlur] = useSetting(backgroundBlurSettings);
-  const videoTrackPromise = useRef<
-    undefined | Promise<LocalTrackPublication | undefined>
-  >(undefined);
-
-  useEffect(() => {
-    // Don't even try if we cannot blur on this platform
-    if (!blur) return;
-    if (!room || videoTrackPromise.current) return;
-    const update = async (): Promise<void> => {
-      let publishCallback: undefined | ((track: LocalTrackPublication) => void);
-      videoTrackPromise.current = new Promise<
-        LocalTrackPublication | undefined
-      >((resolve) => {
-        const videoTrack = Array.from(
-          room.localParticipant.videoTrackPublications.values(),
-        ).find((v) => v.source === Track.Source.Camera);
-        if (videoTrack) {
-          resolve(videoTrack);
-        }
-        publishCallback = (videoTrack: LocalTrackPublication): void => {
-          if (videoTrack.source === Track.Source.Camera) {
-            resolve(videoTrack);
-          }
-        };
-        room.on(RoomEvent.LocalTrackPublished, publishCallback);
-      });
-
-      const videoTrack = await videoTrackPromise.current;
-
-      if (publishCallback)
-        room.off(RoomEvent.LocalTrackPublished, publishCallback);
-
-      if (videoTrack !== undefined) {
-        if (
-          showBackgroundBlur &&
-          videoTrack.track?.getProcessor()?.name !== "background-blur"
-        ) {
-          logger.info("Blur: set blur");
-
-          void videoTrack.track?.setProcessor(blur);
-        } else if (
-          videoTrack.track?.getProcessor()?.name === "background-blur"
-        ) {
-          void videoTrack.track?.stopProcessor();
-        }
-      }
-      videoTrackPromise.current = undefined;
-    };
-    void update();
-  }, [blur, room, showBackgroundBlur]);
-
   useEffect(() => {
     // Sync the requested mute states with LiveKit's mute states. We do it this
     // way around rather than using LiveKit as the source of truth, so that the
@@ -261,6 +213,7 @@ export function useLiveKit(
                 audioMuteUpdating.current = true;
                 trackPublication = await participant.setMicrophoneEnabled(
                   buttonEnabled.current.audio,
+                  room.options.audioCaptureDefaults,
                 );
                 audioMuteUpdating.current = false;
                 break;
@@ -268,6 +221,7 @@ export function useLiveKit(
                 videoMuteUpdating.current = true;
                 trackPublication = await participant.setCameraEnabled(
                   buttonEnabled.current.video,
+                  room.options.videoCaptureDefaults,
                 );
                 videoMuteUpdating.current = false;
                 break;