implement noise reduction based on DeepFilterNet3

package.json

@@ -145,5 +145,8 @@
     "qs": "^6.14.1",
     "js-yaml": "^4.1.1"
   },
-  "packageManager": "yarn@4.7.0"
+  "packageManager": "yarn@4.7.0",
+  "dependencies": {
+    "deepfilternet3-noise-filter": "^1.2.1"
+  }
 }

src/livekit/NoiseSuppressionTransformer.ts

@@ -0,0 +1,147 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { DeepFilterNoiseFilterProcessor } from "deepfilternet3-noise-filter";
+import { logger } from "matrix-js-sdk/lib/logger";
+
+/**
+ * Wrapper for DeepFilterNet3 Noise Suppression Processor.
+ * Integrates with LiveKit audio track processing.
+ */
+export class NoiseSuppressionTransformer {
+  private processor: DeepFilterNoiseFilterProcessor | null = null;
+  private initialized = false;
+  private readonly sampleRate: number = 48000;
+
+  /**
+   * Initialize the noise suppression processor
+   * @param level - Noise reduction level (0-1)
+   * @param enabled - Whether noise suppression is enabled
+   */
+  public async initialize(
+    level: number = 0.75,
+    enabled: boolean = true,
+  ): Promise<void> {
+    if (this.initialized) {
+      return;
+    }
+
+    try {
+      // Clamp level between 0-1
+      const clampedLevel = Math.max(0, Math.min(1, level));
+
+      // Determine asset URL based on environment
+      // In development, use local proxy to avoid CORS issues
+      // In production, use direct CDN or custom assetConfig
+      const isProduction = import.meta.env.PROD;
+      const assetUrl = isProduction
+        ? process.env.VITE_NOISE_SUPPRESSION_CDN_URL ||
+          "https://cdn.mezon.ai/AI/models/datas/noise_suppression/deepfilternet3"
+        : `${window.location.origin}/assets/deepfilternet3`;
+
+      this.processor = new DeepFilterNoiseFilterProcessor({
+        sampleRate: this.sampleRate,
+        noiseReductionLevel: clampedLevel * 100,
+        enabled,
+        assetConfig: {
+          cdnUrl: assetUrl,
+        },
+      });
+
+      this.initialized = true;
+      logger.log(
+        `[NoiseSuppressionTransformer] Initialized with level=${clampedLevel}, enabled=${enabled}, assetUrl=${assetUrl}`,
+      );
+    } catch (error) {
+      logger.error(
+        "[NoiseSuppressionTransformer] Initialization failed:",
+        error,
+      );
+      throw error;
+    }
+  }
+
+  /**
+   * Get the underlying processor instance
+   */
+  public getProcessor(): DeepFilterNoiseFilterProcessor | null {
+    return this.processor;
+  }
+
+  /**
+   * Set the noise reduction level (0-1)
+   */
+  public setSuppressionLevel(level: number): void {
+    if (!this.processor) {
+      logger.warn(
+        "[NoiseSuppressionTransformer] Processor not initialized, cannot set suppression level",
+      );
+      return;
+    }
+
+    const clampedLevel = Math.max(0, Math.min(1, level));
+    try {
+      this.processor.setSuppressionLevel(clampedLevel * 100);
+      logger.log(
+        `[NoiseSuppressionTransformer] Suppression level set to ${clampedLevel}`,
+      );
+    } catch (error) {
+      logger.error(
+        "[NoiseSuppressionTransformer] Failed to set suppression level:",
+        error,
+      );
+    }
+  }
+
+  /**
+   * Enable or disable noise suppression
+   */
+  public setEnabled(enabled: boolean): void {
+    if (!this.processor) {
+      logger.warn(
+        "[NoiseSuppressionTransformer] Processor not initialized, cannot set enabled state",
+      );
+      return;
+    }
+
+    try {
+      this.processor.setEnabled(enabled);
+      logger.log(
+        `[NoiseSuppressionTransformer] Noise suppression ${enabled ? "enabled" : "disabled"}`,
+      );
+      // Log processor state for debugging
+      const processorState = (this.processor as any).enabled;
+      logger.debug(
+        `[NoiseSuppressionTransformer] Processor internal state: enabled=${processorState}`,
+      );
+    } catch (error) {
+      logger.error(
+        "[NoiseSuppressionTransformer] Failed to set enabled state:",
+        error,
+      );
+    }
+  }
+
+  /**
+   * Clean up resources
+   */
+  public destroy(): void {
+    if (this.processor) {
+      try {
+        // Note: DeepFilterNoiseFilterProcessor may have a destroy method
+        // Call it if available
+        if (typeof (this.processor as any).destroy === "function") {
+          (this.processor as any).destroy();
+        }
+      } catch (error) {
+        logger.error("[NoiseSuppressionTransformer] Cleanup failed:", error);
+      }
+      this.processor = null;
+      this.initialized = false;
+    }
+  }
+}

src/livekit/audioTrackNoiseSuppressionSync.ts

@@ -0,0 +1,120 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import type { LocalAudioTrack } from "livekit-client";
+import { combineLatest } from "rxjs";
+import { logger } from "matrix-js-sdk/lib/logger";
+
+import { noiseSuppressionEnabled, noiseSuppressionLevel } from "../settings/settings";
+import { getUrlParams } from "../UrlParams";
+import type { Behavior } from "../state/Behavior";
+import type { ObservableScope } from "../state/ObservableScope";
+import { NoiseSuppressionTransformer } from "./NoiseSuppressionTransformer";
+
+/**
+ * Synchronizes the noise suppression processor with audio tracks and settings.
+ * This function manages the lifecycle of the NoiseSuppressionTransformer
+ * and ensures it's applied to the audio track when settings change.
+ * URL parameters can override user settings if provided.
+ *
+ * @param scope - The ObservableScope for managing subscriptions
+ * @param audioTrack$ - Observable of the local audio track
+ */
+export const audioTrackNoiseSuppressionSync = (
+  scope: ObservableScope,
+  audioTrack$: Behavior<LocalAudioTrack | null>,
+): void => {
+  // Create a single transformer instance shared across all subscriptions
+  let transformer: NoiseSuppressionTransformer | null = null;
+  let hasInitialized = false;
+  // Get URL parameters for noise suppression (only used for initial setup)
+  const urlParams = getUrlParams();
+
+  combineLatest([
+    audioTrack$,
+    noiseSuppressionEnabled.value$,
+    noiseSuppressionLevel.value$,
+  ])
+    .pipe(scope.bind())
+    .subscribe(async ([audioTrack, settingEnabled, settingLevel]) => {
+      try {
+        // On first initialization, use URL parameters if provided, otherwise use settings
+        // After that, always use settings (user can change them at runtime)
+        let enabledValue = settingEnabled;
+        let levelValue = settingLevel;
+
+        if (!hasInitialized) {
+          // First time: use URL params as overrides if provided
+          if (urlParams.noiseSuppressionEnabled !== undefined) {
+            enabledValue = urlParams.noiseSuppressionEnabled;
+          }
+          if (urlParams.noiseSuppressionLevel !== undefined) {
+            levelValue = urlParams.noiseSuppressionLevel;
+          }
+          hasInitialized = true;
+          logger.debug(
+            "[audioTrackNoiseSuppressionSync] Initialized from URL params: enabled=" +
+              enabledValue +
+              ", level=" +
+              levelValue,
+          );
+        }
+
+        // Initialize transformer on first use
+        if (!transformer) {
+          transformer = new NoiseSuppressionTransformer();
+          await transformer.initialize(levelValue, enabledValue);
+          logger.debug(
+            "[audioTrackNoiseSuppressionSync] Transformer initialized with enabled=" +
+              enabledValue +
+              ", level=" +
+              levelValue,
+          );
+        }
+
+        const processor = transformer.getProcessor();
+        if (!processor) {
+          logger.error("[audioTrackNoiseSuppressionSync] Processor not initialized");
+          return;
+        }
+
+        // Apply processor to audio track if track exists
+        if (audioTrack) {
+          if (!audioTrack.getProcessor()) {
+            logger.debug(
+              "[audioTrackNoiseSuppressionSync] Setting noise suppression processor on audio track",
+            );
+            await audioTrack.setProcessor(processor);
+          }
+          // Update processor state - with small delay to ensure processor is ready
+          Promise.resolve().then(() => {
+            transformer!.setEnabled(enabledValue);
+            transformer!.setSuppressionLevel(levelValue);
+            logger.debug(
+              "[audioTrackNoiseSuppressionSync] Updated: enabled=" +
+                enabledValue +
+                ", level=" +
+                levelValue,
+            );
+          });
+        } else {
+          // Track was removed - stop processor if applicable
+          logger.debug("[audioTrackNoiseSuppressionSync] Audio track not available");
+        }
+      } catch (error) {
+        logger.error("[audioTrackNoiseSuppressionSync] Error:", error);
+      }
+    });
+
+  // Cleanup on scope end
+  scope.onEnd(() => {
+    if (transformer) {
+      transformer.destroy();
+      transformer = null;
+    }
+  });
+};

src/livekit/useNoiseSuppressionTransformer.ts

@@ -0,0 +1,57 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { useEffect, useRef } from "react";
+
+import { noiseSuppressionEnabled, noiseSuppressionLevel } from "../settings/settings";
+import { useBehavior } from "../useBehavior";
+import { NoiseSuppressionTransformer } from "../livekit/NoiseSuppressionTransformer";
+
+/**
+ * Hook to manage the NoiseSuppressionTransformer instance.
+ * Synchronizes the transformer with the noise suppression settings.
+ * Returns the transformer instance for use in Publishers.
+ */
+export const useNoiseSuppressionTransformer = (): NoiseSuppressionTransformer => {
+  const transformerRef = useRef<NoiseSuppressionTransformer | null>(null);
+  const enabledValue = useBehavior(noiseSuppressionEnabled.value$);
+  const levelValue = useBehavior(noiseSuppressionLevel.value$);
+
+  // Initialize transformer on first mount
+  useEffect(() => {
+    if (!transformerRef.current) {
+      transformerRef.current = new NoiseSuppressionTransformer();
+      // Initialize with current settings
+      void transformerRef.current.initialize(levelValue, enabledValue);
+    }
+  }, []);
+
+  // Sync enabled state when setting changes
+  useEffect(() => {
+    if (transformerRef.current) {
+      transformerRef.current.setEnabled(enabledValue);
+    }
+  }, [enabledValue]);
+
+  // Sync level when setting changes
+  useEffect(() => {
+    if (transformerRef.current) {
+      transformerRef.current.setSuppressionLevel(levelValue);
+    }
+  }, [levelValue]);
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      if (transformerRef.current) {
+        transformerRef.current.destroy();
+      }
+    };
+  }, []);
+
+  return transformerRef.current!;
+};

src/state/CallViewModel/localMember/Publisher.ts

@@ -9,6 +9,7 @@ import {
   ConnectionState as LivekitConnectionState,
   type LocalTrackPublication,
   LocalVideoTrack,
+  LocalAudioTrack,
   ParticipantEvent,
   type Room as LivekitRoom,
   Track,
@@ -29,6 +30,7 @@ import {
   type ProcessorState,
   trackProcessorSync,
 } from "../../../livekit/TrackProcessorContext.tsx";
+import { audioTrackNoiseSuppressionSync } from "../../../livekit/audioTrackNoiseSuppressionSync";
 import { getUrlParams } from "../../../UrlParams.ts";
 import { observeTrackReference$ } from "../../observeTrackReference";
 import { type Connection } from "../remoteMembers/Connection.ts";
@@ -73,6 +75,8 @@ export class Publisher {
 
     // Setup track processor syncing (blur)
     this.observeTrackProcessors(this.scope, room, trackerProcessorState$);
+    // Setup audio track processor syncing (noise suppression)
+    this.observeAudioTrackProcessors(this.scope, room);
     // Observe media device changes and update LiveKit active devices accordingly
     this.observeMediaDevices(this.scope, devices, controlledAudioDevices);
 
@@ -416,4 +420,20 @@ export class Publisher {
     );
     trackProcessorSync(scope, track$, trackerProcessorState$);
   }
+
+  private observeAudioTrackProcessors(
+    scope: ObservableScope,
+    room: LivekitRoom,
+  ): void {
+    const track$ = scope.behavior(
+      observeTrackReference$(room.localParticipant, Track.Source.Microphone).pipe(
+        map((trackRef) => {
+          const track = trackRef?.publication.track;
+          return track instanceof LocalAudioTrack ? track : null;
+        }),
+      ),
+      null,
+    );
+    audioTrackNoiseSuppressionSync(scope, track$);
+  }
 }

vite.config.ts

@@ -101,6 +101,15 @@ export default ({
         key: fs.readFileSync("./backend/dev_tls_m.localhost.key"),
         cert: fs.readFileSync("./backend/dev_tls_m.localhost.crt"),
       },
+      proxy: {
+        // Proxy for DeepFilterNet3 assets to avoid CORS issues during development
+        "/assets/deepfilternet3": {
+          target: "https://cdn.mezon.ai/AI/models/datas/noise_suppression/deepfilternet3",
+          changeOrigin: true,
+          rewrite: (path) => path.replace(/^\/assets\/deepfilternet3/, ""),
+          secure: false, // Allow self-signed certs in development
+        },
+      },
     },
     worker: {
       format: "es",

yarn.lock

@@ -8016,6 +8016,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"deepfilternet3-noise-filter@npm:^1.2.1":
+  version: 1.2.1
+  resolution: "deepfilternet3-noise-filter@npm:1.2.1"
+  peerDependencies:
+    livekit-client: ^2.0.0
+  checksum: 10c0/db1488bd202a3e3657105c62c7070d68105029501dfd6bc393f89b7598cf4c26d97afc02caca43e6f3b7cef568a17468f17add9f1f7deb8a63a789f05108e230
+  languageName: node
+  linkType: hard
+
 "define-data-property@npm:^1.0.1, define-data-property@npm:^1.1.4":
   version: 1.1.4
   resolution: "define-data-property@npm:1.1.4"
@@ -8331,6 +8340,7 @@ __metadata:
     babel-plugin-transform-vite-meta-env: "npm:^1.0.3"
     classnames: "npm:^2.3.1"
     copy-to-clipboard: "npm:^3.3.3"
+    deepfilternet3-noise-filter: "npm:^1.2.1"
     eslint: "npm:^8.14.0"
     eslint-config-google: "npm:^0.14.0"
     eslint-config-prettier: "npm:^10.0.0"