implement noise reduction based on DeepFilterNet3

2026-08-08 20:09:19 +00:00 · 2026-03-26 14:25:07 +01:00
parent c703adb14d
commit b6cc810db2
7 changed files with 367 additions and 1 deletions
--- a/src/livekit/audioTrackNoiseSuppressionSync.ts
+++ b/src/livekit/audioTrackNoiseSuppressionSync.ts
@@ -0,0 +1,120 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import type { LocalAudioTrack } from "livekit-client";
+import { combineLatest } from "rxjs";
+import { logger } from "matrix-js-sdk/lib/logger";
+
+import { noiseSuppressionEnabled, noiseSuppressionLevel } from "../settings/settings";
+import { getUrlParams } from "../UrlParams";
+import type { Behavior } from "../state/Behavior";
+import type { ObservableScope } from "../state/ObservableScope";
+import { NoiseSuppressionTransformer } from "./NoiseSuppressionTransformer";
+
+/**
+ * Synchronizes the noise suppression processor with audio tracks and settings.
+ * This function manages the lifecycle of the NoiseSuppressionTransformer
+ * and ensures it's applied to the audio track when settings change.
+ * URL parameters can override user settings if provided.
+ *
+ * @param scope - The ObservableScope for managing subscriptions
+ * @param audioTrack$ - Observable of the local audio track
+ */
+export const audioTrackNoiseSuppressionSync = (
+  scope: ObservableScope,
+  audioTrack$: Behavior<LocalAudioTrack | null>,
+): void => {
+  // Create a single transformer instance shared across all subscriptions
+  let transformer: NoiseSuppressionTransformer | null = null;
+  let hasInitialized = false;
+  // Get URL parameters for noise suppression (only used for initial setup)
+  const urlParams = getUrlParams();
+
+  combineLatest([
+    audioTrack$,
+    noiseSuppressionEnabled.value$,
+    noiseSuppressionLevel.value$,
+  ])
+    .pipe(scope.bind())
+    .subscribe(async ([audioTrack, settingEnabled, settingLevel]) => {
+      try {
+        // On first initialization, use URL parameters if provided, otherwise use settings
+        // After that, always use settings (user can change them at runtime)
+        let enabledValue = settingEnabled;
+        let levelValue = settingLevel;
+
+        if (!hasInitialized) {
+          // First time: use URL params as overrides if provided
+          if (urlParams.noiseSuppressionEnabled !== undefined) {
+            enabledValue = urlParams.noiseSuppressionEnabled;
+          }
+          if (urlParams.noiseSuppressionLevel !== undefined) {
+            levelValue = urlParams.noiseSuppressionLevel;
+          }
+          hasInitialized = true;
+          logger.debug(
+            "[audioTrackNoiseSuppressionSync] Initialized from URL params: enabled=" +
+              enabledValue +
+              ", level=" +
+              levelValue,
+          );
+        }
+
+        // Initialize transformer on first use
+        if (!transformer) {
+          transformer = new NoiseSuppressionTransformer();
+          await transformer.initialize(levelValue, enabledValue);
+          logger.debug(
+            "[audioTrackNoiseSuppressionSync] Transformer initialized with enabled=" +
+              enabledValue +
+              ", level=" +
+              levelValue,
+          );
+        }
+
+        const processor = transformer.getProcessor();
+        if (!processor) {
+          logger.error("[audioTrackNoiseSuppressionSync] Processor not initialized");
+          return;
+        }
+
+        // Apply processor to audio track if track exists
+        if (audioTrack) {
+          if (!audioTrack.getProcessor()) {
+            logger.debug(
+              "[audioTrackNoiseSuppressionSync] Setting noise suppression processor on audio track",
+            );
+            await audioTrack.setProcessor(processor);
+          }
+          // Update processor state - with small delay to ensure processor is ready
+          Promise.resolve().then(() => {
+            transformer!.setEnabled(enabledValue);
+            transformer!.setSuppressionLevel(levelValue);
+            logger.debug(
+              "[audioTrackNoiseSuppressionSync] Updated: enabled=" +
+                enabledValue +
+                ", level=" +
+                levelValue,
+            );
+          });
+        } else {
+          // Track was removed - stop processor if applicable
+          logger.debug("[audioTrackNoiseSuppressionSync] Audio track not available");
+        }
+      } catch (error) {
+        logger.error("[audioTrackNoiseSuppressionSync] Error:", error);
+      }
+    });
+
+  // Cleanup on scope end
+  scope.onEnd(() => {
+    if (transformer) {
+      transformer.destroy();
+      transformer = null;
+    }
+  });
+};