From b6cc810db22487160eb57f5a8d96da8cdc23e687 Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 26 Mar 2026 14:25:07 +0100 Subject: [PATCH] implement noise reduction based on DeepFilterNet3 --- package.json | 5 +- src/livekit/NoiseSuppressionTransformer.ts | 147 ++++++++++++++++++ src/livekit/audioTrackNoiseSuppressionSync.ts | 120 ++++++++++++++ src/livekit/useNoiseSuppressionTransformer.ts | 57 +++++++ .../CallViewModel/localMember/Publisher.ts | 20 +++ vite.config.ts | 9 ++ yarn.lock | 10 ++ 7 files changed, 367 insertions(+), 1 deletion(-) create mode 100644 src/livekit/NoiseSuppressionTransformer.ts create mode 100644 src/livekit/audioTrackNoiseSuppressionSync.ts create mode 100644 src/livekit/useNoiseSuppressionTransformer.ts diff --git a/package.json b/package.json index cc8a36eb..c206ce9a 100644 --- a/package.json +++ b/package.json @@ -145,5 +145,8 @@ "qs": "^6.14.1", "js-yaml": "^4.1.1" }, - "packageManager": "yarn@4.7.0" + "packageManager": "yarn@4.7.0", + "dependencies": { + "deepfilternet3-noise-filter": "^1.2.1" + } } diff --git a/src/livekit/NoiseSuppressionTransformer.ts b/src/livekit/NoiseSuppressionTransformer.ts new file mode 100644 index 00000000..2df562fe --- /dev/null +++ b/src/livekit/NoiseSuppressionTransformer.ts @@ -0,0 +1,147 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { DeepFilterNoiseFilterProcessor } from "deepfilternet3-noise-filter"; +import { logger } from "matrix-js-sdk/lib/logger"; + +/** + * Wrapper for DeepFilterNet3 Noise Suppression Processor. + * Integrates with LiveKit audio track processing. + */ +export class NoiseSuppressionTransformer { + private processor: DeepFilterNoiseFilterProcessor | null = null; + private initialized = false; + private readonly sampleRate: number = 48000; + + /** + * Initialize the noise suppression processor + * @param level - Noise reduction level (0-1) + * @param enabled - Whether noise suppression is enabled + */ + public async initialize( + level: number = 0.75, + enabled: boolean = true, + ): Promise { + if (this.initialized) { + return; + } + + try { + // Clamp level between 0-1 + const clampedLevel = Math.max(0, Math.min(1, level)); + + // Determine asset URL based on environment + // In development, use local proxy to avoid CORS issues + // In production, use direct CDN or custom assetConfig + const isProduction = import.meta.env.PROD; + const assetUrl = isProduction + ? process.env.VITE_NOISE_SUPPRESSION_CDN_URL || + "https://cdn.mezon.ai/AI/models/datas/noise_suppression/deepfilternet3" + : `${window.location.origin}/assets/deepfilternet3`; + + this.processor = new DeepFilterNoiseFilterProcessor({ + sampleRate: this.sampleRate, + noiseReductionLevel: clampedLevel * 100, + enabled, + assetConfig: { + cdnUrl: assetUrl, + }, + }); + + this.initialized = true; + logger.log( + `[NoiseSuppressionTransformer] Initialized with level=${clampedLevel}, enabled=${enabled}, assetUrl=${assetUrl}`, + ); + } catch (error) { + logger.error( + "[NoiseSuppressionTransformer] Initialization failed:", + error, + ); + throw error; + } + } + + /** + * Get the underlying processor instance + */ + public getProcessor(): DeepFilterNoiseFilterProcessor | null { + return this.processor; + } + + /** + * Set the noise reduction level (0-1) + */ + public setSuppressionLevel(level: number): void { + if (!this.processor) { + logger.warn( + "[NoiseSuppressionTransformer] Processor not initialized, cannot set suppression level", + ); + return; + } + + const clampedLevel = Math.max(0, Math.min(1, level)); + try { + this.processor.setSuppressionLevel(clampedLevel * 100); + logger.log( + `[NoiseSuppressionTransformer] Suppression level set to ${clampedLevel}`, + ); + } catch (error) { + logger.error( + "[NoiseSuppressionTransformer] Failed to set suppression level:", + error, + ); + } + } + + /** + * Enable or disable noise suppression + */ + public setEnabled(enabled: boolean): void { + if (!this.processor) { + logger.warn( + "[NoiseSuppressionTransformer] Processor not initialized, cannot set enabled state", + ); + return; + } + + try { + this.processor.setEnabled(enabled); + logger.log( + `[NoiseSuppressionTransformer] Noise suppression ${enabled ? "enabled" : "disabled"}`, + ); + // Log processor state for debugging + const processorState = (this.processor as any).enabled; + logger.debug( + `[NoiseSuppressionTransformer] Processor internal state: enabled=${processorState}`, + ); + } catch (error) { + logger.error( + "[NoiseSuppressionTransformer] Failed to set enabled state:", + error, + ); + } + } + + /** + * Clean up resources + */ + public destroy(): void { + if (this.processor) { + try { + // Note: DeepFilterNoiseFilterProcessor may have a destroy method + // Call it if available + if (typeof (this.processor as any).destroy === "function") { + (this.processor as any).destroy(); + } + } catch (error) { + logger.error("[NoiseSuppressionTransformer] Cleanup failed:", error); + } + this.processor = null; + this.initialized = false; + } + } +} diff --git a/src/livekit/audioTrackNoiseSuppressionSync.ts b/src/livekit/audioTrackNoiseSuppressionSync.ts new file mode 100644 index 00000000..6ee9e07f --- /dev/null +++ b/src/livekit/audioTrackNoiseSuppressionSync.ts @@ -0,0 +1,120 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import type { LocalAudioTrack } from "livekit-client"; +import { combineLatest } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { noiseSuppressionEnabled, noiseSuppressionLevel } from "../settings/settings"; +import { getUrlParams } from "../UrlParams"; +import type { Behavior } from "../state/Behavior"; +import type { ObservableScope } from "../state/ObservableScope"; +import { NoiseSuppressionTransformer } from "./NoiseSuppressionTransformer"; + +/** + * Synchronizes the noise suppression processor with audio tracks and settings. + * This function manages the lifecycle of the NoiseSuppressionTransformer + * and ensures it's applied to the audio track when settings change. + * URL parameters can override user settings if provided. + * + * @param scope - The ObservableScope for managing subscriptions + * @param audioTrack$ - Observable of the local audio track + */ +export const audioTrackNoiseSuppressionSync = ( + scope: ObservableScope, + audioTrack$: Behavior, +): void => { + // Create a single transformer instance shared across all subscriptions + let transformer: NoiseSuppressionTransformer | null = null; + let hasInitialized = false; + // Get URL parameters for noise suppression (only used for initial setup) + const urlParams = getUrlParams(); + + combineLatest([ + audioTrack$, + noiseSuppressionEnabled.value$, + noiseSuppressionLevel.value$, + ]) + .pipe(scope.bind()) + .subscribe(async ([audioTrack, settingEnabled, settingLevel]) => { + try { + // On first initialization, use URL parameters if provided, otherwise use settings + // After that, always use settings (user can change them at runtime) + let enabledValue = settingEnabled; + let levelValue = settingLevel; + + if (!hasInitialized) { + // First time: use URL params as overrides if provided + if (urlParams.noiseSuppressionEnabled !== undefined) { + enabledValue = urlParams.noiseSuppressionEnabled; + } + if (urlParams.noiseSuppressionLevel !== undefined) { + levelValue = urlParams.noiseSuppressionLevel; + } + hasInitialized = true; + logger.debug( + "[audioTrackNoiseSuppressionSync] Initialized from URL params: enabled=" + + enabledValue + + ", level=" + + levelValue, + ); + } + + // Initialize transformer on first use + if (!transformer) { + transformer = new NoiseSuppressionTransformer(); + await transformer.initialize(levelValue, enabledValue); + logger.debug( + "[audioTrackNoiseSuppressionSync] Transformer initialized with enabled=" + + enabledValue + + ", level=" + + levelValue, + ); + } + + const processor = transformer.getProcessor(); + if (!processor) { + logger.error("[audioTrackNoiseSuppressionSync] Processor not initialized"); + return; + } + + // Apply processor to audio track if track exists + if (audioTrack) { + if (!audioTrack.getProcessor()) { + logger.debug( + "[audioTrackNoiseSuppressionSync] Setting noise suppression processor on audio track", + ); + await audioTrack.setProcessor(processor); + } + // Update processor state - with small delay to ensure processor is ready + Promise.resolve().then(() => { + transformer!.setEnabled(enabledValue); + transformer!.setSuppressionLevel(levelValue); + logger.debug( + "[audioTrackNoiseSuppressionSync] Updated: enabled=" + + enabledValue + + ", level=" + + levelValue, + ); + }); + } else { + // Track was removed - stop processor if applicable + logger.debug("[audioTrackNoiseSuppressionSync] Audio track not available"); + } + } catch (error) { + logger.error("[audioTrackNoiseSuppressionSync] Error:", error); + } + }); + + // Cleanup on scope end + scope.onEnd(() => { + if (transformer) { + transformer.destroy(); + transformer = null; + } + }); +}; diff --git a/src/livekit/useNoiseSuppressionTransformer.ts b/src/livekit/useNoiseSuppressionTransformer.ts new file mode 100644 index 00000000..9e268159 --- /dev/null +++ b/src/livekit/useNoiseSuppressionTransformer.ts @@ -0,0 +1,57 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { useEffect, useRef } from "react"; + +import { noiseSuppressionEnabled, noiseSuppressionLevel } from "../settings/settings"; +import { useBehavior } from "../useBehavior"; +import { NoiseSuppressionTransformer } from "../livekit/NoiseSuppressionTransformer"; + +/** + * Hook to manage the NoiseSuppressionTransformer instance. + * Synchronizes the transformer with the noise suppression settings. + * Returns the transformer instance for use in Publishers. + */ +export const useNoiseSuppressionTransformer = (): NoiseSuppressionTransformer => { + const transformerRef = useRef(null); + const enabledValue = useBehavior(noiseSuppressionEnabled.value$); + const levelValue = useBehavior(noiseSuppressionLevel.value$); + + // Initialize transformer on first mount + useEffect(() => { + if (!transformerRef.current) { + transformerRef.current = new NoiseSuppressionTransformer(); + // Initialize with current settings + void transformerRef.current.initialize(levelValue, enabledValue); + } + }, []); + + // Sync enabled state when setting changes + useEffect(() => { + if (transformerRef.current) { + transformerRef.current.setEnabled(enabledValue); + } + }, [enabledValue]); + + // Sync level when setting changes + useEffect(() => { + if (transformerRef.current) { + transformerRef.current.setSuppressionLevel(levelValue); + } + }, [levelValue]); + + // Cleanup on unmount + useEffect(() => { + return () => { + if (transformerRef.current) { + transformerRef.current.destroy(); + } + }; + }, []); + + return transformerRef.current!; +}; diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index b7841c49..1f63bfc8 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -9,6 +9,7 @@ import { ConnectionState as LivekitConnectionState, type LocalTrackPublication, LocalVideoTrack, + LocalAudioTrack, ParticipantEvent, type Room as LivekitRoom, Track, @@ -29,6 +30,7 @@ import { type ProcessorState, trackProcessorSync, } from "../../../livekit/TrackProcessorContext.tsx"; +import { audioTrackNoiseSuppressionSync } from "../../../livekit/audioTrackNoiseSuppressionSync"; import { getUrlParams } from "../../../UrlParams.ts"; import { observeTrackReference$ } from "../../observeTrackReference"; import { type Connection } from "../remoteMembers/Connection.ts"; @@ -73,6 +75,8 @@ export class Publisher { // Setup track processor syncing (blur) this.observeTrackProcessors(this.scope, room, trackerProcessorState$); + // Setup audio track processor syncing (noise suppression) + this.observeAudioTrackProcessors(this.scope, room); // Observe media device changes and update LiveKit active devices accordingly this.observeMediaDevices(this.scope, devices, controlledAudioDevices); @@ -416,4 +420,20 @@ export class Publisher { ); trackProcessorSync(scope, track$, trackerProcessorState$); } + + private observeAudioTrackProcessors( + scope: ObservableScope, + room: LivekitRoom, + ): void { + const track$ = scope.behavior( + observeTrackReference$(room.localParticipant, Track.Source.Microphone).pipe( + map((trackRef) => { + const track = trackRef?.publication.track; + return track instanceof LocalAudioTrack ? track : null; + }), + ), + null, + ); + audioTrackNoiseSuppressionSync(scope, track$); + } } diff --git a/vite.config.ts b/vite.config.ts index 97d643ec..3836d5ec 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -101,6 +101,15 @@ export default ({ key: fs.readFileSync("./backend/dev_tls_m.localhost.key"), cert: fs.readFileSync("./backend/dev_tls_m.localhost.crt"), }, + proxy: { + // Proxy for DeepFilterNet3 assets to avoid CORS issues during development + "/assets/deepfilternet3": { + target: "https://cdn.mezon.ai/AI/models/datas/noise_suppression/deepfilternet3", + changeOrigin: true, + rewrite: (path) => path.replace(/^\/assets\/deepfilternet3/, ""), + secure: false, // Allow self-signed certs in development + }, + }, }, worker: { format: "es", diff --git a/yarn.lock b/yarn.lock index cbbbf32f..ef1d4fc0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8016,6 +8016,15 @@ __metadata: languageName: node linkType: hard +"deepfilternet3-noise-filter@npm:^1.2.1": + version: 1.2.1 + resolution: "deepfilternet3-noise-filter@npm:1.2.1" + peerDependencies: + livekit-client: ^2.0.0 + checksum: 10c0/db1488bd202a3e3657105c62c7070d68105029501dfd6bc393f89b7598cf4c26d97afc02caca43e6f3b7cef568a17468f17add9f1f7deb8a63a789f05108e230 + languageName: node + linkType: hard + "define-data-property@npm:^1.0.1, define-data-property@npm:^1.1.4": version: 1.1.4 resolution: "define-data-property@npm:1.1.4" @@ -8331,6 +8340,7 @@ __metadata: babel-plugin-transform-vite-meta-env: "npm:^1.0.3" classnames: "npm:^2.3.1" copy-to-clipboard: "npm:^3.3.3" + deepfilternet3-noise-filter: "npm:^1.2.1" eslint: "npm:^8.14.0" eslint-config-google: "npm:^0.14.0" eslint-config-prettier: "npm:^10.0.0"