diff --git a/package.json b/package.json index 36e03468..30f967e8 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "@livekit/components-core": "^0.11.0", "@livekit/components-react": "^2.0.0", "@livekit/track-processors": "^0.3.2", + "@mediapipe/tasks-vision": "^0.10.18", "@opentelemetry/api": "^1.4.0", "@opentelemetry/core": "^1.25.1", "@opentelemetry/exporter-trace-otlp-http": "^0.55.0", diff --git a/src/livekit/BlurBackgroundTransformer.ts b/src/livekit/BlurBackgroundTransformer.ts new file mode 100644 index 00000000..7273e487 --- /dev/null +++ b/src/livekit/BlurBackgroundTransformer.ts @@ -0,0 +1,62 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { + BackgroundTransformer, + VideoTransformer, + VideoTransformerInitOptions, +} from "@livekit/track-processors"; +import { ImageSegmenter } from "@mediapipe/tasks-vision"; + +interface WasmFileset { + /** The path to the Wasm loader script. */ + wasmLoaderPath: string; + /** The path to the Wasm binary. */ + wasmBinaryPath: string; +} + +// n.b. this only includes the SIMD versions of the WASM files which have good support: +// https://caniuse.com/?search=simd +const wasmFileset: WasmFileset = { + wasmLoaderPath: new URL( + "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js", + import.meta.url, + ).href, + wasmBinaryPath: new URL( + "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm", + import.meta.url, + ).href, +}; + +const modelAssetPath = new URL( + "../mediapipe/imageSegmenter/selfie_segmenter.tflite", + import.meta.url, +).href; + +export class BlurBackgroundTransformer extends BackgroundTransformer { + public async init({ + outputCanvas, + inputElement: inputVideo, + }: VideoTransformerInitOptions): Promise { + // call super.super.init() + await VideoTransformer.prototype.init.call(this, { + outputCanvas, + inputElement: inputVideo, + }); + + this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, { + baseOptions: { + modelAssetPath, + delegate: "GPU", + ...this.options.segmenterOptions, + }, + runningMode: "VIDEO", + outputCategoryMask: true, + outputConfidenceMasks: false, + }); + } +} diff --git a/src/livekit/TrackProcessorContext.tsx b/src/livekit/TrackProcessorContext.tsx index c2bc3826..71714b6e 100644 --- a/src/livekit/TrackProcessorContext.tsx +++ b/src/livekit/TrackProcessorContext.tsx @@ -5,11 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { - BackgroundBlur as backgroundBlur, - BackgroundOptions, - ProcessorWrapper, -} from "@livekit/track-processors"; +import { BackgroundOptions, ProcessorWrapper } from "@livekit/track-processors"; import { createContext, FC, @@ -26,6 +22,7 @@ import { backgroundBlur as backgroundBlurSettings, useSetting, } from "../settings/settings"; +import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer"; type ProcessorState = { supported: boolean | undefined; @@ -87,7 +84,12 @@ export const ProcessorProvider: FC = ({ children }) => { useEffect(() => { if (!shouldCheckSupport) return; try { - if (!blur.current) blur.current = backgroundBlur(15, { delegate: "GPU" }); + if (!blur.current) { + blur.current = new ProcessorWrapper( + new BlurBackgroundTransformer({ blurRadius: 15 }), + "background-blur", + ); + } setProcessorState({ checkSupported, supported: true, diff --git a/src/mediapipe/imageSegmenter/README.md b/src/mediapipe/imageSegmenter/README.md new file mode 100644 index 00000000..39bea2d8 --- /dev/null +++ b/src/mediapipe/imageSegmenter/README.md @@ -0,0 +1,5 @@ +# Google AI Edge MediaPipe Selfie Segmentation + +- See: https://ai.google.dev/edge/mediapipe/solutions/vision/image_segmenter +- Latest: https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite +- License: Apache 2.0 as per https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Selfie%20Segmentation.pdf diff --git a/src/mediapipe/imageSegmenter/selfie_segmenter.tflite b/src/mediapipe/imageSegmenter/selfie_segmenter.tflite new file mode 100644 index 00000000..a4ebd477 Binary files /dev/null and b/src/mediapipe/imageSegmenter/selfie_segmenter.tflite differ diff --git a/yarn.lock b/yarn.lock index b4fc53aa..0b07a82d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1840,6 +1840,11 @@ resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.9.tgz#fbd669f50ac2e888b2c64c9c9863927c111da02f" integrity sha512-/gFguyJm1ng4Qr7VVH2vKO+zZcQd8wc3YafUfvBuYFX0Y5+CvrV+VNPEVkl5W/gUZF5KNKNZAiaHPULGPCIjyQ== +"@mediapipe/tasks-vision@^0.10.18": + version "0.10.18" + resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.18.tgz#9ea0f0bf7506378c55ee661fa70aa0910f21f9b5" + integrity sha512-NRIlyqhGUz1Jdgcs6YybwPRhLK6dgeGAqAMXepIczEQ7FmA/0ouFtgMO1g9SPf/HaDSO8pNVdP54dAb9s9wj/Q== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"