diff --git a/knip.ts b/knip.ts index becafc2e..5ffe03c4 100644 --- a/knip.ts +++ b/knip.ts @@ -24,6 +24,10 @@ export default { // then Knip will flag it as a false positive // https://github.com/webpro-nl/knip/issues/766 "@vector-im/compound-web", + // We need this so the eslint is happy with @livekit/track-processors. + // This might be a bug in the livekit repo but for now we fix it on the + // element call side. + "@types/dom-mediacapture-transform", "matrix-widget-api", ], ignoreExportsUsedInFile: true, diff --git a/locales/en-GB/app.json b/locales/en-GB/app.json index bc37dd7b..899660bb 100644 --- a/locales/en-GB/app.json +++ b/locales/en-GB/app.json @@ -150,6 +150,9 @@ "effect_volume_description": "Adjust the volume at which reactions and hand raised effects play", "effect_volume_label": "Sound effect volume" }, + "background_blur_header": "Background", + "background_blur_label": "Enable background blurring", + "blur_not_supported_by_browser": "(Background blur is not supported by this browser.)", "developer_settings_label": "Developer Settings", "developer_settings_label_description": "Expose developer settings in the settings window.", "developer_tab_title": "Developer", diff --git a/package.json b/package.json index ebaa59ff..ed422c65 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "@formatjs/intl-segmenter": "^11.7.3", "@livekit/components-core": "^0.11.0", "@livekit/components-react": "^2.0.0", + "@livekit/track-processors": "^0.3.2", "@opentelemetry/api": "^1.4.0", "@opentelemetry/core": "^1.25.1", "@opentelemetry/exporter-trace-otlp-http": "^0.55.0", @@ -50,6 +51,7 @@ "@testing-library/react-hooks": "^8.0.1", "@testing-library/user-event": "^14.5.1", "@types/content-type": "^1.1.5", + "@types/dom-mediacapture-transform": "^0.1.10", "@types/grecaptcha": "^3.0.9", "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", diff --git a/src/livekit/useLiveKit.ts b/src/livekit/useLiveKit.ts index 4645fab7..85f3178b 100644 --- a/src/livekit/useLiveKit.ts +++ b/src/livekit/useLiveKit.ts @@ -9,7 +9,9 @@ import { ConnectionState, E2EEOptions, ExternalE2EEKeyProvider, + LocalTrackPublication, Room, + RoomEvent, RoomOptions, Track, } from "livekit-client"; @@ -17,6 +19,7 @@ import { useEffect, useMemo, useRef } from "react"; import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { logger } from "matrix-js-sdk/src/logger"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { BackgroundBlur as backgroundBlur } from "@livekit/track-processors"; import { defaultLiveKitOptions } from "./options"; import { SFUConfig } from "./openIDSFU"; @@ -26,6 +29,7 @@ import { MediaDevices, useMediaDevices, } from "./MediaDevicesContext"; +import { backgroundBlur as backgroundBlurSettings } from "../settings/settings"; import { ECConnectionState, useECConnectionState, @@ -33,6 +37,7 @@ import { import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; import { E2eeType } from "../e2ee/e2eeType"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; +import { useSetting } from "../settings/settings"; interface UseLivekitResult { livekitRoom?: Room; @@ -78,13 +83,22 @@ export function useLiveKit( const initialMuteStates = useRef(muteStates); const devices = useMediaDevices(); const initialDevices = useRef(devices); - + const blur = useMemo(() => { + let b = undefined; + try { + b = backgroundBlur(15, { delegate: "GPU" }); + } catch (e) { + logger.error("disable background blur", e); + } + return b; + }, []); const roomOptions = useMemo( (): RoomOptions => ({ ...defaultLiveKitOptions, videoCaptureDefaults: { ...defaultLiveKitOptions.videoCaptureDefaults, deviceId: initialDevices.current.videoInput.selectedId, + processor: blur, }, audioCaptureDefaults: { ...defaultLiveKitOptions.audioCaptureDefaults, @@ -95,7 +109,7 @@ export function useLiveKit( }, e2ee: e2eeOptions, }), - [e2eeOptions], + [blur, e2eeOptions], ); // Store if audio/video are currently updating. If to prohibit unnecessary calls @@ -129,6 +143,58 @@ export function useLiveKit( sfuConfig, ); + const [showBackgroundBlur] = useSetting(backgroundBlurSettings); + const videoTrackPromise = useRef< + undefined | Promise + >(undefined); + + useEffect(() => { + // Don't even try if we cannot blur on this platform + if (!blur) return; + if (!room || videoTrackPromise.current) return; + const update = async (): Promise => { + let publishCallback: undefined | ((track: LocalTrackPublication) => void); + videoTrackPromise.current = new Promise< + LocalTrackPublication | undefined + >((resolve) => { + const videoTrack = Array.from( + room.localParticipant.videoTrackPublications.values(), + ).find((v) => v.source === Track.Source.Camera); + if (videoTrack) { + resolve(videoTrack); + } + publishCallback = (videoTrack: LocalTrackPublication): void => { + if (videoTrack.source === Track.Source.Camera) { + resolve(videoTrack); + } + }; + room.on(RoomEvent.LocalTrackPublished, publishCallback); + }); + + const videoTrack = await videoTrackPromise.current; + + if (publishCallback) + room.off(RoomEvent.LocalTrackPublished, publishCallback); + + if (videoTrack !== undefined) { + if ( + showBackgroundBlur && + videoTrack.track?.getProcessor()?.name !== "background-blur" + ) { + logger.info("Blur: set blur"); + + void videoTrack.track?.setProcessor(blur); + } else if ( + videoTrack.track?.getProcessor()?.name === "background-blur" + ) { + void videoTrack.track?.stopProcessor(); + } + } + videoTrackPromise.current = undefined; + }; + void update(); + }, [blur, room, showBackgroundBlur]); + useEffect(() => { // Sync the requested mute states with LiveKit's mute states. We do it this // way around rather than using LiveKit as the source of truth, so that the diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index fd3df0c8..6acd99af 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { FC, useCallback, useMemo, useState } from "react"; +import { FC, useCallback, useEffect, useMemo, useState } from "react"; import { useTranslation } from "react-i18next"; import { MatrixClient } from "matrix-js-sdk/src/matrix"; import { Button } from "@vector-im/compound-web"; @@ -16,6 +16,7 @@ import { usePreviewTracks } from "@livekit/components-react"; import { LocalVideoTrack, Track } from "livekit-client"; import { useObservable } from "observable-hooks"; import { map } from "rxjs"; +import { BackgroundBlur as backgroundBlur } from "@livekit/track-processors"; import inCallStyles from "./InCallView.module.css"; import styles from "./LobbyView.module.css"; @@ -32,12 +33,14 @@ import { VideoButton, } from "../button/Button"; import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal"; +import { backgroundBlur as backgroundBlurSettings } from "../settings/settings"; import { useMediaQuery } from "../useMediaQuery"; import { E2eeType } from "../e2ee/e2eeType"; import { Link } from "../button/Link"; import { useMediaDevices } from "../livekit/MediaDevicesContext"; import { useInitial } from "../useInitial"; -import { useSwitchCamera } from "./useSwitchCamera"; +import { useSwitchCamera as useShowSwitchCamera } from "./useSwitchCamera"; +import { useSetting } from "../settings/settings"; interface Props { client: MatrixClient; @@ -108,6 +111,20 @@ export const LobbyView: FC = ({ muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId }, ); + // eslint-disable-next-line new-cap + const blur = useMemo(() => { + let b = undefined; + try { + b = backgroundBlur(15, { delegate: "GPU" }); + } catch (e) { + logger.error( + "disable background blur because its not supported by the platform.", + e, + ); + } + return b; + }, []); + const localTrackOptions = useMemo( () => ({ // The only reason we request audio here is to get the audio permission @@ -119,12 +136,14 @@ export const LobbyView: FC = ({ audio: Object.assign({}, initialAudioOptions), video: muteStates.video.enabled && { deviceId: devices.videoInput.selectedId, + // It should be possible to set a processor here: + // processor: blur, }, }), [ initialAudioOptions, - devices.videoInput.selectedId, muteStates.video.enabled, + devices.videoInput.selectedId, ], ); @@ -146,7 +165,22 @@ export const LobbyView: FC = ({ [tracks], ); - const switchCamera = useSwitchCamera( + const [showBackgroundBlur] = useSetting(backgroundBlurSettings); + + useEffect(() => { + // Fon't even try if we cannot blur on this platform + if (!blur) return; + const updateBlur = async (showBlur: boolean): Promise => { + if (showBlur && !videoTrack?.getProcessor()) { + await videoTrack?.setProcessor(blur); + } else { + await videoTrack?.stopProcessor(); + } + }; + if (videoTrack) void updateBlur(showBackgroundBlur); + }, [videoTrack, showBackgroundBlur, blur]); + + const showSwitchCamera = useShowSwitchCamera( useObservable( (inputs) => inputs.pipe(map(([video]) => video)), [videoTrack], @@ -208,7 +242,9 @@ export const LobbyView: FC = ({ onClick={onVideoPress} disabled={muteStates.video.setEnabled === null} /> - {switchCamera && } + {showSwitchCamera && ( + + )} {!confineToRoom && } diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 45b32ae3..c864ac74 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -5,10 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { ChangeEvent, FC, useCallback } from "react"; +import { ChangeEvent, FC, ReactNode, useCallback } from "react"; import { Trans, useTranslation } from "react-i18next"; import { MatrixClient } from "matrix-js-sdk/src/matrix"; -import { Root as Form, Text } from "@vector-im/compound-web"; +import { Root as Form, Separator, Text } from "@vector-im/compound-web"; +import { BackgroundBlur } from "@livekit/track-processors"; +import { logger } from "matrix-js-sdk/src/logger"; import { Modal } from "../Modal"; import styles from "./SettingsModal.module.css"; @@ -26,6 +28,7 @@ import { useSetting, developerSettingsTab as developerSettingsTabSetting, duplicateTiles as duplicateTilesSetting, + backgroundBlur as backgroundBlurSetting, useOptInAnalytics, soundEffectVolumeSetting, } from "./settings"; @@ -70,6 +73,42 @@ export const SettingsModal: FC = ({ ); const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting); + // Generate a `Checkbox` input to turn blur on or off. + const BlurCheckbox: React.FC = (): ReactNode => { + const [blur, setBlur] = useSetting(backgroundBlurSetting); + let canBlur = true; + try { + // eslint-disable-next-line new-cap + BackgroundBlur(15); + } catch (e) { + logger.debug( + "Cannot blur, so we do not show the option in settings. error: ", + e, + ); + canBlur = false; + setBlur(false); + } + return ( + <> +

{t("settings.background_blur_header")}

+ + + setBlur(b.target.checked)} + disabled={!canBlur} + /> + + + ); + }; + const optInDescription = ( @@ -123,12 +162,16 @@ export const SettingsModal: FC = ({ key: "video", name: t("common.video"), content: ( -
- - + <> +
+ + + + + ), }; diff --git a/src/settings/settings.ts b/src/settings/settings.ts index b2da1674..27d28011 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -88,6 +88,8 @@ export const videoInput = new Setting( undefined, ); +export const backgroundBlur = new Setting("background-blur", false); + export const showHandRaisedTimer = new Setting( "hand-raised-show-timer", false, diff --git a/yarn.lock b/yarn.lock index 5b75ebb0..27e6ab19 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1812,6 +1812,14 @@ dependencies: "@bufbuild/protobuf" "^1.10.0" +"@livekit/track-processors@^0.3.2": + version "0.3.2" + resolved "https://registry.yarnpkg.com/@livekit/track-processors/-/track-processors-0.3.2.tgz#eaff6a48b556c25e85f5dd2c4daf6dcf1bc3b143" + integrity sha512-4JUCzb7yIKoVsTo8J6FTzLZJHcI6DihfX/pGRDg0SOGaxprcDPrt8jaDBBTsnGBSXHeMxl2ugN+xQjdCWzLKEA== + dependencies: + "@mediapipe/holistic" "0.5.1675471629" + "@mediapipe/tasks-vision" "0.10.9" + "@matrix-org/matrix-sdk-crypto-wasm@^9.0.0": version "9.1.0" resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-9.1.0.tgz#f889653eb4fafaad2a963654d586bd34de62acd5" @@ -1822,6 +1830,16 @@ resolved "https://registry.yarnpkg.com/@matrix-org/olm/-/olm-3.2.15.tgz#55f3c1b70a21bbee3f9195cecd6846b1083451ec" integrity sha512-S7lOrndAK9/8qOtaTq/WhttJC/o4GAzdfK0MUPpo8ApzsJEC0QjtwrkC3KBXdFP1cD1MXi/mlKR7aaoVMKgs6Q== +"@mediapipe/holistic@0.5.1675471629": + version "0.5.1675471629" + resolved "https://registry.yarnpkg.com/@mediapipe/holistic/-/holistic-0.5.1675471629.tgz#f1127d43161ff27e8889d5d39aaea164f9730980" + integrity sha512-qY+cxtDeSOvVtevrLgnodiwXYaAtPi7dHZtNv/bUCGEjFicAOYtMmrZSqMmbPkTB2+4jLnPF1vgshkAqQRSYAw== + +"@mediapipe/tasks-vision@0.10.9": + version "0.10.9" + resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.9.tgz#fbd669f50ac2e888b2c64c9c9863927c111da02f" + integrity sha512-/gFguyJm1ng4Qr7VVH2vKO+zZcQd8wc3YafUfvBuYFX0Y5+CvrV+VNPEVkl5W/gUZF5KNKNZAiaHPULGPCIjyQ== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" @@ -2981,6 +2999,18 @@ resolved "https://registry.yarnpkg.com/@types/content-type/-/content-type-1.1.8.tgz#319644d07ee6b4bfc734483008393b89b99f0219" integrity sha512-1tBhmVUeso3+ahfyaKluXe38p+94lovUZdoVfQ3OnJo9uJC42JT7CBoN3k9HYhAae+GwiBYmHu+N9FZhOG+2Pg== +"@types/dom-mediacapture-transform@^0.1.10": + version "0.1.10" + resolved "https://registry.yarnpkg.com/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.10.tgz#93e2c04284c95dd2faa2992216417599ee9d27a6" + integrity sha512-zUxMN2iShu7p3Fz5sqfvLp93qW/3sLs+RwXWWOkMb969hsuoVqUUokqrENjXqTMNmEEcVXKoHuMMbIGcWyrVVA== + dependencies: + "@types/dom-webcodecs" "*" + +"@types/dom-webcodecs@*": + version "0.1.13" + resolved "https://registry.yarnpkg.com/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz#d8be5da4f01b20721307b08ad2cca903ccf4f47f" + integrity sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ== + "@types/estree@1.0.6", "@types/estree@^1.0.0": version "1.0.6" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.6.tgz#628effeeae2064a1b4e79f78e81d87b7e5fc7b50"