mirror of
https://github.com/vector-im/element-call.git
synced 2026-11-07 00:52:28 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Timo K.
|
cc994abeb3 | ||
|
fkwpandClaude Opus 5.5
|
33f4d5043f | ||
|
fkwp
|
1e17a6411c | ||
|
Timo K.andClaude Opus 5.5
|
0d60eabef2 | ||
|
Timo K.andClaude Opus 5.5
|
22083299fa | ||
|
Timo K.andClaude Opus 5.5
|
52d5b69487 | ||
|
fkwpandClaude Fable 5.1
|
a0aaa8e09a | ||
|
fkwpandClaude Fable 5.1
|
146dd3aea1 | ||
|
fkwpandClaude Fable 5.1
|
3f5f707ca7 | ||
|
Timo
|
922a9bbbab | ||
|
Timo K.andClaude Fable 5.1
|
69c57bd534 | ||
|
Timo K.andClaude Fable 5.1
|
734ecbbecf | ||
|
Timo
|
00463c3ede | ||
|
renovate[bot]
|
f930f73c40 | ||
|
Johannes Marbach
|
3914aec53f | ||
|
Johannes Marbach
|
321340a192 | ||
|
Johannes MarbachandTimo
|
517edc12ac | ||
|
Johannes Marbach
|
94ee13c639 |
+11
-12
@@ -72,9 +72,9 @@ import {
|
||||
useUrlParams,
|
||||
} from "../src/UrlParams";
|
||||
import { MediaDevicesContext } from "../src/MediaDevicesContext";
|
||||
import { MediaDevices } from "../src/state/MediaDevices";
|
||||
import { AppViewModel } from "../src/state/AppViewModel";
|
||||
import { ObservableScope } from "../src/state/ObservableScope";
|
||||
import { ProcessorProvider } from "../src/tracks/TrackProcessorContext";
|
||||
import { BackgroundEffectsProvider } from "../src/tracks/BackgroundEffectsContext";
|
||||
import { Config } from "../src/config/Config";
|
||||
import { type ConfigOptions } from "../src/config/ConfigOptions";
|
||||
import { i18n } from "../src/utils/i18n";
|
||||
@@ -194,14 +194,12 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
// observers running for the rest of the page's life. Null until then, which
|
||||
// is one render.
|
||||
const { controlledAudioDevices, callIntent } = params;
|
||||
const [mediaDevices, setMediaDevices] = useState<MediaDevices | null>(null);
|
||||
const [vm, setVm] = useState<AppViewModel | null>(null);
|
||||
useEffect(() => {
|
||||
const scope = new ObservableScope();
|
||||
setMediaDevices(
|
||||
new MediaDevices(scope, { controlledAudioDevices, callIntent }),
|
||||
);
|
||||
setVm(new AppViewModel(scope, { controlledAudioDevices, callIntent }));
|
||||
return (): void => {
|
||||
setMediaDevices(null);
|
||||
setVm(null);
|
||||
scope.end();
|
||||
};
|
||||
}, [controlledAudioDevices, callIntent]);
|
||||
@@ -222,8 +220,7 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
// is told that Element Call has loaded, as the widget tells its client once
|
||||
// its own initialisation is over. Once per mount, however often the pieces
|
||||
// are later swapped out.
|
||||
const ready =
|
||||
container !== null && rtcSession !== null && mediaDevices !== null;
|
||||
const ready = container !== null && rtcSession !== null && vm !== null;
|
||||
const announcedLoaded = useRef(false);
|
||||
useEffect(() => {
|
||||
if (!ready || announcedLoaded.current) return;
|
||||
@@ -254,8 +251,10 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
<Decoration>
|
||||
<TooltipProvider>
|
||||
<ClientProvider client={client}>
|
||||
<MediaDevicesContext value={mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<MediaDevicesContext value={vm.mediaDevices}>
|
||||
<BackgroundEffectsProvider
|
||||
effects={vm.backgroundEffects}
|
||||
>
|
||||
<CallView
|
||||
client={client}
|
||||
rtcSession={rtcSession}
|
||||
@@ -264,7 +263,7 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
preload={params.preload}
|
||||
skipLobby={params.skipLobby}
|
||||
/>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</ClientProvider>
|
||||
</TooltipProvider>
|
||||
|
||||
@@ -19,7 +19,7 @@ services:
|
||||
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
|
||||
element-web:
|
||||
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:fc21073b3c20c5a80aab6787dcbf1ddb3ed416eabf7fdfd32a7f4cabc47b7846
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7
|
||||
element-web-1:
|
||||
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:fc21073b3c20c5a80aab6787dcbf1ddb3ed416eabf7fdfd32a7f4cabc47b7846
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:a79801206c628a912d57fa6f355d5c5ab9c6c40c2d1b6208d4192678fb6e80c7
|
||||
|
||||
+1
-2
@@ -57,7 +57,6 @@
|
||||
"@formatjs/intl-durationformat": "^0.10.0",
|
||||
"@formatjs/intl-segmenter": "^11.7.3",
|
||||
"@livekit/components-core": "^0.12.0",
|
||||
"@livekit/components-react": "^2.0.0",
|
||||
"@livekit/track-processors": "^0.7.1",
|
||||
"@mediapipe/tasks-vision": "^0.10.18",
|
||||
"@playwright/test": "^1.60.0",
|
||||
@@ -107,7 +106,7 @@
|
||||
"livekit-client": "^2.18.1",
|
||||
"lodash-es": "^4.17.21",
|
||||
"loglevel": "^1.9.1",
|
||||
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#3f05dc4d104a3dfc753c5e3549a18c74fab4b076",
|
||||
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01",
|
||||
"matrix-widget-api": "^1.18.0",
|
||||
"node-stdlib-browser": "^1.3.1",
|
||||
"normalize.css": "^8.0.1",
|
||||
|
||||
Generated
+16
-71
@@ -40,9 +40,6 @@ importers:
|
||||
'@livekit/components-core':
|
||||
specifier: ^0.12.0
|
||||
version: 0.12.15(livekit-client@2.23.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)
|
||||
'@livekit/components-react':
|
||||
specifier: ^2.0.0
|
||||
version: 2.9.24(livekit-client@2.23.0(@types/dom-mediacapture-record@1.0.22))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(tslib@2.8.1)
|
||||
'@livekit/track-processors':
|
||||
specifier: ^0.7.1
|
||||
version: 0.7.2(@types/dom-mediacapture-transform@0.1.11)(livekit-client@2.23.0(@types/dom-mediacapture-record@1.0.22))
|
||||
@@ -66,7 +63,7 @@ importers:
|
||||
version: 10.1.2(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
|
||||
'@rolldown/plugin-babel':
|
||||
specifier: ^0.2.3
|
||||
version: 0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.0)(rolldown@1.2.15)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))
|
||||
version: 0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.8)(rolldown@1.2.15)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))
|
||||
'@sentry/react':
|
||||
specifier: ^8.0.0
|
||||
version: 8.55.2(react@19.2.8)
|
||||
@@ -138,7 +135,7 @@ importers:
|
||||
version: 10.5.0(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@11.3.1(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
|
||||
'@vitejs/plugin-react':
|
||||
specifier: ^6.0.2
|
||||
version: 6.1.1(@rolldown/plugin-babel@0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.0)(rolldown@1.2.15)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0)))(babel-plugin-react-compiler@1.0.0)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))
|
||||
version: 6.1.1(@rolldown/plugin-babel@0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.8)(rolldown@1.2.15)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0)))(babel-plugin-react-compiler@1.0.0)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))
|
||||
'@vitest/browser-playwright':
|
||||
specifier: ^4.1.5
|
||||
version: 4.1.11(playwright@1.62.1)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))(vitest@4.1.11)
|
||||
@@ -191,8 +188,8 @@ importers:
|
||||
specifier: ^1.9.1
|
||||
version: 1.9.2
|
||||
matrix-js-sdk:
|
||||
specifier: github:matrix-org/matrix-js-sdk#3f05dc4d104a3dfc753c5e3549a18c74fab4b076
|
||||
version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/3f05dc4d104a3dfc753c5e3549a18c74fab4b076
|
||||
specifier: github:matrix-org/matrix-js-sdk#a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01
|
||||
version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01
|
||||
matrix-widget-api:
|
||||
specifier: ^1.18.0
|
||||
version: 1.20.0
|
||||
@@ -397,8 +394,8 @@ packages:
|
||||
resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==}
|
||||
engines: {node: '>=6.9.0'}
|
||||
|
||||
'@babel/runtime@8.0.0':
|
||||
resolution: {integrity: sha512-sL6cvO2IfkSu/iU+zs2S/w01B7A8V7suXSIKEN4hPFFdZoiPGxrj5pAG0lCaqLWiEIrjKzdznIWuaLcxPR53qw==}
|
||||
'@babel/runtime@8.0.8':
|
||||
resolution: {integrity: sha512-dEu8nGlRqjsJsvviUUoSMfcRHpwomCiRlXsc2mPfJ79+ffTpDm+JtfUtTMzUrqCApHQnCsVPD3gT1ZT8RgBRCQ==}
|
||||
|
||||
'@babel/template@7.29.7':
|
||||
resolution: {integrity: sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==}
|
||||
@@ -1180,19 +1177,6 @@ packages:
|
||||
livekit-client: ^2.20.1
|
||||
tslib: ^2.6.2
|
||||
|
||||
'@livekit/components-react@2.9.24':
|
||||
resolution: {integrity: sha512-qw5Oy1EfPg1f/xrKYrFielSrmtxY8BZ9mDt6299G5aikotQIztmldBfTNosrVS7VSNit7qeYfOFSyKaML20W6Q==}
|
||||
engines: {node: '>=18'}
|
||||
peerDependencies:
|
||||
'@livekit/krisp-noise-filter': ^0.2.12 || ^0.3.0 || ^0.4.0
|
||||
livekit-client: ^2.20.1
|
||||
react: '>=18'
|
||||
react-dom: '>=18'
|
||||
tslib: ^2.6.2
|
||||
peerDependenciesMeta:
|
||||
'@livekit/krisp-noise-filter':
|
||||
optional: true
|
||||
|
||||
'@livekit/mutex@1.1.1':
|
||||
resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==}
|
||||
|
||||
@@ -3688,10 +3672,6 @@ packages:
|
||||
cliui@6.0.0:
|
||||
resolution: {integrity: sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==}
|
||||
|
||||
clsx@2.1.1:
|
||||
resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
color-convert@2.0.1:
|
||||
resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
|
||||
engines: {node: '>=7.0.0'}
|
||||
@@ -4527,9 +4507,6 @@ packages:
|
||||
jose@6.2.13:
|
||||
resolution: {integrity: sha512-7TIg10ZE1x82YediN+lcrPNvGMKge9yFHQD+Qho/b8y2adfugU3AsvvHfxV0fzyFllT6l9xCjf4RWY9g/Z9PJQ==}
|
||||
|
||||
jose@6.2.9:
|
||||
resolution: {integrity: sha512-XrchZOFZUl/T3vTwRe8XK+cJrGtMF4th1ARnDfwbBXFKThGhlsxEE4Zu03AD/bjJSt/9jT/mxrOCkJWOg77aPA==}
|
||||
|
||||
js-tokens@10.0.0:
|
||||
resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==}
|
||||
|
||||
@@ -4685,9 +4662,6 @@ packages:
|
||||
lodash-es@4.18.1:
|
||||
resolution: {integrity: sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==}
|
||||
|
||||
lodash.debounce@4.0.8:
|
||||
resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==}
|
||||
|
||||
lodash.merge@4.6.2:
|
||||
resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==}
|
||||
|
||||
@@ -4750,9 +4724,9 @@ packages:
|
||||
matrix-events-sdk@0.0.1:
|
||||
resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==}
|
||||
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/3f05dc4d104a3dfc753c5e3549a18c74fab4b076:
|
||||
resolution: {gitHosted: true, integrity: sha512-Ly8eS7fNNKDo9KmPvTkE49KvD8iMUmrVWJMNERNuxc8UksMLMG36NLz3fs8HytdwrB7AKPfyVo392VZsxILDIg==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/3f05dc4d104a3dfc753c5e3549a18c74fab4b076}
|
||||
version: 44.0.0
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01:
|
||||
resolution: {gitHosted: true, integrity: sha512-ZAhn/zB44MgrXqUzi9ffSFcmZY3+2nx7bEgfEJR7chYDFyT/XmlmfCJx7Ar4rOTAYZJRVnMxrEwiEHGDapUi0Q==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01}
|
||||
version: 45.0.0
|
||||
engines: {node: '>=22.0.0'}
|
||||
|
||||
matrix-widget-api@1.20.0:
|
||||
@@ -5979,12 +5953,6 @@ packages:
|
||||
peerDependencies:
|
||||
react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
|
||||
|
||||
usehooks-ts@3.1.1:
|
||||
resolution: {integrity: sha512-I4diPp9Cq6ieSUH2wu+fDAVQO43xwtulo+fKEidHUwZPnYImbtkTjzIJYcDcJqxgmX31GVqNFURodvcgHcW0pA==}
|
||||
engines: {node: '>=16.15.0'}
|
||||
peerDependencies:
|
||||
react: ^16.8.0 || ^17 || ^18 || ^19 || ^19.0.0-rc
|
||||
|
||||
util-deprecate@1.0.2:
|
||||
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
|
||||
|
||||
@@ -6424,7 +6392,7 @@ snapshots:
|
||||
|
||||
'@babel/runtime@7.29.7': {}
|
||||
|
||||
'@babel/runtime@8.0.0': {}
|
||||
'@babel/runtime@8.0.8': {}
|
||||
|
||||
'@babel/template@7.29.7':
|
||||
dependencies:
|
||||
@@ -7182,18 +7150,6 @@ snapshots:
|
||||
rxjs: 7.8.2
|
||||
tslib: 2.8.1
|
||||
|
||||
'@livekit/components-react@2.9.24(livekit-client@2.23.0(@types/dom-mediacapture-record@1.0.22))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(tslib@2.8.1)':
|
||||
dependencies:
|
||||
'@livekit/components-core': 0.12.15(livekit-client@2.23.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)
|
||||
clsx: 2.1.1
|
||||
events: 3.3.0
|
||||
jose: 6.2.9
|
||||
livekit-client: 2.23.0(@types/dom-mediacapture-record@1.0.22)
|
||||
react: 19.2.8
|
||||
react-dom: 19.2.8(react@19.2.8)
|
||||
tslib: 2.8.1
|
||||
usehooks-ts: 3.1.1(react@19.2.8)
|
||||
|
||||
'@livekit/mutex@1.1.1': {}
|
||||
|
||||
'@livekit/protocol@1.52.1':
|
||||
@@ -8146,13 +8102,13 @@ snapshots:
|
||||
'@rolldown/binding-win32-x64-msvc@1.2.15':
|
||||
optional: true
|
||||
|
||||
'@rolldown/plugin-babel@0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.0)(rolldown@1.2.15)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))':
|
||||
'@rolldown/plugin-babel@0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.8)(rolldown@1.2.15)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))':
|
||||
dependencies:
|
||||
'@babel/core': 7.29.7(supports-color@7.2.0)
|
||||
picomatch: 4.0.4
|
||||
rolldown: 1.2.15
|
||||
optionalDependencies:
|
||||
'@babel/runtime': 8.0.0
|
||||
'@babel/runtime': 8.0.8
|
||||
vite: 8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0)
|
||||
|
||||
'@rolldown/pluginutils@1.0.1': {}
|
||||
@@ -8904,12 +8860,12 @@ snapshots:
|
||||
- '@types/react-dom'
|
||||
- react-dom
|
||||
|
||||
'@vitejs/plugin-react@6.1.1(@rolldown/plugin-babel@0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.0)(rolldown@1.2.15)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0)))(babel-plugin-react-compiler@1.0.0)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))':
|
||||
'@vitejs/plugin-react@6.1.1(@rolldown/plugin-babel@0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.8)(rolldown@1.2.15)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0)))(babel-plugin-react-compiler@1.0.0)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))':
|
||||
dependencies:
|
||||
'@rolldown/pluginutils': 1.0.1
|
||||
vite: 8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0)
|
||||
optionalDependencies:
|
||||
'@rolldown/plugin-babel': 0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.0)(rolldown@1.2.15)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))
|
||||
'@rolldown/plugin-babel': 0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.8)(rolldown@1.2.15)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))
|
||||
babel-plugin-react-compiler: 1.0.0
|
||||
|
||||
'@vitest/browser-playwright@4.1.11(playwright@1.62.1)(vite@8.2.4(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))(vitest@4.1.11)':
|
||||
@@ -9331,8 +9287,6 @@ snapshots:
|
||||
strip-ansi: 6.0.1
|
||||
wrap-ansi: 6.2.0
|
||||
|
||||
clsx@2.1.1: {}
|
||||
|
||||
color-convert@2.0.1:
|
||||
dependencies:
|
||||
color-name: 1.1.4
|
||||
@@ -10224,8 +10178,6 @@ snapshots:
|
||||
|
||||
jose@6.2.13: {}
|
||||
|
||||
jose@6.2.9: {}
|
||||
|
||||
js-tokens@10.0.0: {}
|
||||
|
||||
js-tokens@4.0.0: {}
|
||||
@@ -10386,8 +10338,6 @@ snapshots:
|
||||
|
||||
lodash-es@4.18.1: {}
|
||||
|
||||
lodash.debounce@4.0.8: {}
|
||||
|
||||
lodash.merge@4.6.2: {}
|
||||
|
||||
log-symbols@7.0.1:
|
||||
@@ -10445,9 +10395,9 @@ snapshots:
|
||||
|
||||
matrix-events-sdk@0.0.1: {}
|
||||
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/3f05dc4d104a3dfc753c5e3549a18c74fab4b076:
|
||||
matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/a9ef09fbf2f1644c8663d0aeaf6a921cd16e7b01:
|
||||
dependencies:
|
||||
'@babel/runtime': 8.0.0
|
||||
'@babel/runtime': 8.0.8
|
||||
'@matrix-org/matrix-sdk-crypto-wasm': 18.9.0
|
||||
another-json: 0.2.0
|
||||
bs58: 6.0.0
|
||||
@@ -11846,11 +11796,6 @@ snapshots:
|
||||
dependencies:
|
||||
react: 19.2.8
|
||||
|
||||
usehooks-ts@3.1.1(react@19.2.8):
|
||||
dependencies:
|
||||
lodash.debounce: 4.0.8
|
||||
react: 19.2.8
|
||||
|
||||
util-deprecate@1.0.2: {}
|
||||
|
||||
util@0.12.5:
|
||||
|
||||
+10
-5
@@ -126,10 +126,15 @@ probe that decides whether the service that issued the token can take over the
|
||||
delayed leave, which picks the delayed leave timings the join uses. The join
|
||||
waits for the publisher to hold its token, so that the LiveKit room exists
|
||||
before a subscriber, who lacks the permission to create it, asks for a token
|
||||
of its own. Until `join()` the tracks exist but their upstream is paused: a host
|
||||
can show a preview without anyone hearing it. The upstream flows exactly while the
|
||||
membership is `joined`: a sync outage pauses it, and `leave()` pauses it for good
|
||||
and sends the leave.
|
||||
of its own. The tracks are created once the membership is `joined`, with their
|
||||
upstream paused until then. A lobby that wants them earlier captures them with
|
||||
`createLocalMedia`, on the browser's devices alone so that any backend fits, and
|
||||
hands the media to `join()` as `publish`: the tracks are moved, not shared — they
|
||||
leave the lobby's `tracks$`, their objects are finished, and the backend
|
||||
publishes the very same device tracks and the processor already running on them,
|
||||
so nothing is captured or initialised again at the join. The upstream flows
|
||||
exactly while the membership is `joined`: a sync outage pauses it, and `leave()`
|
||||
pauses it for good and sends the leave.
|
||||
|
||||
## Vocabulary
|
||||
|
||||
@@ -538,7 +543,7 @@ backend is handed the whole session's view and gives back media:
|
||||
type MemberMediaFields = Pick<RTCMember, "tracks$" | "encryptionError$">;
|
||||
|
||||
export interface MediaBackend {
|
||||
readonly transportType: string; // "livekit"
|
||||
readonly transportTypes: readonly string[]; // ["m.livekit", "livekit"]
|
||||
/** Authenticate with the transport the client found; the probe for delegation runs here. */
|
||||
prepareLocalTransport(
|
||||
transport: Transport,
|
||||
|
||||
+7
-2
@@ -61,8 +61,13 @@ export {
|
||||
createLivekitBackend,
|
||||
type LivekitBackendOptions,
|
||||
} from "./src/media-backend/livekit/LivekitBackend";
|
||||
// For a host that creates LiveKit tracks itself, such as a preview before joining
|
||||
export { convertToLivekitProcessor } from "./src/media-backend/livekit/videoProcessor";
|
||||
// Tracks for a lobby, captured ahead of the join and handed to it
|
||||
export { createLocalMedia } from "./src/LocalMedia";
|
||||
// For a host whose processor is a livekit-client one, such as @livekit/track-processors
|
||||
export {
|
||||
fromLivekitProcessor,
|
||||
toLivekitProcessor,
|
||||
} from "./src/media-backend/livekit/videoProcessor";
|
||||
// For a developer panel to check a transport before using it
|
||||
export {
|
||||
getSFUConfig as authenticateWithTransport,
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { createLocalMedia, type GetUserMedia } from "./LocalMedia";
|
||||
import { type LocalVideoMediaTrack, type VideoProcessor } from "./media-api";
|
||||
|
||||
describe("createLocalMedia", () => {
|
||||
it("opens the microphone and the camera in one request", async () => {
|
||||
const getUserMedia = fakeGetUserMedia();
|
||||
const media = createLocalMedia(getUserMedia);
|
||||
|
||||
const tracks = await media.capture([
|
||||
{ source: "microphone", deviceId: "mic" },
|
||||
{
|
||||
source: "camera",
|
||||
capture: { resolution: { width: 640, height: 480 } },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(getUserMedia).toHaveBeenCalledExactlyOnceWith({
|
||||
audio: { deviceId: "mic" },
|
||||
video: { width: 640, height: 480, frameRate: undefined },
|
||||
});
|
||||
expect(tracks.map((t) => t.source)).toEqual(["microphone", "camera"]);
|
||||
expect(media.tracks$.value).toEqual(tracks);
|
||||
expect(media.requests.map((r) => r.source)).toEqual([
|
||||
"microphone",
|
||||
"camera",
|
||||
]);
|
||||
});
|
||||
|
||||
it("joins an identical capture still in flight rather than opening the devices again", async () => {
|
||||
const getUserMedia = fakeGetUserMedia();
|
||||
const media = createLocalMedia(getUserMedia);
|
||||
const requests = [{ source: "camera" as const }];
|
||||
|
||||
const [first, second] = await Promise.all([
|
||||
media.capture(requests),
|
||||
media.capture(requests),
|
||||
]);
|
||||
|
||||
expect(getUserMedia).toHaveBeenCalledOnce();
|
||||
expect(second).toEqual(first);
|
||||
expect(media.tracks$.value).toEqual(first);
|
||||
});
|
||||
|
||||
it("shows the processor's output, initialised once", async () => {
|
||||
const processor = fakeProcessor();
|
||||
const media = createLocalMedia(fakeGetUserMedia());
|
||||
const [camera] = await media.capture([{ source: "camera", processor }]);
|
||||
const element = document.createElement("video");
|
||||
|
||||
camera.attach(element);
|
||||
|
||||
expect(processor.init).toHaveBeenCalledOnce();
|
||||
expect(tracksOn(element)).toEqual([processor.output]);
|
||||
});
|
||||
|
||||
it("hands the tracks over as they are: no second capture, no second init", async () => {
|
||||
const getUserMedia = fakeGetUserMedia();
|
||||
const processor = fakeProcessor();
|
||||
const media = createLocalMedia(getUserMedia);
|
||||
const [microphone, camera] = await media.capture([
|
||||
{ source: "microphone" },
|
||||
{ source: "camera", processor },
|
||||
]);
|
||||
|
||||
const taken = media.takeTracks();
|
||||
|
||||
expect(taken.get("microphone")?.track).toBe(trackBehind(microphone));
|
||||
expect(taken.get("camera")).toEqual({
|
||||
track: trackBehind(camera),
|
||||
constraints: {},
|
||||
muted: false,
|
||||
processor: { processor, processedTrack: processor.output },
|
||||
});
|
||||
expect(media.tracks$.value).toEqual([]);
|
||||
expect(getUserMedia).toHaveBeenCalledOnce();
|
||||
expect(processor.init).toHaveBeenCalledOnce();
|
||||
expect(processor.destroy).not.toHaveBeenCalled();
|
||||
media.stop();
|
||||
expect(taken.get("camera")!.track.stop).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("finishes a handed-over track", async () => {
|
||||
const media = createLocalMedia(fakeGetUserMedia());
|
||||
const [camera] = await media.capture([{ source: "camera" }]);
|
||||
const element = document.createElement("video");
|
||||
camera.attach(element);
|
||||
|
||||
media.takeTracks();
|
||||
|
||||
expect(element.srcObject).toBeNull();
|
||||
await expect(camera.setEnabled(false)).rejects.toThrow("handed over");
|
||||
expect(() => camera.attach(element)).toThrow("handed over");
|
||||
});
|
||||
|
||||
it("mutes by disabling the track and hands the mute over", async () => {
|
||||
const media = createLocalMedia(fakeGetUserMedia());
|
||||
const [microphone] = await media.capture([{ source: "microphone" }]);
|
||||
|
||||
await expect(microphone.setEnabled(false)).resolves.toBe(false);
|
||||
|
||||
expect(microphone.muted$.value).toBe(true);
|
||||
expect(trackBehind(microphone).enabled).toBe(false);
|
||||
expect(media.takeTracks().get("microphone")?.muted).toBe(true);
|
||||
});
|
||||
|
||||
it("switches the device by opening it anew", async () => {
|
||||
const getUserMedia = fakeGetUserMedia();
|
||||
const processor = fakeProcessor();
|
||||
const media = createLocalMedia(getUserMedia);
|
||||
const [camera] = await media.capture([{ source: "camera", processor }]);
|
||||
const first = trackBehind(camera);
|
||||
const element = document.createElement("video");
|
||||
camera.attach(element);
|
||||
|
||||
await camera.setDevice("other");
|
||||
|
||||
expect(getUserMedia).toHaveBeenLastCalledWith({
|
||||
video: { deviceId: "other" },
|
||||
});
|
||||
expect(first.stop).toHaveBeenCalled();
|
||||
expect(processor.restart).toHaveBeenCalledOnce();
|
||||
const [restartedOn, input] = vi.mocked(processor.restart).mock.calls[0];
|
||||
expect(restartedOn).not.toBe(first);
|
||||
expect(input).toBe(vi.mocked(processor.init).mock.calls[0][1]);
|
||||
expect(tracksOn(element)).toEqual([processor.output]);
|
||||
expect(media.takeTracks().get("camera")?.constraints).toEqual({
|
||||
deviceId: "other",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports the facing mode and switches it", async () => {
|
||||
const getUserMedia = fakeGetUserMedia({ facingMode: "user" });
|
||||
const media = createLocalMedia(getUserMedia);
|
||||
const [camera] = (await media.capture([{ source: "camera" }])) as [
|
||||
LocalVideoMediaTrack,
|
||||
];
|
||||
expect(camera.facingMode$.value).toBe("user");
|
||||
|
||||
await camera.switchFacingMode();
|
||||
|
||||
expect(getUserMedia).toHaveBeenLastCalledWith({
|
||||
video: { facingMode: "environment" },
|
||||
});
|
||||
});
|
||||
|
||||
it("releases one device and stops the rest", async () => {
|
||||
const media = createLocalMedia(fakeGetUserMedia());
|
||||
const [microphone, camera] = await media.capture([
|
||||
{ source: "microphone" },
|
||||
{ source: "camera" },
|
||||
]);
|
||||
|
||||
media.release("camera");
|
||||
expect(trackBehind(camera).stop).toHaveBeenCalled();
|
||||
expect(media.tracks$.value).toEqual([microphone]);
|
||||
|
||||
media.stop();
|
||||
expect(trackBehind(microphone).stop).toHaveBeenCalled();
|
||||
expect(media.tracks$.value).toEqual([]);
|
||||
});
|
||||
|
||||
it("refuses a screen share", async () => {
|
||||
const media = createLocalMedia(fakeGetUserMedia());
|
||||
await expect(media.capture([{ source: "screenShare" }])).rejects.toThrow(
|
||||
"screen share",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
class FakeMediaStream {
|
||||
public constructor(private readonly tracks: MediaStreamTrack[]) {}
|
||||
public getTracks(): MediaStreamTrack[] {
|
||||
return this.tracks;
|
||||
}
|
||||
}
|
||||
|
||||
beforeAll(() => {
|
||||
globalThis.MediaStream = FakeMediaStream as unknown as typeof MediaStream;
|
||||
});
|
||||
|
||||
let nextId = 0;
|
||||
const tracksById = new Map<string, MediaStreamTrack>();
|
||||
|
||||
function fakeTrack(
|
||||
kind: "audio" | "video",
|
||||
settings: MediaTrackSettings = {},
|
||||
): MediaStreamTrack {
|
||||
const track = {
|
||||
kind,
|
||||
id: `${kind}-${nextId++}`,
|
||||
enabled: true,
|
||||
stop: vi.fn(),
|
||||
getSettings: () => settings,
|
||||
} as unknown as MediaStreamTrack;
|
||||
tracksById.set(track.id, track);
|
||||
return track;
|
||||
}
|
||||
|
||||
function fakeGetUserMedia(videoSettings?: MediaTrackSettings): GetUserMedia {
|
||||
return vi.fn(async (constraints: MediaStreamConstraints) => {
|
||||
const tracks: MediaStreamTrack[] = [];
|
||||
if (constraints.audio) tracks.push(fakeTrack("audio"));
|
||||
if (constraints.video) tracks.push(fakeTrack("video", videoSettings));
|
||||
return Promise.resolve(
|
||||
new FakeMediaStream(tracks) as unknown as MediaStream,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function fakeProcessor(): VideoProcessor & { output: MediaStreamTrack } {
|
||||
const output = fakeTrack("video");
|
||||
return {
|
||||
name: "blur",
|
||||
output,
|
||||
init: vi.fn(async () => Promise.resolve(output)),
|
||||
restart: vi.fn(async () => Promise.resolve(output)),
|
||||
destroy: vi.fn(async () => Promise.resolve()),
|
||||
};
|
||||
}
|
||||
|
||||
/** The device's track, which a media track was created on and keeps the id of. */
|
||||
function trackBehind(mediaTrack: { id: string }): MediaStreamTrack {
|
||||
return tracksById.get(mediaTrack.id)!;
|
||||
}
|
||||
|
||||
function tracksOn(element: HTMLMediaElement): MediaStreamTrack[] {
|
||||
return (element.srcObject as unknown as FakeMediaStream).getTracks();
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { BehaviorSubject, type Observable, of } from "rxjs";
|
||||
|
||||
import { MatrixRTCError } from "./errors";
|
||||
import {
|
||||
type HandedOverTrack,
|
||||
type LocalAudioMediaTrack,
|
||||
type LocalMedia,
|
||||
type LocalVideoMediaTrack,
|
||||
type MediaSource,
|
||||
type MediaStreamStats,
|
||||
type PublishRequest,
|
||||
type VideoProcessor,
|
||||
} from "./media-api";
|
||||
import { constant } from "./reactive/Behavior";
|
||||
import { inTurn } from "./utils/processorTurns";
|
||||
|
||||
type Source = "microphone" | "camera";
|
||||
type Request = Extract<PublishRequest, { source: Source }>;
|
||||
export type GetUserMedia = (
|
||||
constraints: MediaStreamConstraints,
|
||||
) => Promise<MediaStream>;
|
||||
|
||||
/**
|
||||
* Local media on the browser's devices alone, for a lobby ahead of a join:
|
||||
* no backend is involved, so whichever backend joins can take the tracks
|
||||
* over. Activity and level are not measured; stats are none.
|
||||
*/
|
||||
export function createLocalMedia(
|
||||
getUserMedia: GetUserMedia = async (constraints) =>
|
||||
navigator.mediaDevices.getUserMedia(constraints),
|
||||
): LocalMedia {
|
||||
const capturedTracks = new Map<Source, CapturedTrack>();
|
||||
const requests = new Map<Source, Request>();
|
||||
const tracks$ = new BehaviorSubject<
|
||||
(LocalAudioMediaTrack | LocalVideoMediaTrack)[]
|
||||
>([]);
|
||||
const publishCapturedTracks = (): void =>
|
||||
tracks$.next(
|
||||
[...capturedTracks.values()].map(({ mediaTrack }) => mediaTrack),
|
||||
);
|
||||
|
||||
// A second identical capture while the first is open joins it: a host that
|
||||
// mounts twice (React's strict mode) must not open the devices twice and
|
||||
// replace the tracks it is already showing
|
||||
const pending = new Map<
|
||||
string,
|
||||
Promise<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]>
|
||||
>();
|
||||
const capture = async (
|
||||
wanted: PublishRequest[],
|
||||
): Promise<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]> => {
|
||||
const key = JSON.stringify(
|
||||
wanted.map((request) => ({ ...request, processor: undefined })),
|
||||
);
|
||||
const running = pending.get(key);
|
||||
if (running) return running;
|
||||
const started = captureNow(wanted).finally(() => pending.delete(key));
|
||||
pending.set(key, started);
|
||||
return started;
|
||||
};
|
||||
|
||||
const captureNow = async (
|
||||
wanted: PublishRequest[],
|
||||
): Promise<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]> => {
|
||||
const bySource = new Map<Source, Request>();
|
||||
for (const request of wanted) {
|
||||
if (request.source === "screenShare")
|
||||
throw new MatrixRTCError(
|
||||
"A screen share cannot be captured ahead of a join",
|
||||
);
|
||||
bySource.set(request.source, request);
|
||||
}
|
||||
const constraints: MediaStreamConstraints = {};
|
||||
for (const request of bySource.values())
|
||||
constraints[request.source === "microphone" ? "audio" : "video"] =
|
||||
trackConstraints(request);
|
||||
const stream = await getUserMedia(constraints);
|
||||
const captured: CapturedTrack[] = [];
|
||||
for (const track of stream.getTracks()) {
|
||||
const source: Source = track.kind === "audio" ? "microphone" : "camera";
|
||||
const request = bySource.get(source)!;
|
||||
capturedTracks.get(source)?.stop();
|
||||
const captured_ = new CapturedTrack(
|
||||
source,
|
||||
track,
|
||||
trackConstraints(request),
|
||||
getUserMedia,
|
||||
);
|
||||
capturedTracks.set(source, captured_);
|
||||
requests.set(source, request);
|
||||
if (request.source === "camera" && request.processor)
|
||||
await captured_.setProcessor(request.processor);
|
||||
captured.push(captured_);
|
||||
}
|
||||
publishCapturedTracks();
|
||||
return captured.map(({ mediaTrack }) => mediaTrack);
|
||||
};
|
||||
|
||||
const release = (source: MediaSource): void => {
|
||||
if (source !== "microphone" && source !== "camera") return;
|
||||
capturedTracks.get(source)?.stop();
|
||||
capturedTracks.delete(source);
|
||||
publishCapturedTracks();
|
||||
};
|
||||
|
||||
const stop = (): void => {
|
||||
for (const track of capturedTracks.values()) track.stop();
|
||||
capturedTracks.clear();
|
||||
publishCapturedTracks();
|
||||
};
|
||||
|
||||
const takeTracks = (): Map<Source, HandedOverTrack> => {
|
||||
const taken = new Map<Source, HandedOverTrack>();
|
||||
for (const [source, track] of capturedTracks)
|
||||
taken.set(source, track.handOver());
|
||||
capturedTracks.clear();
|
||||
publishCapturedTracks();
|
||||
return taken;
|
||||
};
|
||||
|
||||
return {
|
||||
get requests(): PublishRequest[] {
|
||||
return [...requests.values()];
|
||||
},
|
||||
tracks$,
|
||||
capture,
|
||||
release,
|
||||
stop,
|
||||
takeTracks,
|
||||
};
|
||||
}
|
||||
|
||||
/** The constraints a request means to the browser. */
|
||||
function trackConstraints(request: Request): MediaTrackConstraints {
|
||||
const device = request.deviceId && { deviceId: request.deviceId };
|
||||
if (request.source === "microphone") return { ...device, ...request.capture };
|
||||
const resolution = request.capture?.resolution;
|
||||
return {
|
||||
...device,
|
||||
...(resolution && {
|
||||
width: resolution.width,
|
||||
height: resolution.height,
|
||||
frameRate: resolution.frameRate,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal Class that is the owner of the tracks.
|
||||
* Will get stored in `capturedTracks` of `createLocalMedia`.
|
||||
* To the outside this class will return the `LocalMedia` interface.
|
||||
*/
|
||||
class CapturedTrack {
|
||||
public readonly mediaTrack: LocalAudioMediaTrack | LocalVideoMediaTrack;
|
||||
private readonly muted$ = new BehaviorSubject(false);
|
||||
private readonly facingMode$: BehaviorSubject<
|
||||
"user" | "environment" | undefined
|
||||
>;
|
||||
private readonly elements = new Set<HTMLMediaElement>();
|
||||
private processor: HandedOverTrack["processor"];
|
||||
/** What the processor reads frames from; a processor needs a playing element. */
|
||||
private processorInput?: HTMLVideoElement;
|
||||
private finished = false;
|
||||
|
||||
public constructor(
|
||||
public readonly source: Source,
|
||||
private track: MediaStreamTrack,
|
||||
private constraints: MediaTrackConstraints,
|
||||
private readonly getUserMedia: GetUserMedia,
|
||||
) {
|
||||
this.facingMode$ = new BehaviorSubject(facingModeOf(track));
|
||||
this.mediaTrack = this.asMediaTrack();
|
||||
}
|
||||
|
||||
public handOver(): HandedOverTrack {
|
||||
const { track, processor } = this;
|
||||
// The device actually in use, so that a backend asked for it again
|
||||
// recognises it rather than restarting the track
|
||||
const { deviceId } = track.getSettings();
|
||||
const constraints = { ...this.constraints, ...(deviceId && { deviceId }) };
|
||||
this.finish();
|
||||
return { track, constraints, muted: this.muted$.value, processor };
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
const { processor } = this;
|
||||
if (processor)
|
||||
void inTurn(processor.processor, async () =>
|
||||
processor.processor.destroy(),
|
||||
);
|
||||
this.processor?.processedTrack.stop();
|
||||
this.track.stop();
|
||||
this.finish();
|
||||
}
|
||||
|
||||
public async setProcessor(
|
||||
processor: VideoProcessor | undefined,
|
||||
): Promise<void> {
|
||||
this.live();
|
||||
if (processor === this.processor?.processor) return;
|
||||
const previous = this.processor?.processor;
|
||||
if (previous) await inTurn(previous, async () => previous.destroy());
|
||||
this.processor = processor && {
|
||||
processor,
|
||||
processedTrack: await inTurn(processor, async () =>
|
||||
processor.init(this.track, this.input()),
|
||||
),
|
||||
};
|
||||
this.show();
|
||||
}
|
||||
|
||||
private input(): HTMLVideoElement {
|
||||
this.processorInput ??= document.createElement("video");
|
||||
this.processorInput.muted = true;
|
||||
this.processorInput.playsInline = true;
|
||||
play(this.processorInput, this.track);
|
||||
return this.processorInput;
|
||||
}
|
||||
|
||||
private finish(): void {
|
||||
this.finished = true;
|
||||
for (const element of this.elements) element.srcObject = null;
|
||||
this.elements.clear();
|
||||
}
|
||||
|
||||
private live(): void {
|
||||
if (this.finished)
|
||||
throw new MatrixRTCError(
|
||||
`The ${this.source} was handed over to a participation`,
|
||||
);
|
||||
}
|
||||
|
||||
private shown(): MediaStreamTrack {
|
||||
return this.processor?.processedTrack ?? this.track;
|
||||
}
|
||||
|
||||
private show(): void {
|
||||
for (const element of this.elements) play(element, this.shown());
|
||||
}
|
||||
|
||||
private async reacquire(constraints: MediaTrackConstraints): Promise<void> {
|
||||
this.live();
|
||||
this.constraints = { ...this.constraints, ...constraints };
|
||||
const stream = await this.getUserMedia({
|
||||
[this.track.kind]: this.constraints,
|
||||
});
|
||||
this.track.stop();
|
||||
this.track = stream.getTracks()[0];
|
||||
this.track.enabled = !this.muted$.value;
|
||||
const { processor } = this;
|
||||
if (processor)
|
||||
processor.processedTrack = await inTurn(processor.processor, async () =>
|
||||
processor.processor.restart(this.track, this.input()),
|
||||
);
|
||||
this.facingMode$.next(facingModeOf(this.track));
|
||||
this.show();
|
||||
}
|
||||
|
||||
private asMediaTrack(): LocalAudioMediaTrack | LocalVideoMediaTrack {
|
||||
const base = {
|
||||
source: this.source,
|
||||
id: this.track.id,
|
||||
muted$: this.muted$,
|
||||
// Nothing leaves the device before the track is published, so there is
|
||||
// nothing unencrypted to report
|
||||
encrypted: true,
|
||||
stats$: (): Observable<MediaStreamStats> => of(undefined),
|
||||
attach: (element: HTMLMediaElement): void => {
|
||||
this.live();
|
||||
this.elements.add(element);
|
||||
play(element, this.shown());
|
||||
},
|
||||
detach: (element: HTMLMediaElement): void => {
|
||||
if (this.elements.delete(element)) element.srcObject = null;
|
||||
},
|
||||
setEnabled: async (enabled: boolean): Promise<boolean> => {
|
||||
this.live();
|
||||
this.track.enabled = enabled;
|
||||
this.muted$.next(!enabled);
|
||||
return Promise.resolve(enabled);
|
||||
},
|
||||
setDevice: async (deviceId: string): Promise<void> => {
|
||||
if (this.track.getSettings().deviceId === deviceId) return;
|
||||
await this.reacquire({ deviceId });
|
||||
},
|
||||
};
|
||||
if (this.source === "microphone")
|
||||
return {
|
||||
...base,
|
||||
kind: "audio",
|
||||
isActive$: constant(false),
|
||||
audioLevel$: constant(0),
|
||||
setAudioContext: () => {},
|
||||
setVolume: () => {},
|
||||
};
|
||||
return {
|
||||
...base,
|
||||
kind: "video",
|
||||
facingMode$: this.facingMode$,
|
||||
switchFacingMode: async () => {
|
||||
const facingMode = this.facingMode$.value;
|
||||
if (facingMode === undefined) return;
|
||||
await this.reacquire({
|
||||
facingMode: facingMode === "user" ? "environment" : "user",
|
||||
});
|
||||
return this.track.getSettings().deviceId;
|
||||
},
|
||||
setProcessor: async (processor) => this.setProcessor(processor),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function play(element: HTMLMediaElement, track: MediaStreamTrack): void {
|
||||
element.srcObject = new MediaStream([track]);
|
||||
// Autoplay is not a given for a stream set after mount; a test DOM returns
|
||||
// nothing from play
|
||||
const playing = element.play() as Promise<void> | undefined;
|
||||
playing?.catch(() => undefined);
|
||||
}
|
||||
|
||||
function facingModeOf(
|
||||
track: MediaStreamTrack,
|
||||
): "user" | "environment" | undefined {
|
||||
const { facingMode } = track.getSettings();
|
||||
return facingMode === "user" || facingMode === "environment"
|
||||
? facingMode
|
||||
: undefined;
|
||||
}
|
||||
@@ -164,7 +164,7 @@ function setup(
|
||||
context: MediaBackendContext,
|
||||
): MediaBackend => {
|
||||
const backend: MediaBackend = {
|
||||
transportType: "livekit",
|
||||
transportTypes: ["m.livekit", "livekit"],
|
||||
prepareLocalTransport: async () => fake.prepared.promise,
|
||||
delegateDelayedLeave: fake.delegateDelayedLeave,
|
||||
local: {
|
||||
|
||||
@@ -136,7 +136,8 @@ export function createRTCParticipation(
|
||||
// fallback: another member's SFU may let us publish (MSC4195). LiveKit only,
|
||||
// so it stays out of the backend interface
|
||||
const transportFromBackends = async (): Promise<Transport | null> =>
|
||||
backend.transportType === "livekit"
|
||||
backend.transportTypes.includes("m.livekit") ||
|
||||
backend.transportTypes.includes("livekit")
|
||||
? findRemotePublishLivekitTransport({
|
||||
client,
|
||||
// A snapshot: if nobody with a usable SFU is here yet, the user has
|
||||
@@ -152,7 +153,7 @@ export function createRTCParticipation(
|
||||
const preparedTransport$ = from(
|
||||
discoverLocalTransport(
|
||||
client,
|
||||
backend.transportType,
|
||||
backend.transportTypes,
|
||||
options,
|
||||
transportFromBackends,
|
||||
logger,
|
||||
|
||||
+10
-6
@@ -20,6 +20,7 @@ import { type Observable } from "rxjs";
|
||||
import { type Behavior } from "./reactive/Behavior";
|
||||
import {
|
||||
type LocalAudioMediaTrack,
|
||||
type LocalMedia,
|
||||
type LocalMemberMedia,
|
||||
type LocalVideoMediaTrack,
|
||||
type MediaSource,
|
||||
@@ -84,12 +85,15 @@ export interface RTCSlot {
|
||||
|
||||
export interface RTCParticipationOptions {
|
||||
/**
|
||||
* Published at the join; `publish` on the participation adds to it from then
|
||||
* on. Kept as an option, rather than only the method, because the LiveKit
|
||||
* backend reads the initial requests to build the room's default capture
|
||||
* options before the first connection exists.
|
||||
* What the local member publishes at the join: requests the SDK captures
|
||||
* itself, or the media a lobby captured ahead of the join, which the
|
||||
* participation takes over (its tracks leave `media.tracks$`) and publishes
|
||||
* in place of capturing. `publish` on the participation adds to either from
|
||||
* then on. Kept as an option, rather than only the method, because the
|
||||
* LiveKit backend reads it to build the room's default capture options
|
||||
* before the first connection exists.
|
||||
*/
|
||||
publish: PublishRequest[];
|
||||
publish: PublishRequest[] | LocalMedia;
|
||||
/**
|
||||
* LEGACY
|
||||
*
|
||||
@@ -230,7 +234,7 @@ export interface DataMessage {
|
||||
* backend-specific developer panel and for connection diagnostics.
|
||||
*/
|
||||
export interface TransportMetadata {
|
||||
/** `"livekit"` today. */
|
||||
/** `"m.livekit"` or `"livekit"` today. */
|
||||
type: string;
|
||||
/** Stable key, unique per transport in the slot: the raw transport, serialised with sorted keys. */
|
||||
id: string;
|
||||
|
||||
@@ -48,7 +48,7 @@ describe("discoverLocalTransport", () => {
|
||||
await expect(
|
||||
discoverLocalTransport(
|
||||
c,
|
||||
"livekit",
|
||||
["m.livekit", "livekit"],
|
||||
{ transport: hostTransport },
|
||||
fromBackends(),
|
||||
logger,
|
||||
@@ -61,7 +61,7 @@ describe("discoverLocalTransport", () => {
|
||||
await expect(
|
||||
discoverLocalTransport(
|
||||
client([homeserverTransport]),
|
||||
"livekit",
|
||||
["m.livekit", "livekit"],
|
||||
{ fallbackTransport: hostTransport },
|
||||
fromBackends(),
|
||||
logger,
|
||||
@@ -73,7 +73,7 @@ describe("discoverLocalTransport", () => {
|
||||
await expect(
|
||||
discoverLocalTransport(
|
||||
client([{ type: "not_livekit" }]),
|
||||
"livekit",
|
||||
["m.livekit", "livekit"],
|
||||
{ fallbackTransport: hostTransport },
|
||||
fromBackends(),
|
||||
logger,
|
||||
@@ -87,7 +87,7 @@ describe("discoverLocalTransport", () => {
|
||||
await expect(
|
||||
discoverLocalTransport(
|
||||
client([homeserverTransport]),
|
||||
"livekit",
|
||||
["m.livekit", "livekit"],
|
||||
{},
|
||||
backends,
|
||||
logger,
|
||||
@@ -99,7 +99,7 @@ describe("discoverLocalTransport", () => {
|
||||
it("fails naming the homeserver where there is no transport at all", async () => {
|
||||
const promise = discoverLocalTransport(
|
||||
client([]),
|
||||
"livekit",
|
||||
["m.livekit", "livekit"],
|
||||
{},
|
||||
fromBackends(),
|
||||
logger,
|
||||
|
||||
@@ -15,14 +15,15 @@ import { TransportAutoDiscovery } from "./TransportAutoDiscovery";
|
||||
|
||||
/**
|
||||
* The transport the local member advertises: the one the host names, else the
|
||||
* homeserver's first of the type the backend serves, else the host's fallback,
|
||||
* else the first the backends can provide, which are asked only then.
|
||||
* homeserver's first of the most preferred type the backend serves, else the
|
||||
* host's fallback, else the first the backends can provide, which are asked
|
||||
* only then.
|
||||
*
|
||||
* @throws {MatrixRTCTransportMissingError} where none of these exists.
|
||||
*/
|
||||
export async function discoverLocalTransport(
|
||||
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports">,
|
||||
transportType: string,
|
||||
transportTypes: readonly string[],
|
||||
{
|
||||
transport: override,
|
||||
fallbackTransport,
|
||||
@@ -34,7 +35,7 @@ export async function discoverLocalTransport(
|
||||
override ??
|
||||
(await new TransportAutoDiscovery({
|
||||
client,
|
||||
transportType,
|
||||
transportTypes,
|
||||
fallbackTransport,
|
||||
logger,
|
||||
}).discoverPreferredTransport()) ??
|
||||
|
||||
@@ -61,7 +61,7 @@ describe("TransportAutoDiscovery", () => {
|
||||
|
||||
const discovery = new TransportAutoDiscovery({
|
||||
client,
|
||||
transportType: "livekit",
|
||||
transportTypes: ["m.livekit", "livekit"],
|
||||
fallbackTransport: configTransport,
|
||||
logger: rootLogger,
|
||||
});
|
||||
@@ -75,6 +75,29 @@ describe("TransportAutoDiscovery", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("prefers a transport of an earlier type over one listed before it", async () => {
|
||||
const mLivekit: Transport = {
|
||||
type: "m.livekit",
|
||||
url: "wss://sfu.example.org",
|
||||
};
|
||||
const client = makeClient();
|
||||
client._unstable_getRTCTransports.mockResolvedValue([
|
||||
backendTransport,
|
||||
mLivekit,
|
||||
]);
|
||||
|
||||
const discovery = new TransportAutoDiscovery({
|
||||
client,
|
||||
transportTypes: ["m.livekit", "livekit"],
|
||||
fallbackTransport: configTransport,
|
||||
logger: rootLogger,
|
||||
});
|
||||
|
||||
await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual(
|
||||
mLivekit,
|
||||
);
|
||||
});
|
||||
|
||||
it("Retries limit_exceeded backend transport", async () => {
|
||||
const client = makeClient();
|
||||
client._unstable_getRTCTransports
|
||||
@@ -92,7 +115,7 @@ describe("TransportAutoDiscovery", () => {
|
||||
|
||||
const discovery = new TransportAutoDiscovery({
|
||||
client,
|
||||
transportType: "livekit",
|
||||
transportTypes: ["m.livekit", "livekit"],
|
||||
fallbackTransport: configTransport,
|
||||
logger: rootLogger,
|
||||
});
|
||||
@@ -116,7 +139,7 @@ describe("TransportAutoDiscovery", () => {
|
||||
|
||||
const discovery = new TransportAutoDiscovery({
|
||||
client,
|
||||
transportType: "livekit",
|
||||
transportTypes: ["m.livekit", "livekit"],
|
||||
fallbackTransport: configTransport,
|
||||
logger: rootLogger,
|
||||
});
|
||||
@@ -133,7 +156,7 @@ describe("TransportAutoDiscovery", () => {
|
||||
|
||||
const discovery = new TransportAutoDiscovery({
|
||||
client,
|
||||
transportType: "livekit",
|
||||
transportTypes: ["m.livekit", "livekit"],
|
||||
fallbackTransport: undefined,
|
||||
logger: rootLogger,
|
||||
});
|
||||
|
||||
@@ -17,27 +17,27 @@ type TransportDiscoveryClient = Pick<
|
||||
|
||||
export interface TransportAutoDiscoveryProps {
|
||||
client: TransportDiscoveryClient;
|
||||
/** The transport type the media backend serves. */
|
||||
transportType: string;
|
||||
/** A transport to use when the homeserver advertises none of the type. */
|
||||
/** The transport types we want to discover. Those are the ones, the local media backend supports. most preferred first. */
|
||||
transportTypes: readonly string[];
|
||||
/** A transport to use when the homeserver advertises none of the types. */
|
||||
fallbackTransport?: Transport;
|
||||
logger: Logger;
|
||||
}
|
||||
|
||||
export class TransportAutoDiscovery {
|
||||
private readonly client: TransportDiscoveryClient;
|
||||
private readonly transportType: string;
|
||||
private readonly transportTypes: readonly string[];
|
||||
private readonly fallbackTransport: Transport | undefined;
|
||||
private readonly logger: Logger;
|
||||
|
||||
public constructor({
|
||||
client,
|
||||
transportType,
|
||||
transportTypes,
|
||||
fallbackTransport,
|
||||
logger,
|
||||
}: TransportAutoDiscoveryProps) {
|
||||
this.client = client;
|
||||
this.transportType = transportType;
|
||||
this.transportTypes = transportTypes;
|
||||
this.fallbackTransport = fallbackTransport;
|
||||
this.logger = logger.getChild("[TransportAutoDiscovery]");
|
||||
}
|
||||
@@ -64,7 +64,7 @@ export class TransportAutoDiscovery {
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the first transport of the backend's type from the homeserver.
|
||||
* Fetches the first transport of the backend's most preferred type from the homeserver.
|
||||
* This will not throw errors, but instead just log them and return null if the expected config is not found or malformed.
|
||||
* @private
|
||||
*/
|
||||
@@ -76,17 +76,16 @@ export class TransportAutoDiscovery {
|
||||
const transportList = await doNetworkOperationWithRetry(async () =>
|
||||
client._unstable_getRTCTransports(),
|
||||
);
|
||||
const first = transportList.find(
|
||||
(transport) => transport.type === this.transportType,
|
||||
);
|
||||
if (first) {
|
||||
return first;
|
||||
} else {
|
||||
this.logger.info(
|
||||
`No ${this.transportType} transport found in getRTCTransports end point`,
|
||||
transportList,
|
||||
for (const type of this.transportTypes) {
|
||||
const first = transportList.find(
|
||||
(transport) => transport.type === type,
|
||||
);
|
||||
if (first) return first;
|
||||
}
|
||||
this.logger.info(
|
||||
`No ${this.transportTypes.join(" or ")} transport found in getRTCTransports end point`,
|
||||
transportList,
|
||||
);
|
||||
} catch (ex) {
|
||||
this.logger.info(`Failed to use getRTCTransports end point: ${ex}`);
|
||||
}
|
||||
|
||||
@@ -185,3 +185,45 @@ export interface MemberMedia {
|
||||
export interface LocalMemberMedia extends MemberMedia {
|
||||
tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tracks captured ahead of a join, for a lobby: the same local tracks a
|
||||
* participation hands out, on nothing but the browser's devices, so they fit
|
||||
* whichever backend joins. Handed to `join` as `publish`, they are moved once
|
||||
* the participation publishes them: they leave `tracks$`, their objects are
|
||||
* finished (controls reject, elements detached), and the same devices come
|
||||
* back under `localMember$.tracks$` as new objects. `stop` releases only what
|
||||
* was not moved; `capture` works again afterwards.
|
||||
*/
|
||||
export interface LocalMedia {
|
||||
/** What was captured with, one per source; a join reads its defaults from here. */
|
||||
readonly requests: PublishRequest[];
|
||||
tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]>;
|
||||
/**
|
||||
* Opens the devices, in one permission prompt where the browser asks for
|
||||
* one. Rejects where a device cannot be opened, having opened none of them.
|
||||
* A source captured already is replaced.
|
||||
*/
|
||||
capture(
|
||||
requests: PublishRequest[],
|
||||
): Promise<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]>;
|
||||
/** Closes the device behind a source; nothing where it was not captured. */
|
||||
release(source: MediaSource): void;
|
||||
/** Closes every device still held. */
|
||||
stop(): void;
|
||||
/**
|
||||
* For a media backend, as it starts publishing: moves every track out,
|
||||
* finishing its object here. Not for a host; the participation calls it.
|
||||
*/
|
||||
takeTracks(): Map<"microphone" | "camera", HandedOverTrack>;
|
||||
}
|
||||
|
||||
/** One track as a backend receives it: the device's own track and what runs on it. */
|
||||
export interface HandedOverTrack {
|
||||
track: MediaStreamTrack;
|
||||
/** What it was opened with, for a backend that restarts it. */
|
||||
constraints: MediaTrackConstraints;
|
||||
muted: boolean;
|
||||
/** Where a processor runs: it and its output, to adopt rather than initialise again. */
|
||||
processor?: { processor: VideoProcessor; processedTrack: MediaStreamTrack };
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import { type Observable } from "rxjs";
|
||||
import { type ResolvedTransport, type RTCMember } from "../api";
|
||||
import {
|
||||
type LocalAudioMediaTrack,
|
||||
type LocalMedia,
|
||||
type LocalMemberMedia,
|
||||
type LocalVideoMediaTrack,
|
||||
type MediaSource,
|
||||
@@ -56,8 +57,8 @@ export interface MediaBackendContext {
|
||||
/** The slot the session is in, `application#id`; a token is issued for it. */
|
||||
slotId: string;
|
||||
ownMembershipIdentity: CallMembershipIdentityParts;
|
||||
/** What the local member publishes at the join. */
|
||||
publish: PublishRequest[];
|
||||
/** What the local member publishes at the join: requests, or a lobby's media. */
|
||||
publish: PublishRequest[] | LocalMedia;
|
||||
encryptionSystem: EncryptionSystem;
|
||||
/**
|
||||
* Every media key the session knows: the current ones replayed on
|
||||
@@ -113,8 +114,11 @@ export interface DataPacket {
|
||||
}
|
||||
|
||||
export interface MediaBackend {
|
||||
/** The membership transport type this backend serves: "livekit". */
|
||||
readonly transportType: string;
|
||||
/**
|
||||
* The membership transport types this backend serves, ordered decreasingly
|
||||
* by preference.
|
||||
*/
|
||||
readonly transportTypes: readonly string[];
|
||||
|
||||
/**
|
||||
* Makes `transport` usable as the one the local member publishes on. The
|
||||
|
||||
@@ -92,7 +92,7 @@ export function createLivekitBackend(
|
||||
roomId,
|
||||
slotId,
|
||||
ownMembershipIdentity,
|
||||
publish,
|
||||
publish: requestsOrMedia,
|
||||
encryptionSystem,
|
||||
mediaKeys$,
|
||||
timings,
|
||||
@@ -101,6 +101,9 @@ export function createLivekitBackend(
|
||||
{ client, mediaQuality }: LivekitBackendOptions,
|
||||
): MediaBackend {
|
||||
const logger = parentLogger.getChild("[LivekitBackend]");
|
||||
const [publish, media] = Array.isArray(requestsOrMedia)
|
||||
? [requestsOrMedia, undefined]
|
||||
: [requestsOrMedia.requests, requestsOrMedia];
|
||||
const keyProvider = createKeyProvider(encryptionSystem, mediaKeys$, logger);
|
||||
const transports = createTransportRegistry(scope);
|
||||
const localTransport$ = new BehaviorSubject<TransportLocator | null>(null);
|
||||
@@ -137,6 +140,7 @@ export function createLivekitBackend(
|
||||
connectionMap$,
|
||||
localTransport$,
|
||||
publish,
|
||||
media,
|
||||
audioOutputDeviceId$,
|
||||
encryptionSystem,
|
||||
logger,
|
||||
@@ -279,7 +283,7 @@ export function createLivekitBackend(
|
||||
});
|
||||
|
||||
return {
|
||||
transportType: "livekit",
|
||||
transportTypes: ["m.livekit", "livekit"],
|
||||
prepareLocalTransport,
|
||||
delegateDelayedLeave: delegateLeave,
|
||||
local: localMedia_,
|
||||
|
||||
@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
import { EventEmitter } from "events";
|
||||
import {
|
||||
type LocalParticipant,
|
||||
LocalVideoTrack,
|
||||
ParticipantEvent,
|
||||
RemoteAudioTrack,
|
||||
type RemoteParticipant,
|
||||
@@ -17,10 +18,22 @@ import {
|
||||
TrackEvent,
|
||||
type TrackPublication,
|
||||
} from "livekit-client";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
onTestFinished,
|
||||
vi,
|
||||
} from "vitest";
|
||||
|
||||
import { ObservableScope } from "../../reactive/ObservableScope";
|
||||
import { type AudioMediaTrack } from "../../media-api";
|
||||
import {
|
||||
type AudioMediaTrack,
|
||||
type LocalVideoMediaTrack,
|
||||
type VideoProcessor,
|
||||
} from "../../media-api";
|
||||
import {
|
||||
createLivekitMediaTrack,
|
||||
createLocalLivekitMediaTrack,
|
||||
@@ -205,8 +218,55 @@ describe("createLivekitMediaTrack", () => {
|
||||
await expect(track.setEnabled(false)).resolves.toBe(false);
|
||||
expect(setEnabled).toHaveBeenCalledWith("microphone", false);
|
||||
await track.setDevice("mic-2");
|
||||
expect(switchActiveDevice).toHaveBeenCalledWith("audioinput", "mic-2");
|
||||
expect(switchActiveDevice).toHaveBeenCalledWith(
|
||||
"audioinput",
|
||||
"mic-2",
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
// LiveKit stops a camera that is turned off; the processor is built when it
|
||||
// is turned on again
|
||||
it.each([
|
||||
{ state: "is off", isMuted: true, takes: true },
|
||||
{ state: "has stopped for good", isMuted: false, takes: false },
|
||||
])(
|
||||
"gives a processor to a camera that $state: $takes",
|
||||
async ({ isMuted, takes }) => {
|
||||
// jsdom has none; LiveKit's facing mode checks against it
|
||||
vi.stubGlobal("MediaStreamTrack", class {});
|
||||
onTestFinished(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
|
||||
isMuted,
|
||||
_mediaStreamTrack: {
|
||||
readyState: "ended",
|
||||
label: "",
|
||||
getSettings: () => ({}),
|
||||
},
|
||||
getProcessor: () => undefined,
|
||||
setProcessor: vi.fn().mockResolvedValue(undefined),
|
||||
}) as LocalVideoTrack;
|
||||
const { participant, publication, room } = fakes({
|
||||
track: camera as unknown as FakeTrack,
|
||||
});
|
||||
const track = createLocalLivekitMediaTrack(
|
||||
scope,
|
||||
Object.assign(participant, {
|
||||
isLocal: true,
|
||||
}) as unknown as LocalParticipant,
|
||||
publication as unknown as TrackPublication,
|
||||
room,
|
||||
vi.fn(),
|
||||
) as LocalVideoMediaTrack;
|
||||
|
||||
await track.setProcessor({ name: "blur" } as VideoProcessor);
|
||||
expect(vi.mocked(camera.setProcessor).mock.calls.length).toBe(
|
||||
takes ? 1 : 0,
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
type FakeTrack = Track & {
|
||||
|
||||
@@ -50,7 +50,7 @@ import {
|
||||
type VideoMediaTrack,
|
||||
} from "../../media-api";
|
||||
import { MatrixRTCError } from "../../errors";
|
||||
import { convertToLivekitProcessor } from "./videoProcessor";
|
||||
import { toLivekitProcessor, type LivekitProcessor } from "./videoProcessor";
|
||||
|
||||
export const mediaSources: Record<Track.Source, MediaSource> = {
|
||||
[Track.Source.Microphone]: "microphone",
|
||||
@@ -197,9 +197,19 @@ export function createLocalLivekitMediaTrack(
|
||||
setDevice: async (deviceId: string): Promise<void> => {
|
||||
if (base.source !== "microphone" && base.source !== "camera")
|
||||
throw new MatrixRTCError(`A ${base.source} track has no device`);
|
||||
// LiveKit decides whether to restart by the constraints it keeps, which
|
||||
// a browser that does not echo the device id back (Firefox) leaves
|
||||
// without one; a track already on the device is left alone here
|
||||
const current = publication.track;
|
||||
if (
|
||||
current instanceof LocalTrack &&
|
||||
current.mediaStreamTrack.getSettings().deviceId === deviceId
|
||||
)
|
||||
return;
|
||||
await room.switchActiveDevice(
|
||||
base.kind === "audio" ? "audioinput" : "videoinput",
|
||||
deviceId,
|
||||
false,
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -228,11 +238,17 @@ export function createLocalLivekitMediaTrack(
|
||||
},
|
||||
setProcessor: async (processor) => {
|
||||
const track = videoTrack();
|
||||
// A processor cannot be built on a track that has already ended
|
||||
if (!track || track.mediaStreamTrack.readyState === "ended") return;
|
||||
if (processor)
|
||||
await track.setProcessor(convertToLivekitProcessor(processor));
|
||||
else if (track.getProcessor()) await track.stopProcessor();
|
||||
// A camera that is off has a stopped track and takes the processor on
|
||||
// turning on again; one stopped for good takes none
|
||||
if (
|
||||
!track ||
|
||||
(track.mediaStreamTrack.readyState === "ended" && !track.isMuted)
|
||||
)
|
||||
return;
|
||||
const current = track.getProcessor() as LivekitProcessor | undefined;
|
||||
if (processor === current?.source) return;
|
||||
if (processor) await track.setProcessor(toLivekitProcessor(processor));
|
||||
else if (current) await track.stopProcessor();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
|
||||
import {
|
||||
type LocalAudioMediaTrack,
|
||||
type LocalMedia,
|
||||
type LocalVideoMediaTrack,
|
||||
type MediaSource,
|
||||
type PublishRequest,
|
||||
@@ -57,6 +58,8 @@ interface Props {
|
||||
/** The transport we publish on; null until the session has found it. */
|
||||
localTransport$: Behavior<TransportLocator | null>;
|
||||
publish: PublishRequest[];
|
||||
/** Tracks the host captured ahead of the join; the publisher takes them over. */
|
||||
media?: LocalMedia;
|
||||
/** Read by the connection factory for the rooms it creates from now on. */
|
||||
audioOutputDeviceId$: BehaviorSubject<string | undefined>;
|
||||
encryptionSystem: EncryptionSystem;
|
||||
@@ -79,6 +82,7 @@ export function createLivekitLocalMedia({
|
||||
connectionMap$,
|
||||
localTransport$,
|
||||
publish: initialRequests,
|
||||
media,
|
||||
audioOutputDeviceId$,
|
||||
encryptionSystem,
|
||||
logger: parentLogger,
|
||||
@@ -119,6 +123,7 @@ export function createLivekitLocalMedia({
|
||||
connection.livekitRoom,
|
||||
desired,
|
||||
logger.getChild(`[Publisher ${JSON.stringify(connection.transport)}]`),
|
||||
media,
|
||||
);
|
||||
publisher$.next(publisher);
|
||||
return Promise.resolve(async (): Promise<void> => {
|
||||
|
||||
@@ -11,16 +11,32 @@ import {
|
||||
LocalParticipant,
|
||||
type LocalTrack,
|
||||
type LocalTrackPublication,
|
||||
LocalVideoTrack,
|
||||
ParticipantEvent,
|
||||
type Room as LivekitRoom,
|
||||
Track,
|
||||
type TrackPublishOptions,
|
||||
} from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
afterEach,
|
||||
beforeAll,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from "vitest";
|
||||
|
||||
import { type PublishRequest } from "../../media-api";
|
||||
import {
|
||||
type HandedOverTrack,
|
||||
type LocalMedia,
|
||||
type PublishRequest,
|
||||
type VideoProcessor,
|
||||
} from "../../media-api";
|
||||
import { flushPromises } from "../../utils/test";
|
||||
import { type DesiredMedia, Publisher } from "./Publisher";
|
||||
import { type LivekitProcessor } from "./videoProcessor";
|
||||
|
||||
describe("Publisher", () => {
|
||||
describe("setAudioOutputDevice", () => {
|
||||
@@ -104,6 +120,24 @@ describe("Publisher", () => {
|
||||
expect(participant.createTracks).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("creates both with the options of requests made since the join", async () => {
|
||||
const processor: VideoProcessor = {
|
||||
name: "effect",
|
||||
init: vi.fn(),
|
||||
restart: vi.fn(),
|
||||
destroy: vi.fn(),
|
||||
};
|
||||
want(microphone, { source: "camera", deviceId: "front", processor });
|
||||
|
||||
publisher.start();
|
||||
await flushPromises();
|
||||
|
||||
expect(room.options.videoCaptureDefaults).toMatchObject({
|
||||
deviceId: "front",
|
||||
processor: expect.objectContaining({ source: processor }),
|
||||
});
|
||||
});
|
||||
|
||||
it("asks for the camera and the microphone in one call", async () => {
|
||||
const enableBoth = vi.spyOn(participant, "enableCameraAndMicrophone");
|
||||
want(microphone, camera);
|
||||
@@ -327,6 +361,143 @@ function fakeRoom(
|
||||
return { room, switchActiveDevice };
|
||||
}
|
||||
|
||||
// A lobby's tracks come as the device's own MediaStreamTracks; LiveKit's
|
||||
// track classes are real here so that what is published is what LiveKit will
|
||||
// restart and stop as its own
|
||||
describe("adopting a lobby's tracks", () => {
|
||||
// LiveKit wraps every track in a MediaStream, which jsdom does not have
|
||||
beforeAll(() => {
|
||||
globalThis.MediaStream = class {
|
||||
public constructor(private readonly tracks: MediaStreamTrack[]) {}
|
||||
public getTracks(): MediaStreamTrack[] {
|
||||
return this.tracks;
|
||||
}
|
||||
} as unknown as typeof MediaStream;
|
||||
});
|
||||
|
||||
function setup(
|
||||
source: "microphone" | "camera",
|
||||
handed: HandedOverTrack,
|
||||
): { participant: LocalParticipant; publisher: Publisher } {
|
||||
const participant = fakeLocalParticipant(async () => Promise.resolve());
|
||||
const room = {
|
||||
options: {},
|
||||
setE2EEEnabled: () => undefined,
|
||||
localParticipant: participant,
|
||||
} as unknown as LivekitRoom;
|
||||
const desired: DesiredMedia = new Map([
|
||||
[source, { request: { source }, enabled: true }],
|
||||
]);
|
||||
const media = {
|
||||
takeTracks: () => new Map([[source, handed]]),
|
||||
} as unknown as LocalMedia;
|
||||
const publisher = new Publisher(room, desired, logger, media);
|
||||
return { participant, publisher };
|
||||
}
|
||||
|
||||
it("publishes the lobby's track as its own rather than capturing", async () => {
|
||||
const handed = handedOver("video");
|
||||
const { participant, publisher } = setup("camera", handed);
|
||||
|
||||
publisher.start();
|
||||
await flushPromises();
|
||||
|
||||
expect(participant.createTracks).not.toHaveBeenCalled();
|
||||
const [published, options] = vi.mocked(participant.publishTrack).mock
|
||||
.calls[0] as [LocalVideoTrack, TrackPublishOptions];
|
||||
expect(published).toBeInstanceOf(LocalVideoTrack);
|
||||
expect(published.mediaStreamTrack).toBe(handed.track);
|
||||
expect(published.isUserProvided).toBe(false);
|
||||
expect(options.source).toBe(Track.Source.Camera);
|
||||
});
|
||||
|
||||
it("adopts the processor's output without initialising it again", async () => {
|
||||
const setProcessor = vi
|
||||
.spyOn(LocalVideoTrack.prototype, "setProcessor")
|
||||
.mockResolvedValue(undefined);
|
||||
const processor = {
|
||||
name: "blur",
|
||||
init: vi.fn(),
|
||||
restart: vi.fn(),
|
||||
destroy: vi.fn(),
|
||||
} as unknown as VideoProcessor;
|
||||
const processedTrack = fakeMediaStreamTrack("video");
|
||||
const handed = handedOver("video", {
|
||||
processor: { processor, processedTrack },
|
||||
});
|
||||
const { publisher } = setup("camera", handed);
|
||||
|
||||
publisher.start();
|
||||
await flushPromises();
|
||||
|
||||
const adapter = setProcessor.mock.calls[0][0] as LivekitProcessor;
|
||||
await adapter.init({ kind: Track.Kind.Video, track: handed.track });
|
||||
expect(adapter.processedTrack).toBe(processedTrack);
|
||||
expect(adapter.source).toBe(processor);
|
||||
expect(processor.init).not.toHaveBeenCalled();
|
||||
setProcessor.mockRestore();
|
||||
});
|
||||
|
||||
// A development build mounts, unmounts and mounts the call again: the first
|
||||
// publisher must not have taken the tracks the second one needs
|
||||
it("leaves the lobby's tracks alone until it starts", async () => {
|
||||
const participant = fakeLocalParticipant(async () => Promise.resolve());
|
||||
const room = {
|
||||
options: {},
|
||||
setE2EEEnabled: () => undefined,
|
||||
localParticipant: participant,
|
||||
} as unknown as LivekitRoom;
|
||||
const takeTracks = vi.fn(() => new Map());
|
||||
const media = { takeTracks } as unknown as LocalMedia;
|
||||
|
||||
const publisher = new Publisher(room, new Map(), logger, media);
|
||||
await publisher.destroy();
|
||||
|
||||
expect(takeTracks).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("joins muted where the lobby muted", async () => {
|
||||
const { participant, publisher } = setup(
|
||||
"microphone",
|
||||
handedOver("audio", { muted: true }),
|
||||
);
|
||||
|
||||
publisher.start();
|
||||
await flushPromises();
|
||||
|
||||
const [published] = vi.mocked(participant.publishTrack).mock.calls[0];
|
||||
expect((published as LocalTrack).isMuted).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
function fakeMediaStreamTrack(kind: "audio" | "video"): MediaStreamTrack {
|
||||
return {
|
||||
kind,
|
||||
id: `${kind}-track`,
|
||||
label: "",
|
||||
enabled: true,
|
||||
readyState: "live",
|
||||
getSettings: () => ({}),
|
||||
getConstraints: () => ({}),
|
||||
getCapabilities: () => ({}),
|
||||
addEventListener: vi.fn(),
|
||||
removeEventListener: vi.fn(),
|
||||
stop: vi.fn(),
|
||||
} as unknown as MediaStreamTrack;
|
||||
}
|
||||
|
||||
function handedOver(
|
||||
kind: "audio" | "video",
|
||||
extra: Partial<HandedOverTrack> = {},
|
||||
): HandedOverTrack {
|
||||
return {
|
||||
track: fakeMediaStreamTrack(kind),
|
||||
constraints: { deviceId: "lobby" },
|
||||
muted: false,
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
function fakeLocalTrack(source: Track.Source): LocalTrack {
|
||||
const state = { isMuted: false, isUpstreamPaused: false };
|
||||
const track = {
|
||||
|
||||
@@ -6,15 +6,23 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
LocalAudioTrack,
|
||||
type LocalTrackPublication,
|
||||
LocalVideoTrack,
|
||||
ParticipantEvent,
|
||||
type Room as LivekitRoom,
|
||||
type Track,
|
||||
} from "livekit-client";
|
||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { type MediaSource, type PublishRequest } from "../../media-api";
|
||||
import {
|
||||
type HandedOverTrack,
|
||||
type LocalMedia,
|
||||
type MediaSource,
|
||||
type PublishRequest,
|
||||
} from "../../media-api";
|
||||
import { livekitSources, mediaSources } from "./LivekitMediaTrack";
|
||||
import { toLivekitProcessor } from "./videoProcessor";
|
||||
import {
|
||||
audioCaptureOptions,
|
||||
screenShareCaptureOptions,
|
||||
@@ -31,6 +39,9 @@ export interface DesiredPublication {
|
||||
|
||||
export type DesiredMedia = Map<MediaSource, DesiredPublication>;
|
||||
|
||||
/** A lobby's tracks, moved out of its media, until each is published. */
|
||||
export type HandedOverTracks = ReturnType<LocalMedia["takeTracks"]>;
|
||||
|
||||
/**
|
||||
* Publishes the local media on one LiveKit room, following the host's
|
||||
* requests.
|
||||
@@ -46,10 +57,19 @@ export class Publisher {
|
||||
public started = false;
|
||||
private readonly room: LivekitRoom;
|
||||
|
||||
/** A lobby's tracks, taken over at `start`; each leaves the map as it is published. */
|
||||
private captured: HandedOverTracks = new Map();
|
||||
|
||||
/**
|
||||
* @param media - What a lobby opened ahead of the join, published in place
|
||||
* of capturing. Taken over only at `start`, so that a publisher torn down
|
||||
* before it publishes leaves the tracks to the next one.
|
||||
*/
|
||||
public constructor(
|
||||
room: LivekitRoom,
|
||||
private readonly desired: DesiredMedia,
|
||||
private readonly logger: Logger,
|
||||
private readonly media?: LocalMedia,
|
||||
) {
|
||||
this.room = room;
|
||||
room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
|
||||
@@ -79,33 +99,106 @@ export class Publisher {
|
||||
public start(): void {
|
||||
if (this.started) return;
|
||||
this.started = true;
|
||||
this.captured = this.media?.takeTracks() ?? new Map();
|
||||
// A track muted in the lobby joins muted
|
||||
for (const [source, { muted }] of this.captured) {
|
||||
const wanted = this.desired.get(source);
|
||||
if (wanted && muted) wanted.enabled = false;
|
||||
}
|
||||
const requests = [...this.desired.values()]
|
||||
.filter(({ enabled }) => enabled)
|
||||
.filter(
|
||||
({ request, enabled }) => enabled || this.hasCaptured(request.source),
|
||||
)
|
||||
.map(({ request }) => request);
|
||||
const sources = new Set(requests.map((request) => request.source));
|
||||
const toCapture = new Set(
|
||||
requests
|
||||
.map((request) => request.source)
|
||||
.filter((source) => !this.hasCaptured(source)),
|
||||
);
|
||||
// Microphone and camera in one call so that the browser asks for
|
||||
// permission once; the room defaults carry their options. LiveKit
|
||||
// resolves these once the track is published, which may block on the
|
||||
// connection; LocalTrackPublished is what tells us a track exists.
|
||||
if (sources.has("microphone") && sources.has("camera"))
|
||||
const captureBoth = toCapture.has("microphone") && toCapture.has("camera");
|
||||
if (captureBoth) {
|
||||
this.takeCaptureDefaults(requests);
|
||||
void this.room.localParticipant.enableCameraAndMicrophone();
|
||||
}
|
||||
for (const request of requests)
|
||||
if (
|
||||
request.source === "screenShare" ||
|
||||
!(sources.has("microphone") && sources.has("camera"))
|
||||
this.hasCaptured(request.source) ||
|
||||
!captureBoth
|
||||
)
|
||||
this.publish(request).catch((e) => {
|
||||
this.logger.error(`Failed to publish the ${request.source}`, e);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes the requests the room's capture defaults. The room took its defaults
|
||||
* from the requests at the join, but one made since, such as a camera turned
|
||||
* on before the start, has options of its own: a video processor dropped
|
||||
* here would let the room see the camera before it is applied.
|
||||
*/
|
||||
private takeCaptureDefaults(requests: PublishRequest[]): void {
|
||||
const options = this.room.options;
|
||||
for (const request of requests)
|
||||
if (request.source === "microphone")
|
||||
options.audioCaptureDefaults = {
|
||||
...options.audioCaptureDefaults,
|
||||
...audioCaptureOptions(request),
|
||||
};
|
||||
else if (request.source === "camera")
|
||||
options.videoCaptureDefaults = {
|
||||
...options.videoCaptureDefaults,
|
||||
...videoCaptureOptions(request),
|
||||
};
|
||||
}
|
||||
|
||||
private hasCaptured(source: MediaSource): boolean {
|
||||
return (
|
||||
(source === "microphone" || source === "camera") &&
|
||||
this.captured.has(source)
|
||||
);
|
||||
}
|
||||
|
||||
private takeCaptured(request: PublishRequest): HandedOverTrack | undefined {
|
||||
if (request.source === "screenShare") return undefined;
|
||||
const track = this.captured.get(request.source);
|
||||
this.captured.delete(request.source);
|
||||
return track;
|
||||
}
|
||||
|
||||
/** A lobby's track as if this participant had captured it: LiveKit restarts and stops it as its own. */
|
||||
private async adopt(
|
||||
{ track, constraints, muted, processor }: HandedOverTrack,
|
||||
request: PublishRequest,
|
||||
): Promise<LocalTrackPublication> {
|
||||
const local =
|
||||
track.kind === "audio"
|
||||
? new LocalAudioTrack(track, constraints, false)
|
||||
: new LocalVideoTrack(track, constraints, false);
|
||||
if (muted) await local.mute();
|
||||
if (processor && local instanceof LocalVideoTrack)
|
||||
await local.setProcessor(
|
||||
toLivekitProcessor(processor.processor, processor.processedTrack),
|
||||
);
|
||||
return this.room.localParticipant.publishTrack(local, {
|
||||
source: livekitSources[request.source],
|
||||
...(request.source === "camera" && videoPublishOptions(request.capture)),
|
||||
});
|
||||
}
|
||||
|
||||
/** Publishes one request and resolves with its publication once LiveKit has it. */
|
||||
public async publish(
|
||||
request: PublishRequest,
|
||||
): Promise<LocalTrackPublication | undefined> {
|
||||
const participant = this.room.localParticipant;
|
||||
let publication: LocalTrackPublication | undefined;
|
||||
if (request.source === "microphone")
|
||||
const captured = this.takeCaptured(request);
|
||||
if (captured) publication = await this.adopt(captured, request);
|
||||
else if (request.source === "microphone")
|
||||
publication = await participant.setMicrophoneEnabled(
|
||||
true,
|
||||
audioCaptureOptions(request),
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
|
||||
|
||||
import { serverNameOfTransport } from "../../utils/serverName";
|
||||
import { isCsApiUnavailable } from "./auth";
|
||||
import * as csApi from "./auth/csApi";
|
||||
import { decodeSFUJWTPayload } from "./auth/types";
|
||||
@@ -59,7 +60,7 @@ export async function findRemotePublishLivekitTransport({
|
||||
if (!isUnstableLivekitTransport(transport) || !("url" in transport))
|
||||
continue;
|
||||
const { url } = transport;
|
||||
const serverName = membership.getTransportServerName(transport);
|
||||
const serverName = serverNameOfTransport(transport, membership.userId);
|
||||
const key = JSON.stringify([url, serverName]);
|
||||
if (tried.has(key)) continue;
|
||||
tried.add(key);
|
||||
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
type PublishRequest,
|
||||
type VideoCaptureSettings,
|
||||
} from "../../media-api";
|
||||
import { convertToLivekitProcessor } from "./videoProcessor";
|
||||
import { toLivekitProcessor } from "./videoProcessor";
|
||||
|
||||
export type PublishRequestFor<S extends PublishRequest["source"]> = Extract<
|
||||
PublishRequest,
|
||||
@@ -40,7 +40,7 @@ export function videoCaptureOptions({
|
||||
if (!deviceId && !processor && !capture?.resolution) return undefined;
|
||||
return {
|
||||
...(deviceId && { deviceId }),
|
||||
...(processor && { processor: convertToLivekitProcessor(processor) }),
|
||||
...(processor && { processor: toLivekitProcessor(processor) }),
|
||||
...(capture?.resolution && { resolution: capture.resolution }),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -5,13 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { Track, type VideoProcessorOptions } from "livekit-client";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
LocalVideoTrack,
|
||||
Track,
|
||||
type TrackProcessor,
|
||||
type VideoProcessorOptions,
|
||||
} from "livekit-client";
|
||||
import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { type VideoProcessor } from "../../media-api";
|
||||
import { convertToLivekitProcessor } from "./videoProcessor";
|
||||
import { fromLivekitProcessor, toLivekitProcessor } from "./videoProcessor";
|
||||
|
||||
describe("convertToLivekitProcessor", () => {
|
||||
describe("toLivekitProcessor", () => {
|
||||
const captured = { id: "captured" } as MediaStreamTrack;
|
||||
const processed = { id: "processed" } as MediaStreamTrack;
|
||||
const restarted = { id: "restarted" } as MediaStreamTrack;
|
||||
@@ -33,7 +38,7 @@ describe("convertToLivekitProcessor", () => {
|
||||
|
||||
it("holds the processed track where LiveKit reads it", async () => {
|
||||
const processor = fakeProcessor();
|
||||
const adapter = convertToLivekitProcessor(processor);
|
||||
const adapter = toLivekitProcessor(processor);
|
||||
expect(adapter.name).toBe("blur");
|
||||
expect(adapter.processedTrack).toBeUndefined();
|
||||
|
||||
@@ -47,4 +52,202 @@ describe("convertToLivekitProcessor", () => {
|
||||
await adapter.destroy();
|
||||
expect(processor.destroy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// A stopped track tears its processor down without waiting, while the track
|
||||
// replacing it initialises the same one
|
||||
it("runs a destroy and the next init on one processor in turn, across adapters", async () => {
|
||||
const processor = fakeProcessor();
|
||||
let finishDestroy = (): void => {};
|
||||
vi.mocked(processor.destroy).mockImplementation(
|
||||
async () =>
|
||||
new Promise<void>((resolve) => {
|
||||
finishDestroy = resolve;
|
||||
}),
|
||||
);
|
||||
const previous = toLivekitProcessor(processor);
|
||||
const next = toLivekitProcessor(processor);
|
||||
await previous.init(options);
|
||||
vi.mocked(processor.init).mockClear();
|
||||
|
||||
const destroying = previous.destroy();
|
||||
const initialising = next.init(options);
|
||||
await Promise.resolve();
|
||||
expect(processor.init).not.toHaveBeenCalled();
|
||||
|
||||
finishDestroy();
|
||||
await Promise.all([destroying, initialising]);
|
||||
expect(processor.init).toHaveBeenCalledOnce();
|
||||
expect(next.processedTrack).toBe(processed);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a processor set on a camera that is off", () => {
|
||||
const ended = { id: "ended", readyState: "ended" } as MediaStreamTrack;
|
||||
|
||||
function fakeProcessor(): VideoProcessor {
|
||||
return {
|
||||
name: "blur",
|
||||
init: vi.fn().mockResolvedValue({ id: "processed" }),
|
||||
restart: vi.fn().mockResolvedValue({ id: "restarted" }),
|
||||
destroy: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
it("builds on the restart that turns the camera on, not before", async () => {
|
||||
const processor = fakeProcessor();
|
||||
const adapter = toLivekitProcessor(processor);
|
||||
|
||||
await adapter.init({ kind: Track.Kind.Video, track: ended });
|
||||
expect(processor.init).not.toHaveBeenCalled();
|
||||
expect(adapter.processedTrack).toBeUndefined();
|
||||
|
||||
const live = { id: "live", readyState: "live" } as MediaStreamTrack;
|
||||
await adapter.restart({ kind: Track.Kind.Video, track: live });
|
||||
expect(processor.init).toHaveBeenCalledWith(live, undefined);
|
||||
expect(processor.restart).not.toHaveBeenCalled();
|
||||
expect(adapter.processedTrack).toEqual({ id: "processed" });
|
||||
});
|
||||
|
||||
// The processor may be the app's one pipeline, running on another track
|
||||
it("leaves the processor alone when let go before it was built", async () => {
|
||||
const processor = fakeProcessor();
|
||||
const adapter = toLivekitProcessor(processor);
|
||||
await adapter.init({ kind: Track.Kind.Video, track: ended });
|
||||
await adapter.destroy();
|
||||
expect(processor.destroy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// LiveKit's own camera track, so that the order of its restart is what is
|
||||
// tested: a raw frame handed to the sender would reach the room
|
||||
describe("on LiveKit's camera track", () => {
|
||||
// jsdom has no MediaStream; LiveKit wraps every track in one
|
||||
beforeAll(() => {
|
||||
const original = globalThis.MediaStream;
|
||||
globalThis.MediaStream = class {
|
||||
public constructor(private readonly tracks: MediaStreamTrack[] = []) {}
|
||||
public getTracks(): MediaStreamTrack[] {
|
||||
return this.tracks;
|
||||
}
|
||||
public getVideoTracks(): MediaStreamTrack[] {
|
||||
return this.tracks.filter(({ kind }) => kind === "video");
|
||||
}
|
||||
public getAudioTracks(): MediaStreamTrack[] {
|
||||
return this.tracks.filter(({ kind }) => kind === "audio");
|
||||
}
|
||||
public addTrack(track: MediaStreamTrack): void {
|
||||
this.tracks.push(track);
|
||||
}
|
||||
public removeTrack(track: MediaStreamTrack): void {
|
||||
this.tracks.splice(this.tracks.indexOf(track), 1);
|
||||
}
|
||||
} as unknown as typeof MediaStream;
|
||||
// LiveKit plays its processor element and catches a refusal
|
||||
const play = vi
|
||||
.spyOn(HTMLMediaElement.prototype, "play")
|
||||
.mockResolvedValue(undefined);
|
||||
return (): void => {
|
||||
globalThis.MediaStream = original;
|
||||
play.mockRestore();
|
||||
};
|
||||
});
|
||||
|
||||
function deviceTrack(id: string): MediaStreamTrack {
|
||||
const track = {
|
||||
kind: "video",
|
||||
id,
|
||||
label: "",
|
||||
enabled: true,
|
||||
readyState: "live",
|
||||
getSettings: () => ({}),
|
||||
getConstraints: () => ({}),
|
||||
getCapabilities: () => ({}),
|
||||
applyConstraints: vi.fn().mockResolvedValue(undefined),
|
||||
addEventListener: vi.fn(),
|
||||
removeEventListener: vi.fn(),
|
||||
stop: vi.fn(() => {
|
||||
Object.assign(track, { readyState: "ended" });
|
||||
}),
|
||||
};
|
||||
return track as unknown as MediaStreamTrack;
|
||||
}
|
||||
|
||||
it("hands the sender only processed frames when the camera turns on", async () => {
|
||||
const reacquired = deviceTrack("reacquired");
|
||||
Object.defineProperty(navigator, "mediaDevices", {
|
||||
configurable: true,
|
||||
value: {
|
||||
getUserMedia: vi.fn().mockResolvedValue({
|
||||
getTracks: () => [reacquired],
|
||||
}),
|
||||
},
|
||||
});
|
||||
const processedTrack = { id: "processed" } as MediaStreamTrack;
|
||||
const processor = {
|
||||
...fakeProcessor(),
|
||||
init: vi.fn().mockResolvedValue(processedTrack),
|
||||
};
|
||||
const camera = new LocalVideoTrack(deviceTrack("first"), {}, false);
|
||||
camera.source = Track.Source.Camera;
|
||||
const replaceTrack = vi.fn().mockResolvedValue(undefined);
|
||||
Object.assign(camera, {
|
||||
sender: {
|
||||
replaceTrack,
|
||||
getParameters: () => ({ encodings: [] }),
|
||||
setParameters: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
});
|
||||
|
||||
await camera.mute();
|
||||
expect(camera.mediaStreamTrack.readyState).toBe("ended");
|
||||
// What the camera sent before it was turned off is not in question
|
||||
replaceTrack.mockClear();
|
||||
await camera.setProcessor(toLivekitProcessor(processor));
|
||||
await camera.unmute();
|
||||
|
||||
expect(processor.init).toHaveBeenCalledWith(
|
||||
reacquired,
|
||||
expect.anything(),
|
||||
);
|
||||
expect(replaceTrack.mock.calls).toEqual([[processedTrack]]);
|
||||
// LiveKit plays attached elements again on a timer
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("fromLivekitProcessor", () => {
|
||||
const captured = { id: "captured" } as MediaStreamTrack;
|
||||
const processed = { id: "processed" } as MediaStreamTrack;
|
||||
|
||||
function fakeLivekitProcessor(
|
||||
processedTrack: MediaStreamTrack | undefined,
|
||||
): TrackProcessor<Track.Kind.Video> {
|
||||
return {
|
||||
name: "background-blur",
|
||||
processedTrack,
|
||||
init: vi.fn().mockResolvedValue(undefined),
|
||||
restart: vi.fn().mockResolvedValue(undefined),
|
||||
destroy: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
it("resolves with the track the LiveKit processor produced", async () => {
|
||||
const livekitProcessor = fakeLivekitProcessor(processed);
|
||||
const processor = fromLivekitProcessor(livekitProcessor);
|
||||
expect(processor.name).toBe("background-blur");
|
||||
await expect(processor.init(captured)).resolves.toBe(processed);
|
||||
expect(livekitProcessor.init).toHaveBeenCalledWith({
|
||||
kind: Track.Kind.Video,
|
||||
track: captured,
|
||||
});
|
||||
await expect(processor.restart(captured)).resolves.toBe(processed);
|
||||
await processor.destroy();
|
||||
expect(livekitProcessor.destroy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects where the LiveKit processor produced no track", async () => {
|
||||
await expect(
|
||||
fromLivekitProcessor(fakeLivekitProcessor(undefined)).init(captured),
|
||||
).rejects.toThrow("produced no track");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,30 +5,84 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type Track, type TrackProcessor } from "livekit-client";
|
||||
import { Track, type TrackProcessor } from "livekit-client";
|
||||
|
||||
import { type VideoProcessor } from "../../media-api";
|
||||
import { inTurn } from "../../utils/processorTurns";
|
||||
|
||||
export interface LivekitProcessor extends TrackProcessor<Track.Kind.Video> {
|
||||
/** The processor this adapts, so that setting the same one again is a no-op. */
|
||||
source: VideoProcessor;
|
||||
}
|
||||
|
||||
/**
|
||||
* A `VideoProcessor` as livekit-client wants it. LiveKit reads the processed
|
||||
* track off the processor after `init` and `restart` rather than taking it as
|
||||
* their result, so the adapter keeps it there.
|
||||
* A `VideoProcessor` as a livekit-client processor. LiveKit reads the
|
||||
* processed track off the processor after `init` and `restart` rather than
|
||||
* taking it as their result, so the adapter keeps it there. A processor a
|
||||
* lobby already runs on the track is adopted with its output, `running`, so
|
||||
* that the join does not initialise it again. Every step takes its turn on
|
||||
* the processor, which a stopped track tears down without waiting for the
|
||||
* next one's init.
|
||||
*
|
||||
* Set on a camera that is off, whose track LiveKit has stopped, the build
|
||||
* waits for the restart that turns it back on. LiveKit restarts a processor
|
||||
* before it hands the new track to the sender, so the camera's first frame
|
||||
* out is already processed.
|
||||
*/
|
||||
export function convertToLivekitProcessor(
|
||||
export function toLivekitProcessor(
|
||||
processor: VideoProcessor,
|
||||
): TrackProcessor<Track.Kind.Video> {
|
||||
const adapter: TrackProcessor<Track.Kind.Video> = {
|
||||
running?: MediaStreamTrack,
|
||||
): LivekitProcessor {
|
||||
let built = false;
|
||||
const adapter: LivekitProcessor = {
|
||||
name: processor.name,
|
||||
init: async (opts) => {
|
||||
adapter.processedTrack = await processor.init(opts.track, opts.element);
|
||||
},
|
||||
restart: async (opts) => {
|
||||
adapter.processedTrack = await processor.restart(
|
||||
opts.track,
|
||||
opts.element,
|
||||
);
|
||||
},
|
||||
destroy: async () => processor.destroy(),
|
||||
source: processor,
|
||||
init: async (opts) =>
|
||||
inTurn(processor, async () => {
|
||||
if (running === undefined && opts.track.readyState === "ended") return;
|
||||
adapter.processedTrack =
|
||||
running ?? (await processor.init(opts.track, opts.element));
|
||||
running = undefined;
|
||||
built = true;
|
||||
}),
|
||||
restart: async (opts) =>
|
||||
inTurn(processor, async () => {
|
||||
adapter.processedTrack = built
|
||||
? await processor.restart(opts.track, opts.element)
|
||||
: await processor.init(opts.track, opts.element);
|
||||
built = true;
|
||||
}),
|
||||
destroy: async () =>
|
||||
inTurn(processor, async () => {
|
||||
if (built) await processor.destroy();
|
||||
}),
|
||||
};
|
||||
return adapter;
|
||||
}
|
||||
|
||||
/**
|
||||
* A livekit-client processor, such as one from `@livekit/track-processors`,
|
||||
* as a `VideoProcessor`: `init` and `restart` resolve with the track it
|
||||
* leaves on itself.
|
||||
*/
|
||||
export function fromLivekitProcessor(
|
||||
processor: TrackProcessor<Track.Kind.Video>,
|
||||
): VideoProcessor {
|
||||
const processedTrack = (): MediaStreamTrack => {
|
||||
if (processor.processedTrack === undefined)
|
||||
throw new Error(`Processor ${processor.name} produced no track`);
|
||||
return processor.processedTrack;
|
||||
};
|
||||
return {
|
||||
name: processor.name,
|
||||
init: async (track, element) => {
|
||||
await processor.init({ kind: Track.Kind.Video, track, element });
|
||||
return processedTrack();
|
||||
},
|
||||
restart: async (track, element) => {
|
||||
await processor.restart({ kind: Track.Kind.Video, track, element });
|
||||
return processedTrack();
|
||||
},
|
||||
destroy: async () => processor.destroy(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type VideoProcessor } from "../media-api";
|
||||
|
||||
const turns = new WeakMap<VideoProcessor, Promise<unknown>>();
|
||||
|
||||
/**
|
||||
* One operation at a time on a processor. A host shares one across the
|
||||
* tracks it captures and publishes, and a second init while the first still
|
||||
* loads its model, or a destroy while an init runs, leaves it broken: the
|
||||
* backend's own locking is per track, and a track that is stopped tears its
|
||||
* processor down without waiting.
|
||||
*/
|
||||
export async function inTurn<T>(
|
||||
processor: VideoProcessor,
|
||||
run: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const turn = (turns.get(processor) ?? Promise.resolve()).then(run, run);
|
||||
turns.set(processor, turn);
|
||||
return turn;
|
||||
}
|
||||
+3
-3
@@ -35,7 +35,7 @@ import { ErrorPage, LoadingPage } from "./FullScreenView";
|
||||
import { Initializer } from "./initializer";
|
||||
import { type WidgetHelpers } from "./widget";
|
||||
import { useTheme } from "./useTheme";
|
||||
import { ProcessorProvider } from "./tracks/TrackProcessorContext";
|
||||
import { BackgroundEffectsProvider } from "./tracks/BackgroundEffectsContext";
|
||||
import { type AppViewModel } from "./state/AppViewModel";
|
||||
import { MediaDevicesContext } from "./MediaDevicesContext";
|
||||
import {
|
||||
@@ -148,7 +148,7 @@ export const App: FC<Props> = ({ vm, widget }) => {
|
||||
loaded && clientReady ? (
|
||||
<ClientProvider client={widgetClient}>
|
||||
<MediaDevicesContext value={vm.mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider effects={vm.backgroundEffects}>
|
||||
<Sentry.ErrorBoundary
|
||||
fallback={(error) => <ErrorPage error={error} />}
|
||||
>
|
||||
@@ -159,7 +159,7 @@ export const App: FC<Props> = ({ vm, widget }) => {
|
||||
<SentryRoute path="*" element={<RoomPage />} />
|
||||
</Routes>
|
||||
</Sentry.ErrorBoundary>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</ClientProvider>
|
||||
) : (
|
||||
|
||||
@@ -13,7 +13,10 @@ import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import type { Layout } from "../state/layout-types";
|
||||
import type { SpotlightTileViewModel } from "../state/TileViewModel";
|
||||
import type { DeviceLabel } from "../state/MediaDevices";
|
||||
import { createCallFooterViewModel } from "./CallFooterViewModel";
|
||||
import {
|
||||
createCallFooterViewModel,
|
||||
createLobbyFooterViewModel,
|
||||
} from "./CallFooterViewModel";
|
||||
import { HeaderStyle } from "../UrlParams";
|
||||
|
||||
const platformMock = vi.hoisted(() => vi.fn(() => "desktop"));
|
||||
@@ -23,10 +26,10 @@ vi.mock("../Platform", () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
// Prevent supportsBackgroundProcessors from throwing in jsdom – it is not
|
||||
// exercised by these tests (only used in `videoToggles`, not `videoOptions`).
|
||||
// The SDK's own check needs WebGL and canvas APIs jsdom does not have.
|
||||
const sdkSupportMock = vi.hoisted(() => vi.fn(() => false));
|
||||
vi.mock("@livekit/track-processors", () => ({
|
||||
supportsBackgroundProcessors: (): boolean => false,
|
||||
supportsBackgroundProcessors: (): boolean => sdkSupportMock(),
|
||||
}));
|
||||
|
||||
const outputSelectionMock = vi.hoisted(() => vi.fn(() => true));
|
||||
@@ -217,4 +220,55 @@ describe("createCallFooterViewModel", () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("background effects", () => {
|
||||
// The lobby, because in a call the whole device switcher is already
|
||||
// withheld on a phone, whatever the verdict says.
|
||||
function lobbyOn(
|
||||
platform: string,
|
||||
): ReturnType<typeof createLobbyFooterViewModel> {
|
||||
platformMock.mockReturnValue(platform);
|
||||
return createLobbyFooterViewModel(
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
);
|
||||
}
|
||||
|
||||
it("offers nothing on a phone, where the pipeline would refuse them", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
expect(lobbyOn("ios").toggleBlur$.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("offers them where the pipeline will honour them", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
expect(lobbyOn("desktop").toggleBlur$.value).toBeDefined();
|
||||
});
|
||||
|
||||
it("offers nothing where the browser itself cannot run them", () => {
|
||||
sdkSupportMock.mockReturnValue(false);
|
||||
expect(lobbyOn("desktop").toggleBlur$.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("availability is the same before and during a call", () => {
|
||||
for (const supported of [true, false]) {
|
||||
sdkSupportMock.mockReturnValue(supported);
|
||||
const lobby = lobbyOn("desktop");
|
||||
const inCall = createCallFooterViewModel(
|
||||
testScope(),
|
||||
buildMinimalCallViewModel(gridLayout),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
const offeredInLobby = lobby.toggleBlur$.value !== undefined;
|
||||
expect(inCall.toggleBlur$.value !== undefined).toBe(offeredInLobby);
|
||||
expect(offeredInLobby).toBe(supported);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,14 +6,16 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, map, type Observable, switchMap } from "rxjs";
|
||||
import { supportsBackgroundProcessors } from "@livekit/track-processors";
|
||||
import { supportsAudioOutputSelection } from "livekit-client";
|
||||
|
||||
import { supportsBackgroundProcessors } from "../tracks/backgroundProcessing";
|
||||
import { parseEffect } from "../tracks/backgroundEffectCatalog";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { type MenuOptions } from "./MediaMuteAndSwitchButton";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
import {
|
||||
backgroundBlur as backgroundBlurSettings,
|
||||
backgroundEffect as backgroundEffectSetting,
|
||||
debugTileLayout as debugTileLayoutSetting,
|
||||
} from "../settings/settings";
|
||||
import { type Behavior, constant } from "../state/Behavior";
|
||||
@@ -92,6 +94,11 @@ function buildDeviceBehaviors(
|
||||
),
|
||||
);
|
||||
|
||||
const blurOn$ = scope.behavior(
|
||||
backgroundEffectSetting.value$.pipe(
|
||||
map((raw) => parseEffect(raw).kind === "blur"),
|
||||
),
|
||||
);
|
||||
return {
|
||||
audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)),
|
||||
selectedAudio$: scope.behavior(
|
||||
@@ -117,17 +124,17 @@ function buildDeviceBehaviors(
|
||||
),
|
||||
selectVideoButtonOption$: constant(mediaDevices.videoInput.select),
|
||||
toggleBlur$: scope.behavior(
|
||||
combineLatest([backgroundBlurSettings.value$, disableSwitcher$]).pipe(
|
||||
combineLatest([blurOn$, disableSwitcher$]).pipe(
|
||||
map(([current, switcherDisabled]) => {
|
||||
return !switcherDisabled && supportsBackgroundProcessors()
|
||||
? (): void => {
|
||||
backgroundBlurSettings.setValue(!current);
|
||||
backgroundEffectSetting.setValue(current ? "none" : "blur");
|
||||
}
|
||||
: undefined;
|
||||
}),
|
||||
),
|
||||
),
|
||||
videoBlurEnabled$: backgroundBlurSettings.value$,
|
||||
videoBlurEnabled$: blurOn$,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 22 KiB |
@@ -50,6 +50,7 @@ import {
|
||||
mockMediaDevices,
|
||||
mockRtcMembership,
|
||||
MockRTCSession,
|
||||
testScope,
|
||||
} from "../utils/test";
|
||||
import { CallView } from "./CallView";
|
||||
import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary";
|
||||
@@ -61,7 +62,10 @@ import {
|
||||
} from "../HostBridge";
|
||||
import { type JoinCallData } from "../widget";
|
||||
import { MatrixRTCTransportMissingError } from "../utils/errors";
|
||||
import { ProcessorProvider } from "../tracks/TrackProcessorContext";
|
||||
import {
|
||||
BackgroundEffectsProvider,
|
||||
createAppBackgroundEffects,
|
||||
} from "../tracks/BackgroundEffectsContext";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { constant } from "../state/Behavior";
|
||||
|
||||
@@ -193,7 +197,9 @@ function createCallView(
|
||||
<HostBridgeProvider value={hostBridge}>
|
||||
<TooltipProvider>
|
||||
<MediaDevicesContext value={mockMediaDevices({})}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider
|
||||
effects={createAppBackgroundEffects(testScope())}
|
||||
>
|
||||
{options.withErrorBoundary ? (
|
||||
<GroupCallErrorBoundary recoveryActionHandler={vi.fn()}>
|
||||
{callView}
|
||||
@@ -201,7 +207,7 @@ function createCallView(
|
||||
) : (
|
||||
callView
|
||||
)}
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</TooltipProvider>
|
||||
</HostBridgeProvider>
|
||||
|
||||
@@ -72,6 +72,7 @@ import { useRootElement } from "../RootElementContext.ts";
|
||||
import { useHostBridge } from "../HostBridge.ts";
|
||||
import { useMuteStates } from "../state/useMuteStates.ts";
|
||||
import { useLeaveToHome } from "../LeaveToHomeContext.ts";
|
||||
import { useLocalMedia } from "../useLocalMedia.ts";
|
||||
|
||||
/**
|
||||
* If there already are this many participants in the call, we automatically mute
|
||||
@@ -273,6 +274,8 @@ const LoadedCallView: FC<LoadedProps> = ({
|
||||
|
||||
const mediaDevices = useMediaDevices();
|
||||
const latestMuteStates = useLatest(muteStates);
|
||||
// Opened by the lobby, published by the call: one capture for both
|
||||
const media = useLocalMedia();
|
||||
|
||||
const enterRTCSessionOrError = useCallback(
|
||||
async (rtcSession: MatrixRTCSession): Promise<void> => {
|
||||
@@ -500,6 +503,7 @@ const LoadedCallView: FC<LoadedProps> = ({
|
||||
<LobbyView
|
||||
client={client}
|
||||
matrixInfo={matrixInfo}
|
||||
media={media}
|
||||
muteStates={muteStates}
|
||||
onEnter={() => setJoined(true)}
|
||||
confineToRoom={confineToRoom}
|
||||
@@ -529,6 +533,7 @@ const LoadedCallView: FC<LoadedProps> = ({
|
||||
rtcSession={rtcSession as MatrixRTCSession}
|
||||
matrixRoom={room}
|
||||
onLeft={onLeft}
|
||||
media={media}
|
||||
muteStates={muteStates}
|
||||
e2eeSystem={e2eeSystem}
|
||||
//otelGroupCallMembership={otelGroupCallMembership}
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
type MockRTCSession,
|
||||
mockRTCSlot,
|
||||
mockRTCMember,
|
||||
testScope,
|
||||
} from "../utils/test";
|
||||
import { E2eeType } from "../e2ee/e2eeType";
|
||||
import {
|
||||
@@ -48,7 +49,10 @@ import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type MediaDevices as ECMediaDevices } from "../state/MediaDevices";
|
||||
import { AppBar } from "../AppBar";
|
||||
import { type MatrixInfo } from "./VideoPreview";
|
||||
import { ProcessorProvider } from "../tracks/TrackProcessorContext";
|
||||
import {
|
||||
BackgroundEffectsProvider,
|
||||
createAppBackgroundEffects,
|
||||
} from "../tracks/BackgroundEffectsContext";
|
||||
import { initializeWidget } from "../widget";
|
||||
import { RootElementProvider } from "../RootElementContext";
|
||||
|
||||
@@ -227,6 +231,18 @@ describe("InCallView", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// Media that opened nothing: the join captures as it always did
|
||||
function fakeLocalMedia(): MatrixRTCSdk.LocalMedia {
|
||||
return {
|
||||
requests: [],
|
||||
tracks$: constant([]),
|
||||
capture: vi.fn(async () => Promise.resolve([])),
|
||||
release: vi.fn(),
|
||||
stop: vi.fn(),
|
||||
takeTracks: () => new Map(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("ActiveCall", () => {
|
||||
it("creates the view models and renders the call", async () => {
|
||||
const mediaDevices = mockMediaDevices({});
|
||||
@@ -234,7 +250,9 @@ describe("ActiveCall", () => {
|
||||
const { findByTestId } = render(
|
||||
<BrowserRouter>
|
||||
<MediaDevicesContext value={mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider
|
||||
effects={createAppBackgroundEffects(testScope())}
|
||||
>
|
||||
<TooltipProvider>
|
||||
<ActiveCall
|
||||
client={matrixRoom.client}
|
||||
@@ -244,10 +262,11 @@ describe("ActiveCall", () => {
|
||||
matrixInfo={matrixInfo}
|
||||
onShareClick={null}
|
||||
e2eeSystem={{ kind: E2eeType.NONE }}
|
||||
media={fakeLocalMedia()}
|
||||
onLeft={(): void => {}}
|
||||
/>
|
||||
</TooltipProvider>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</BrowserRouter>,
|
||||
);
|
||||
@@ -282,7 +301,9 @@ describe("ActiveCall", () => {
|
||||
<BrowserRouter>
|
||||
<RootElementProvider value={root}>
|
||||
<MediaDevicesContext value={mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider
|
||||
effects={createAppBackgroundEffects(testScope())}
|
||||
>
|
||||
<TooltipProvider>
|
||||
<ActiveCall
|
||||
client={matrixRoom.client}
|
||||
@@ -292,10 +313,11 @@ describe("ActiveCall", () => {
|
||||
matrixInfo={matrixInfo}
|
||||
onShareClick={null}
|
||||
e2eeSystem={{ kind: E2eeType.NONE }}
|
||||
media={fakeLocalMedia()}
|
||||
onLeft={(): void => {}}
|
||||
/>
|
||||
</TooltipProvider>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</RootElementProvider>
|
||||
</BrowserRouter>,
|
||||
|
||||
+19
-6
@@ -50,6 +50,7 @@ import { Grid, type TileProps } from "../grid/Grid";
|
||||
import { SpotlightTile } from "../tile/SpotlightTile";
|
||||
import {
|
||||
type EncryptionSystem,
|
||||
type LocalMedia,
|
||||
type MatrixRTCMode,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { E2eeType } from "../e2ee/e2eeType";
|
||||
@@ -109,7 +110,7 @@ import { useBehavior } from "../useBehavior.ts";
|
||||
import { useValueBehavior } from "../useValueBehavior.ts";
|
||||
import { Toast } from "../Toast.tsx";
|
||||
import overlayStyles from "../Overlay.module.css";
|
||||
import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext.tsx";
|
||||
import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext";
|
||||
import { type Layout } from "../state/layout-types.ts";
|
||||
import { ObservableScope } from "../state/ObservableScope.ts";
|
||||
import { CallFooter, type FooterSnapshot } from "../components/CallFooter.tsx";
|
||||
@@ -133,6 +134,8 @@ export interface ActiveCallProps extends Omit<
|
||||
"vm" | "footerVm" | "developerSettingsVm"
|
||||
> {
|
||||
e2eeSystem: EncryptionSystem;
|
||||
/** The tracks the lobby opened, for the join to publish. */
|
||||
media: LocalMedia;
|
||||
// TODO refactor those reasons into an enum
|
||||
onLeft: (
|
||||
reason: "user" | "timeout" | "decline" | "allOthersLeft" | "error",
|
||||
@@ -150,7 +153,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
const urlParams = useUrlParams();
|
||||
const hostBridge = useHostBridge();
|
||||
const mediaDevices = useMediaDevices();
|
||||
const trackProcessorState$ = useTrackProcessorState$();
|
||||
const backgroundEffectsState$ = useBackgroundEffectsState$();
|
||||
// The element we have to draw the call in: the page, or the container a host
|
||||
// gave us. Its size, not the window's, decides how the call is laid out.
|
||||
const rootElement = useRootElement();
|
||||
@@ -170,7 +173,16 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
);
|
||||
const rtcParticipation = slot.join({
|
||||
...rtcParticipationOptions(urlParams),
|
||||
publish: initialPublishRequests(props.muteStates, mediaDevices, capture),
|
||||
// Without a lobby nothing was captured and the join captures itself
|
||||
publish:
|
||||
props.media.requests.length > 0
|
||||
? props.media
|
||||
: initialPublishRequests(
|
||||
props.muteStates,
|
||||
mediaDevices,
|
||||
capture,
|
||||
backgroundEffectsState$.value.processor,
|
||||
),
|
||||
});
|
||||
|
||||
const vm = createCallViewModel$(
|
||||
@@ -184,7 +196,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
encryptionSystem: props.e2eeSystem,
|
||||
capture,
|
||||
videoProcessor$: scope.behavior(
|
||||
trackProcessorState$.pipe(map((state) => state.processor)),
|
||||
backgroundEffectsState$.pipe(map((state) => state.processor)),
|
||||
),
|
||||
hostBridge,
|
||||
autoLeaveWhenOthersLeft,
|
||||
@@ -214,8 +226,9 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
urlParams,
|
||||
hostBridge,
|
||||
mediaDevices,
|
||||
trackProcessorState$,
|
||||
backgroundEffectsState$,
|
||||
props.client,
|
||||
props.media,
|
||||
rootElement,
|
||||
]);
|
||||
|
||||
@@ -245,7 +258,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
props.onLeft,
|
||||
urlParams,
|
||||
mediaDevices,
|
||||
trackProcessorState$,
|
||||
backgroundEffectsState$,
|
||||
props.client,
|
||||
vm,
|
||||
]);
|
||||
|
||||
@@ -10,23 +10,41 @@ import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { TooltipProvider } from "@vector-im/compound-web";
|
||||
import { type MatrixClient, type RoomSummary } from "matrix-js-sdk";
|
||||
import {
|
||||
type Behavior,
|
||||
constant,
|
||||
type LocalMedia,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import { KnockLobbyView } from "./KnockLobbyView";
|
||||
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type ProcessorState } from "../tracks/TrackProcessorContext";
|
||||
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
|
||||
import { mockMediaDevices } from "../utils/test";
|
||||
|
||||
vi.mock("@livekit/components-react", () => ({
|
||||
usePreviewTracks: (): unknown[] => [],
|
||||
// One instance each: the lobby keys effects on them
|
||||
const stable = vi.hoisted(() => ({
|
||||
processorState$: undefined as unknown,
|
||||
media: undefined as unknown,
|
||||
}));
|
||||
|
||||
vi.mock("../tracks/TrackProcessorContext", () => ({
|
||||
useTrackProcessor: (): ProcessorState => ({
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
}),
|
||||
useTrackProcessorSync: (): void => {},
|
||||
vi.mock("../tracks/BackgroundEffectsContext", () => ({
|
||||
useBackgroundEffectsState$: (): Behavior<BackgroundEffectsState> =>
|
||||
(stable.processorState$ ??= constant({
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
})) as Behavior<BackgroundEffectsState>,
|
||||
}));
|
||||
// Media that opens no device: the view is what is under test
|
||||
vi.mock("../useLocalMedia", () => ({
|
||||
useLocalMedia: (): LocalMedia =>
|
||||
(stable.media ??= {
|
||||
requests: [],
|
||||
tracks$: constant([]),
|
||||
capture: vi.fn(async () => Promise.resolve([])),
|
||||
release: vi.fn(),
|
||||
stop: vi.fn(),
|
||||
takeTracks: () => new Map(),
|
||||
}) as LocalMedia,
|
||||
}));
|
||||
|
||||
vi.mock("react-use-measure", () => ({
|
||||
|
||||
@@ -13,6 +13,7 @@ import { CheckIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
import { LobbyView } from "./LobbyView";
|
||||
import { E2eeType } from "../e2ee/e2eeType";
|
||||
import { useMuteStates } from "../state/useMuteStates";
|
||||
import { useLocalMedia } from "../useLocalMedia";
|
||||
|
||||
interface Props {
|
||||
client: MatrixClient;
|
||||
@@ -49,6 +50,7 @@ export const KnockLobbyView: FC<Props> = ({
|
||||
}): ReactNode => {
|
||||
const { t } = useTranslation();
|
||||
const muteStates = useMuteStates();
|
||||
const media = useLocalMedia();
|
||||
|
||||
if (muteStates === null) return null;
|
||||
|
||||
@@ -85,6 +87,7 @@ export const KnockLobbyView: FC<Props> = ({
|
||||
confineToRoom={confineToRoom}
|
||||
hideHeader={hideHeader}
|
||||
participantCount={null}
|
||||
media={media}
|
||||
muteStates={muteStates}
|
||||
onShareClick={null}
|
||||
/>
|
||||
|
||||
+26
-10
@@ -12,6 +12,11 @@ import { of } from "rxjs";
|
||||
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
|
||||
import { TooltipProvider } from "@vector-im/compound-web";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
import {
|
||||
type Behavior,
|
||||
constant,
|
||||
type LocalMedia,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { axe } from "vitest-axe";
|
||||
import {
|
||||
ArrowLeftIcon,
|
||||
@@ -28,7 +33,7 @@ import {
|
||||
} from "../utils/test";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type ProcessorState } from "../tracks/TrackProcessorContext";
|
||||
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
|
||||
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
|
||||
import lobbyStyles from "./LobbyView.module.css";
|
||||
import headerStyles from "../Header.module.css";
|
||||
@@ -37,17 +42,27 @@ import { AppBar } from "../AppBar";
|
||||
// Somewhere to go home to, so that the lobby offers the way back
|
||||
const leaveToHome = vi.fn();
|
||||
|
||||
vi.mock("@livekit/components-react", () => ({
|
||||
usePreviewTracks: (): unknown[] => [],
|
||||
// One instance: the view keys an effect on it
|
||||
const processorState$ = vi.hoisted(() => ({ current: undefined as unknown }));
|
||||
vi.mock("../tracks/BackgroundEffectsContext", () => ({
|
||||
useBackgroundEffectsState$: (): Behavior<BackgroundEffectsState> =>
|
||||
(processorState$.current ??= constant({
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
})) as Behavior<BackgroundEffectsState>,
|
||||
}));
|
||||
|
||||
vi.mock("../tracks/TrackProcessorContext", () => ({
|
||||
useTrackProcessor: (): ProcessorState => ({
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
}),
|
||||
useTrackProcessorSync: (): void => {},
|
||||
}));
|
||||
// Media that opens no device: the view is what is under test
|
||||
function fakeLocalMedia(): LocalMedia {
|
||||
return {
|
||||
requests: [],
|
||||
tracks$: constant([]),
|
||||
capture: vi.fn(async () => Promise.resolve([])),
|
||||
release: vi.fn(),
|
||||
stop: vi.fn(),
|
||||
takeTracks: () => new Map(),
|
||||
};
|
||||
}
|
||||
|
||||
vi.mock("react-use-measure", () => ({
|
||||
default: (): [() => void, object] => [(): void => {}, {}],
|
||||
@@ -95,6 +110,7 @@ function renderLobbyView(
|
||||
<LobbyView
|
||||
client={mockClient}
|
||||
matrixInfo={matrixInfo}
|
||||
media={fakeLocalMedia()}
|
||||
muteStates={muteStates}
|
||||
onEnter={() => {}}
|
||||
confineToRoom={false}
|
||||
|
||||
+38
-71
@@ -5,26 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type FC,
|
||||
useCallback,
|
||||
useMemo,
|
||||
useState,
|
||||
type JSX,
|
||||
useEffect,
|
||||
} from "react";
|
||||
import { type FC, useCallback, useState, type JSX, useEffect } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
import { Button } from "@vector-im/compound-web";
|
||||
import classNames from "classnames";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { usePreviewTracks } from "@livekit/components-react";
|
||||
import {
|
||||
type CreateLocalTracksOptions,
|
||||
type LocalVideoTrack,
|
||||
Track,
|
||||
} from "livekit-client";
|
||||
import { convertToLivekitProcessor } from "@element-hq/matrixrtc-sdk";
|
||||
constant,
|
||||
type LocalMedia,
|
||||
type LocalVideoMediaTrack,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { map } from "rxjs";
|
||||
|
||||
import inCallStyles from "./InCallView.module.css";
|
||||
import styles from "./LobbyView.module.css";
|
||||
@@ -39,22 +31,26 @@ import { LeaveToHomeLink } from "../button/LeaveToHomeLink";
|
||||
import { useLeaveToHome } from "../LeaveToHomeContext";
|
||||
import { useMediaDevices } from "../MediaDevicesContext";
|
||||
import { ObservableScope } from "../state/ObservableScope";
|
||||
import { useInitial } from "../useInitial";
|
||||
import {
|
||||
useTrackProcessor,
|
||||
useTrackProcessorSync,
|
||||
} from "../tracks/TrackProcessorContext";
|
||||
import { getValue } from "../utils/observable";
|
||||
import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { CallFooter, type FooterSnapshot } from "../components/CallFooter";
|
||||
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
|
||||
import { createLobbyFooterViewModel } from "../components/CallFooterViewModel";
|
||||
import { type ViewModel } from "../state/ViewModel";
|
||||
import { useAppBarPrimaryButtonIconKind } from "../AppBar";
|
||||
import {
|
||||
createLobbyMediaViewModel,
|
||||
type LobbyMediaViewModel,
|
||||
} from "../state/LobbyMediaViewModel";
|
||||
import { captureSettings } from "./InCallView";
|
||||
|
||||
const noVideoTrack = constant<LocalVideoMediaTrack | null>(null);
|
||||
|
||||
interface Props {
|
||||
client: MatrixClient;
|
||||
matrixInfo: MatrixInfo;
|
||||
/** The media to open the devices on; the join takes it over. */
|
||||
media: LocalMedia;
|
||||
muteStates: MuteStates;
|
||||
onEnter: () => void;
|
||||
enterLabel?: JSX.Element | string;
|
||||
@@ -68,6 +64,7 @@ interface Props {
|
||||
export const LobbyView: FC<Props> = ({
|
||||
client,
|
||||
matrixInfo,
|
||||
media,
|
||||
muteStates,
|
||||
onEnter,
|
||||
enterLabel,
|
||||
@@ -87,7 +84,6 @@ export const LobbyView: FC<Props> = ({
|
||||
const { t } = useTranslation();
|
||||
|
||||
useAppBarPrimaryButtonIconKind("back");
|
||||
const audioEnabled = useBehavior(muteStates.audio.enabled$);
|
||||
const videoEnabled = useBehavior(muteStates.video.enabled$);
|
||||
const toggleAudio = useBehavior(muteStates.audio.toggle$);
|
||||
const toggleVideo = useBehavior(muteStates.video.toggle$);
|
||||
@@ -126,55 +122,28 @@ export const LobbyView: FC<Props> = ({
|
||||
|
||||
const devices = useMediaDevices();
|
||||
const videoInputId = useBehavior(devices.videoInput.selected$)?.id;
|
||||
const backgroundEffectsState$ = useBackgroundEffectsState$();
|
||||
|
||||
// Capture the audio options as they were when we first mounted, because
|
||||
// we're not doing anything with the audio anyway so we don't need to
|
||||
// re-open the devices when they change (see below).
|
||||
const initialAudioOptions = useInitial(
|
||||
() =>
|
||||
audioEnabled && {
|
||||
deviceId: getValue(devices.audioInput.selected$)?.id,
|
||||
},
|
||||
);
|
||||
|
||||
const { processor } = useTrackProcessor();
|
||||
|
||||
const initialProcessor = useInitial(() => processor);
|
||||
const localTrackOptions = useMemo<CreateLocalTracksOptions>(
|
||||
() => ({
|
||||
// The only reason we request audio here is to get the audio permission
|
||||
// request over with at the same time. But changing the audio settings
|
||||
// shouldn't cause this hook to recreate the track, which is why we
|
||||
// reference the initial values here.
|
||||
// We also pass in a clone because livekit mutates the object passed in,
|
||||
// which would cause the devices to be re-opened on the next render.
|
||||
audio: Object.assign({}, initialAudioOptions),
|
||||
video: videoEnabled && {
|
||||
deviceId: videoInputId,
|
||||
processor:
|
||||
initialProcessor && convertToLivekitProcessor(initialProcessor),
|
||||
},
|
||||
}),
|
||||
[initialAudioOptions, videoEnabled, videoInputId, initialProcessor],
|
||||
);
|
||||
|
||||
const onError = useCallback(
|
||||
(error: Error) => {
|
||||
logger.error("Error while creating preview Tracks:", error);
|
||||
muteStates.audio.setEnabled$.value?.(false);
|
||||
muteStates.video.setEnabled$.value?.(false);
|
||||
},
|
||||
[muteStates],
|
||||
);
|
||||
|
||||
const tracks = usePreviewTracks(localTrackOptions, onError);
|
||||
|
||||
const videoTrack = useMemo(
|
||||
() =>
|
||||
(tracks?.find((t) => t.kind === Track.Kind.Video) ??
|
||||
null) as LocalVideoTrack | null,
|
||||
[tracks],
|
||||
);
|
||||
const [mediaVm, setMediaVm] = useState<LobbyMediaViewModel | null>(null);
|
||||
useEffect(() => {
|
||||
const mediaScope = new ObservableScope();
|
||||
setMediaVm(
|
||||
createLobbyMediaViewModel(
|
||||
mediaScope,
|
||||
media,
|
||||
muteStates,
|
||||
devices,
|
||||
captureSettings(),
|
||||
mediaScope.behavior(
|
||||
backgroundEffectsState$.pipe(map((state) => state.processor)),
|
||||
),
|
||||
),
|
||||
);
|
||||
return (): void => {
|
||||
mediaScope.end();
|
||||
};
|
||||
}, [media, muteStates, devices, backgroundEffectsState$]);
|
||||
const videoTrack = useBehavior(mediaVm?.videoTrack$ ?? noVideoTrack);
|
||||
|
||||
useEffect(() => {
|
||||
if (videoTrack && videoInputId === undefined) {
|
||||
@@ -185,8 +154,6 @@ export const LobbyView: FC<Props> = ({
|
||||
}
|
||||
}, [devices, videoInputId, videoTrack]);
|
||||
|
||||
useTrackProcessorSync(videoTrack);
|
||||
|
||||
const [footerVm, setFooterVm] = useState<ViewModel<FooterSnapshot> | null>(
|
||||
null,
|
||||
);
|
||||
|
||||
+11
-15
@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { useEffect, useMemo, useRef, type FC, type ReactNode } from "react";
|
||||
import useMeasure from "react-use-measure";
|
||||
import { facingModeFromLocalTrack, type LocalVideoTrack } from "livekit-client";
|
||||
import { constant, type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
|
||||
import classNames from "classnames";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
@@ -15,6 +15,9 @@ import { TileAvatar } from "../tile/TileAvatar";
|
||||
import styles from "./VideoPreview.module.css";
|
||||
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
|
||||
import videoPlaceholder from "../graphics/video-placeholder.gif";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
|
||||
const noFacingMode = constant(undefined);
|
||||
|
||||
export type MatrixInfo = {
|
||||
userId: string;
|
||||
@@ -30,7 +33,7 @@ export type MatrixInfo = {
|
||||
interface Props {
|
||||
matrixInfo: MatrixInfo;
|
||||
videoEnabled: boolean;
|
||||
videoTrack: LocalVideoTrack | null;
|
||||
videoTrack: LocalVideoMediaTrack | null;
|
||||
children: ReactNode;
|
||||
}
|
||||
|
||||
@@ -46,14 +49,12 @@ export const VideoPreview: FC<Props> = ({
|
||||
const videoEl = useRef<HTMLVideoElement | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
// Effect to connect the videoTrack with the video element.
|
||||
if (videoEl.current) {
|
||||
videoTrack?.attach(videoEl.current);
|
||||
}
|
||||
return (): void => {
|
||||
videoTrack?.detach();
|
||||
};
|
||||
const element = videoEl.current;
|
||||
if (!element || !videoTrack) return;
|
||||
videoTrack.attach(element);
|
||||
return (): void => videoTrack.detach(element);
|
||||
}, [videoTrack]);
|
||||
const facingMode = useBehavior(videoTrack?.facingMode$ ?? noFacingMode);
|
||||
|
||||
const cameraIsStarting = useMemo(
|
||||
() => videoEnabled && !videoTrack,
|
||||
@@ -63,12 +64,7 @@ export const VideoPreview: FC<Props> = ({
|
||||
return (
|
||||
<div className={classNames(styles.preview)} ref={previewRef}>
|
||||
<video
|
||||
className={
|
||||
videoTrack &&
|
||||
facingModeFromLocalTrack(videoTrack).facingMode === "user"
|
||||
? styles.mirror
|
||||
: undefined
|
||||
}
|
||||
className={facingMode === "user" ? styles.mirror : undefined}
|
||||
ref={videoEl}
|
||||
muted
|
||||
playsInline
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
import { TooltipProvider } from "@vector-im/compound-web";
|
||||
|
||||
import { SettingsModal } from "./SettingsModal";
|
||||
import { backgroundEffect } from "./settings";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
|
||||
import { mockMediaDevices } from "../utils/test";
|
||||
import { constant } from "../state/Behavior";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
|
||||
vi.mock("../tracks/BackgroundEffectsContext", () => ({
|
||||
useBackgroundEffects: (): BackgroundEffectsState => ({
|
||||
supported: true,
|
||||
processor: undefined,
|
||||
}),
|
||||
}));
|
||||
|
||||
// Behaviors, as the device menus read them.
|
||||
const noDevices = {
|
||||
available$: constant(new Map()),
|
||||
selected$: constant(undefined),
|
||||
select: vi.fn(),
|
||||
};
|
||||
|
||||
function renderVideoTab(): void {
|
||||
render(
|
||||
<TooltipProvider>
|
||||
<MediaDevicesContext
|
||||
value={mockMediaDevices({
|
||||
requestDeviceNames: vi.fn(),
|
||||
audioInput: noDevices,
|
||||
audioOutput: noDevices,
|
||||
videoInput: noDevices,
|
||||
} as unknown as Partial<MediaDevices>)}
|
||||
>
|
||||
<SettingsModal
|
||||
open
|
||||
onDismiss={vi.fn()}
|
||||
tab="video"
|
||||
onTabChange={vi.fn()}
|
||||
client={
|
||||
{
|
||||
getUserId: () => "@user:example.org",
|
||||
getUser: () => null,
|
||||
} as unknown as MatrixClient
|
||||
}
|
||||
/>
|
||||
</MediaDevicesContext>
|
||||
</TooltipProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
const blurControl = (): HTMLElement =>
|
||||
screen.getByRole("checkbox", { name: "Blur the background of the video" });
|
||||
|
||||
describe("SettingsModal", () => {
|
||||
afterEach(() => backgroundEffect.setValue("none"));
|
||||
|
||||
it("blur control turns blur on and off", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderVideoTab();
|
||||
expect(blurControl()).not.toBeChecked();
|
||||
|
||||
await user.click(blurControl());
|
||||
expect(backgroundEffect.getValue()).toBe("blur");
|
||||
expect(blurControl()).toBeChecked();
|
||||
|
||||
await user.click(blurControl());
|
||||
expect(backgroundEffect.getValue()).toBe("none");
|
||||
expect(blurControl()).not.toBeChecked();
|
||||
});
|
||||
|
||||
it("blur control reflects and replaces an image background", async () => {
|
||||
const user = userEvent.setup();
|
||||
backgroundEffect.setValue("image:arc");
|
||||
renderVideoTab();
|
||||
expect(blurControl()).not.toBeChecked();
|
||||
|
||||
await user.click(blurControl());
|
||||
expect(backgroundEffect.getValue()).toBe("blur");
|
||||
});
|
||||
});
|
||||
@@ -21,13 +21,17 @@ import { useHostBridge } from "../HostBridge";
|
||||
import {
|
||||
useSetting,
|
||||
soundEffectVolume as soundEffectVolumeSetting,
|
||||
backgroundBlur as backgroundBlurSetting,
|
||||
backgroundEffect as backgroundEffectSetting,
|
||||
developerMode,
|
||||
} from "./settings";
|
||||
import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
|
||||
import { Slider } from "../Slider";
|
||||
import { DeviceSelection } from "./DeviceSelection";
|
||||
import { useTrackProcessor } from "../tracks/TrackProcessorContext";
|
||||
import { useBackgroundEffects } from "../tracks/BackgroundEffectsContext";
|
||||
import {
|
||||
parseEffect,
|
||||
serializeEffect,
|
||||
} from "../tracks/backgroundEffectCatalog";
|
||||
import {
|
||||
DeveloperSettingsTab,
|
||||
type DeveloperSettingsSnapshot,
|
||||
@@ -78,9 +82,11 @@ export const SettingsModal: FC<Props> = ({
|
||||
|
||||
// Generate a `Checkbox` input to turn blur on or off.
|
||||
const BlurCheckbox: React.FC = (): ReactNode => {
|
||||
const { supported } = useTrackProcessor();
|
||||
const { supported } = useBackgroundEffects();
|
||||
|
||||
const [blurActive, setBlurActive] = useSetting(backgroundBlurSetting);
|
||||
// The camera menu's setting, of which this control sees only blur.
|
||||
const [effect, setEffect] = useSetting(backgroundEffectSetting);
|
||||
const blurActive = parseEffect(effect).kind === "blur";
|
||||
|
||||
return (
|
||||
<>
|
||||
@@ -94,8 +100,12 @@ export const SettingsModal: FC<Props> = ({
|
||||
supported ? "" : t("settings.blur_not_supported_by_browser")
|
||||
}
|
||||
type="checkbox"
|
||||
checked={!!blurActive}
|
||||
onChange={(b): void => setBlurActive(b.target.checked)}
|
||||
checked={blurActive}
|
||||
onChange={(b): void =>
|
||||
setEffect(
|
||||
serializeEffect({ kind: b.target.checked ? "blur" : "none" }),
|
||||
)
|
||||
}
|
||||
disabled={!supported}
|
||||
/>
|
||||
</FieldRow>
|
||||
|
||||
@@ -13,6 +13,7 @@ import { type Behavior } from "../state/Behavior";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { MatrixRTCMode } from "../config/ConfigOptions";
|
||||
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type EffectId } from "../tracks/backgroundEffectCatalog";
|
||||
|
||||
export class Setting<T> {
|
||||
public constructor(
|
||||
@@ -116,7 +117,14 @@ export const videoInput = new Setting<string | undefined>(
|
||||
undefined,
|
||||
);
|
||||
|
||||
export const backgroundBlur = new Setting<boolean>("background-blur", false);
|
||||
// Only read, to carry blur over for anyone who had it on.
|
||||
const backgroundBlur = new Setting<boolean>("background-blur", false);
|
||||
|
||||
/** The chosen background effect, in the form `parseEffect` reads. */
|
||||
export const backgroundEffect = new Setting<EffectId>(
|
||||
"background-effect",
|
||||
backgroundBlur.getValue() ? "blur" : "none",
|
||||
);
|
||||
|
||||
export const showHandRaisedTimer = new Setting<boolean>(
|
||||
"hand-raised-show-timer",
|
||||
|
||||
@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type AudioOutputOptions, MediaDevices } from "./MediaDevices";
|
||||
import { type ObservableScope } from "./ObservableScope";
|
||||
import { createAppBackgroundEffects } from "../tracks/BackgroundEffectsContext";
|
||||
|
||||
/**
|
||||
* The top-level state holder for the application.
|
||||
@@ -16,6 +17,7 @@ export class AppViewModel {
|
||||
this.scope,
|
||||
this.audioOutputOptions,
|
||||
);
|
||||
public readonly backgroundEffects = createAppBackgroundEffects(this.scope);
|
||||
|
||||
// TODO: Move more application logic here. The CallViewModel, at the very
|
||||
// least, ought to be accessible from this object.
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { BehaviorSubject, distinctUntilChanged, map } from "rxjs";
|
||||
import { type BackgroundProcessorWrapper } from "@livekit/track-processors";
|
||||
|
||||
import {
|
||||
type BackgroundEffects,
|
||||
createBackgroundEffects,
|
||||
type BackgroundEffectsOptions,
|
||||
type BackgroundEffectsState,
|
||||
} from "./BackgroundEffects";
|
||||
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
|
||||
import { flushPromises, testScope, withTestScheduler } from "../utils/test";
|
||||
|
||||
/** A pipeline that records what it is switched to. */
|
||||
function fakePipeline(): {
|
||||
pipeline: BackgroundProcessorWrapper;
|
||||
switches: unknown[];
|
||||
/** Holds the next switch until it resolves, as a picture loading does. */
|
||||
holdNext: () => () => void;
|
||||
} {
|
||||
const switches: unknown[] = [];
|
||||
let slow: Promise<void> | undefined;
|
||||
const pipeline = {
|
||||
switchTo: vi.fn(async (options: unknown) => {
|
||||
switches.push(options);
|
||||
const held = slow;
|
||||
slow = undefined;
|
||||
await held;
|
||||
}),
|
||||
} as unknown as BackgroundProcessorWrapper;
|
||||
return {
|
||||
pipeline,
|
||||
switches,
|
||||
holdNext: () => {
|
||||
let release!: () => void;
|
||||
slow = new Promise((resolve) => (release = resolve));
|
||||
return release;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** One letter per state: idle, attached. */
|
||||
function letter(state: BackgroundEffectsState): string {
|
||||
return state.processor === undefined ? "i" : "a";
|
||||
}
|
||||
|
||||
describe("the pipeline's state", () => {
|
||||
function testState({
|
||||
effect,
|
||||
expected,
|
||||
}: {
|
||||
effect: string;
|
||||
expected: string;
|
||||
}): void {
|
||||
withTestScheduler(({ behavior, expectObservable }) => {
|
||||
const effects = createBackgroundEffects(testScope(), {
|
||||
supported: true,
|
||||
effect$: behavior(effect, { n: "none", b: "blur" }),
|
||||
pipeline: fakePipeline().pipeline,
|
||||
});
|
||||
expectObservable(
|
||||
effects.state$.pipe(map(letter), distinctUntilChanged()),
|
||||
).toBe(expected);
|
||||
});
|
||||
}
|
||||
|
||||
it("defaults to no effect", () => testState({ effect: "n", expected: "i" }));
|
||||
|
||||
it("attaches on first use and stays attached", () =>
|
||||
testState({ effect: "nbn", expected: "ia" }));
|
||||
});
|
||||
|
||||
describe("background effects", () => {
|
||||
let effect$: BehaviorSubject<string>;
|
||||
let fake: ReturnType<typeof fakePipeline>;
|
||||
|
||||
function build(
|
||||
options: Partial<BackgroundEffectsOptions> = {},
|
||||
): BackgroundEffects {
|
||||
return createBackgroundEffects(testScope(), {
|
||||
supported: true,
|
||||
effect$,
|
||||
pipeline: fake.pipeline,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
const choose = async (raw: string): Promise<void> => {
|
||||
effect$.next(raw);
|
||||
await flushPromises();
|
||||
};
|
||||
const blur = async (on: boolean): Promise<void> =>
|
||||
choose(on ? "blur" : "none");
|
||||
|
||||
beforeEach(() => {
|
||||
effect$ = new BehaviorSubject("none");
|
||||
fake = fakePipeline();
|
||||
});
|
||||
|
||||
it("puts a shipped background on as that picture", async () => {
|
||||
build();
|
||||
await choose(`image:${shippedBackgrounds[0].id}`);
|
||||
expect(fake.switches).toEqual([
|
||||
{
|
||||
mode: "virtual-background",
|
||||
imagePath: shippedBackgrounds[0].imagePath,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("switches in place rather than reattaching", async () => {
|
||||
const effects = build();
|
||||
await blur(true);
|
||||
const pipeline = effects.state$.value.processor;
|
||||
await blur(false);
|
||||
await blur(true);
|
||||
|
||||
expect(effects.state$.value.processor).toBe(pipeline);
|
||||
expect(fake.switches).toEqual([
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
{ mode: "disabled" },
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps switching after a switch fails", async () => {
|
||||
vi.mocked(fake.pipeline.switchTo).mockRejectedValueOnce(new Error("lost"));
|
||||
build();
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
expect(fake.switches).toEqual([{ mode: "disabled" }]);
|
||||
});
|
||||
|
||||
it("switches one at a time, skipping those overtaken", async () => {
|
||||
const finished = fake.holdNext();
|
||||
build();
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
expect(fake.switches).toHaveLength(1);
|
||||
|
||||
finished();
|
||||
await flushPromises();
|
||||
expect(fake.switches).toEqual([
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
{ mode: "disabled" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,115 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, distinctUntilChanged, map, scan } from "rxjs";
|
||||
import {
|
||||
type BackgroundProcessorWrapper,
|
||||
type SwitchBackgroundProcessorOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { deepCompare } from "matrix-js-sdk/lib/utils";
|
||||
import {
|
||||
fromLivekitProcessor,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import { type Behavior } from "./Behavior";
|
||||
import { type ObservableScope } from "./ObservableScope";
|
||||
import {
|
||||
type BackgroundEffect,
|
||||
imagePathFor,
|
||||
parseEffect,
|
||||
} from "../tracks/backgroundEffectCatalog";
|
||||
|
||||
const blurRadius = 15;
|
||||
|
||||
export interface BackgroundEffectsState {
|
||||
supported: boolean | undefined;
|
||||
processor: undefined | VideoProcessor;
|
||||
}
|
||||
|
||||
export interface BackgroundEffectsOptions {
|
||||
/** Whether this browser can run a pipeline at all. */
|
||||
supported: boolean;
|
||||
/** The effect chosen, as the setting stores it. */
|
||||
effect$: Behavior<string>;
|
||||
/** The background processor pipeline to be switched between effects. */
|
||||
pipeline: BackgroundProcessorWrapper;
|
||||
}
|
||||
|
||||
/** The background effect pipeline, as the camera tracks and the menus see it. */
|
||||
export interface BackgroundEffects {
|
||||
readonly state$: Behavior<BackgroundEffectsState>;
|
||||
}
|
||||
|
||||
/** Switches the pipeline as the choice changes, for as long as the scope lasts. */
|
||||
export function createBackgroundEffects(
|
||||
scope: ObservableScope,
|
||||
{ supported, effect$, pipeline }: BackgroundEffectsOptions,
|
||||
): BackgroundEffects {
|
||||
// Wrapped once, so the call and the preview see one processor throughout.
|
||||
const processor = fromLivekitProcessor(pipeline);
|
||||
const choice$ = effect$.pipe(map(parseEffect));
|
||||
const wanted$ = choice$.pipe(
|
||||
map((effect) => effect.kind !== "none"),
|
||||
distinctUntilChanged(),
|
||||
);
|
||||
|
||||
const state$ = scope.behavior(
|
||||
wanted$.pipe(
|
||||
scan<boolean, BackgroundEffectsState>(
|
||||
(previous, wanted) => {
|
||||
// Attached the first time an effect is wanted and never detached
|
||||
// after, so someone who never turns one on pays for none of it.
|
||||
const enable =
|
||||
previous.processor !== undefined || (supported && wanted);
|
||||
return { supported, processor: enable ? processor : undefined };
|
||||
},
|
||||
{ supported, processor: undefined },
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const switchOptions$ = scope.behavior<
|
||||
SwitchBackgroundProcessorOptions | undefined
|
||||
>(
|
||||
combineLatest([state$, choice$]).pipe(
|
||||
map(([{ processor }, effect]) =>
|
||||
processor === undefined ? undefined : switchOptionsFor(effect),
|
||||
),
|
||||
distinctUntilChanged(deepCompare),
|
||||
),
|
||||
);
|
||||
// In turn: a picture's switch ends once it loads, so an earlier one could land last.
|
||||
scope.reconcile(switchOptions$, async (options) => {
|
||||
if (options === undefined) return;
|
||||
try {
|
||||
await pipeline.switchTo(options);
|
||||
} catch (e) {
|
||||
logger.warn("Failed to switch background effect", e);
|
||||
}
|
||||
});
|
||||
|
||||
return { state$ };
|
||||
}
|
||||
|
||||
function switchOptionsFor(
|
||||
effect: BackgroundEffect,
|
||||
): SwitchBackgroundProcessorOptions {
|
||||
switch (effect.kind) {
|
||||
case "blur":
|
||||
return { mode: "background-blur", blurRadius };
|
||||
case "shipped": {
|
||||
const imagePath = imagePathFor(effect.id);
|
||||
return imagePath
|
||||
? { mode: "virtual-background", imagePath }
|
||||
: { mode: "disabled" };
|
||||
}
|
||||
default:
|
||||
return { mode: "disabled" };
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,8 @@ import { type Layout } from "../layout-types.ts";
|
||||
import {
|
||||
mockMatrixRoomMember,
|
||||
mockMediaDevices,
|
||||
mockMuteStates,
|
||||
flushPromises,
|
||||
mockRemoteParticipant,
|
||||
withTestScheduler,
|
||||
mockRtcMembership,
|
||||
@@ -62,7 +64,15 @@ import {
|
||||
import { MatrixRTCMode } from "../../config/ConfigOptions.ts";
|
||||
import { initializeWidget } from "../../widget.ts";
|
||||
import { computeUrlParams } from "../../UrlParams.ts";
|
||||
import { callViewModelOptionsFromParams } from "./CallViewModel.ts";
|
||||
import {
|
||||
callViewModelOptionsFromParams,
|
||||
initialPublishRequests,
|
||||
} from "./CallViewModel.ts";
|
||||
import { type Handler, MuteState } from "../MuteStates.ts";
|
||||
import {
|
||||
type RTCParticipation,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
initializeWidget();
|
||||
|
||||
@@ -1664,3 +1674,50 @@ describe("callViewModelOptionsFromParams", () => {
|
||||
expect(callViewModelOptionsFromParams(params).hideScreensharing).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a camera with a video processor", () => {
|
||||
const withCallViewModel = withCallViewModelInMode(MatrixRTCMode.Matrix_2_0);
|
||||
const processor = { name: "effect" } as VideoProcessor;
|
||||
|
||||
/** The handler the view model gives the camera's mute switch. */
|
||||
const cameraHandler = (setHandler: {
|
||||
mock: { calls: [Handler][] };
|
||||
}): Handler => setHandler.mock.calls[1][0];
|
||||
|
||||
it("starts each camera track with the processor", () => {
|
||||
const muteStates = mockMuteStates();
|
||||
vi.spyOn(muteStates.video.enabled$, "value", "get").mockReturnValue(true);
|
||||
expect(
|
||||
initialPublishRequests(
|
||||
muteStates,
|
||||
mockMediaDevices({}),
|
||||
{},
|
||||
processor,
|
||||
).find((request) => request.source === "camera"),
|
||||
).toMatchObject({ processor });
|
||||
});
|
||||
|
||||
// The SDK builds the processor on the camera's restart, before the room
|
||||
// sees a frame, so the track is unmuted rather than published anew
|
||||
it("turns on a camera turned off before the processor was chosen", async () => {
|
||||
const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
|
||||
onTestFinished(() => setHandler.mockRestore());
|
||||
const videoProcessor$ = new BehaviorSubject<VideoProcessor | undefined>(
|
||||
undefined,
|
||||
);
|
||||
let participation: RTCParticipation | undefined;
|
||||
withCallViewModel(
|
||||
{},
|
||||
(_vm, _session, { rtcParticipation }) => {
|
||||
participation = rtcParticipation;
|
||||
},
|
||||
{ videoProcessor$ },
|
||||
);
|
||||
videoProcessor$.next(processor);
|
||||
await cameraHandler(setHandler)(true);
|
||||
await flushPromises();
|
||||
|
||||
expect(participation!.unpublish).not.toHaveBeenCalled();
|
||||
expect(participation!.publish).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,7 +14,6 @@ import {
|
||||
type EncryptionSystem,
|
||||
generateItems,
|
||||
type LocalAudioMediaTrack,
|
||||
type LocalMediaTrack,
|
||||
type LocalRTCMember,
|
||||
type LocalVideoMediaTrack,
|
||||
type RTCParticipation,
|
||||
@@ -79,8 +78,14 @@ import {
|
||||
type ReactionOption,
|
||||
} from "../../reactions";
|
||||
import { shallowEquals as arrayShallowEquals } from "../../utils/array";
|
||||
import { type MediaDevice, type MediaDevices } from "../MediaDevices";
|
||||
import { type Handler, type MuteStates } from "../MuteStates";
|
||||
import { type MediaDevices } from "../MediaDevices";
|
||||
import { type MuteStates } from "../MuteStates";
|
||||
import {
|
||||
followDevice,
|
||||
followMuteState,
|
||||
followProcessor,
|
||||
type TrackSupply,
|
||||
} from "../localMediaControls";
|
||||
import { HeaderStyle, type UrlParams } from "../../UrlParams";
|
||||
import { type HostBridge, nullHostBridge } from "../../HostBridge";
|
||||
import {
|
||||
@@ -135,14 +140,16 @@ export interface CaptureSettings {
|
||||
}
|
||||
|
||||
/**
|
||||
* What to publish for a microphone or camera: the device the user picked and
|
||||
* the host's capture settings. The video processor is not in here; the call
|
||||
* attaches it to the camera track once that exists.
|
||||
* What to publish for a microphone or camera: the device the user picked, the
|
||||
* host's capture settings and, for the camera, the video processor, so that
|
||||
* the track starts with it rather than letting the room through while it is
|
||||
* built.
|
||||
*/
|
||||
export function publishRequest(
|
||||
source: "microphone" | "camera",
|
||||
mediaDevices: MediaDevices,
|
||||
capture: CaptureSettings,
|
||||
processor?: VideoProcessor,
|
||||
): PublishRequest {
|
||||
return source === "microphone"
|
||||
? {
|
||||
@@ -154,6 +161,7 @@ export function publishRequest(
|
||||
source,
|
||||
deviceId: mediaDevices.videoInput.selected$.value?.id,
|
||||
capture: capture.camera,
|
||||
...(processor && { processor }),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -162,12 +170,13 @@ export function initialPublishRequests(
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
capture: CaptureSettings,
|
||||
processor?: VideoProcessor,
|
||||
): PublishRequest[] {
|
||||
const requests: PublishRequest[] = [];
|
||||
if (muteStates.audio.enabled$.value)
|
||||
requests.push(publishRequest("microphone", mediaDevices, capture));
|
||||
if (muteStates.video.enabled$.value)
|
||||
requests.push(publishRequest("camera", mediaDevices, capture));
|
||||
requests.push(publishRequest("camera", mediaDevices, capture, processor));
|
||||
return requests;
|
||||
}
|
||||
|
||||
@@ -503,72 +512,41 @@ export function createCallViewModel$(
|
||||
"screenShare",
|
||||
) as Behavior<LocalVideoMediaTrack | undefined>;
|
||||
|
||||
// The mute switches drive our tracks, which report back what the device
|
||||
// allowed, so a denied permission flips the switch back. A source is muted
|
||||
// and unmuted while its track exists and published when it does not.
|
||||
const followMuteState = (
|
||||
muteState: {
|
||||
setHandler: (handler: Handler) => void;
|
||||
enabled$: Behavior<boolean>;
|
||||
},
|
||||
track$: Behavior<LocalMediaTrack | undefined>,
|
||||
source: "microphone" | "camera",
|
||||
): void => {
|
||||
// What the switch asks for, ahead of what it shows: the switch follows
|
||||
// only once the handler has resolved
|
||||
let desired = muteState.enabled$.value;
|
||||
muteState.setHandler(async (enabled) => {
|
||||
desired = enabled;
|
||||
const track = track$.value;
|
||||
if (track !== undefined) {
|
||||
desired = await track.setEnabled(enabled);
|
||||
return desired;
|
||||
}
|
||||
if (!enabled) return false;
|
||||
try {
|
||||
await rtcParticipation.publish(
|
||||
publishRequest(source, mediaDevices, capture),
|
||||
);
|
||||
return true;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to publish the ${source}`, e);
|
||||
desired = false;
|
||||
return false;
|
||||
}
|
||||
});
|
||||
// A track that arrives after its switch was flipped off catches up with it
|
||||
track$.pipe(scope.bind()).subscribe((track) => {
|
||||
if (track !== undefined && !desired)
|
||||
track.setEnabled(false).catch((e) => {
|
||||
logger.error(`Failed to mute the ${source}`, e);
|
||||
});
|
||||
});
|
||||
};
|
||||
followMuteState(muteStates.audio, microphone$, "microphone");
|
||||
followMuteState(muteStates.video, camera$, "camera");
|
||||
scope.onEnd(() => {
|
||||
muteStates.audio.unsetHandler();
|
||||
muteStates.video.unsetHandler();
|
||||
const supply = (source: "microphone" | "camera"): TrackSupply => ({
|
||||
publish: async () =>
|
||||
rtcParticipation.publish(
|
||||
publishRequest(
|
||||
source,
|
||||
mediaDevices,
|
||||
capture,
|
||||
source === "camera" ? videoProcessor$.value : undefined,
|
||||
),
|
||||
),
|
||||
});
|
||||
|
||||
// The device the user picked follows onto the track as it is made and as the
|
||||
// choice changes; the SDK does not watch the settings itself
|
||||
const followDevice = (
|
||||
track$: Behavior<LocalMediaTrack | undefined>,
|
||||
device: MediaDevice<unknown, { id: string }>,
|
||||
source: "microphone" | "camera",
|
||||
): void => {
|
||||
combineLatest([track$, device.selected$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([track, selected]) => {
|
||||
if (track === undefined || selected === undefined) return;
|
||||
track.setDevice(selected.id).catch((e) => {
|
||||
logger.error(`Failed to switch the ${source} device`, e);
|
||||
});
|
||||
});
|
||||
};
|
||||
followDevice(microphone$, mediaDevices.audioInput, "microphone");
|
||||
followDevice(camera$, mediaDevices.videoInput, "camera");
|
||||
followMuteState(
|
||||
scope,
|
||||
muteStates.audio,
|
||||
microphone$,
|
||||
supply("microphone"),
|
||||
"microphone",
|
||||
logger,
|
||||
);
|
||||
followMuteState(
|
||||
scope,
|
||||
muteStates.video,
|
||||
camera$,
|
||||
supply("camera"),
|
||||
"camera",
|
||||
logger,
|
||||
);
|
||||
followDevice(
|
||||
scope,
|
||||
microphone$,
|
||||
mediaDevices.audioInput,
|
||||
"microphone",
|
||||
logger,
|
||||
);
|
||||
followDevice(scope, camera$, mediaDevices.videoInput, "camera", logger);
|
||||
|
||||
// The output device follows the selection onto the client the same way. A
|
||||
// host that routes audio itself picks the output outside the browser, so the
|
||||
@@ -583,15 +561,7 @@ export function createCallViewModel$(
|
||||
});
|
||||
});
|
||||
|
||||
// Attached once the camera track exists rather than carried in the publish
|
||||
// request, so that a new track and a change of processor take the same path
|
||||
combineLatest([camera$, videoProcessor$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([camera, processor]) => {
|
||||
camera?.setProcessor(processor).catch((e) => {
|
||||
logger.error("Failed to set the video processor", e);
|
||||
});
|
||||
});
|
||||
followProcessor(scope, camera$, videoProcessor$, logger);
|
||||
|
||||
const members$ = scope.behavior<(LocalRTCMember | RemoteRTCMember)[]>(
|
||||
combineLatest(
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { BehaviorSubject, of } from "rxjs";
|
||||
import {
|
||||
constant,
|
||||
type LocalAudioMediaTrack,
|
||||
type LocalMedia,
|
||||
type LocalVideoMediaTrack,
|
||||
type PublishRequest,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import { createLobbyMediaViewModel } from "./LobbyMediaViewModel";
|
||||
import { type MediaDevices } from "./MediaDevices";
|
||||
import { MuteStates } from "./MuteStates";
|
||||
import { nullHostBridge } from "../HostBridge";
|
||||
import { flushPromises, mockMediaDevices, testScope } from "../utils/test";
|
||||
|
||||
describe("LobbyMediaViewModel", () => {
|
||||
it("opens what starts enabled in one capture and follows the devices", async () => {
|
||||
const { media, vm, mediaDevices } = setup();
|
||||
await flushPromises();
|
||||
|
||||
expect(media.capture).toHaveBeenCalledExactlyOnceWith([
|
||||
expect.objectContaining({ source: "microphone", deviceId: "mic" }),
|
||||
// With the processor, so that the camera starts with it
|
||||
expect.objectContaining({ source: "camera", deviceId: "cam", processor }),
|
||||
]);
|
||||
const camera = vm.videoTrack$.value!;
|
||||
expect(camera.source).toBe("camera");
|
||||
expect(camera.setDevice).toHaveBeenCalledWith("cam");
|
||||
expect(camera.setProcessor).toHaveBeenCalledWith(processor);
|
||||
mediaDevices.videoInput.selected$ satisfies unknown;
|
||||
});
|
||||
|
||||
it("closes the camera when its switch goes off and reopens it when on", async () => {
|
||||
const { media, vm, muteStates } = setup();
|
||||
await flushPromises();
|
||||
|
||||
muteStates.video.setEnabled$.value?.(false);
|
||||
await flushPromises();
|
||||
expect(media.release).toHaveBeenCalledWith("camera");
|
||||
expect(vm.videoTrack$.value).toBeNull();
|
||||
|
||||
muteStates.video.setEnabled$.value?.(true);
|
||||
await flushPromises();
|
||||
expect(media.capture).toHaveBeenLastCalledWith([
|
||||
expect.objectContaining({ source: "camera" }),
|
||||
]);
|
||||
expect(vm.videoTrack$.value?.source).toBe("camera");
|
||||
});
|
||||
|
||||
it("mutes the microphone in place", async () => {
|
||||
const { media, muteStates } = setup();
|
||||
await flushPromises();
|
||||
const microphone = media.tracks$.value.find(
|
||||
(t) => t.source === "microphone",
|
||||
)!;
|
||||
|
||||
muteStates.audio.setEnabled$.value?.(false);
|
||||
await flushPromises();
|
||||
|
||||
expect(microphone.setEnabled).toHaveBeenCalledWith(false);
|
||||
expect(media.release).not.toHaveBeenCalled();
|
||||
expect(muteStates.audio.enabled$.value).toBe(false);
|
||||
});
|
||||
|
||||
it("leaves both switches off where the devices cannot be opened", async () => {
|
||||
const { muteStates } = setup(async () =>
|
||||
Promise.reject(new Error("NotAllowedError")),
|
||||
);
|
||||
await flushPromises();
|
||||
|
||||
expect(muteStates.audio.enabled$.value).toBe(false);
|
||||
expect(muteStates.video.enabled$.value).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
const processor = { name: "blur" } as VideoProcessor;
|
||||
|
||||
function setup(
|
||||
capture: (
|
||||
requests: PublishRequest[],
|
||||
) => Promise<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]> = async (
|
||||
requests,
|
||||
) => Promise.resolve(requests.map(fakeTrack)),
|
||||
): {
|
||||
media: LocalMedia;
|
||||
vm: ReturnType<typeof createLobbyMediaViewModel>;
|
||||
muteStates: MuteStates;
|
||||
mediaDevices: MediaDevices;
|
||||
} {
|
||||
const scope = testScope();
|
||||
const tracks$ = new BehaviorSubject<
|
||||
(LocalAudioMediaTrack | LocalVideoMediaTrack)[]
|
||||
>([]);
|
||||
const media: LocalMedia = {
|
||||
requests: [],
|
||||
tracks$,
|
||||
capture: vi.fn(async (requests: PublishRequest[]) => {
|
||||
const tracks = await capture(requests);
|
||||
tracks$.next([
|
||||
...tracks$.value.filter(
|
||||
(t) => !tracks.some((n) => n.source === t.source),
|
||||
),
|
||||
...tracks,
|
||||
]);
|
||||
return tracks;
|
||||
}),
|
||||
release: vi.fn((source) =>
|
||||
tracks$.next(tracks$.value.filter((t) => t.source !== source)),
|
||||
),
|
||||
stop: vi.fn(),
|
||||
takeTracks: () => new Map(),
|
||||
};
|
||||
const mediaDevices = mockMediaDevices({
|
||||
audioInput: device("mic"),
|
||||
videoInput: device("cam"),
|
||||
} as Partial<MediaDevices>);
|
||||
const muteStates = new MuteStates(
|
||||
scope,
|
||||
mediaDevices,
|
||||
{ audioEnabled: true, videoEnabled: true },
|
||||
nullHostBridge,
|
||||
);
|
||||
const vm = createLobbyMediaViewModel(
|
||||
scope,
|
||||
media,
|
||||
muteStates,
|
||||
mediaDevices,
|
||||
{},
|
||||
constant(processor),
|
||||
);
|
||||
return { media, vm, muteStates, mediaDevices };
|
||||
}
|
||||
|
||||
function device(id: string): unknown {
|
||||
return {
|
||||
available$: of(new Map([[id, { type: "name", name: id }]])),
|
||||
selected$: new BehaviorSubject({ id }),
|
||||
select: (): void => {},
|
||||
};
|
||||
}
|
||||
|
||||
function fakeTrack(
|
||||
request: PublishRequest,
|
||||
): LocalAudioMediaTrack | LocalVideoMediaTrack {
|
||||
const muted$ = new BehaviorSubject(false);
|
||||
return {
|
||||
source: request.source,
|
||||
kind: request.source === "microphone" ? "audio" : "video",
|
||||
id: request.source,
|
||||
muted$,
|
||||
setEnabled: vi.fn(async (enabled: boolean) => {
|
||||
muted$.next(!enabled);
|
||||
return Promise.resolve(enabled);
|
||||
}),
|
||||
setDevice: vi.fn(async () => Promise.resolve()),
|
||||
setProcessor: vi.fn(async () => Promise.resolve()),
|
||||
} as unknown as LocalAudioMediaTrack | LocalVideoMediaTrack;
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type Behavior,
|
||||
type LocalAudioMediaTrack,
|
||||
type LocalMedia,
|
||||
type LocalVideoMediaTrack,
|
||||
type ObservableScope,
|
||||
type PublishRequest,
|
||||
trackBySource$,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import { combineLatest, debounceTime, map } from "rxjs";
|
||||
|
||||
import {
|
||||
type CaptureSettings,
|
||||
publishRequest,
|
||||
} from "./CallViewModel/CallViewModel";
|
||||
import { type MediaDevices } from "./MediaDevices";
|
||||
import { type MuteStates } from "./MuteStates";
|
||||
import { followDevice, followProcessor } from "./localMediaControls";
|
||||
|
||||
export interface LobbyMediaViewModel {
|
||||
/** The camera, once open, for the preview. */
|
||||
videoTrack$: Behavior<LocalVideoMediaTrack | null>;
|
||||
}
|
||||
|
||||
/**
|
||||
* The local media of the lobby: the devices the mute switches ask for, opened
|
||||
* ahead of the join on the media the join is then given, so that the call
|
||||
* publishes the very tracks the user previewed.
|
||||
*/
|
||||
export function createLobbyMediaViewModel(
|
||||
scope: ObservableScope,
|
||||
media: LocalMedia,
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
capture: CaptureSettings,
|
||||
videoProcessor$: Behavior<VideoProcessor | undefined>,
|
||||
): LobbyMediaViewModel {
|
||||
const logger = rootLogger.getChild("[LobbyMedia]");
|
||||
const microphone$ = trackBySource$(
|
||||
scope,
|
||||
media.tracks$,
|
||||
"microphone",
|
||||
) as Behavior<LocalAudioMediaTrack | undefined>;
|
||||
const camera$ = trackBySource$(scope, media.tracks$, "camera") as Behavior<
|
||||
LocalVideoMediaTrack | undefined
|
||||
>;
|
||||
|
||||
// One could ask why this is not just done via followMuteStates.
|
||||
// The main purpose why this code exists is to handle the native browser permission
|
||||
// prompts properly.
|
||||
// no prompt if everything is muted, shared prompt if both are requested.
|
||||
const opening = new Set<PublishRequest["source"]>();
|
||||
combineLatest([
|
||||
muteStates.audio.enabled$,
|
||||
muteStates.video.enabled$,
|
||||
microphone$,
|
||||
camera$,
|
||||
])
|
||||
.pipe(debounceTime(0), scope.bind())
|
||||
.subscribe(([audio, video, microphone, camera]) => {
|
||||
const open: PublishRequest[] = [];
|
||||
if (audio && microphone === undefined && !opening.has("microphone"))
|
||||
open.push(publishRequest("microphone", mediaDevices, capture));
|
||||
if (video && camera === undefined && !opening.has("camera"))
|
||||
open.push(
|
||||
publishRequest(
|
||||
"camera",
|
||||
mediaDevices,
|
||||
capture,
|
||||
videoProcessor$.value,
|
||||
),
|
||||
);
|
||||
if (open.length > 0) {
|
||||
for (const { source } of open) opening.add(source);
|
||||
media
|
||||
.capture(open)
|
||||
.catch((e) => {
|
||||
logger.error("Failed to open the devices", e);
|
||||
// A device that cannot be opened leaves its switch off
|
||||
for (const { source } of open)
|
||||
(source === "microphone"
|
||||
? muteStates.audio
|
||||
: muteStates.video
|
||||
).setEnabled$.value?.(false);
|
||||
})
|
||||
.finally(() => {
|
||||
for (const { source } of open) opening.delete(source);
|
||||
});
|
||||
}
|
||||
// The camera is closed rather than muted, so that its light goes off
|
||||
if (!video && camera !== undefined) media.release("camera");
|
||||
if (microphone !== undefined && microphone.muted$.value === audio)
|
||||
microphone.setEnabled(audio).catch((e) => {
|
||||
logger.error("Failed to mute the microphone", e);
|
||||
});
|
||||
});
|
||||
|
||||
followDevice(
|
||||
scope,
|
||||
microphone$,
|
||||
mediaDevices.audioInput,
|
||||
"microphone",
|
||||
logger,
|
||||
);
|
||||
followDevice(scope, camera$, mediaDevices.videoInput, "camera", logger);
|
||||
followProcessor(scope, camera$, videoProcessor$, logger);
|
||||
|
||||
return {
|
||||
videoTrack$: scope.behavior(camera$.pipe(map((camera) => camera ?? null))),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type Behavior,
|
||||
type LocalMediaTrack,
|
||||
type LocalVideoMediaTrack,
|
||||
type ObservableScope,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
import { combineLatest } from "rxjs";
|
||||
|
||||
import { type MediaDevice } from "./MediaDevices";
|
||||
import { type Handler } from "./MuteStates";
|
||||
|
||||
/** How a mute switch gets a track where there is none. */
|
||||
export interface TrackSupply {
|
||||
publish: () => Promise<unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* The mute switch drives the track, which reports back what the device
|
||||
* allowed, so a denied permission flips the switch back. A source is muted
|
||||
* and unmuted while its track exists and published when it does not.
|
||||
*/
|
||||
export function followMuteState(
|
||||
scope: ObservableScope,
|
||||
muteState: {
|
||||
setHandler: (handler: Handler) => void;
|
||||
unsetHandler: () => void;
|
||||
enabled$: Behavior<boolean>;
|
||||
},
|
||||
track$: Behavior<LocalMediaTrack | undefined>,
|
||||
supply: TrackSupply,
|
||||
source: "microphone" | "camera",
|
||||
logger: Logger,
|
||||
): void {
|
||||
// What the switch asks for, ahead of what it shows: the switch follows
|
||||
// only once the handler has resolved
|
||||
let desired = muteState.enabled$.value;
|
||||
muteState.setHandler(async (enabled) => {
|
||||
desired = enabled;
|
||||
const track = track$.value;
|
||||
if (track !== undefined) {
|
||||
desired = await track.setEnabled(enabled);
|
||||
return desired;
|
||||
}
|
||||
if (!enabled) return false;
|
||||
try {
|
||||
await supply.publish();
|
||||
return true;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to publish the ${source}`, e);
|
||||
desired = false;
|
||||
return false;
|
||||
}
|
||||
});
|
||||
scope.onEnd(() => muteState.unsetHandler());
|
||||
// A track that arrives after its switch was flipped off catches up with it
|
||||
track$.pipe(scope.bind()).subscribe((track) => {
|
||||
if (track !== undefined && !desired)
|
||||
track.setEnabled(false).catch((e) => {
|
||||
logger.error(`Failed to mute the ${source}`, e);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The device the user picked follows onto the track as it is made and as the
|
||||
* choice changes; the SDK does not watch the settings itself.
|
||||
*/
|
||||
export function followDevice(
|
||||
scope: ObservableScope,
|
||||
track$: Behavior<LocalMediaTrack | undefined>,
|
||||
device: MediaDevice<unknown, { id: string }>,
|
||||
source: "microphone" | "camera",
|
||||
logger: Logger,
|
||||
): void {
|
||||
combineLatest([track$, device.selected$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([track, selected]) => {
|
||||
if (track === undefined || selected === undefined) return;
|
||||
track.setDevice(selected.id).catch((e) => {
|
||||
logger.error(`Failed to switch the ${source} device`, e);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Attached once the camera track exists rather than carried in the publish
|
||||
* request, so that a new track and a change of processor take the same path.
|
||||
*/
|
||||
export function followProcessor(
|
||||
scope: ObservableScope,
|
||||
camera$: Behavior<LocalVideoMediaTrack | undefined>,
|
||||
videoProcessor$: Behavior<VideoProcessor | undefined>,
|
||||
logger: Logger,
|
||||
): void {
|
||||
combineLatest([camera$, videoProcessor$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([camera, processor]) => {
|
||||
camera?.setProcessor(processor).catch((e) => {
|
||||
logger.error("Failed to set the video processor", e);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
BackgroundTransformer,
|
||||
VideoTransformer,
|
||||
type VideoTransformerInitOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
||||
|
||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||
|
||||
describe("BackgroundEffectTransformer", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("loads a background chosen before it was built", async () => {
|
||||
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
|
||||
vi.spyOn(ImageSegmenter, "createFromOptions").mockResolvedValue(
|
||||
{} as ImageSegmenter,
|
||||
);
|
||||
const load = vi
|
||||
.spyOn(BackgroundTransformer.prototype, "loadAndSetBackground")
|
||||
.mockResolvedValue();
|
||||
const transformer = new BackgroundEffectTransformer({
|
||||
backgroundDisabled: true,
|
||||
});
|
||||
// Chosen before the camera is processed, as the first effect always is:
|
||||
// loaded then, there is nothing yet to draw it with.
|
||||
await transformer.update({ imagePath: "/background.jpg" });
|
||||
load.mockClear();
|
||||
|
||||
await transformer.init({} as VideoTransformerInitOptions);
|
||||
expect(load).toHaveBeenCalledWith("/background.jpg");
|
||||
});
|
||||
|
||||
it("passes frames on untouched once disabled, even after blurring", async () => {
|
||||
const transformer = new BackgroundEffectTransformer({});
|
||||
await transformer.update({ blurRadius: 15, backgroundDisabled: false });
|
||||
await transformer.update({
|
||||
imagePath: undefined,
|
||||
backgroundDisabled: true,
|
||||
});
|
||||
// With neither, the library passes a frame on without segmenting it.
|
||||
expect(transformer.options.blurRadius).toBeUndefined();
|
||||
expect(transformer.options.imagePath).toBeUndefined();
|
||||
});
|
||||
});
|
||||
+25
-1
@@ -1,5 +1,6 @@
|
||||
/*
|
||||
Copyright 2024-2025 New Vector Ltd.
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
@@ -9,7 +10,9 @@ import {
|
||||
BackgroundTransformer,
|
||||
VideoTransformer,
|
||||
type VideoTransformerInitOptions,
|
||||
type BackgroundOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
||||
|
||||
import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
|
||||
@@ -49,7 +52,21 @@ const wasmFileset: WasmFileset = {
|
||||
* loads the segmentation models from our own bundle rather than as an external
|
||||
* resource fetched from the public internet.
|
||||
*/
|
||||
export class BlurBackgroundTransformer extends BackgroundTransformer {
|
||||
export class BackgroundEffectTransformer extends BackgroundTransformer {
|
||||
/**
|
||||
* As the library's, except that disabling also drops the blur radius. The
|
||||
* library skips a frame on `isDisabled ?? backgroundDisabled`, and
|
||||
* `isDisabled` is never undefined, so a disabled effect that keeps its
|
||||
* radius has every frame segmented and drawn for nothing. With neither a
|
||||
* radius nor a picture, frames are passed on untouched.
|
||||
* TODO: drop once fixed upstream in livekit/track-processors-js (still in 0.8.2).
|
||||
*/
|
||||
public override async update(opts: BackgroundOptions): Promise<void> {
|
||||
await super.update(
|
||||
opts.backgroundDisabled ? { ...opts, blurRadius: undefined } : opts,
|
||||
);
|
||||
}
|
||||
|
||||
public async init({
|
||||
outputCanvas,
|
||||
inputElement: inputVideo,
|
||||
@@ -73,8 +90,15 @@ export class BlurBackgroundTransformer extends BackgroundTransformer {
|
||||
outputConfidenceMasks: false,
|
||||
});
|
||||
|
||||
// BackgroundTransformer's own init applies these, and this one replaces it.
|
||||
if (this.options.imagePath) {
|
||||
await this.loadAndSetBackground(this.options.imagePath).catch((e) =>
|
||||
logger.warn("Failed to load the background image", e),
|
||||
);
|
||||
}
|
||||
if (this.options.blurRadius) {
|
||||
this.gl?.setBlurRadius(this.options.blurRadius);
|
||||
}
|
||||
this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { act, createElement, type FC } from "react";
|
||||
import { render } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
BackgroundEffectsProvider,
|
||||
useBackgroundEffects,
|
||||
createAppBackgroundEffects,
|
||||
} from "./BackgroundEffectsContext";
|
||||
import {
|
||||
type BackgroundEffects,
|
||||
type BackgroundEffectsState,
|
||||
} from "../state/BackgroundEffects";
|
||||
import { backgroundEffect } from "../settings/settings";
|
||||
import { flushPromises, testScope } from "../utils/test";
|
||||
|
||||
const pipelines = vi.hoisted(() => ({
|
||||
built: 0,
|
||||
destroyed: 0,
|
||||
switches: [] as unknown[],
|
||||
}));
|
||||
|
||||
// The pipeline itself needs WebGL and MediaPipe; what these tests are about is
|
||||
// what the provider asks of it.
|
||||
vi.mock("@livekit/track-processors", () => ({
|
||||
BackgroundProcessorWrapper: vi.fn(function (this: Record<string, unknown>) {
|
||||
pipelines.built += 1;
|
||||
this.switchTo = vi.fn(async (options: unknown) => {
|
||||
pipelines.switches.push(options);
|
||||
return Promise.resolve();
|
||||
});
|
||||
this.destroy = vi.fn(async () => {
|
||||
pipelines.destroyed += 1;
|
||||
return Promise.resolve();
|
||||
});
|
||||
}),
|
||||
supportsBackgroundProcessors: (): boolean => true,
|
||||
}));
|
||||
const platformMock = vi.hoisted(() => ({ platform: "desktop" }));
|
||||
vi.mock("../Platform", () => platformMock);
|
||||
vi.mock("./BackgroundEffectTransformer", () => ({
|
||||
BackgroundEffectTransformer: vi.fn(),
|
||||
}));
|
||||
|
||||
describe("BackgroundEffectsProvider", () => {
|
||||
let seen: BackgroundEffectsState[];
|
||||
const Surface: FC = () => {
|
||||
seen.push(useBackgroundEffects());
|
||||
return null;
|
||||
};
|
||||
// Made on first render, once a test has set up the platform.
|
||||
let effects: BackgroundEffects | undefined;
|
||||
const surface = (): ReturnType<typeof createElement> =>
|
||||
createElement(BackgroundEffectsProvider, {
|
||||
effects: (effects ??= createAppBackgroundEffects(testScope())),
|
||||
children: createElement(Surface),
|
||||
});
|
||||
const blur = async (on: boolean): Promise<void> => {
|
||||
await act(async () => {
|
||||
backgroundEffect.setValue(on ? "blur" : "none");
|
||||
await flushPromises();
|
||||
});
|
||||
};
|
||||
const latest = (): BackgroundEffectsState => seen[seen.length - 1];
|
||||
|
||||
beforeEach(() => {
|
||||
seen = [];
|
||||
effects = undefined;
|
||||
pipelines.built = 0;
|
||||
pipelines.destroyed = 0;
|
||||
pipelines.switches = [];
|
||||
platformMock.platform = "desktop";
|
||||
backgroundEffect.setValue("none");
|
||||
});
|
||||
afterEach(() => backgroundEffect.setValue("none"));
|
||||
|
||||
// A pipeline primes itself when it is built and when it is destroyed, and a
|
||||
// primed pipeline lets one frame through untouched: so the frame is spent
|
||||
// once, as long as it is built once and never taken off the camera.
|
||||
it("builds one pipeline on first use and keeps it through every change", async () => {
|
||||
render(surface());
|
||||
for (const on of [true, false, true, false, true]) await blur(on);
|
||||
|
||||
expect(pipelines.built).toBe(1);
|
||||
expect(pipelines.destroyed).toBe(0);
|
||||
expect(latest().processor).toBeDefined();
|
||||
});
|
||||
|
||||
it("does not run on a phone, even where its browser could", async () => {
|
||||
platformMock.platform = "ios";
|
||||
render(surface());
|
||||
await blur(true);
|
||||
|
||||
expect(latest().supported).toBe(false);
|
||||
expect(latest().processor).toBeUndefined();
|
||||
});
|
||||
|
||||
it("switches to an effect chosen while the camera is off", async () => {
|
||||
render(surface());
|
||||
await blur(true);
|
||||
|
||||
// Already switched to blur, with nothing yet to attach it to
|
||||
expect(pipelines.switches).toEqual([
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
]);
|
||||
expect(pipelines.built).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
Copyright 2024-2026 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { BackgroundProcessorWrapper } from "@livekit/track-processors";
|
||||
import { createContext, type FC, type JSX, use } from "react";
|
||||
|
||||
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
|
||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||
import { supportsBackgroundProcessors } from "./backgroundProcessing";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
import { type ObservableScope } from "../state/ObservableScope";
|
||||
import {
|
||||
type BackgroundEffects,
|
||||
type BackgroundEffectsState,
|
||||
createBackgroundEffects,
|
||||
} from "../state/BackgroundEffects";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
|
||||
const BackgroundEffectsContext = createContext<BackgroundEffects | undefined>(
|
||||
undefined,
|
||||
);
|
||||
|
||||
export function useBackgroundEffects(): BackgroundEffectsState {
|
||||
return useBehavior(useBackgroundEffectsState$());
|
||||
}
|
||||
|
||||
export function useBackgroundEffectsState$(): Behavior<BackgroundEffectsState> {
|
||||
const effects = use(BackgroundEffectsContext);
|
||||
if (effects === undefined)
|
||||
throw new Error(
|
||||
"useBackgroundEffects must be used within a BackgroundEffectsProvider",
|
||||
);
|
||||
return effects.state$;
|
||||
}
|
||||
|
||||
/**
|
||||
* The app's one pipeline, shared by every camera track it opens. Its builds
|
||||
* and teardowns take their turns through the SDK, which runs one operation
|
||||
* at a time on a processor.
|
||||
*/
|
||||
export function createAppBackgroundEffects(
|
||||
scope: ObservableScope,
|
||||
): BackgroundEffects {
|
||||
return createBackgroundEffects(scope, {
|
||||
supported: supportsBackgroundProcessors(),
|
||||
effect$: backgroundEffectSetting.value$,
|
||||
pipeline: new BackgroundProcessorWrapper(
|
||||
new BackgroundEffectTransformer({ backgroundDisabled: true }),
|
||||
"background-effect",
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
interface Props {
|
||||
effects: BackgroundEffects;
|
||||
children: JSX.Element;
|
||||
}
|
||||
|
||||
export const BackgroundEffectsProvider: FC<Props> = ({ effects, children }) => (
|
||||
<BackgroundEffectsContext value={effects}>
|
||||
{children}
|
||||
</BackgroundEffectsContext>
|
||||
);
|
||||
@@ -1,133 +0,0 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type LocalVideoTrack } from "livekit-client";
|
||||
import {
|
||||
type BackgroundOptions,
|
||||
type ProcessorWrapper,
|
||||
} from "@livekit/track-processors";
|
||||
import { type VideoProcessor } from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import {
|
||||
applyProcessor,
|
||||
fromLivekitProcessor,
|
||||
trackProcessorSync,
|
||||
} from "./TrackProcessorContext";
|
||||
import { constant } from "../state/Behavior";
|
||||
import { flushPromises, testScope } from "../utils/test";
|
||||
|
||||
const processor = { name: "blur" } as VideoProcessor;
|
||||
|
||||
function mockTrack(
|
||||
readyState: MediaStreamTrackState,
|
||||
current?: VideoProcessor,
|
||||
): LocalVideoTrack {
|
||||
return {
|
||||
mediaStreamTrack: { readyState },
|
||||
getProcessor: vi.fn().mockReturnValue(current),
|
||||
setProcessor: vi.fn().mockResolvedValue(undefined),
|
||||
stopProcessor: vi.fn().mockResolvedValue(undefined),
|
||||
} as unknown as LocalVideoTrack;
|
||||
}
|
||||
|
||||
describe("applyProcessor", () => {
|
||||
it("attaches the processor to a live track", () => {
|
||||
const track = mockTrack("live");
|
||||
applyProcessor(track, processor);
|
||||
expect(track.setProcessor).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ name: "blur" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not attach the processor to an ended track", () => {
|
||||
const track = mockTrack("ended");
|
||||
applyProcessor(track, processor);
|
||||
expect(track.setProcessor).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not surface a rejected setProcessor", async () => {
|
||||
const track = mockTrack("live");
|
||||
vi.mocked(track.setProcessor).mockRejectedValue(
|
||||
new TypeError("Input track cannot be ended"),
|
||||
);
|
||||
const unhandled = vi.fn();
|
||||
process.on("unhandledRejection", unhandled);
|
||||
applyProcessor(track, processor);
|
||||
await flushPromises();
|
||||
process.off("unhandledRejection", unhandled);
|
||||
expect(unhandled).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("stops the processor when none is wanted", () => {
|
||||
const track = mockTrack("live", processor);
|
||||
applyProcessor(track, undefined);
|
||||
expect(track.stopProcessor).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not surface a rejected stopProcessor", async () => {
|
||||
const track = mockTrack("live", processor);
|
||||
vi.mocked(track.stopProcessor).mockRejectedValue(new Error("nope"));
|
||||
const unhandled = vi.fn();
|
||||
process.on("unhandledRejection", unhandled);
|
||||
applyProcessor(track, undefined);
|
||||
await flushPromises();
|
||||
process.off("unhandledRejection", unhandled);
|
||||
expect(unhandled).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("trackProcessorSync", () => {
|
||||
it("applies the processor to the current track", () => {
|
||||
const track = mockTrack("live");
|
||||
trackProcessorSync(
|
||||
testScope(),
|
||||
constant(track),
|
||||
constant({ supported: true, processor }),
|
||||
);
|
||||
expect(track.setProcessor).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ name: "blur" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fromLivekitProcessor", () => {
|
||||
const captured = { id: "captured" } as MediaStreamTrack;
|
||||
const processed = { id: "processed" } as MediaStreamTrack;
|
||||
|
||||
function mockWrapper(
|
||||
processedTrack: MediaStreamTrack | undefined,
|
||||
): ProcessorWrapper<BackgroundOptions> {
|
||||
return {
|
||||
name: "background-blur",
|
||||
processedTrack,
|
||||
init: vi.fn().mockResolvedValue(undefined),
|
||||
restart: vi.fn().mockResolvedValue(undefined),
|
||||
destroy: vi.fn().mockResolvedValue(undefined),
|
||||
} as unknown as ProcessorWrapper<BackgroundOptions>;
|
||||
}
|
||||
|
||||
it("resolves with the track the wrapper produced", async () => {
|
||||
const wrapper = mockWrapper(processed);
|
||||
const sdkProcessor = fromLivekitProcessor(wrapper);
|
||||
expect(sdkProcessor.name).toBe("background-blur");
|
||||
await expect(sdkProcessor.init(captured)).resolves.toBe(processed);
|
||||
expect(wrapper.init).toHaveBeenCalledWith({
|
||||
kind: "video",
|
||||
track: captured,
|
||||
});
|
||||
await expect(sdkProcessor.restart(captured)).resolves.toBe(processed);
|
||||
await sdkProcessor.destroy();
|
||||
expect(wrapper.destroy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects where the wrapper produced no track", async () => {
|
||||
await expect(
|
||||
fromLivekitProcessor(mockWrapper(undefined)).init(captured),
|
||||
).rejects.toThrow("produced no track");
|
||||
});
|
||||
});
|
||||
@@ -1,178 +0,0 @@
|
||||
/*
|
||||
Copyright 2024-2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
ProcessorWrapper,
|
||||
supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk,
|
||||
type BackgroundOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import {
|
||||
createContext,
|
||||
type FC,
|
||||
type JSX,
|
||||
use,
|
||||
useEffect,
|
||||
useMemo,
|
||||
} from "react";
|
||||
import { type LocalVideoTrack, Track } from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { combineLatest } from "rxjs";
|
||||
import {
|
||||
convertToLivekitProcessor,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import {
|
||||
backgroundBlur as backgroundBlurSettings,
|
||||
useSetting,
|
||||
} from "../settings/settings";
|
||||
import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
import { type ObservableScope } from "../state/ObservableScope";
|
||||
import { platform } from "../Platform";
|
||||
import { useValueBehavior } from "../useValueBehavior";
|
||||
|
||||
//TODO-MULTI-SFU: This is not yet fully there.
|
||||
// it is a combination of exposing observable and react hooks.
|
||||
// preferably we should not make this a context anymore and instead just a vm?
|
||||
|
||||
export type ProcessorState = {
|
||||
supported: boolean | undefined;
|
||||
processor: undefined | VideoProcessor;
|
||||
};
|
||||
|
||||
/**
|
||||
* This allows to use a LiveKit processor for the matrixRTCSDK (VideoProcessor).
|
||||
* They have slightly different apis so we need to convert them with this
|
||||
* function before we can use them in the matrixRTCSDK.
|
||||
*/
|
||||
export function fromLivekitProcessor(
|
||||
wrapper: ProcessorWrapper<BackgroundOptions>,
|
||||
): VideoProcessor {
|
||||
const processedTrack = (): MediaStreamTrack => {
|
||||
if (wrapper.processedTrack === undefined)
|
||||
throw new Error(`Processor ${wrapper.name} produced no track`);
|
||||
return wrapper.processedTrack;
|
||||
};
|
||||
return {
|
||||
name: wrapper.name,
|
||||
init: async (track: MediaStreamTrack, element?: HTMLMediaElement) => {
|
||||
await wrapper.init({ kind: Track.Kind.Video, track, element });
|
||||
return processedTrack();
|
||||
},
|
||||
restart: async (track: MediaStreamTrack, element?: HTMLMediaElement) => {
|
||||
await wrapper.restart({ kind: Track.Kind.Video, track, element });
|
||||
return processedTrack();
|
||||
},
|
||||
destroy: async () => wrapper.destroy(),
|
||||
};
|
||||
}
|
||||
|
||||
const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
|
||||
|
||||
export function useTrackProcessor(): ProcessorState {
|
||||
const state = use(ProcessorContext);
|
||||
if (state === undefined)
|
||||
throw new Error(
|
||||
"useTrackProcessor must be used within a ProcessorProvider",
|
||||
);
|
||||
return state;
|
||||
}
|
||||
|
||||
export function useTrackProcessorState$(): Behavior<ProcessorState> {
|
||||
return useValueBehavior(useTrackProcessor());
|
||||
}
|
||||
|
||||
/**
|
||||
* Attaches or detaches the processor so that the track matches the desired
|
||||
* state, without throwing.
|
||||
*/
|
||||
export function applyProcessor(
|
||||
videoTrack: LocalVideoTrack,
|
||||
processor: VideoProcessor | undefined,
|
||||
): void {
|
||||
if (processor && !videoTrack.getProcessor()) {
|
||||
// A MediaStreamTrackProcessor cannot be constructed on an ended track
|
||||
// (e.g. the camera was stopped while the processor was being applied),
|
||||
// and setProcessor rejects with a TypeError. The track is going away
|
||||
// anyway, so there is nothing to attach to.
|
||||
if (videoTrack.mediaStreamTrack.readyState === "ended") {
|
||||
logger.debug("Not attaching video processor to an ended track");
|
||||
return;
|
||||
}
|
||||
videoTrack.setProcessor(convertToLivekitProcessor(processor)).catch((e) => {
|
||||
logger.warn("Failed to attach video processor", e);
|
||||
});
|
||||
}
|
||||
if (!processor && videoTrack.getProcessor()) {
|
||||
videoTrack.stopProcessor().catch((e) => {
|
||||
logger.warn("Failed to stop video processor", e);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates your video tracks to always use the given processor.
|
||||
*/
|
||||
export const trackProcessorSync = (
|
||||
scope: ObservableScope,
|
||||
videoTrack$: Behavior<LocalVideoTrack | null>,
|
||||
processor$: Behavior<ProcessorState>,
|
||||
): void => {
|
||||
combineLatest([videoTrack$, processor$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([videoTrack, processorState]) => {
|
||||
if (!processorState) return;
|
||||
if (!videoTrack) return;
|
||||
applyProcessor(videoTrack, processorState.processor);
|
||||
});
|
||||
};
|
||||
|
||||
export const useTrackProcessorSync = (
|
||||
videoTrack: LocalVideoTrack | null,
|
||||
): void => {
|
||||
const { processor } = useTrackProcessor();
|
||||
useEffect(() => {
|
||||
if (!videoTrack) return;
|
||||
applyProcessor(videoTrack, processor);
|
||||
}, [processor, videoTrack]);
|
||||
};
|
||||
|
||||
interface Props {
|
||||
children: JSX.Element;
|
||||
}
|
||||
|
||||
function supportsBackgroundProcessors(): boolean {
|
||||
return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop";
|
||||
}
|
||||
|
||||
export const ProcessorProvider: FC<Props> = ({ children }) => {
|
||||
// The setting the user wants to have
|
||||
const [blurActivated] = useSetting(backgroundBlurSettings);
|
||||
const supported = useMemo(() => supportsBackgroundProcessors(), []);
|
||||
const blur = useMemo(
|
||||
() =>
|
||||
fromLivekitProcessor(
|
||||
new ProcessorWrapper(
|
||||
new BlurBackgroundTransformer({ blurRadius: 15 }),
|
||||
"background-blur",
|
||||
),
|
||||
),
|
||||
[],
|
||||
);
|
||||
|
||||
// This is the actual state exposed through the context
|
||||
const processorState = useMemo(
|
||||
() => ({
|
||||
supported,
|
||||
processor: supported && blurActivated ? blur : undefined,
|
||||
}),
|
||||
[supported, blurActivated, blur],
|
||||
);
|
||||
|
||||
return <ProcessorContext value={processorState}>{children}</ProcessorContext>;
|
||||
};
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import {
|
||||
type BackgroundEffect,
|
||||
imagePathFor,
|
||||
parseEffect,
|
||||
serializeEffect,
|
||||
shippedBackgrounds,
|
||||
} from "./backgroundEffectCatalog";
|
||||
|
||||
describe("the chosen background effect", () => {
|
||||
test("round-trips through its stored form", () => {
|
||||
const effects: BackgroundEffect[] = [
|
||||
{ kind: "none" },
|
||||
{ kind: "blur" },
|
||||
...shippedBackgrounds.map(({ id }) => ({ kind: "shipped" as const, id })),
|
||||
];
|
||||
for (const effect of effects)
|
||||
expect(parseEffect(serializeEffect(effect))).toEqual(effect);
|
||||
});
|
||||
|
||||
test("falls back to no effect when the stored form names nothing we ship", () => {
|
||||
expect(parseEffect("image:gone")).toEqual({ kind: "none" });
|
||||
expect(parseEffect("")).toEqual({ kind: "none" });
|
||||
});
|
||||
|
||||
test("reads a stored value that is not a string as no effect", () => {
|
||||
for (const raw of [null, 1, {}, ["blur"]])
|
||||
expect(parseEffect(raw)).toEqual({ kind: "none" });
|
||||
});
|
||||
|
||||
test("gives every shipped background an image to draw", () => {
|
||||
for (const { id } of shippedBackgrounds)
|
||||
expect(imagePathFor(id)).toBeTruthy();
|
||||
});
|
||||
|
||||
// A background has to cover what is behind it: JPEG has no alpha channel.
|
||||
test("ships only opaque backgrounds", async () => {
|
||||
for (const { imagePath } of shippedBackgrounds) {
|
||||
const name = imagePath.split("/").pop()!.split("?")[0];
|
||||
const bytes = await readFile(`src/graphics/${name}`);
|
||||
expect([bytes[0], bytes[1], bytes[2]]).toEqual([0xff, 0xd8, 0xff]);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import arc from "../graphics/background-arc.jpg?url";
|
||||
import glow from "../graphics/background-glow.jpg?url";
|
||||
|
||||
export interface ShippedBackground {
|
||||
/** Stored in the user's settings when chosen, so it must not change. */
|
||||
id: string;
|
||||
imagePath: string;
|
||||
}
|
||||
|
||||
export const shippedBackgrounds: ShippedBackground[] = [
|
||||
{ id: "arc", imagePath: arc },
|
||||
{ id: "glow", imagePath: glow },
|
||||
];
|
||||
|
||||
export type BackgroundEffect =
|
||||
| { kind: "none" }
|
||||
| { kind: "blur" }
|
||||
| { kind: "shipped"; id: string };
|
||||
|
||||
/** An effect as the setting stores it, and as the menu names its option. */
|
||||
export type EffectId = "none" | "blur" | `image:${string}`;
|
||||
|
||||
export const noEffect: BackgroundEffect = { kind: "none" };
|
||||
|
||||
/**
|
||||
* Reads a stored effect; one we no longer ship reads as no effect. Stored by
|
||||
* hand or by another build, the value need not even be a string.
|
||||
*/
|
||||
export function parseEffect(raw: unknown): BackgroundEffect {
|
||||
if (typeof raw !== "string") return noEffect;
|
||||
if (raw === "blur") return { kind: "blur" };
|
||||
if (raw.startsWith("image:")) {
|
||||
const id = raw.slice("image:".length);
|
||||
if (shippedBackgrounds.some((b) => b.id === id))
|
||||
return { kind: "shipped", id };
|
||||
}
|
||||
return noEffect;
|
||||
}
|
||||
|
||||
export function serializeEffect(effect: BackgroundEffect): EffectId {
|
||||
switch (effect.kind) {
|
||||
case "blur":
|
||||
return "blur";
|
||||
case "shipped":
|
||||
return `image:${effect.id}`;
|
||||
default:
|
||||
return "none";
|
||||
}
|
||||
}
|
||||
|
||||
export function imagePathFor(id: string): string | undefined {
|
||||
return shippedBackgrounds.find((b) => b.id === id)?.imagePath;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk } from "@livekit/track-processors";
|
||||
|
||||
import { platform } from "../Platform";
|
||||
|
||||
/** One answer for the controls and the pipeline, so neither offers what the other refuses. */
|
||||
export function supportsBackgroundProcessors(): boolean {
|
||||
return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop";
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { useEffect } from "react";
|
||||
import { createLocalMedia, type LocalMedia } from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import { useInitial } from "./useInitial";
|
||||
|
||||
/**
|
||||
* Local media for the lobby to open and the join to take over, released with
|
||||
* the component where the join never comes.
|
||||
*/
|
||||
export function useLocalMedia(): LocalMedia {
|
||||
const media = useInitial(() => createLocalMedia());
|
||||
useEffect(() => (): void => media.stop(), [media]);
|
||||
return media;
|
||||
}
|
||||
+35
-4
@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import {
|
||||
loadEnv,
|
||||
type Plugin,
|
||||
type PluginOption,
|
||||
searchForWorkspaceRoot,
|
||||
type ConfigEnv,
|
||||
@@ -43,10 +44,12 @@ export const vitePluginsConfig = ({
|
||||
}),
|
||||
react(),
|
||||
wasm(),
|
||||
nodePolyfills({
|
||||
// Enables the 'events' module, which is required by the matrix-js-sdk
|
||||
include: ["events"],
|
||||
}),
|
||||
exceptMediaPipeWasm(
|
||||
nodePolyfills({
|
||||
// Enables the 'events' module, which is required by the matrix-js-sdk
|
||||
include: ["events"],
|
||||
}),
|
||||
),
|
||||
svgrPlugin({
|
||||
svgrOptions: {
|
||||
// This enables ref forwarding on SVGR components, which is needed, for
|
||||
@@ -191,3 +194,31 @@ export default ({
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Keeps a plugin's transform off MediaPipe's WASM loader. The loader is a
|
||||
* classic script that sets `self.ModuleFactory`; transformed into a module it
|
||||
* sets nothing, and the segmenter fails with "ModuleFactory not set".
|
||||
*/
|
||||
function exceptMediaPipeWasm(plugins: PluginOption): PluginOption {
|
||||
const isLoader = (id: string): boolean =>
|
||||
id.includes("tasks-vision") && id.includes("wasm");
|
||||
return (Array.isArray(plugins) ? plugins : [plugins]).map((plugin) => {
|
||||
if (
|
||||
!plugin ||
|
||||
typeof plugin !== "object" ||
|
||||
!("transform" in plugin) ||
|
||||
typeof plugin.transform !== "function"
|
||||
)
|
||||
return plugin;
|
||||
const transform = plugin.transform;
|
||||
const wrapped: Plugin = {
|
||||
...plugin,
|
||||
transform(code, id, options) {
|
||||
if (isLoader(id)) return null;
|
||||
return transform.call(this, code, id, options);
|
||||
},
|
||||
};
|
||||
return wrapped;
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user