From 797a86815bca87150b2893fb853624c2a8037748 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Fri, 6 Nov 2026 14:34:14 +0100 Subject: [PATCH 01/16] Ship the component's assets as files beside the module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The component module was 18 MB, of which 15 MB was MediaPipe's runtime as a base64 data URL, with the segmentation model, every sound and the background pictures inlined the same way: a library build inlines every asset unless its import says `no-inline`. Every host downloaded all of it before the first call, whether or not a background effect was ever switched on, and a host with a Content Security Policy without `script-src data:` could not run effects at all, since MediaPipe injects its loader as a script tag. The imports now say `no-inline`, and the build renders each reference as `new URL("./assets/…", import.meta.url)`, which webpack 5, Vite and Rollup follow as an asset of the host's own; the `./` is spelled out because a bare `assets/…` is read as a package name. The module is 2.3 MB. The runtime is fetched when an effect is first switched on and cached on its own. MediaPipe's files come through the package's `exports` rather than a path into node_modules. MediaPipe's JavaScript (160 kB) stays in the module: LiveKit's track processors import it statically, so a dynamic import moves nothing. Co-Authored-By: Claude Fable 5.1 --- README.md | 7 +++ src/reactions/index.ts | 48 +++++++++---------- src/room/CallEventAudioRenderer.tsx | 24 +++++----- src/room/RingingAudioRenderer.tsx | 4 +- .../BackgroundEffectTransformer.test.ts | 18 +++++++ src/tracks/BackgroundEffectTransformer.ts | 29 ++++------- src/tracks/backgroundEffectCatalog.ts | 4 +- vite-component.config.ts | 25 ++++++++++ 8 files changed, 98 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 17eae35d9..ebc26f936 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,13 @@ widget's `theme` URL parameter: `light`, `dark`, `light-high-contrast` or `dark-high-contrast`. Both can change while a call is running without disturbing it. +The component's sounds, pictures and the MediaPipe runtime behind background +effects are files beside the module rather than data URLs inside it, each +referenced relative to the module (`new URL("./assets/…", import.meta.url)`), +so a host's bundler copies them as assets of its own — webpack 5, Vite and +Rollup all do — and the browser fetches the MediaPipe runtime, the largest of +them by far, only when an effect is first switched on. + A host must call and await `initializeElementCall(config)` once before rendering the component: it loads the `Intl` polyfills, applies the deployment-wide `config.json`-style configuration and sets up translations. diff --git a/src/reactions/index.ts b/src/reactions/index.ts index 26ca7f142..ffc14d43e 100644 --- a/src/reactions/index.ts +++ b/src/reactions/index.ts @@ -7,30 +7,30 @@ Please see LICENSE in the repository root for full details. import { type RelationType } from "matrix-js-sdk"; -import catSoundOgg from "../sound/reactions/cat.ogg?url"; -import catSoundMp3 from "../sound/reactions/cat.mp3?url"; -import clapSoundOgg from "../sound/reactions/clap.ogg?url"; -import clapSoundMp3 from "../sound/reactions/clap.mp3?url"; -import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url"; -import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url"; -import dogSoundOgg from "../sound/reactions/dog.ogg?url"; -import dogSoundMp3 from "../sound/reactions/dog.mp3?url"; -import genericSoundOgg from "../sound/reactions/generic.ogg?url"; -import genericSoundMp3 from "../sound/reactions/generic.mp3?url"; -import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url"; -import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url"; -import partySoundOgg from "../sound/reactions/party.ogg?url"; -import partySoundMp3 from "../sound/reactions/party.mp3?url"; -import deerSoundOgg from "../sound/reactions/deer.ogg?url"; -import deerSoundMp3 from "../sound/reactions/deer.mp3?url"; -import rockSoundOgg from "../sound/reactions/rock.ogg?url"; -import rockSoundMp3 from "../sound/reactions/rock.mp3?url"; -import waveSoundOgg from "../sound/reactions/wave.ogg?url"; -import waveSoundMp3 from "../sound/reactions/wave.mp3?url"; -import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url"; -import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url"; -import bellSoundOgg from "../sound/reactions/bell.ogg?url"; -import bellSoundMp3 from "../sound/reactions/bell.mp3?url"; +import catSoundOgg from "../sound/reactions/cat.ogg?url&no-inline"; +import catSoundMp3 from "../sound/reactions/cat.mp3?url&no-inline"; +import clapSoundOgg from "../sound/reactions/clap.ogg?url&no-inline"; +import clapSoundMp3 from "../sound/reactions/clap.mp3?url&no-inline"; +import cricketsSoundOgg from "../sound/reactions/crickets.ogg?url&no-inline"; +import cricketsSoundMp3 from "../sound/reactions/crickets.mp3?url&no-inline"; +import dogSoundOgg from "../sound/reactions/dog.ogg?url&no-inline"; +import dogSoundMp3 from "../sound/reactions/dog.mp3?url&no-inline"; +import genericSoundOgg from "../sound/reactions/generic.ogg?url&no-inline"; +import genericSoundMp3 from "../sound/reactions/generic.mp3?url&no-inline"; +import lightbulbSoundOgg from "../sound/reactions/lightbulb.ogg?url&no-inline"; +import lightbulbSoundMp3 from "../sound/reactions/lightbulb.mp3?url&no-inline"; +import partySoundOgg from "../sound/reactions/party.ogg?url&no-inline"; +import partySoundMp3 from "../sound/reactions/party.mp3?url&no-inline"; +import deerSoundOgg from "../sound/reactions/deer.ogg?url&no-inline"; +import deerSoundMp3 from "../sound/reactions/deer.mp3?url&no-inline"; +import rockSoundOgg from "../sound/reactions/rock.ogg?url&no-inline"; +import rockSoundMp3 from "../sound/reactions/rock.mp3?url&no-inline"; +import waveSoundOgg from "../sound/reactions/wave.ogg?url&no-inline"; +import waveSoundMp3 from "../sound/reactions/wave.mp3?url&no-inline"; +import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url&no-inline"; +import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url&no-inline"; +import bellSoundOgg from "../sound/reactions/bell.ogg?url&no-inline"; +import bellSoundMp3 from "../sound/reactions/bell.mp3?url&no-inline"; export const ElementCallReactionEventType = "io.element.call.reaction"; diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index d33f3b843..fa14bcd77 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -8,18 +8,18 @@ Please see LICENSE in the repository root for full details. import { type ReactNode, useEffect } from "react"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; -import joinCallSoundMp3 from "../sound/join_call.mp3"; -import joinCallSoundOgg from "../sound/join_call.ogg"; -import leftCallSoundMp3 from "../sound/left_call.mp3"; -import leftCallSoundOgg from "../sound/left_call.ogg"; -import handSoundOgg from "../sound/raise_hand.ogg"; -import handSoundMp3 from "../sound/raise_hand.mp3"; -import screenShareStartedOgg from "../sound/screen_share_started.ogg"; -import screenShareStartedMp3 from "../sound/screen_share_started.mp3"; -import declineMp3 from "../sound/call_declined.mp3?url"; -import declineOgg from "../sound/call_declined.ogg?url"; -import timeoutMp3 from "../sound/call_timeout.mp3?url"; -import timeoutOgg from "../sound/call_timeout.ogg?url"; +import joinCallSoundMp3 from "../sound/join_call.mp3?url&no-inline"; +import joinCallSoundOgg from "../sound/join_call.ogg?url&no-inline"; +import leftCallSoundMp3 from "../sound/left_call.mp3?url&no-inline"; +import leftCallSoundOgg from "../sound/left_call.ogg?url&no-inline"; +import handSoundOgg from "../sound/raise_hand.ogg?url&no-inline"; +import handSoundMp3 from "../sound/raise_hand.mp3?url&no-inline"; +import screenShareStartedOgg from "../sound/screen_share_started.ogg?url&no-inline"; +import screenShareStartedMp3 from "../sound/screen_share_started.mp3?url&no-inline"; +import declineMp3 from "../sound/call_declined.mp3?url&no-inline"; +import declineOgg from "../sound/call_declined.ogg?url&no-inline"; +import timeoutMp3 from "../sound/call_timeout.mp3?url&no-inline"; +import timeoutOgg from "../sound/call_timeout.ogg?url&no-inline"; import { useAudioContext } from "../useAudioContext"; import { prefetchSounds } from "../soundUtils"; import { useLatest } from "../useLatest"; diff --git a/src/room/RingingAudioRenderer.tsx b/src/room/RingingAudioRenderer.tsx index c0fe45d5b..f0ebd8386 100644 --- a/src/room/RingingAudioRenderer.tsx +++ b/src/room/RingingAudioRenderer.tsx @@ -12,8 +12,8 @@ import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel import { useBehavior } from "../useBehavior"; import { useInitial } from "../useInitial"; import { prefetchSounds } from "../soundUtils"; -import ringtoneMp3 from "../sound/ringtone.mp3?url"; -import ringtoneOgg from "../sound/ringtone.ogg?url"; +import ringtoneMp3 from "../sound/ringtone.mp3?url&no-inline"; +import ringtoneOgg from "../sound/ringtone.ogg?url&no-inline"; import { type UseAudioContext, useAudioContext } from "../useAudioContext"; import { useLatest } from "../useLatest"; diff --git a/src/tracks/BackgroundEffectTransformer.test.ts b/src/tracks/BackgroundEffectTransformer.test.ts index 400a50416..4e7ed8947 100644 --- a/src/tracks/BackgroundEffectTransformer.test.ts +++ b/src/tracks/BackgroundEffectTransformer.test.ts @@ -41,6 +41,24 @@ describe("BackgroundEffectTransformer", () => { expect(load).toHaveBeenCalledWith("/background.jpg"); }); + it("runs MediaPipe from its own bundle", async () => { + vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue(); + const create = vi + .spyOn(ImageSegmenter, "createFromOptions") + .mockResolvedValue({} as ImageSegmenter); + + await new BackgroundEffectTransformer({ backgroundDisabled: true }).init( + {} as VideoTransformerInitOptions, + ); + + const [fileset, options] = create.mock.calls[0]; + expect(fileset.wasmLoaderPath).toMatch(/vision_wasm_internal\.js/); + expect(fileset.wasmBinaryPath).toMatch(/vision_wasm_internal\.wasm/); + expect(options.baseOptions?.modelAssetPath).toMatch( + /selfie_segmenter\.tflite/, + ); + }); + it("reports the first frame carrying an effect, once", async () => { vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation( async function (this: BackgroundTransformer) { diff --git a/src/tracks/BackgroundEffectTransformer.ts b/src/tracks/BackgroundEffectTransformer.ts index e3c0a9183..8d7221a6d 100644 --- a/src/tracks/BackgroundEffectTransformer.ts +++ b/src/tracks/BackgroundEffectTransformer.ts @@ -15,7 +15,13 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { ImageSegmenter } from "@mediapipe/tasks-vision"; -import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url"; +// Only the SIMD build of MediaPipe's runtime: every browser Element Call +// supports has WebAssembly SIMD. As files rather than data URLs, so that a +// host's bundler serves them as assets of its own, and a browser fetches them +// only when an effect is first switched on. +import wasmLoaderPath from "@mediapipe/tasks-vision/vision_wasm_internal.js?url&no-inline"; +import wasmBinaryPath from "@mediapipe/tasks-vision/vision_wasm_internal.wasm?url&no-inline"; +import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url&no-inline"; interface WasmFileset { /** The path to the Wasm loader script. */ @@ -24,26 +30,7 @@ interface WasmFileset { wasmBinaryPath: string; } -// The MediaPipe package, by default, ships some alternative versions of the -// WASM files which avoid SIMD for compatibility with older browsers. But SIMD -// in WASM is actually fine by our support policy, so we include just the SIMD -// versions. -// It's really not ideal that we have to reference these internal files from -// MediaPipe and depend on node_modules having this specific structure. It's -// easy to see this breaking if our dependencies changed and MediaPipe were -// no longer hoisted, or if we switched to another dependency loader such as -// yarn PnP. -// https://github.com/google-ai-edge/mediapipe/issues/5961 -const wasmFileset: WasmFileset = { - wasmLoaderPath: new URL( - "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js", - import.meta.url, - ).href, - wasmBinaryPath: new URL( - "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm", - import.meta.url, - ).href, -}; +const wasmFileset: WasmFileset = { wasmLoaderPath, wasmBinaryPath }; /** * Track processor that applies effects such as blurring to a user's background. diff --git a/src/tracks/backgroundEffectCatalog.ts b/src/tracks/backgroundEffectCatalog.ts index 71c9e2d5d..9dd083ce1 100644 --- a/src/tracks/backgroundEffectCatalog.ts +++ b/src/tracks/backgroundEffectCatalog.ts @@ -5,8 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import arc from "../graphics/background-arc.jpg?url"; -import glow from "../graphics/background-glow.jpg?url"; +import arc from "../graphics/background-arc.jpg?url&no-inline"; +import glow from "../graphics/background-glow.jpg?url&no-inline"; export interface ShippedBackground { /** Stored in the user's settings when chosen, so it must not change. */ diff --git a/vite-component.config.ts b/vite-component.config.ts index 63183c781..fc0c9ac4a 100644 --- a/vite-component.config.ts +++ b/vite-component.config.ts @@ -34,6 +34,22 @@ export default defineConfig(({ mode }) => { // whatever is in `public` — including the developer's own config.json, which // is not in the repository — into the output we would publish. publicDir: false, + // A library does not know where it will be served from, so every file it + // emits beside itself — the sounds, the pictures, MediaPipe's runtime — is + // referenced relative to the module's own URL. (A library build inlines + // each asset as a data URL unless its import says `no-inline`, so the + // imports do.) + base: "./", + experimental: { + // Spelled `./assets/…`, not `assets/…` as the relative base would have it: + // a host's bundler reads a bare name as a package to look up. + renderBuiltUrl: (filename, { hostType }) => + hostType === "js" + ? { + runtime: `new URL(${JSON.stringify(`./${filename}`)}, import.meta.url).href`, + } + : { relative: true }, + }, // A host's document is not ours to style: everything in the stylesheet is // confined to the element Element Call is mounted in css: { postcss: { plugins: [scopeStylesToRoot()] } }, @@ -64,6 +80,15 @@ export default defineConfig(({ mode }) => { cssFileName: "element-call", }, rollupOptions: { + output: { + // Hashed, as the app's are, so that a file a host serves as it is + // stays correct across versions of the package. Not the stylesheet, + // which the package exports by name (see `cssFileName`). + assetFileNames: ({ names }) => + names.some((name) => name.endsWith(".css")) + ? "[name][extname]" + : "assets/[name]-[hash][extname]", + }, // The host already has these, and a second copy of any of them does not // merely bloat the bundle: React would hold two sets of hooks, the // Matrix client would run two sync loops, and a second Compound would From 51fac690f4fd3f813f902c6b727bc76494189f06 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Fri, 6 Nov 2026 13:30:36 +0100 Subject: [PATCH 02/16] Hand the camera track to the footer view model --- src/components/CallFooter.stories.tsx | 1 + src/components/CallFooter.tsx | 3 ++ src/components/CallFooterViewModel.test.ts | 43 ++++++++++++++++++++++ src/components/CallFooterViewModel.tsx | 20 ++++++++++ src/room/LobbyView.tsx | 3 ++ src/state/CallViewModel/CallViewModel.ts | 3 ++ 6 files changed, 73 insertions(+) diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index d7b1dcf90..8d6cc2a77 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -141,6 +141,7 @@ export const Default: Story = { backgroundImageRefusal: undefined, addBackgroundImage: fn(), removeBackgroundEffect: fn(), + cameraTrack: null, hangup: fn(), buttonSize: "lg", showFooter: true, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index fce982347..f09109357 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -34,6 +34,7 @@ import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; import { LayoutSwitch } from "../room/LayoutSwitch"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; import { type UnusableReason } from "../tracks/backgroundImages"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; export interface BackgroundImageRefusal { reason: UnusableReason | "not-kept"; @@ -92,6 +93,8 @@ export interface FooterState { backgroundEffectSettling: boolean; /** Why the last file offered couldn't be kept, if it couldn't. */ backgroundImageRefusal: BackgroundImageRefusal | undefined; + /** The camera to preview, with the effect in force; null while it is off. */ + cameraTrack: LocalVideoMediaTrack | null; showFooter: boolean; /* This is needed for WindowMode = "flat" */ diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index a1b174462..7036db800 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -7,12 +7,14 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; import { flushPromises, testScope, mockMuteStates, mockMediaDevices, + mockVideoTrack, } from "../utils/test"; import { constant } from "../state/Behavior"; import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; @@ -78,6 +80,7 @@ vi.mock("livekit-client", () => ({ function buildMinimalCallViewModel(layout: Layout): CallViewModel { return { layout$: constant(layout), + camera$: constant(undefined), edgeToEdge$: constant(false), showHeader$: constant(false), hangup: (): void => {}, @@ -272,6 +275,7 @@ describe("createCallFooterViewModel", () => { mockMuteStates(), twoMicsAndOneCamMediaDevices, constant(false), + /* camera$ */ constant(null), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, @@ -362,6 +366,7 @@ describe("createCallFooterViewModel", () => { mockMuteStates(), twoMicsAndOneCamMediaDevices, settling$, + /* camera$ */ constant(null), /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, @@ -427,6 +432,44 @@ describe("createCallFooterViewModel", () => { expect(store.remove).toHaveBeenCalledWith("spare"); }); + it("previews the lobby's camera while there is one", () => { + const track = mockVideoTrack({}); + const camera$ = new BehaviorSubject(track); + const vm = createLobbyFooterViewModel( + testScope(), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + constant(false), + camera$, + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.cameraTrack$.value).toBe(track); + // The lobby releases the camera when it is turned off + camera$.next(null); + expect(vm.cameraTrack$.value).toBeNull(); + }); + + it("previews the call's camera only while it is not muted", () => { + const muted$ = new BehaviorSubject(false); + const track = mockVideoTrack({ muted$ }); + const vm = createCallFooterViewModel( + testScope(), + { ...buildMinimalCallViewModel(gridLayout), camera$: constant(track) }, + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + constant(false), + /* reactionIdentifier */ undefined, + { showControls: true, header: HeaderStyle.Standard }, + ); + expect(vm.cameraTrack$.value).toBe(track); + muted$.next(true); + expect(vm.cameraTrack$.value).toBeNull(); + muted$.next(false); + expect(vm.cameraTrack$.value).toBe(track); + }); + it("availability is the same before and during a call", () => { for (const supported of [true, false]) { sdkSupportMock.mockReturnValue(supported); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 8a22aa52e..07f6dc0b1 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -11,9 +11,11 @@ import { distinctUntilChanged, map, type Observable, + of, switchMap, } from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; +import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; import { supportsBackgroundProcessors, supportsModernBackgroundProcessors, @@ -88,6 +90,8 @@ function buildDeviceBehaviors( /** return empty arrays for audioOptions and videoOptions*/ disableSwitcher$: Behavior, backgroundEffectSettling$: Observable, + /** The camera as the pipeline has it, for the menu's preview. */ + camera$: Observable, ): Pick< ViewModel, | "audioOptions$" @@ -107,6 +111,7 @@ function buildDeviceBehaviors( | "addBackgroundImage$" | "backgroundImageRefusal$" | "removeBackgroundEffect$" + | "cameraTrack$" > { const options$ = ( available$: Behavior>, @@ -208,6 +213,17 @@ function buildDeviceBehaviors( ), ), backgroundImageRefusal$: refusal$, + // None while muted: the call mutes its track rather than releasing it. + cameraTrack$: scope.behavior( + camera$.pipe( + switchMap((camera) => + camera + ? camera.muted$.pipe(map((muted) => (muted ? null : camera))) + : of(null), + ), + ), + null, + ), // Never the one in force: taking it away would leave nothing chosen. removeBackgroundEffect$: scope.behavior( offered$.pipe( @@ -269,6 +285,7 @@ export function createCallFooterViewModel( mediaDevices, disableDeviceSwitcher$, backgroundEffectSettling$, + callModel.camera$, ), // candidat to move into the FooterViewModel showFooter$: callModel.showFooter$, @@ -333,6 +350,7 @@ export function createCallFooterViewModel( * @param mediaDevices - Available and selected input devices. * @param backgroundEffectSettling$ - Whether the first effect chosen is still * being prepared. + * @param camera$ - The lobby's camera, for the camera menu's preview. * @param openSettings - Callback to open the settings modal, or undefined. * @param hangup - Callback to leave/cancel, or undefined (hides the button). * @param showLogo - Whether to show the Element Call logo. @@ -342,6 +360,7 @@ export function createLobbyFooterViewModel( muteStates: MuteStates, mediaDevices: MediaDevices, backgroundEffectSettling$: Observable, + camera$: Behavior, openSettings: (() => void) | undefined, hangup: (() => void) | undefined, showLogo: boolean, @@ -390,6 +409,7 @@ export function createLobbyFooterViewModel( mediaDevices, constant(false), backgroundEffectSettling$, + camera$, ), }; } diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index 1a197b8d0..a72c04965 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -158,6 +158,7 @@ export const LobbyView: FC = ({ null, ); useEffect(() => { + if (mediaVm === null) return; const footerScope = new ObservableScope(); setFooterVm( createLobbyFooterViewModel( @@ -165,6 +166,7 @@ export const LobbyView: FC = ({ muteStates, devices, backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)), + mediaVm.videoTrack$, openSettings, hangup, // Logo and header are connected: only show the logo in SPA with header. @@ -179,6 +181,7 @@ export const LobbyView: FC = ({ backgroundEffectsState$, hangup, hideHeader, + mediaVm, muteStates, openSettings, ]); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 9b1e853a3..ba83fad68 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -363,6 +363,8 @@ export interface CallViewModel { keyRotationSuppressed$: Behavior; /** The local member, once our membership has been seen in the room. */ localMember$: Behavior; + /** Our own camera track, while one is published. */ + camera$: Behavior; remoteMembers$: Behavior; /** The transports the client holds a connection to, for the developer panel. */ connectedTransports$: Behavior; @@ -1508,6 +1510,7 @@ export function createCallViewModel$( leave, toggleScreenSharing: toggleScreenSharing, sharingScreen$: sharingScreen$, + camera$, tapScreen: (): void => screenTap$.next(), tapControls: (): void => controlsTap$.next(), From f2f67019b41250e1f70b1c33cd08fdb8b3612cd9 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Fri, 6 Nov 2026 13:30:36 +0100 Subject: [PATCH 03/16] Show the self preview in the video mute button context menu --- playwright/background-effects.spec.ts | 60 +++++++ src/components/CallFooter.tsx | 3 + .../MediaMuteAndSwitchButton.module.css | 32 ++++ .../MediaMuteAndSwitchButton.stories.tsx | 159 +++++++++++++++++- src/components/MediaMuteAndSwitchButton.tsx | 60 ++++++- src/components/SelfPreview.module.css | 10 ++ src/components/SelfPreview.test.tsx | 47 ++++++ src/components/SelfPreview.tsx | 48 ++++++ 8 files changed, 412 insertions(+), 7 deletions(-) create mode 100644 src/components/SelfPreview.module.css create mode 100644 src/components/SelfPreview.test.tsx create mode 100644 src/components/SelfPreview.tsx diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index 4d01c2d72..019ccfd43 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -106,6 +106,66 @@ async function drawsPerSecond(page: Page): Promise { }); } +test.describe("the camera menu's preview", () => { + test("shows the camera with the effect in force", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Preview user", "Preview in the menu"); + const camera = await cameraColour(page.locator("video").first()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const preview = page.getByRole("menu").locator("video"); + await expect(preview).toBeVisible(); + await expect + .poll(async () => distance(await averageColour(preview), camera), { + timeout: 20_000, + }) + .toBeLessThan(30); + + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); + + test("shows the call's own camera once joined", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall( + page, + "Call preview user", + "Preview in the call's menu", + true, + ); + await expect(page.getByTestId("incall_videomute")).toBeEnabled({ + timeout: 10_000, + }); + const camera = await cameraColour(page.locator("video").first()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const preview = page.getByRole("menu").locator("video"); + await expect(preview).toBeVisible(); + await expect + .poll(async () => distance(await averageColour(preview), camera), { + timeout: 20_000, + }) + .toBeLessThan(30); + + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); +}); + test.describe("background effects section", () => { test("section stays within the call area at every size", async ({ page }) => { test.setTimeout(120_000); diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index f09109357..4d13b4c6a 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -33,6 +33,7 @@ import { useBehavior } from "../useBehavior"; import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; import { LayoutSwitch } from "../room/LayoutSwitch"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; import { type UnusableReason } from "../tracks/backgroundImages"; import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk"; @@ -184,6 +185,7 @@ export const CallFooter: FC = ({ const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); const addBackgroundImage = useBehavior(vm.addBackgroundImage$); const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$); + const cameraTrack = useBehavior(vm.cameraTrack$); const refusal = useBehavior(vm.backgroundImageRefusal$); const refusalMessage = useMemo(() => { switch (refusal?.reason) { @@ -267,6 +269,7 @@ export const CallFooter: FC = ({ backgroundEffectSettling={backgroundEffectSettling} onAddBackgroundImage={addBackgroundImage} onRemoveBackgroundEffect={removeBackgroundEffect} + selfPreview={} backgroundImageRefusal={refusalMessage} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" diff --git a/src/components/MediaMuteAndSwitchButton.module.css b/src/components/MediaMuteAndSwitchButton.module.css index 10b3267f5..2e8c60f3f 100644 --- a/src/components/MediaMuteAndSwitchButton.module.css +++ b/src/components/MediaMuteAndSwitchButton.module.css @@ -97,3 +97,35 @@ Please see LICENSE in the repository root for full details. .refusal { margin: var(--cpd-space-2x) var(--cpd-space-4x) 0; } + +.selfPreview { + position: relative; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-2x); + overflow: clip; + border-start-start-radius: var(--cpd-space-3x); + border-start-end-radius: var(--cpd-space-3x); + background: var(--cpd-color-bg-subtle-secondary); + color: var(--cpd-color-icon-secondary); +} + +.selfPreview > video { + position: absolute; + inset: 0; + inline-size: 100%; + block-size: 100%; + object-fit: cover; + /* Rounded itself too: a video can be drawn on a layer of its own, which the + frame's rounded clip doesn't always reach. */ + border-radius: inherit; +} + +/* Up to the menu's frame, over the padding at its top. */ +.selfPreviewPinned, +.deviceListLeadsWithPreview { + margin-block-start: calc(-1 * var(--cpd-space-5x)); +} diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 38ace8aee..1d9447773 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -12,6 +12,7 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton"; import styles from "./MediaMuteAndSwitchButton.module.css"; import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog"; import meterStyles from "./MicrophoneLevelMeter.module.css"; import { MediaDevicesContext } from "../MediaDevicesContext"; @@ -60,6 +61,50 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => { return <>{children}; }; +/** A camera for the preview: a figure against a room, drawn to a stream. */ +const FakeCamera: FC = () => { + const [stream, setStream] = useState(); + useEffect(() => { + const canvas = document.createElement("canvas"); + canvas.width = 640; + canvas.height = 360; + const context = canvas.getContext("2d")!; + const draw = (): void => { + context.fillStyle = "#6b7f8e"; + context.fillRect(0, 0, 640, 360); + context.fillStyle = "#c9a27e"; + context.beginPath(); + context.arc(320, 150, 62, 0, 2 * Math.PI); + context.fill(); + context.fillStyle = "#34495e"; + context.beginPath(); + context.ellipse(320, 380, 150, 140, 0, Math.PI, 2 * Math.PI); + context.fill(); + }; + draw(); + // Redrawn, so the stream keeps delivering frames. + const timer = setInterval(draw, 200); + const captured = canvas.captureStream(10); + setStream(captured); + return (): void => { + clearInterval(timer); + captured.getTracks().forEach((track) => track.stop()); + }; + }, []); + return ( +