From 989bdc68157767dd00049546acfd36bb8b401421 Mon Sep 17 00:00:00 2001 From: fkwp Date: Thu, 24 Sep 2026 19:28:00 +0200 Subject: [PATCH] Show the user their camera at the top of the camera menu - The camera menu leads with the user's own camera, with the effect in force, full width at sixteen by nine: a background is judged by how it looks rather than by its thumbnail. - It shows the pipeline's own track again, not a second capture. Each provider keeps the camera its pipeline is synced to, before joining and after, and hands it out with its state, so two calls on a page keep their own. It is reported again only when it changes, which keeps the preview from blanking on every change of state. - It and the pre-join preview share useAttachedTrack, which detaches only its own element: the pre-join preview detached every element showing its track, the menu's among them. - Paid for out of the list's share of the call, so a menu with it is no taller: at the top where the list keeps its floor after paying, else first among the options, scrolling away with them. Its size comes from the tokens the stylesheets draw it with, so it is placed once. - While the first effect is prepared the preview shows the wait; with the camera off it says so, and an effect can still be chosen. - The video is rounded itself as well as clipped by its frame: Firefox composites it on a layer of its own, which the frame's rounded clip didn't reach at 100% zoom. Confirmed by hand, as headless Firefox draws it rounded either way. - Every camera story shows the preview, with a stand-in camera drawn to a stream. End to end on Chromium and Firefox, the lobby's menu shows the camera and then the chosen picture. Co-Authored-By: Claude Opus 5.5 (1M context) --- playwright/background-effects.spec.ts | 27 +++ src/components/CallFooter.stories.tsx | 1 + src/components/CallFooter.tsx | 6 + src/components/CallFooterViewModel.test.ts | 58 ++++++- src/components/CallFooterViewModel.tsx | 38 +++-- .../MediaMuteAndSwitchButton.module.css | 32 ++++ .../MediaMuteAndSwitchButton.stories.tsx | 159 +++++++++++++++++- src/components/MediaMuteAndSwitchButton.tsx | 60 ++++++- src/components/SelfPreview.module.css | 10 ++ src/components/SelfPreview.tsx | 36 ++++ src/livekit/BackgroundEffectsContext.test.ts | 71 ++++++++ src/livekit/BackgroundEffectsContext.tsx | 35 +++- src/livekit/cameraTrack.ts | 28 +++ src/room/InCallView.tsx | 3 +- src/room/LobbyView.tsx | 3 +- src/room/VideoPreview.tsx | 26 +-- src/state/BackgroundEffects.ts | 4 + src/useAttachedTrack.test.tsx | 59 +++++++ src/useAttachedTrack.ts | 34 ++++ src/utils/test-viewmodel.ts | 2 +- 20 files changed, 640 insertions(+), 52 deletions(-) create mode 100644 src/components/SelfPreview.module.css create mode 100644 src/components/SelfPreview.tsx create mode 100644 src/livekit/cameraTrack.ts create mode 100644 src/useAttachedTrack.test.tsx create mode 100644 src/useAttachedTrack.ts diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index fb0a7516c..f47c59cba 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -106,6 +106,33 @@ async function drawsPerSecond(page: Page): Promise { }); } +test.describe("the camera menu's preview", () => { + test("shows the camera with the effect in force", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Preview user", "Preview in the menu"); + const camera = await cameraColour(page.locator("video").first()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const preview = page.getByRole("menu").locator("video"); + await expect(preview).toBeVisible(); + await expect + .poll(async () => distance(await averageColour(preview), camera), { + timeout: 20_000, + }) + .toBeLessThan(30); + + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); +}); + test.describe("background effects section", () => { test("section stays within the call area at every size", async ({ page }) => { test.setTimeout(120_000); diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index 767aec39a..23b4059a1 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -141,6 +141,7 @@ export const Default: Story = { backgroundImageRefusal: undefined, addBackgroundImage: fn(), removeBackgroundEffect: fn(), + cameraTrack: null, hangup: fn(), buttonSize: "lg", showFooter: true, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index d49771bc8..3873719ca 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { type FC, type JSX, type Ref, useMemo } from "react"; import classNames from "classnames"; +import { type LocalVideoTrack } from "livekit-client"; import { useTranslation } from "react-i18next"; import LogoMark from "../icons/LogoMark.svg?react"; @@ -33,6 +34,7 @@ import { useBehavior } from "../useBehavior"; import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; import { LayoutSwitch } from "../room/LayoutSwitch"; import { type BackgroundEffectOption } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; import { type UnusableReason } from "../livekit/backgroundImages"; export interface BackgroundImageRefusal { @@ -102,6 +104,8 @@ export interface FooterState { backgroundEffectSettling: boolean; /** Why the last file offered couldn't be kept, if it couldn't. */ backgroundImageRefusal: BackgroundImageRefusal | undefined; + /** The camera to preview, with the effect in force; null while it is off. */ + cameraTrack: LocalVideoTrack | null; showFooter: boolean; /* This is needed for WindowMode = "flat" */ @@ -191,6 +195,7 @@ export const CallFooter: FC = ({ const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); const addBackgroundImage = useBehavior(vm.addBackgroundImage$); const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$); + const cameraTrack = useBehavior(vm.cameraTrack$); const refusal = useBehavior(vm.backgroundImageRefusal$); const refusalMessage = useMemo(() => { switch (refusal?.reason) { @@ -276,6 +281,7 @@ export const CallFooter: FC = ({ backgroundEffectSettling={backgroundEffectSettling} onAddBackgroundImage={addBackgroundImage} onRemoveBackgroundEffect={removeBackgroundEffect} + selfPreview={} backgroundImageRefusal={refusalMessage} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index 2d2cbd30f..a4cf9b8b3 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; +import { type LocalVideoTrack } from "livekit-client"; import { flushPromises, @@ -24,6 +25,9 @@ import { createLobbyFooterViewModel, } from "./CallFooterViewModel"; import { HeaderStyle } from "../UrlParams"; +import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext"; +import { SyncedCameraTrack } from "../livekit/cameraTrack"; +import { type MuteStates } from "../state/MuteStates"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { shippedBackgrounds } from "../livekit/backgroundEffects"; import type * as BackgroundImages from "../livekit/backgroundImages"; @@ -113,6 +117,11 @@ const pipLayout: Layout = { spotlight: {} as SpotlightTileViewModel, }; +const noPipeline = constant({ + supported: false, + processor: undefined, +}); + const twoMicsAndOneCamMediaDevices = mockMediaDevices({ audioInput: { available$: constant( @@ -144,7 +153,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -187,7 +196,7 @@ describe("createCallFooterViewModel", () => { select: vi.fn(), }, }), - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -206,7 +215,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(layout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -229,7 +238,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -275,7 +284,7 @@ describe("createCallFooterViewModel", () => { testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, @@ -360,18 +369,22 @@ describe("createCallFooterViewModel", () => { it("passes on the wait for the first effect", () => { platformMock.mockReturnValue("desktop"); - const settling$ = new BehaviorSubject(true); + const pipeline$ = new BehaviorSubject({ + supported: true, + processor: undefined, + settling: true, + }); const vm = createLobbyFooterViewModel( testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, - settling$, + pipeline$, /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, ); expect(vm.backgroundEffectSettling$.value).toBe(true); - settling$.next(false); + pipeline$.next({ ...pipeline$.value, settling: false }); expect(vm.backgroundEffectSettling$.value).toBe(false); }); @@ -431,6 +444,33 @@ describe("createCallFooterViewModel", () => { expect(store.remove).toHaveBeenCalledWith("spare"); }); + it("previews the camera only while it is on", () => { + const camera = new SyncedCameraTrack(); + const track = {} as LocalVideoTrack; + camera.report(track); + const videoEnabled$ = new BehaviorSubject(true); + const muteStates = mockMuteStates(); + const vm = createLobbyFooterViewModel( + testScope(), + { + ...muteStates, + video: { ...muteStates.video, enabled$: videoEnabled$ }, + } as unknown as MuteStates, + twoMicsAndOneCamMediaDevices, + constant({ + supported: true, + processor: undefined, + cameraTrack: camera, + }), + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.cameraTrack$.value).toBe(track); + videoEnabled$.next(false); + expect(vm.cameraTrack$.value).toBeNull(); + }); + it("availability is the same before and during a call", () => { for (const supported of [true, false]) { sdkSupportMock.mockReturnValue(supported); @@ -440,7 +480,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 7702e6526..ca9f780de 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -11,6 +11,7 @@ import { distinctUntilChanged, map, type Observable, + of, switchMap, } from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; @@ -25,6 +26,7 @@ import { serializeEffect, shippedBackgrounds, } from "../livekit/backgroundEffects"; +import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext"; import { type AddedBackground, addedBackgrounds, @@ -90,7 +92,8 @@ function buildDeviceBehaviors( mediaDevices: MediaDevices, /** return empty arrays for audioOptions and videoOptions*/ disableSwitcher$: Behavior, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, + videoEnabled$: Behavior, ): Pick< ViewModel, | "audioOptions$" @@ -110,6 +113,7 @@ function buildDeviceBehaviors( | "addBackgroundImage$" | "backgroundImageRefusal$" | "removeBackgroundEffect$" + | "cameraTrack$" > { const options$ = ( available$: Behavior>, @@ -189,7 +193,10 @@ function buildDeviceBehaviors( ), ), backgroundEffectSettling$: scope.behavior( - backgroundEffectSettling$.pipe(distinctUntilChanged()), + backgroundEffectsState$.pipe( + map(({ settling }) => settling ?? false), + distinctUntilChanged(), + ), false, ), // Kept and offered, not put on: that waits for the user to choose it. @@ -211,6 +218,15 @@ function buildDeviceBehaviors( ), ), backgroundImageRefusal$: refusal$, + cameraTrack$: scope.behavior( + combineLatest([ + backgroundEffectsState$.pipe( + switchMap(({ cameraTrack }) => cameraTrack?.track$ ?? of(null)), + ), + videoEnabled$, + ]).pipe(map(([track, videoEnabled]) => (videoEnabled ? track : null))), + null, + ), // Never the one in force: taking it away would leave nothing chosen. removeBackgroundEffect$: scope.behavior( offered$.pipe( @@ -240,8 +256,8 @@ function buildDeviceBehaviors( * @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. - * @param backgroundEffectSettling$ - Whether the first effect chosen is still - * being prepared. + * @param backgroundEffectsState$ - The background pipeline's state: whether effects + * run here, the wait for the first, and the camera it is synced to. * @param reactionIdentifier - The local user's reaction identifier string, or * undefined when reactions are not supported (hides the reaction button). * @param options - `showControls`: whether the call controls should be shown. @@ -252,7 +268,7 @@ export function createCallFooterViewModel( callModel: CallViewModel, muteStates: MuteStates, mediaDevices: MediaDevices, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, reactionIdentifier: string | undefined, options: { showControls: boolean; header: HeaderStyle }, ): ViewModel { @@ -271,7 +287,8 @@ export function createCallFooterViewModel( scope, mediaDevices, disableDeviceSwitcher$, - backgroundEffectSettling$, + backgroundEffectsState$, + muteStates.video.enabled$, ), // candidat to move into the FooterViewModel showFooter$: callModel.showFooter$, @@ -334,8 +351,8 @@ export function createCallFooterViewModel( * @param scope - ObservableScope that bounds the lifetime of derived behaviors. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. - * @param backgroundEffectSettling$ - Whether the first effect chosen is still - * being prepared. + * @param backgroundEffectsState$ - The background pipeline's state: whether effects + * run here, the wait for the first, and the camera it is synced to. * @param openSettings - Callback to open the settings modal, or undefined. * @param hangup - Callback to leave/cancel, or undefined (hides the button). * @param showLogo - Whether to show the Element Call logo. @@ -344,7 +361,7 @@ export function createLobbyFooterViewModel( scope: ObservableScope, muteStates: MuteStates, mediaDevices: MediaDevices, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, openSettings: (() => void) | undefined, hangup: (() => void) | undefined, showLogo: boolean, @@ -392,7 +409,8 @@ export function createLobbyFooterViewModel( scope, mediaDevices, constant(false), - backgroundEffectSettling$, + backgroundEffectsState$, + muteStates.video.enabled$, ), }; } diff --git a/src/components/MediaMuteAndSwitchButton.module.css b/src/components/MediaMuteAndSwitchButton.module.css index 73a24f627..9e12cad34 100644 --- a/src/components/MediaMuteAndSwitchButton.module.css +++ b/src/components/MediaMuteAndSwitchButton.module.css @@ -129,3 +129,35 @@ Please see LICENSE in the repository root for full details. .refusal { margin: var(--cpd-space-2x) var(--cpd-space-4x) 0; } + +.selfPreview { + position: relative; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-2x); + overflow: clip; + border-start-start-radius: var(--cpd-space-3x); + border-start-end-radius: var(--cpd-space-3x); + background: var(--cpd-color-bg-subtle-secondary); + color: var(--cpd-color-icon-secondary); +} + +.selfPreview > video { + position: absolute; + inset: 0; + inline-size: 100%; + block-size: 100%; + object-fit: cover; + /* Rounded itself too: a video can be drawn on a layer of its own, which the + frame's rounded clip doesn't always reach. */ + border-radius: inherit; +} + +/* Up to the menu's frame, over the padding at its top. */ +.selfPreviewPinned, +.deviceListLeadsWithPreview { + margin-block-start: calc(-1 * var(--cpd-space-5x)); +} diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index c58eb1464..9dba9a48a 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -12,6 +12,7 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton"; import styles from "./MediaMuteAndSwitchButton.module.css"; import { type BackgroundEffectOption } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; import { shippedBackgrounds } from "../livekit/backgroundEffects"; import meterStyles from "./MicrophoneLevelMeter.module.css"; import { MediaDevicesContext } from "../MediaDevicesContext"; @@ -60,6 +61,50 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => { return <>{children}; }; +/** A camera for the preview: a figure against a room, drawn to a stream. */ +const FakeCamera: FC = () => { + const [stream, setStream] = useState(); + useEffect(() => { + const canvas = document.createElement("canvas"); + canvas.width = 640; + canvas.height = 360; + const context = canvas.getContext("2d")!; + const draw = (): void => { + context.fillStyle = "#6b7f8e"; + context.fillRect(0, 0, 640, 360); + context.fillStyle = "#c9a27e"; + context.beginPath(); + context.arc(320, 150, 62, 0, 2 * Math.PI); + context.fill(); + context.fillStyle = "#34495e"; + context.beginPath(); + context.ellipse(320, 380, 150, 140, 0, Math.PI, 2 * Math.PI); + context.fill(); + }; + draw(); + // Redrawn, so the stream keeps delivering frames. + const timer = setInterval(draw, 200); + const captured = canvas.captureStream(10); + setStream(captured); + return (): void => { + clearInterval(timer); + captured.getTracks().forEach((track) => track.stop()); + }; + }, []); + return ( +