From 45469f3828c809158337c421c191ca2168004eb8 Mon Sep 17 00:00:00 2001 From: fkwp Date: Wed, 23 Sep 2026 09:35:57 +0200 Subject: [PATCH] Show how the user looks while they choose a background - A live self-preview at the top of the camera menu, as design drew it: full width, flush with the top, sixteen by nine. Both sections scroll beneath it, because it is what the choosing below is for. - Only where the call leaves room. It is paid for out of the device list's share rather than on top of it, and only where the list keeps its floor after paying, so the menu is never taller for having one. - Attached to the track the pipeline is synced to, which both the lobby and the call already report, so it shows the effect in force without a capture or a pipeline of its own. Mirrored like the pre-join preview, a camera-off mark while there is no camera, and the wait while the first effect builds. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/components/CallFooter.tsx | 4 +- .../MediaMuteAndSwitchButton.module.css | 49 ++++++++++ .../MediaMuteAndSwitchButton.stories.tsx | 90 +++++++++++++++++-- src/components/MediaMuteAndSwitchButton.tsx | 59 ++++++++++-- src/components/SelfPreview.tsx | 54 +++++++++++ src/livekit/TrackProcessorContext.tsx | 52 ++++++++++- 6 files changed, 294 insertions(+), 14 deletions(-) create mode 100644 src/components/SelfPreview.tsx diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index 942d25d9a..bc99cbec7 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -50,6 +50,7 @@ import { useBackgroundProcessing, } from "../livekit/TrackProcessorContext"; import { usesFallbackProcessing } from "../livekit/backgroundProcessing"; +import { SelfPreview } from "./SelfPreview"; import { type Behavior } from "../state/Behavior"; import { type ViewModel } from "../state/ViewModel"; import { useBehavior } from "../useBehavior"; @@ -190,7 +191,7 @@ export const CallFooter: FC = ({ const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$); const beforeJoining = useBehavior(vm.beforeJoining$); const { added, addBackground, removeBackground } = useAddedBackgrounds(); - const { settling } = useBackgroundProcessing(); + const { settling, cameraTrack } = useBackgroundProcessing(); // Said, not decided. Where only the slow path exists the effect still works, // it costs frames — and the user is the only one who knows whether they would @@ -363,6 +364,7 @@ export const CallFooter: FC = ({ backgroundEffectError={backgroundEffectError} backgroundEffectNotice={backgroundEffectNotice} backgroundEffectSettling={settling} + selfPreview={} />, ); } else { diff --git a/src/components/MediaMuteAndSwitchButton.module.css b/src/components/MediaMuteAndSwitchButton.module.css index 9dffd659b..91282cea4 100644 --- a/src/components/MediaMuteAndSwitchButton.module.css +++ b/src/components/MediaMuteAndSwitchButton.module.css @@ -426,3 +426,52 @@ Please see LICENSE in the repository root for full details. flex-shrink: 0; color: var(--cpd-color-icon-secondary); } + +/* + * How the user looks, at the top of the camera menu. + * + * Edge to edge and flush with the top, as the design draws it: it takes the + * menu's own top padding, which is why it pulls itself up over it. Held one + * border width clear of the frame and rounded to match what is left of the + * frame's corner, because a video paints on a layer of its own and would + * otherwise draw over the outline the menu is framed with. + * + * Sixteen by nine and never taller than its budget, cropped rather than + * letterboxed: a face cut at the edge reads as a self-view, a face in bars + * reads as a broken one. + */ +.selfPreview { + position: relative; + display: flex; + align-items: center; + justify-content: center; + margin-block-start: calc( + -1 * var(--cpd-space-5x) + var(--cpd-border-width-1) + ); + margin-inline: var(--cpd-border-width-1); + margin-block-end: var(--cpd-space-2x); + aspect-ratio: 16 / 9; + max-block-size: 176px; + overflow: hidden; + border-start-start-radius: calc( + var(--cpd-space-3x) - var(--cpd-border-width-1) + ); + border-start-end-radius: calc( + var(--cpd-space-3x) - var(--cpd-border-width-1) + ); + background: var(--cpd-color-bg-subtle-secondary); + color: var(--cpd-color-icon-secondary); +} + +.selfPreview > video, +.selfPreview > img { + inline-size: 100%; + block-size: 100%; + object-fit: cover; +} + +/* The camera faces the user, so the picture is what a mirror would show — + the same as the pre-join preview, or the two would disagree. */ +.selfPreview > .mirrored { + transform: scaleX(-1); +} diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index c6c48ee68..34350432b 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -9,6 +9,8 @@ import { fn, userEvent, waitFor, within, expect } from "storybook/test"; import { useEffect, useState, type FC, type JSX, type ReactNode } from "react"; import { TooltipProvider } from "@vector-im/compound-web"; +import cameraStandIn from "../graphics/background-indoor-standin.jpg?url"; + import type { Meta, StoryObj } from "@storybook/react-vite"; import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton"; import styles from "./MediaMuteAndSwitchButton.module.css"; @@ -80,7 +82,10 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => { * the top of the canvas. Supplying a root is the same courtesy as supplying the * devices: the story stands in for the call, so it has to say how big it is. */ -const WithACallArea: FC<{ children: ReactNode }> = ({ children }) => { +const WithACallArea: FC<{ children: ReactNode; height?: number }> = ({ + children, + height = 720, +}) => { const [callArea, setCallArea] = useState(null); return (
= ({ children }) => { // a share of this, so a small area makes even a two-device menu scroll, // which no real call does. Tall enough to leave the menu room to open // upward and still be wholly on screen in the story's frame. - blockSize: 720, + blockSize: height, display: "flex", alignItems: "flex-end", justifyContent: "center", @@ -108,10 +113,12 @@ const meta = { decorators: [ // The app puts one of these over everything; the remove cross needs it to // be able to name itself. - (Story): JSX.Element => ( + (Story, { parameters }): JSX.Element => ( - + @@ -923,6 +930,71 @@ export const BackgroundEffectsSlowInThisBrowser: Story = { }, }; +/** + * Room for the self-preview: the user sees how they look with the effect in + * force, at the top of the menu, while choosing. + * + * Paid for out of the device list's share of the call rather than on top of + * it, so the menu is no taller for having it. Both sections scroll beneath it. + */ +export const BackgroundEffectsWithPreview: Story = { + args: { + ...BackgroundEffects.args, + selfPreview: , + }, + parameters: { callAreaHeight: 720 }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + await within(document.body).findByRole("menuitemradio", { name: "Blur" }); + + const menu = document.body.querySelector("[role='menu']")!; + const preview = menu.querySelector(`.${styles.selfPreview}`)!; + await expect(preview).toBeInTheDocument(); + + // Flush with the top and the sides, as drawn: it takes the menu's own top + // padding, held clear of the frame by no more than the frame's own line. + const frame = menu.getBoundingClientRect(); + const box = preview.getBoundingClientRect(); + await expect(box.top - frame.top).toBeLessThanOrEqual(2); + await expect(box.left - frame.left).toBeLessThanOrEqual(2); + await expect(frame.right - box.right).toBeLessThanOrEqual(2); + + // Above the list, not in it, so the sections scroll beneath it. + const list = menu.querySelector(`.${styles.deviceList}`)!; + await expect(list.contains(preview)).toBe(false); + + // And paid for out of the list's share: 60% of 720, less the preview. + await expect(getComputedStyle(list).maxBlockSize).toBe( + `${Math.round(720 * 0.6) - 176}px`, + ); + }, +}; + +/** + * No room: a call area as short as a laptop browser often leaves. The list + * could not keep its floor after paying for the preview, so there is none — + * the backgrounds and the devices matter more than a picture of the user. + */ +export const BackgroundEffectsNoRoomForPreview: Story = { + args: BackgroundEffectsWithPreview.args, + parameters: { callAreaHeight: 470 }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + await within(document.body).findByRole("menuitemradio", { name: "Blur" }); + + const menu = document.body.querySelector("[role='menu']")!; + await expect( + menu.querySelector(`.${styles.selfPreview}`), + ).not.toBeInTheDocument(); + const list = menu.querySelector(`.${styles.deviceList}`)!; + await expect(getComputedStyle(list).maxBlockSize).toBe( + `${Math.round(470 * 0.6)}px`, + ); + }, +}; + /** * The sequence a user sees on the first effect of a session. * @@ -942,6 +1014,7 @@ export const BackgroundEffectsSlowInThisBrowser: Story = { export const BackgroundEffectsSettling: Story = { args: { ...BackgroundEffects.args, + selfPreview: , backgroundEffectNotice: "This browser runs background effects slowly, so other people may see your video stutter.", }, @@ -979,10 +1052,15 @@ export const BackgroundEffectsSettling: Story = { const blur = await body.findByRole("menuitemradio", { name: "Blur" }); await userEvent.click(blur); - // The pressed tile is busy, and has no tick yet. + // The pressed tile is busy, and has no tick yet — and the preview shows + // the same wait where the picture will be, rather than the picture from + // before the choice. await waitFor(async () => expect(blur.querySelector(`.${styles.effectBusy}`)).toBeInTheDocument(), ); + const preview = document.body.querySelector(`.${styles.selfPreview}`)!; + await expect(preview.querySelector("img")).not.toBeInTheDocument(); + await expect(preview.querySelector("svg")).toBeInTheDocument(); await expect( await body.findByText(/runs background effects slowly/), ).toBeInTheDocument(); @@ -996,6 +1074,8 @@ export const BackgroundEffectsSettling: Story = { { timeout: 5000 }, ); await expect(blur).toHaveAttribute("aria-checked", "true"); + // And the picture is back. + await expect(preview.querySelector("img")).toBeInTheDocument(); }, }; diff --git a/src/components/MediaMuteAndSwitchButton.tsx b/src/components/MediaMuteAndSwitchButton.tsx index ee12ae4c6..12d8ec3c5 100644 --- a/src/components/MediaMuteAndSwitchButton.tsx +++ b/src/components/MediaMuteAndSwitchButton.tsx @@ -13,6 +13,7 @@ import { useEffect, useRef, type ReactElement, + type ReactNode, } from "react"; import { Alert, @@ -143,6 +144,13 @@ export interface MediaMuteAndSwitchButtonProps { * and where the wait is short a message would only flash. */ backgroundEffectSettling?: boolean; + /** + * How the user looks right now, with whatever effect is in force: drawn at + * the top of the camera menu, where choosing a background can be judged by + * its result rather than by a thumbnail of it. Shown only where the call + * leaves room for it, and scrolled beneath rather than scrolled away. + */ + selfPreview?: ReactNode; /** * For any toggle and option this method will be called. * So toggles need to be implemented by listening here and setting the right toggle item to `enabled` @@ -177,6 +185,17 @@ const LIST_SHARE_OF_CALL = 0.6; */ const MIN_LIST_HEIGHT = 160; +/** + * The most the self-preview may take, which is also what it costs the list. + * + * Drawn at sixteen by nine across the full width of the menu, capped here so + * that a menu widened by a long device name cannot make it taller than its + * budget. The preview is paid for out of the list's share rather than on top + * of it, so the menu is never taller for having one: where the list could not + * keep its floor after paying, there is no preview at all. + */ +const PREVIEW_BLOCK = 176; + export const MediaMuteAndSwitchButton: FC = ({ title, enabled, @@ -198,6 +217,7 @@ export const MediaMuteAndSwitchButton: FC = ({ backgroundEffectError, backgroundEffectNotice, backgroundEffectSettling, + selfPreview, onSelect, }) => { // Which device we have asked for but not yet been given. Carries the kind as @@ -282,7 +302,7 @@ export const MediaMuteAndSwitchButton: FC = ({ // can size it against the call. Measure the call area rather than the window, // or the menu is wrong wherever Element Call is not the whole page. const rootElement = useRootElement(); - const [listMaxHeight, setListMaxHeight] = useState(); + const [listShare, setListShare] = useState(); useEffect(() => { if (!menuOpen) return; // Followed rather than measured once: a host can resize the space Element @@ -292,12 +312,10 @@ export const MediaMuteAndSwitchButton: FC = ({ // resize re-renders only when the bound itself moves. const subscription = observeElementSize$(rootElement) .pipe( - map(({ height }) => - Math.max(MIN_LIST_HEIGHT, Math.round(height * LIST_SHARE_OF_CALL)), - ), + map(({ height }) => Math.round(height * LIST_SHARE_OF_CALL)), distinctUntilChanged(), ) - .subscribe(setListMaxHeight); + .subscribe(setListShare); return (): void => subscription.unsubscribe(); }, [menuOpen, rootElement]); @@ -668,6 +686,22 @@ export const MediaMuteAndSwitchButton: FC = ({ return tiles; }; + // The preview is paid for out of the list's share, and only where the list + // keeps its floor after paying, so a menu with one is no taller than a menu + // without — which is what keeps it inside the call area at any size. + const showPreview = + iconsAndLabels === "video" && + selfPreview !== undefined && + listShare !== undefined && + listShare - PREVIEW_BLOCK >= MIN_LIST_HEIGHT; + const listMaxHeight = + listShare === undefined + ? undefined + : Math.max( + MIN_LIST_HEIGHT, + listShare - (showPreview ? PREVIEW_BLOCK : 0), + ); + return (
= ({ /> } > + {showPreview && ( + // Pinned above the list rather than inside it: the devices and the + // effects scroll beneath it, because it is what the choosing below is + // for. Decoration to assistive technology — the choice is announced + // by the items, and a picture of the user tells them nothing new. +
+ {backgroundEffectSettling ? ( + // The same wait the pressed tile shows, where the picture will + // be, so the eye does not have to go looking for why it is not. + + ) : ( + selfPreview + )} +
+ )}
= ({ track }) => { + const video = useRef(null); + useEffect(() => { + const element = video.current; + if (!track || !element) return; + track.attach(element); + return (): void => { + track.detach(element); + }; + }, [track]); + + // Nothing to show, and saying so: an effect chosen now still applies when + // the camera comes back, so the grid below stays live. + if (!track) return ; + + return ( +