diff --git a/playwright/background-effects.spec.ts b/playwright/background-effects.spec.ts index fb0a7516c..f47c59cba 100644 --- a/playwright/background-effects.spec.ts +++ b/playwright/background-effects.spec.ts @@ -106,6 +106,33 @@ async function drawsPerSecond(page: Page): Promise { }); } +test.describe("the camera menu's preview", () => { + test("shows the camera with the effect in force", async ({ page }) => { + test.setTimeout(120_000); + await page.goto("/"); + await SpaHelpers.createCall(page, "Preview user", "Preview in the menu"); + const camera = await cameraColour(page.locator("video").first()); + + await page.getByRole("button", { name: "Camera", exact: true }).click(); + const preview = page.getByRole("menu").locator("video"); + await expect(preview).toBeVisible(); + await expect + .poll(async () => distance(await averageColour(preview), camera), { + timeout: 20_000, + }) + .toBeLessThan(30); + + const tile = page.getByRole("menuitemradio", { name: "Background 1" }); + await tile.click(); + const picture = await averageColour(tile.locator("img")); + await expect + .poll(async () => distance(await averageColour(preview), picture), { + timeout: 60_000, + }) + .toBeLessThan(distance(camera, picture) / 2); + }); +}); + test.describe("background effects section", () => { test("section stays within the call area at every size", async ({ page }) => { test.setTimeout(120_000); diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index 767aec39a..23b4059a1 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -141,6 +141,7 @@ export const Default: Story = { backgroundImageRefusal: undefined, addBackgroundImage: fn(), removeBackgroundEffect: fn(), + cameraTrack: null, hangup: fn(), buttonSize: "lg", showFooter: true, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index d49771bc8..3873719ca 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { type FC, type JSX, type Ref, useMemo } from "react"; import classNames from "classnames"; +import { type LocalVideoTrack } from "livekit-client"; import { useTranslation } from "react-i18next"; import LogoMark from "../icons/LogoMark.svg?react"; @@ -33,6 +34,7 @@ import { useBehavior } from "../useBehavior"; import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; import { LayoutSwitch } from "../room/LayoutSwitch"; import { type BackgroundEffectOption } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; import { type UnusableReason } from "../livekit/backgroundImages"; export interface BackgroundImageRefusal { @@ -102,6 +104,8 @@ export interface FooterState { backgroundEffectSettling: boolean; /** Why the last file offered couldn't be kept, if it couldn't. */ backgroundImageRefusal: BackgroundImageRefusal | undefined; + /** The camera to preview, with the effect in force; null while it is off. */ + cameraTrack: LocalVideoTrack | null; showFooter: boolean; /* This is needed for WindowMode = "flat" */ @@ -191,6 +195,7 @@ export const CallFooter: FC = ({ const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); const addBackgroundImage = useBehavior(vm.addBackgroundImage$); const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$); + const cameraTrack = useBehavior(vm.cameraTrack$); const refusal = useBehavior(vm.backgroundImageRefusal$); const refusalMessage = useMemo(() => { switch (refusal?.reason) { @@ -276,6 +281,7 @@ export const CallFooter: FC = ({ backgroundEffectSettling={backgroundEffectSettling} onAddBackgroundImage={addBackgroundImage} onRemoveBackgroundEffect={removeBackgroundEffect} + selfPreview={} backgroundImageRefusal={refusalMessage} backgroundEffectNotice={ backgroundEffectNotice === "unavailable" diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index 2d2cbd30f..a4cf9b8b3 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; +import { type LocalVideoTrack } from "livekit-client"; import { flushPromises, @@ -24,6 +25,9 @@ import { createLobbyFooterViewModel, } from "./CallFooterViewModel"; import { HeaderStyle } from "../UrlParams"; +import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext"; +import { SyncedCameraTrack } from "../livekit/cameraTrack"; +import { type MuteStates } from "../state/MuteStates"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { shippedBackgrounds } from "../livekit/backgroundEffects"; import type * as BackgroundImages from "../livekit/backgroundImages"; @@ -113,6 +117,11 @@ const pipLayout: Layout = { spotlight: {} as SpotlightTileViewModel, }; +const noPipeline = constant({ + supported: false, + processor: undefined, +}); + const twoMicsAndOneCamMediaDevices = mockMediaDevices({ audioInput: { available$: constant( @@ -144,7 +153,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -187,7 +196,7 @@ describe("createCallFooterViewModel", () => { select: vi.fn(), }, }), - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -206,7 +215,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(layout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -229,7 +238,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); @@ -275,7 +284,7 @@ describe("createCallFooterViewModel", () => { testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, @@ -360,18 +369,22 @@ describe("createCallFooterViewModel", () => { it("passes on the wait for the first effect", () => { platformMock.mockReturnValue("desktop"); - const settling$ = new BehaviorSubject(true); + const pipeline$ = new BehaviorSubject({ + supported: true, + processor: undefined, + settling: true, + }); const vm = createLobbyFooterViewModel( testScope(), mockMuteStates(), twoMicsAndOneCamMediaDevices, - settling$, + pipeline$, /* openSettings */ undefined, /* hangup */ undefined, /* showLogo */ false, ); expect(vm.backgroundEffectSettling$.value).toBe(true); - settling$.next(false); + pipeline$.next({ ...pipeline$.value, settling: false }); expect(vm.backgroundEffectSettling$.value).toBe(false); }); @@ -431,6 +444,33 @@ describe("createCallFooterViewModel", () => { expect(store.remove).toHaveBeenCalledWith("spare"); }); + it("previews the camera only while it is on", () => { + const camera = new SyncedCameraTrack(); + const track = {} as LocalVideoTrack; + camera.report(track); + const videoEnabled$ = new BehaviorSubject(true); + const muteStates = mockMuteStates(); + const vm = createLobbyFooterViewModel( + testScope(), + { + ...muteStates, + video: { ...muteStates.video, enabled$: videoEnabled$ }, + } as unknown as MuteStates, + twoMicsAndOneCamMediaDevices, + constant({ + supported: true, + processor: undefined, + cameraTrack: camera, + }), + /* openSettings */ undefined, + /* hangup */ undefined, + /* showLogo */ false, + ); + expect(vm.cameraTrack$.value).toBe(track); + videoEnabled$.next(false); + expect(vm.cameraTrack$.value).toBeNull(); + }); + it("availability is the same before and during a call", () => { for (const supported of [true, false]) { sdkSupportMock.mockReturnValue(supported); @@ -440,7 +480,7 @@ describe("createCallFooterViewModel", () => { buildMinimalCallViewModel(gridLayout), mockMuteStates(), twoMicsAndOneCamMediaDevices, - constant(false), + noPipeline, /* reactionIdentifier */ undefined, { showControls: true, header: HeaderStyle.Standard }, ); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 7702e6526..ca9f780de 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -11,6 +11,7 @@ import { distinctUntilChanged, map, type Observable, + of, switchMap, } from "rxjs"; import { supportsAudioOutputSelection } from "livekit-client"; @@ -25,6 +26,7 @@ import { serializeEffect, shippedBackgrounds, } from "../livekit/backgroundEffects"; +import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext"; import { type AddedBackground, addedBackgrounds, @@ -90,7 +92,8 @@ function buildDeviceBehaviors( mediaDevices: MediaDevices, /** return empty arrays for audioOptions and videoOptions*/ disableSwitcher$: Behavior, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, + videoEnabled$: Behavior, ): Pick< ViewModel, | "audioOptions$" @@ -110,6 +113,7 @@ function buildDeviceBehaviors( | "addBackgroundImage$" | "backgroundImageRefusal$" | "removeBackgroundEffect$" + | "cameraTrack$" > { const options$ = ( available$: Behavior>, @@ -189,7 +193,10 @@ function buildDeviceBehaviors( ), ), backgroundEffectSettling$: scope.behavior( - backgroundEffectSettling$.pipe(distinctUntilChanged()), + backgroundEffectsState$.pipe( + map(({ settling }) => settling ?? false), + distinctUntilChanged(), + ), false, ), // Kept and offered, not put on: that waits for the user to choose it. @@ -211,6 +218,15 @@ function buildDeviceBehaviors( ), ), backgroundImageRefusal$: refusal$, + cameraTrack$: scope.behavior( + combineLatest([ + backgroundEffectsState$.pipe( + switchMap(({ cameraTrack }) => cameraTrack?.track$ ?? of(null)), + ), + videoEnabled$, + ]).pipe(map(([track, videoEnabled]) => (videoEnabled ? track : null))), + null, + ), // Never the one in force: taking it away would leave nothing chosen. removeBackgroundEffect$: scope.behavior( offered$.pipe( @@ -240,8 +256,8 @@ function buildDeviceBehaviors( * @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. - * @param backgroundEffectSettling$ - Whether the first effect chosen is still - * being prepared. + * @param backgroundEffectsState$ - The background pipeline's state: whether effects + * run here, the wait for the first, and the camera it is synced to. * @param reactionIdentifier - The local user's reaction identifier string, or * undefined when reactions are not supported (hides the reaction button). * @param options - `showControls`: whether the call controls should be shown. @@ -252,7 +268,7 @@ export function createCallFooterViewModel( callModel: CallViewModel, muteStates: MuteStates, mediaDevices: MediaDevices, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, reactionIdentifier: string | undefined, options: { showControls: boolean; header: HeaderStyle }, ): ViewModel { @@ -271,7 +287,8 @@ export function createCallFooterViewModel( scope, mediaDevices, disableDeviceSwitcher$, - backgroundEffectSettling$, + backgroundEffectsState$, + muteStates.video.enabled$, ), // candidat to move into the FooterViewModel showFooter$: callModel.showFooter$, @@ -334,8 +351,8 @@ export function createCallFooterViewModel( * @param scope - ObservableScope that bounds the lifetime of derived behaviors. * @param muteStates - Audio and video mute state + toggles. * @param mediaDevices - Available and selected input devices. - * @param backgroundEffectSettling$ - Whether the first effect chosen is still - * being prepared. + * @param backgroundEffectsState$ - The background pipeline's state: whether effects + * run here, the wait for the first, and the camera it is synced to. * @param openSettings - Callback to open the settings modal, or undefined. * @param hangup - Callback to leave/cancel, or undefined (hides the button). * @param showLogo - Whether to show the Element Call logo. @@ -344,7 +361,7 @@ export function createLobbyFooterViewModel( scope: ObservableScope, muteStates: MuteStates, mediaDevices: MediaDevices, - backgroundEffectSettling$: Observable, + backgroundEffectsState$: Observable, openSettings: (() => void) | undefined, hangup: (() => void) | undefined, showLogo: boolean, @@ -392,7 +409,8 @@ export function createLobbyFooterViewModel( scope, mediaDevices, constant(false), - backgroundEffectSettling$, + backgroundEffectsState$, + muteStates.video.enabled$, ), }; } diff --git a/src/components/MediaMuteAndSwitchButton.module.css b/src/components/MediaMuteAndSwitchButton.module.css index 73a24f627..9e12cad34 100644 --- a/src/components/MediaMuteAndSwitchButton.module.css +++ b/src/components/MediaMuteAndSwitchButton.module.css @@ -129,3 +129,35 @@ Please see LICENSE in the repository root for full details. .refusal { margin: var(--cpd-space-2x) var(--cpd-space-4x) 0; } + +.selfPreview { + position: relative; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-2x); + overflow: clip; + border-start-start-radius: var(--cpd-space-3x); + border-start-end-radius: var(--cpd-space-3x); + background: var(--cpd-color-bg-subtle-secondary); + color: var(--cpd-color-icon-secondary); +} + +.selfPreview > video { + position: absolute; + inset: 0; + inline-size: 100%; + block-size: 100%; + object-fit: cover; + /* Rounded itself too: a video can be drawn on a layer of its own, which the + frame's rounded clip doesn't always reach. */ + border-radius: inherit; +} + +/* Up to the menu's frame, over the padding at its top. */ +.selfPreviewPinned, +.deviceListLeadsWithPreview { + margin-block-start: calc(-1 * var(--cpd-space-5x)); +} diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index c58eb1464..9dba9a48a 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -12,6 +12,7 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton"; import styles from "./MediaMuteAndSwitchButton.module.css"; import { type BackgroundEffectOption } from "./BackgroundEffectGrid"; +import { SelfPreview } from "./SelfPreview"; import { shippedBackgrounds } from "../livekit/backgroundEffects"; import meterStyles from "./MicrophoneLevelMeter.module.css"; import { MediaDevicesContext } from "../MediaDevicesContext"; @@ -60,6 +61,50 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => { return <>{children}; }; +/** A camera for the preview: a figure against a room, drawn to a stream. */ +const FakeCamera: FC = () => { + const [stream, setStream] = useState(); + useEffect(() => { + const canvas = document.createElement("canvas"); + canvas.width = 640; + canvas.height = 360; + const context = canvas.getContext("2d")!; + const draw = (): void => { + context.fillStyle = "#6b7f8e"; + context.fillRect(0, 0, 640, 360); + context.fillStyle = "#c9a27e"; + context.beginPath(); + context.arc(320, 150, 62, 0, 2 * Math.PI); + context.fill(); + context.fillStyle = "#34495e"; + context.beginPath(); + context.ellipse(320, 380, 150, 140, 0, Math.PI, 2 * Math.PI); + context.fill(); + }; + draw(); + // Redrawn, so the stream keeps delivering frames. + const timer = setInterval(draw, 200); + const captured = canvas.captureStream(10); + setStream(captured); + return (): void => { + clearInterval(timer); + captured.getTracks().forEach((track) => track.stop()); + }; + }, []); + return ( +