Show the user their camera at the top of the camera menu

- The camera menu leads with the user's own camera, with the effect in
  force, full width at sixteen by nine: a background is judged by how it
  looks rather than by its thumbnail.
- It shows the pipeline's own track again, not a second capture. Each
  provider keeps the camera its pipeline is synced to, before joining and
  after, and hands it out with its state, so two calls on a page keep
  their own. It is reported again only when it changes, which keeps the
  preview from blanking on every change of state.
- It and the pre-join preview share useAttachedTrack, which detaches only
  its own element: the pre-join preview detached every element showing
  its track, the menu's among them.
- Paid for out of the list's share of the call, so a menu with it is no
  taller: at the top where the list keeps its floor after paying, else
  first among the options, scrolling away with them. Its size comes from
  the tokens the stylesheets draw it with, so it is placed once.
- While the first effect is prepared the preview shows the wait; with the
  camera off it says so, and an effect can still be chosen.
- The video is rounded itself as well as clipped by its frame: Firefox
  composites it on a layer of its own, which the frame's rounded clip
  didn't reach at 100% zoom. Confirmed by hand, as headless Firefox draws
  it rounded either way.
- Every camera story shows the preview, with a stand-in camera drawn to a
  stream. End to end on Chromium and Firefox, the lobby's menu shows the
  camera and then the chosen picture.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
fkwp
2026-10-08 14:44:24 +02:00
co-authored by Claude Opus 5.5
parent a9fd753fbb
commit 989bdc6815
20 changed files with 640 additions and 52 deletions
+27
View File
@@ -106,6 +106,33 @@ async function drawsPerSecond(page: Page): Promise<number> {
}); });
} }
test.describe("the camera menu's preview", () => {
test("shows the camera with the effect in force", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Preview user", "Preview in the menu");
const camera = await cameraColour(page.locator("video").first());
await page.getByRole("button", { name: "Camera", exact: true }).click();
const preview = page.getByRole("menu").locator("video");
await expect(preview).toBeVisible();
await expect
.poll(async () => distance(await averageColour(preview), camera), {
timeout: 20_000,
})
.toBeLessThan(30);
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
});
});
test.describe("background effects section", () => { test.describe("background effects section", () => {
test("section stays within the call area at every size", async ({ page }) => { test("section stays within the call area at every size", async ({ page }) => {
test.setTimeout(120_000); test.setTimeout(120_000);
+1
View File
@@ -141,6 +141,7 @@ export const Default: Story = {
backgroundImageRefusal: undefined, backgroundImageRefusal: undefined,
addBackgroundImage: fn(), addBackgroundImage: fn(),
removeBackgroundEffect: fn(), removeBackgroundEffect: fn(),
cameraTrack: null,
hangup: fn(), hangup: fn(),
buttonSize: "lg", buttonSize: "lg",
showFooter: true, showFooter: true,
+6
View File
@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
import { type FC, type JSX, type Ref, useMemo } from "react"; import { type FC, type JSX, type Ref, useMemo } from "react";
import classNames from "classnames"; import classNames from "classnames";
import { type LocalVideoTrack } from "livekit-client";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import LogoMark from "../icons/LogoMark.svg?react"; import LogoMark from "../icons/LogoMark.svg?react";
@@ -33,6 +34,7 @@ import { useBehavior } from "../useBehavior";
import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel";
import { LayoutSwitch } from "../room/LayoutSwitch"; import { LayoutSwitch } from "../room/LayoutSwitch";
import { type BackgroundEffectOption } from "./BackgroundEffectGrid"; import { type BackgroundEffectOption } from "./BackgroundEffectGrid";
import { SelfPreview } from "./SelfPreview";
import { type UnusableReason } from "../livekit/backgroundImages"; import { type UnusableReason } from "../livekit/backgroundImages";
export interface BackgroundImageRefusal { export interface BackgroundImageRefusal {
@@ -102,6 +104,8 @@ export interface FooterState {
backgroundEffectSettling: boolean; backgroundEffectSettling: boolean;
/** Why the last file offered couldn't be kept, if it couldn't. */ /** Why the last file offered couldn't be kept, if it couldn't. */
backgroundImageRefusal: BackgroundImageRefusal | undefined; backgroundImageRefusal: BackgroundImageRefusal | undefined;
/** The camera to preview, with the effect in force; null while it is off. */
cameraTrack: LocalVideoTrack | null;
showFooter: boolean; showFooter: boolean;
/* This is needed for WindowMode = "flat" */ /* This is needed for WindowMode = "flat" */
@@ -191,6 +195,7 @@ export const CallFooter: FC<FooterProps> = ({
const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$); const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$);
const addBackgroundImage = useBehavior(vm.addBackgroundImage$); const addBackgroundImage = useBehavior(vm.addBackgroundImage$);
const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$); const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$);
const cameraTrack = useBehavior(vm.cameraTrack$);
const refusal = useBehavior(vm.backgroundImageRefusal$); const refusal = useBehavior(vm.backgroundImageRefusal$);
const refusalMessage = useMemo(() => { const refusalMessage = useMemo(() => {
switch (refusal?.reason) { switch (refusal?.reason) {
@@ -276,6 +281,7 @@ export const CallFooter: FC<FooterProps> = ({
backgroundEffectSettling={backgroundEffectSettling} backgroundEffectSettling={backgroundEffectSettling}
onAddBackgroundImage={addBackgroundImage} onAddBackgroundImage={addBackgroundImage}
onRemoveBackgroundEffect={removeBackgroundEffect} onRemoveBackgroundEffect={removeBackgroundEffect}
selfPreview={<SelfPreview track={cameraTrack} />}
backgroundImageRefusal={refusalMessage} backgroundImageRefusal={refusalMessage}
backgroundEffectNotice={ backgroundEffectNotice={
backgroundEffectNotice === "unavailable" backgroundEffectNotice === "unavailable"
+49 -9
View File
@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
import { afterEach, describe, expect, it, vi } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
import { BehaviorSubject } from "rxjs"; import { BehaviorSubject } from "rxjs";
import { type LocalVideoTrack } from "livekit-client";
import { import {
flushPromises, flushPromises,
@@ -24,6 +25,9 @@ import {
createLobbyFooterViewModel, createLobbyFooterViewModel,
} from "./CallFooterViewModel"; } from "./CallFooterViewModel";
import { HeaderStyle } from "../UrlParams"; import { HeaderStyle } from "../UrlParams";
import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext";
import { SyncedCameraTrack } from "../livekit/cameraTrack";
import { type MuteStates } from "../state/MuteStates";
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
import { shippedBackgrounds } from "../livekit/backgroundEffects"; import { shippedBackgrounds } from "../livekit/backgroundEffects";
import type * as BackgroundImages from "../livekit/backgroundImages"; import type * as BackgroundImages from "../livekit/backgroundImages";
@@ -113,6 +117,11 @@ const pipLayout: Layout = {
spotlight: {} as SpotlightTileViewModel, spotlight: {} as SpotlightTileViewModel,
}; };
const noPipeline = constant<BackgroundEffectsState>({
supported: false,
processor: undefined,
});
const twoMicsAndOneCamMediaDevices = mockMediaDevices({ const twoMicsAndOneCamMediaDevices = mockMediaDevices({
audioInput: { audioInput: {
available$: constant( available$: constant(
@@ -144,7 +153,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout), buildMinimalCallViewModel(gridLayout),
mockMuteStates(), mockMuteStates(),
twoMicsAndOneCamMediaDevices, twoMicsAndOneCamMediaDevices,
constant(false), noPipeline,
/* reactionIdentifier */ undefined, /* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard }, { showControls: true, header: HeaderStyle.Standard },
); );
@@ -187,7 +196,7 @@ describe("createCallFooterViewModel", () => {
select: vi.fn(), select: vi.fn(),
}, },
}), }),
constant(false), noPipeline,
/* reactionIdentifier */ undefined, /* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard }, { showControls: true, header: HeaderStyle.Standard },
); );
@@ -206,7 +215,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(layout), buildMinimalCallViewModel(layout),
mockMuteStates(), mockMuteStates(),
twoMicsAndOneCamMediaDevices, twoMicsAndOneCamMediaDevices,
constant(false), noPipeline,
/* reactionIdentifier */ undefined, /* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard }, { showControls: true, header: HeaderStyle.Standard },
); );
@@ -229,7 +238,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout), buildMinimalCallViewModel(gridLayout),
mockMuteStates(), mockMuteStates(),
twoMicsAndOneCamMediaDevices, twoMicsAndOneCamMediaDevices,
constant(false), noPipeline,
/* reactionIdentifier */ undefined, /* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard }, { showControls: true, header: HeaderStyle.Standard },
); );
@@ -275,7 +284,7 @@ describe("createCallFooterViewModel", () => {
testScope(), testScope(),
mockMuteStates(), mockMuteStates(),
twoMicsAndOneCamMediaDevices, twoMicsAndOneCamMediaDevices,
constant(false), noPipeline,
/* openSettings */ undefined, /* openSettings */ undefined,
/* hangup */ undefined, /* hangup */ undefined,
/* showLogo */ false, /* showLogo */ false,
@@ -360,18 +369,22 @@ describe("createCallFooterViewModel", () => {
it("passes on the wait for the first effect", () => { it("passes on the wait for the first effect", () => {
platformMock.mockReturnValue("desktop"); platformMock.mockReturnValue("desktop");
const settling$ = new BehaviorSubject(true); const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
settling: true,
});
const vm = createLobbyFooterViewModel( const vm = createLobbyFooterViewModel(
testScope(), testScope(),
mockMuteStates(), mockMuteStates(),
twoMicsAndOneCamMediaDevices, twoMicsAndOneCamMediaDevices,
settling$, pipeline$,
/* openSettings */ undefined, /* openSettings */ undefined,
/* hangup */ undefined, /* hangup */ undefined,
/* showLogo */ false, /* showLogo */ false,
); );
expect(vm.backgroundEffectSettling$.value).toBe(true); expect(vm.backgroundEffectSettling$.value).toBe(true);
settling$.next(false); pipeline$.next({ ...pipeline$.value, settling: false });
expect(vm.backgroundEffectSettling$.value).toBe(false); expect(vm.backgroundEffectSettling$.value).toBe(false);
}); });
@@ -431,6 +444,33 @@ describe("createCallFooterViewModel", () => {
expect(store.remove).toHaveBeenCalledWith("spare"); expect(store.remove).toHaveBeenCalledWith("spare");
}); });
it("previews the camera only while it is on", () => {
const camera = new SyncedCameraTrack();
const track = {} as LocalVideoTrack;
camera.report(track);
const videoEnabled$ = new BehaviorSubject(true);
const muteStates = mockMuteStates();
const vm = createLobbyFooterViewModel(
testScope(),
{
...muteStates,
video: { ...muteStates.video, enabled$: videoEnabled$ },
} as unknown as MuteStates,
twoMicsAndOneCamMediaDevices,
constant({
supported: true,
processor: undefined,
cameraTrack: camera,
}),
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
expect(vm.cameraTrack$.value).toBe(track);
videoEnabled$.next(false);
expect(vm.cameraTrack$.value).toBeNull();
});
it("availability is the same before and during a call", () => { it("availability is the same before and during a call", () => {
for (const supported of [true, false]) { for (const supported of [true, false]) {
sdkSupportMock.mockReturnValue(supported); sdkSupportMock.mockReturnValue(supported);
@@ -440,7 +480,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout), buildMinimalCallViewModel(gridLayout),
mockMuteStates(), mockMuteStates(),
twoMicsAndOneCamMediaDevices, twoMicsAndOneCamMediaDevices,
constant(false), noPipeline,
/* reactionIdentifier */ undefined, /* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard }, { showControls: true, header: HeaderStyle.Standard },
); );
+28 -10
View File
@@ -11,6 +11,7 @@ import {
distinctUntilChanged, distinctUntilChanged,
map, map,
type Observable, type Observable,
of,
switchMap, switchMap,
} from "rxjs"; } from "rxjs";
import { supportsAudioOutputSelection } from "livekit-client"; import { supportsAudioOutputSelection } from "livekit-client";
@@ -25,6 +26,7 @@ import {
serializeEffect, serializeEffect,
shippedBackgrounds, shippedBackgrounds,
} from "../livekit/backgroundEffects"; } from "../livekit/backgroundEffects";
import { type BackgroundEffectsState } from "../livekit/BackgroundEffectsContext";
import { import {
type AddedBackground, type AddedBackground,
addedBackgrounds, addedBackgrounds,
@@ -90,7 +92,8 @@ function buildDeviceBehaviors(
mediaDevices: MediaDevices, mediaDevices: MediaDevices,
/** return empty arrays for audioOptions and videoOptions*/ /** return empty arrays for audioOptions and videoOptions*/
disableSwitcher$: Behavior<boolean>, disableSwitcher$: Behavior<boolean>,
backgroundEffectSettling$: Observable<boolean>, backgroundEffectsState$: Observable<BackgroundEffectsState>,
videoEnabled$: Behavior<boolean>,
): Pick< ): Pick<
ViewModel<FooterSnapshot>, ViewModel<FooterSnapshot>,
| "audioOptions$" | "audioOptions$"
@@ -110,6 +113,7 @@ function buildDeviceBehaviors(
| "addBackgroundImage$" | "addBackgroundImage$"
| "backgroundImageRefusal$" | "backgroundImageRefusal$"
| "removeBackgroundEffect$" | "removeBackgroundEffect$"
| "cameraTrack$"
> { > {
const options$ = ( const options$ = (
available$: Behavior<Map<string, MenuOptions["label"]>>, available$: Behavior<Map<string, MenuOptions["label"]>>,
@@ -189,7 +193,10 @@ function buildDeviceBehaviors(
), ),
), ),
backgroundEffectSettling$: scope.behavior( backgroundEffectSettling$: scope.behavior(
backgroundEffectSettling$.pipe(distinctUntilChanged()), backgroundEffectsState$.pipe(
map(({ settling }) => settling ?? false),
distinctUntilChanged(),
),
false, false,
), ),
// Kept and offered, not put on: that waits for the user to choose it. // Kept and offered, not put on: that waits for the user to choose it.
@@ -211,6 +218,15 @@ function buildDeviceBehaviors(
), ),
), ),
backgroundImageRefusal$: refusal$, backgroundImageRefusal$: refusal$,
cameraTrack$: scope.behavior(
combineLatest([
backgroundEffectsState$.pipe(
switchMap(({ cameraTrack }) => cameraTrack?.track$ ?? of(null)),
),
videoEnabled$,
]).pipe(map(([track, videoEnabled]) => (videoEnabled ? track : null))),
null,
),
// Never the one in force: taking it away would leave nothing chosen. // Never the one in force: taking it away would leave nothing chosen.
removeBackgroundEffect$: scope.behavior( removeBackgroundEffect$: scope.behavior(
offered$.pipe( offered$.pipe(
@@ -240,8 +256,8 @@ function buildDeviceBehaviors(
* @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc. * @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc.
* @param muteStates - Audio and video mute state + toggles. * @param muteStates - Audio and video mute state + toggles.
* @param mediaDevices - Available and selected input devices. * @param mediaDevices - Available and selected input devices.
* @param backgroundEffectSettling$ - Whether the first effect chosen is still * @param backgroundEffectsState$ - The background pipeline's state: whether effects
* being prepared. * run here, the wait for the first, and the camera it is synced to.
* @param reactionIdentifier - The local user's reaction identifier string, or * @param reactionIdentifier - The local user's reaction identifier string, or
* undefined when reactions are not supported (hides the reaction button). * undefined when reactions are not supported (hides the reaction button).
* @param options - `showControls`: whether the call controls should be shown. * @param options - `showControls`: whether the call controls should be shown.
@@ -252,7 +268,7 @@ export function createCallFooterViewModel(
callModel: CallViewModel, callModel: CallViewModel,
muteStates: MuteStates, muteStates: MuteStates,
mediaDevices: MediaDevices, mediaDevices: MediaDevices,
backgroundEffectSettling$: Observable<boolean>, backgroundEffectsState$: Observable<BackgroundEffectsState>,
reactionIdentifier: string | undefined, reactionIdentifier: string | undefined,
options: { showControls: boolean; header: HeaderStyle }, options: { showControls: boolean; header: HeaderStyle },
): ViewModel<FooterSnapshot> { ): ViewModel<FooterSnapshot> {
@@ -271,7 +287,8 @@ export function createCallFooterViewModel(
scope, scope,
mediaDevices, mediaDevices,
disableDeviceSwitcher$, disableDeviceSwitcher$,
backgroundEffectSettling$, backgroundEffectsState$,
muteStates.video.enabled$,
), ),
// candidat to move into the FooterViewModel // candidat to move into the FooterViewModel
showFooter$: callModel.showFooter$, showFooter$: callModel.showFooter$,
@@ -334,8 +351,8 @@ export function createCallFooterViewModel(
* @param scope - ObservableScope that bounds the lifetime of derived behaviors. * @param scope - ObservableScope that bounds the lifetime of derived behaviors.
* @param muteStates - Audio and video mute state + toggles. * @param muteStates - Audio and video mute state + toggles.
* @param mediaDevices - Available and selected input devices. * @param mediaDevices - Available and selected input devices.
* @param backgroundEffectSettling$ - Whether the first effect chosen is still * @param backgroundEffectsState$ - The background pipeline's state: whether effects
* being prepared. * run here, the wait for the first, and the camera it is synced to.
* @param openSettings - Callback to open the settings modal, or undefined. * @param openSettings - Callback to open the settings modal, or undefined.
* @param hangup - Callback to leave/cancel, or undefined (hides the button). * @param hangup - Callback to leave/cancel, or undefined (hides the button).
* @param showLogo - Whether to show the Element Call logo. * @param showLogo - Whether to show the Element Call logo.
@@ -344,7 +361,7 @@ export function createLobbyFooterViewModel(
scope: ObservableScope, scope: ObservableScope,
muteStates: MuteStates, muteStates: MuteStates,
mediaDevices: MediaDevices, mediaDevices: MediaDevices,
backgroundEffectSettling$: Observable<boolean>, backgroundEffectsState$: Observable<BackgroundEffectsState>,
openSettings: (() => void) | undefined, openSettings: (() => void) | undefined,
hangup: (() => void) | undefined, hangup: (() => void) | undefined,
showLogo: boolean, showLogo: boolean,
@@ -392,7 +409,8 @@ export function createLobbyFooterViewModel(
scope, scope,
mediaDevices, mediaDevices,
constant(false), constant(false),
backgroundEffectSettling$, backgroundEffectsState$,
muteStates.video.enabled$,
), ),
}; };
} }
@@ -129,3 +129,35 @@ Please see LICENSE in the repository root for full details.
.refusal { .refusal {
margin: var(--cpd-space-2x) var(--cpd-space-4x) 0; margin: var(--cpd-space-2x) var(--cpd-space-4x) 0;
} }
.selfPreview {
position: relative;
display: flex;
align-items: center;
justify-content: center;
flex-shrink: 0;
aspect-ratio: 16 / 9;
margin-block-end: var(--cpd-space-2x);
overflow: clip;
border-start-start-radius: var(--cpd-space-3x);
border-start-end-radius: var(--cpd-space-3x);
background: var(--cpd-color-bg-subtle-secondary);
color: var(--cpd-color-icon-secondary);
}
.selfPreview > video {
position: absolute;
inset: 0;
inline-size: 100%;
block-size: 100%;
object-fit: cover;
/* Rounded itself too: a video can be drawn on a layer of its own, which the
frame's rounded clip doesn't always reach. */
border-radius: inherit;
}
/* Up to the menu's frame, over the padding at its top. */
.selfPreviewPinned,
.deviceListLeadsWithPreview {
margin-block-start: calc(-1 * var(--cpd-space-5x));
}
@@ -12,6 +12,7 @@ import type { Meta, StoryObj } from "@storybook/react-vite";
import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton"; import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton";
import styles from "./MediaMuteAndSwitchButton.module.css"; import styles from "./MediaMuteAndSwitchButton.module.css";
import { type BackgroundEffectOption } from "./BackgroundEffectGrid"; import { type BackgroundEffectOption } from "./BackgroundEffectGrid";
import { SelfPreview } from "./SelfPreview";
import { shippedBackgrounds } from "../livekit/backgroundEffects"; import { shippedBackgrounds } from "../livekit/backgroundEffects";
import meterStyles from "./MicrophoneLevelMeter.module.css"; import meterStyles from "./MicrophoneLevelMeter.module.css";
import { MediaDevicesContext } from "../MediaDevicesContext"; import { MediaDevicesContext } from "../MediaDevicesContext";
@@ -60,6 +61,50 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => {
return <>{children}</>; return <>{children}</>;
}; };
/** A camera for the preview: a figure against a room, drawn to a stream. */
const FakeCamera: FC = () => {
const [stream, setStream] = useState<MediaStream>();
useEffect(() => {
const canvas = document.createElement("canvas");
canvas.width = 640;
canvas.height = 360;
const context = canvas.getContext("2d")!;
const draw = (): void => {
context.fillStyle = "#6b7f8e";
context.fillRect(0, 0, 640, 360);
context.fillStyle = "#c9a27e";
context.beginPath();
context.arc(320, 150, 62, 0, 2 * Math.PI);
context.fill();
context.fillStyle = "#34495e";
context.beginPath();
context.ellipse(320, 380, 150, 140, 0, Math.PI, 2 * Math.PI);
context.fill();
};
draw();
// Redrawn, so the stream keeps delivering frames.
const timer = setInterval(draw, 200);
const captured = canvas.captureStream(10);
setStream(captured);
return (): void => {
clearInterval(timer);
captured.getTracks().forEach((track) => track.stop());
};
}, []);
return (
<video
data-testid="preview-camera"
autoPlay
muted
playsInline
ref={(video): void => {
if (video && stream && video.srcObject !== stream)
video.srcObject = stream;
}}
/>
);
};
/** Supplies a call-sized root. Without one the list is bounded by the whole Storybook frame. */ /** Supplies a call-sized root. Without one the list is bounded by the whole Storybook frame. */
const WithACallArea: FC<{ children: ReactNode; blockSize?: number }> = ({ const WithACallArea: FC<{ children: ReactNode; blockSize?: number }> = ({
children, children,
@@ -167,8 +212,8 @@ export const VideoMute: Story = {
{ label: { type: "name", name: "Camera 1" }, id: "1" }, { label: { type: "name", name: "Camera 1" }, id: "1" },
{ label: { type: "name", name: "Camera 2" }, id: "2" }, { label: { type: "name", name: "Camera 2" }, id: "2" },
], ],
selectedOption: "1", selectedOption: "1",
selfPreview: <SelfPreview track={null} />,
}, },
}; };
@@ -196,6 +241,8 @@ export const VideoUnmute: Story = {
selectedBackgroundEffect: "none", selectedBackgroundEffect: "none",
onSelectBackgroundEffect: fn(), onSelectBackgroundEffect: fn(),
onAddBackgroundImage: fn(), onAddBackgroundImage: fn(),
// As the footer always gives the camera menu one.
selfPreview: <FakeCamera />,
}, },
}; };
@@ -335,6 +382,9 @@ export const BackgroundEffectsSettling: Story = {
checked: false, checked: false,
})) }))
await expect(other).toHaveAttribute("aria-busy", "false"); await expect(other).toHaveAttribute("aria-busy", "false");
await expect(menu.queryByTestId("preview-camera")).toBeNull();
const frames = document.body.querySelectorAll(`.${styles.selfPreview} svg`);
await expect(frames).toHaveLength(1);
}, },
}; };
@@ -494,6 +544,113 @@ export const RemovingWithLiveSelection: Story = {
}, },
}; };
/** Where the call leaves room, the camera holds the top while the options scroll. */
export const BackgroundEffectsWithPreview: Story = {
args: {
...VideoUnmute.args,
// Enough that the list fills its share, so the preview's cost counts.
options: Array.from({ length: 8 }, (_, i) => ({
label: { type: "name" as const, name: `Camera ${i + 1}` },
id: `${i + 1}`,
})),
},
decorators: [
(Story): JSX.Element => (
<WithACallArea blockSize={640}>
<Story />
</WithACallArea>
),
],
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const picture = await within(document.body).findByTestId("preview-camera");
const preview = picture.parentElement!;
const list = document.body.querySelector<HTMLElement>(
`.${styles.deviceList}`,
)!;
const frame = document.body
.querySelector("[role='menu']")!
.getBoundingClientRect();
const box = preview.getBoundingClientRect();
await expect(list.contains(preview)).toBe(false);
await expect(box.top - frame.top).toBeLessThanOrEqual(2);
await expect(frame.width - box.width).toBeLessThanOrEqual(2);
await expect(box.width / box.height).toBeCloseTo(16 / 9, 1);
await expect(box.bottom).toBeLessThanOrEqual(
list.getBoundingClientRect().top,
);
const listShare = Math.round(640 * 0.6);
// The list runs on through the menu's padding at its foot, which isn't
// part of its share.
const listEnd =
list.getBoundingClientRect().bottom -
Number.parseFloat(getComputedStyle(list).paddingBlockEnd);
await expect(listEnd - box.top).toBeLessThanOrEqual(listShare + 1);
await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight);
list.scrollTop = list.scrollHeight;
await expect(preview.getBoundingClientRect().top).toBe(box.top);
},
};
/** With the camera off the preview says so, and effects can still be chosen. */
export const BackgroundEffectsPreviewWithTheCameraOff: Story = {
args: {
...VideoUnmute.args,
enabled: false,
selfPreview: <SelfPreview track={null} />,
},
play: async ({ args, canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const menu = within(document.body);
await menu.findByRole("group", { name: "Background effects" });
const preview = document.body.querySelector(`.${styles.selfPreview}`)!;
await expect(preview.querySelector("video")).toBeNull();
await expect(preview.querySelector("svg")).not.toBeNull();
await userEvent.click(menu.getByRole("menuitemradio", { name: "Blur" }));
await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith("blur");
},
};
/** In a call too short to hold it, the camera leads the options and scrolls away. */
export const BackgroundEffectsPreviewScrollsWithTheList: Story = {
args: VideoUnmute.args,
decorators: [
(Story): JSX.Element => (
<WithACallArea blockSize={300}>
<Story />
</WithACallArea>
),
],
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const picture = await within(document.body).findByTestId("preview-camera");
const preview = picture.parentElement!;
const list = document.body.querySelector<HTMLElement>(
`.${styles.deviceList}`,
)!;
await expect(list.contains(preview)).toBe(true);
const cameras = within(document.body).getByRole("group", {
name: "Camera",
});
await expect(
preview.compareDocumentPosition(cameras) &
Node.DOCUMENT_POSITION_FOLLOWING,
).toBeTruthy();
list.scrollTop = list.scrollHeight;
await expect(preview.getBoundingClientRect().bottom).toBeLessThan(
list.getBoundingClientRect().top,
);
},
};
/** In a short call the effects scroll into view with the list. */ /** In a short call the effects scroll into view with the list. */
export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = { export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = {
args: { args: {
+55 -5
View File
@@ -14,10 +14,12 @@ import {
type FC, type FC,
useEffect, useEffect,
type ReactElement, type ReactElement,
type ReactNode,
} from "react"; } from "react";
import { import {
Alert, Alert,
Button, Button,
InlineSpinner,
Menu, Menu,
MenuItem, MenuItem,
MenuScrollArea, MenuScrollArea,
@@ -87,6 +89,8 @@ export interface MediaMuteAndSwitchButtonProps {
onRemoveBackgroundEffect?: (id: string) => void; onRemoveBackgroundEffect?: (id: string) => void;
/** Why the last file offered couldn't be used; a new object each time. */ /** Why the last file offered couldn't be used; a new object each time. */
backgroundImageRefusal?: { text: string }; backgroundImageRefusal?: { text: string };
/** The user's own camera, shown at the top of the camera menu. */
selfPreview?: ReactNode;
/** /**
* For any toggle and option this method will be called. * For any toggle and option this method will be called.
* So toggles need to be implemented by listening here and setting the right toggle item to `enabled` * So toggles need to be implemented by listening here and setting the right toggle item to `enabled`
@@ -103,6 +107,10 @@ const LIST_SHARE_OF_CALL = 0.6;
/** Smallest device list height in px, so a short call still shows more than one device. */ /** Smallest device list height in px, so a short call still shows more than one device. */
const MIN_LIST_HEIGHT = 160; const MIN_LIST_HEIGHT = 160;
/** The menu's width as its stylesheet sets it, to place the preview before
the menu is drawn. */
const MENU_WIDTH = 296;
/** Space kept between the menu and the call area's sides. */ /** Space kept between the menu and the call area's sides. */
const MENU_MARGIN = 16; const MENU_MARGIN = 16;
@@ -124,6 +132,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
onAddBackgroundImage, onAddBackgroundImage,
onRemoveBackgroundEffect, onRemoveBackgroundEffect,
backgroundImageRefusal, backgroundImageRefusal,
selfPreview,
onSelect, onSelect,
}) => { }) => {
// Requested but not yet selected. Keyed by kind too, since Chrome uses // Requested but not yet selected. Keyed by kind too, since Chrome uses
@@ -202,21 +211,59 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
// Measured on the call area: CSS can't size the portalled menu against it. // Measured on the call area: CSS can't size the portalled menu against it.
const rootElement = useRootElement(); const rootElement = useRootElement();
const [listMaxHeight, setListMaxHeight] = useState<number>(); const [listShare, setListShare] = useState<number>();
const [previewBlock, setPreviewBlock] = useState(0);
useEffect(() => { useEffect(() => {
if (!menuOpen) return; if (!menuOpen) return;
// From the tokens the stylesheets size the preview with, so that it is
// placed once rather than measured and moved.
const tokens = getComputedStyle(rootElement);
const px = (token: string): number =>
parseFloat(tokens.getPropertyValue(token)) || 0;
const below = px("--cpd-space-2x") + px("--cpd-space-1x");
// Followed, since a host can resize the call while the menu is open. // Followed, since a host can resize the call while the menu is open.
const subscription = observeElementSize$(rootElement) const subscription = observeElementSize$(rootElement)
.pipe( .pipe(
map(({ height }) => map(({ width, height }) => ({
Math.max(MIN_LIST_HEIGHT, Math.round(height * LIST_SHARE_OF_CALL)), height: Math.round(height * LIST_SHARE_OF_CALL),
width: Math.min(MENU_WIDTH, Math.round(width - 2 * MENU_MARGIN)),
})),
distinctUntilChanged(
(a, b) => a.height === b.height && a.width === b.width,
), ),
distinctUntilChanged(),
) )
.subscribe(setListMaxHeight); .subscribe(({ height, width }) => {
setListShare(height);
setPreviewBlock(Math.round((width * 9) / 16) + below);
});
return (): void => subscription.unsubscribe(); return (): void => subscription.unsubscribe();
}, [menuOpen, rootElement]); }, [menuOpen, rootElement]);
// Paid for out of the list's share, so a preview makes the menu no taller.
const hasPreview =
iconsAndLabels === "video" &&
selfPreview !== undefined &&
listShare !== undefined;
const previewPinned =
hasPreview && listShare - previewBlock >= MIN_LIST_HEIGHT;
const listMaxHeight =
listShare === undefined
? undefined
: Math.max(
MIN_LIST_HEIGHT,
listShare - (previewPinned ? previewBlock : 0),
);
const preview = hasPreview && (
<div
aria-hidden
className={classNames(styles.selfPreview, {
[styles.selfPreviewPinned]: previewPinned,
})}
>
{backgroundEffectSettling ? <InlineSpinner size={32} /> : selfPreview}
</div>
);
// Kept clear at the list's foot, so a row reached by keyboard isn't under // Kept clear at the list's foot, so a row reached by keyboard isn't under
// the meter. // the meter.
const [meterHeight, meter] = useMeasuredHeight(); const [meterHeight, meter] = useMeasuredHeight();
@@ -409,9 +456,11 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
/> />
} }
> >
{previewPinned && preview}
<MenuScrollArea <MenuScrollArea
ref={trackFocusSource} ref={trackFocusSource}
className={classNames(styles.deviceList, { className={classNames(styles.deviceList, {
[styles.deviceListLeadsWithPreview]: hasPreview && !previewPinned,
[styles.deviceListWithMeter]: iconsAndLabels === "audio", [styles.deviceListWithMeter]: iconsAndLabels === "audio",
})} })}
style={ style={
@@ -429,6 +478,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
} as CSSProperties } as CSSProperties
} }
> >
{hasPreview && !previewPinned && preview}
{iconsAndLabels === "audio" && speakerOptions && ( {iconsAndLabels === "audio" && speakerOptions && (
<> <>
{/* A menu may only contain items, separators and groups, so each {/* A menu may only contain items, separators and groups, so each
+10
View File
@@ -0,0 +1,10 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
.mirrored {
transform: scaleX(-1);
}
+36
View File
@@ -0,0 +1,36 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type FC } from "react";
import { type LocalVideoTrack } from "livekit-client";
import { VideoCallOffIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
import styles from "./SelfPreview.module.css";
import { useAttachedTrack } from "../useAttachedTrack";
interface Props {
/** The camera as the pipeline has it, or none while the camera is off. */
track: LocalVideoTrack | null;
}
/**
* The user's own camera with the effect in force: the pipeline's own track,
* shown a second time rather than captured again.
*/
export const SelfPreview: FC<Props> = ({ track }) => {
const { videoRef, mirrored } = useAttachedTrack(track);
if (!track) return <VideoCallOffIcon width={32} height={32} />;
return (
<video
ref={videoRef}
muted
playsInline
className={mirrored ? styles.mirrored : undefined}
/>
);
};
@@ -9,6 +9,7 @@ import { act, createElement, type FC } from "react";
import { render } from "@testing-library/react"; import { render } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { type LocalVideoTrack } from "livekit-client"; import { type LocalVideoTrack } from "livekit-client";
import { BehaviorSubject } from "rxjs";
import { import {
type BackgroundOptions, type BackgroundOptions,
type ProcessorWrapper, type ProcessorWrapper,
@@ -25,6 +26,7 @@ import {
} from "./BackgroundEffectsContext"; } from "./BackgroundEffectsContext";
import { type BackgroundEffects } from "../state/BackgroundEffects"; import { type BackgroundEffects } from "../state/BackgroundEffects";
import { backgroundEffect } from "../settings/settings"; import { backgroundEffect } from "../settings/settings";
import { SyncedCameraTrack } from "./cameraTrack";
import { type AddedBackground } from "./backgroundImages"; import { type AddedBackground } from "./backgroundImages";
import { constant } from "../state/Behavior"; import { constant } from "../state/Behavior";
import { flushPromises, testScope } from "../utils/test"; import { flushPromises, testScope } from "../utils/test";
@@ -136,6 +138,75 @@ describe("syncBackgroundEffects", () => {
}); });
}); });
describe("the synced camera", () => {
it("is the track last synced, and cleared only while still its own", () => {
const camera = new SyncedCameraTrack();
const pipeline = constant<BackgroundEffectsState>({
supported: false,
processor: undefined,
cameraTrack: camera,
});
const lobby = mockTrack("live");
const call = mockTrack("live");
const lobbyTrack$ = new BehaviorSubject<LocalVideoTrack | null>(lobby);
syncBackgroundEffects(testScope(), lobbyTrack$, pipeline);
expect(camera.track$.value).toBe(lobby);
const callScope = testScope();
syncBackgroundEffects(callScope, constant(call), pipeline);
expect(camera.track$.value).toBe(call);
lobbyTrack$.next(null);
expect(camera.track$.value).toBe(call);
callScope.end();
expect(camera.track$.value).toBeNull();
});
it("reports the camera once, however often the pipeline's state changes", () => {
const camera = new SyncedCameraTrack();
const report = vi.spyOn(camera, "report");
const state$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
cameraTrack: camera,
});
syncBackgroundEffects(testScope(), constant(mockTrack("live")), state$);
state$.next({ ...state$.value, settling: true });
state$.next({ ...state$.value, settling: false });
expect(report).toHaveBeenCalledOnce();
});
it("keeps each call's camera to its own", () => {
const seen: (SyncedCameraTrack | undefined)[] = [];
const Call: FC<{ track: LocalVideoTrack }> = ({ track }) => {
const { cameraTrack } = useBackgroundEffects();
seen.push(cameraTrack);
useSyncBackgroundEffects(track);
return null;
};
const first = mockTrack("live");
const second = mockTrack("live");
render(
createElement(
"div",
null,
createElement(BackgroundEffectsProvider, {
effects: createAppBackgroundEffects(testScope()),
children: createElement(Call, { track: first }),
}),
createElement(BackgroundEffectsProvider, {
effects: createAppBackgroundEffects(testScope()),
children: createElement(Call, { track: second }),
}),
),
);
const [mine, theirs] = [...new Set(seen)];
expect(mine).not.toBe(theirs);
expect(mine?.track$.value).toBe(first);
expect(theirs?.track$.value).toBe(second);
});
});
describe("BackgroundEffectsProvider", () => { describe("BackgroundEffectsProvider", () => {
/** A camera track that holds whatever processor it is given. */ /** A camera track that holds whatever processor it is given. */
function cameraTrack(): LocalVideoTrack { function cameraTrack(): LocalVideoTrack {
+33 -2
View File
@@ -14,10 +14,18 @@ import {
import { createContext, type FC, type JSX, use, useEffect } from "react"; import { createContext, type FC, type JSX, use, useEffect } from "react";
import { type LocalVideoTrack } from "livekit-client"; import { type LocalVideoTrack } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger"; import { logger } from "matrix-js-sdk/lib/logger";
import { combineLatest } from "rxjs"; import {
combineLatest,
distinctUntilChanged,
EMPTY,
map,
Observable,
switchMap,
} from "rxjs";
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings"; import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer"; import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
import { type SyncedCameraTrack } from "./cameraTrack";
import { OneStepPipeline } from "./OneStepPipeline"; import { OneStepPipeline } from "./OneStepPipeline";
import { addedBackgrounds } from "./backgroundImages"; import { addedBackgrounds } from "./backgroundImages";
import { type Behavior } from "../state/Behavior"; import { type Behavior } from "../state/Behavior";
@@ -37,6 +45,8 @@ export type BackgroundEffectsState = {
processor: undefined | ProcessorWrapper<BackgroundOptions>; processor: undefined | ProcessorWrapper<BackgroundOptions>;
/** From the first effect chosen until a frame carrying it is drawn. */ /** From the first effect chosen until a frame carrying it is drawn. */
settling?: boolean; settling?: boolean;
/** The camera this provider's pipeline is synced to. */
cameraTrack?: SyncedCameraTrack;
}; };
const BackgroundEffectsContext = createContext<BackgroundEffects | undefined>( const BackgroundEffectsContext = createContext<BackgroundEffects | undefined>(
@@ -102,16 +112,37 @@ export const syncBackgroundEffects = (
if (!videoTrack) return; if (!videoTrack) return;
applyProcessor(videoTrack, backgroundEffectsState.processor); applyProcessor(videoTrack, backgroundEffectsState.processor);
}); });
// Keyed on the camera alone: reporting again for every change of state would
// blank the preview each time.
combineLatest([
videoTrack$,
backgroundEffectsState$.pipe(
map(({ cameraTrack }) => cameraTrack),
distinctUntilChanged(),
),
])
.pipe(
switchMap(([track, cameraTrack]) =>
track === null || cameraTrack === undefined
? EMPTY
: new Observable<never>(() => cameraTrack.report(track)),
),
scope.bind(),
)
.subscribe();
}; };
export const useSyncBackgroundEffects = ( export const useSyncBackgroundEffects = (
videoTrack: LocalVideoTrack | null, videoTrack: LocalVideoTrack | null,
): void => { ): void => {
const { processor } = useBackgroundEffects(); const { processor, cameraTrack } = useBackgroundEffects();
useEffect(() => { useEffect(() => {
if (!videoTrack) return; if (!videoTrack) return;
applyProcessor(videoTrack, processor); applyProcessor(videoTrack, processor);
}, [processor, videoTrack]); }, [processor, videoTrack]);
useEffect(() => {
if (videoTrack && cameraTrack) return cameraTrack.report(videoTrack);
}, [cameraTrack, videoTrack]);
}; };
/** The app's one pipeline, shared by every camera track it opens. */ /** The app's one pipeline, shared by every camera track it opens. */
+28
View File
@@ -0,0 +1,28 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject } from "rxjs";
import { type LocalVideoTrack } from "livekit-client";
import { type Behavior } from "../state/Behavior";
/**
* The camera track a background pipeline is synced to: the preview's before
* joining, the call's after, none while the camera is off.
*/
export class SyncedCameraTrack {
private readonly subject$ = new BehaviorSubject<LocalVideoTrack | null>(null);
public readonly track$: Behavior<LocalVideoTrack | null> = this.subject$;
public report(track: LocalVideoTrack): () => void {
this.subject$.next(track);
// Only if still this one, so a lobby leaving doesn't blank a call starting.
return (): void => {
if (this.subject$.value === track) this.subject$.next(null);
};
}
}
+1 -2
View File
@@ -22,7 +22,6 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import classNames from "classnames"; import classNames from "classnames";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import { map } from "rxjs";
import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
import { HeaderStyle, useUrlParams } from "../UrlParams"; import { HeaderStyle, useUrlParams } from "../UrlParams";
@@ -185,7 +184,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
vm, vm,
props.muteStates, props.muteStates,
mediaDevices, mediaDevices,
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)), backgroundEffectsState$,
`${props.client.getUserId()}:${props.client.getDeviceId()}`, `${props.client.getUserId()}:${props.client.getDeviceId()}`,
{ showControls: urlParams.showControls, header: urlParams.header }, { showControls: urlParams.showControls, header: urlParams.header },
); );
+1 -2
View File
@@ -24,7 +24,6 @@ import {
type LocalVideoTrack, type LocalVideoTrack,
Track, Track,
} from "livekit-client"; } from "livekit-client";
import { map } from "rxjs";
import inCallStyles from "./InCallView.module.css"; import inCallStyles from "./InCallView.module.css";
import styles from "./LobbyView.module.css"; import styles from "./LobbyView.module.css";
@@ -198,7 +197,7 @@ export const LobbyView: FC<Props> = ({
footerScope, footerScope,
muteStates, muteStates,
devices, devices,
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)), backgroundEffectsState$,
openSettings, openSettings,
hangup, hangup,
// Logo and header are connected: only show the logo in SPA with header. // Logo and header are connected: only show the logo in SPA with header.
+6 -20
View File
@@ -5,9 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details. Please see LICENSE in the repository root for full details.
*/ */
import { useEffect, useMemo, useRef, type FC, type ReactNode } from "react"; import { useMemo, type FC, type ReactNode } from "react";
import useMeasure from "react-use-measure"; import useMeasure from "react-use-measure";
import { facingModeFromLocalTrack, type LocalVideoTrack } from "livekit-client"; import { type LocalVideoTrack } from "livekit-client";
import classNames from "classnames"; import classNames from "classnames";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
@@ -15,6 +15,7 @@ import { TileAvatar } from "../tile/TileAvatar";
import styles from "./VideoPreview.module.css"; import styles from "./VideoPreview.module.css";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import videoPlaceholder from "../graphics/video-placeholder.gif"; import videoPlaceholder from "../graphics/video-placeholder.gif";
import { useAttachedTrack } from "../useAttachedTrack";
export type MatrixInfo = { export type MatrixInfo = {
userId: string; userId: string;
@@ -43,17 +44,7 @@ export const VideoPreview: FC<Props> = ({
const { t } = useTranslation(); const { t } = useTranslation();
const [previewRef, previewBounds] = useMeasure(); const [previewRef, previewBounds] = useMeasure();
const videoEl = useRef<HTMLVideoElement | null>(null); const { videoRef, mirrored } = useAttachedTrack(videoTrack);
useEffect(() => {
// Effect to connect the videoTrack with the video element.
if (videoEl.current) {
videoTrack?.attach(videoEl.current);
}
return (): void => {
videoTrack?.detach();
};
}, [videoTrack]);
const cameraIsStarting = useMemo( const cameraIsStarting = useMemo(
() => videoEnabled && !videoTrack, () => videoEnabled && !videoTrack,
@@ -63,13 +54,8 @@ export const VideoPreview: FC<Props> = ({
return ( return (
<div className={classNames(styles.preview)} ref={previewRef}> <div className={classNames(styles.preview)} ref={previewRef}>
<video <video
className={ className={mirrored ? styles.mirror : undefined}
videoTrack && ref={videoRef}
facingModeFromLocalTrack(videoTrack).facingMode === "user"
? styles.mirror
: undefined
}
ref={videoEl}
muted muted
playsInline playsInline
// There's no reason for this to be focusable // There's no reason for this to be focusable
+4
View File
@@ -30,6 +30,7 @@ import {
parseEffect, parseEffect,
} from "../livekit/backgroundEffects"; } from "../livekit/backgroundEffects";
import { type AddedBackground } from "../livekit/backgroundImages"; import { type AddedBackground } from "../livekit/backgroundImages";
import { SyncedCameraTrack } from "../livekit/cameraTrack";
export interface BackgroundEffectsOptions { export interface BackgroundEffectsOptions {
/** Whether this browser can run a pipeline at all. */ /** Whether this browser can run a pipeline at all. */
@@ -63,6 +64,8 @@ export function createBackgroundEffects(
transformer, transformer,
}: BackgroundEffectsOptions, }: BackgroundEffectsOptions,
): BackgroundEffects { ): BackgroundEffects {
// The camera the pipeline is synced to.
const cameraTrack = new SyncedCameraTrack();
const choice$ = effect$.pipe(map(parseEffect)); const choice$ = effect$.pipe(map(parseEffect));
const wanted$ = choice$.pipe( const wanted$ = choice$.pipe(
map((effect) => effect.kind !== "none"), map((effect) => effect.kind !== "none"),
@@ -104,6 +107,7 @@ export function createBackgroundEffects(
supported, supported,
processor: enable ? pipeline : undefined, processor: enable ? pipeline : undefined,
settling: enable && !drewAFrame, settling: enable && !drewAFrame,
cameraTrack,
}; };
}, },
{ supported, processor: undefined }, { supported, processor: undefined },
+59
View File
@@ -0,0 +1,59 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type FC } from "react";
import { render } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";
import { type LocalVideoTrack } from "livekit-client";
import { useAttachedTrack } from "./useAttachedTrack";
vi.mock("livekit-client", () => ({
facingModeFromLocalTrack: (track: { facing: string }): object => ({
facingMode: track.facing,
}),
}));
function cameraTrack(facing: string): LocalVideoTrack {
return {
facing,
attach: vi.fn(),
detach: vi.fn(),
} as unknown as LocalVideoTrack;
}
const Video: FC<{ track: LocalVideoTrack | null }> = ({ track }) => {
const { videoRef, mirrored } = useAttachedTrack(track);
return <video ref={videoRef} data-mirrored={mirrored} />;
};
describe("useAttachedTrack", () => {
it("detaches only its own element from a track shown twice", () => {
const track = cameraTrack("user");
const first = render(<Video track={track} />);
const second = render(<Video track={track} />);
const [a, b] = [first, second].map((r) =>
r.container.querySelector("video"),
);
expect(track.attach).toHaveBeenCalledWith(a);
expect(track.attach).toHaveBeenCalledWith(b);
first.unmount();
expect(track.detach).toHaveBeenCalledExactlyOnceWith(a);
});
it("mirrors a camera facing the user, and only that", () => {
const { container, rerender } = render(
<Video track={cameraTrack("user")} />,
);
expect(container.querySelector("video")?.dataset.mirrored).toBe("true");
rerender(<Video track={cameraTrack("environment")} />);
expect(container.querySelector("video")?.dataset.mirrored).toBe("false");
rerender(<Video track={null} />);
expect(container.querySelector("video")?.dataset.mirrored).toBe("false");
});
});
+34
View File
@@ -0,0 +1,34 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RefObject, useEffect, useRef } from "react";
import { facingModeFromLocalTrack, type LocalVideoTrack } from "livekit-client";
/**
* Shows a camera track in the video element given the returned ref, and says
* whether to mirror it, as a camera facing the user is. Only that element is
* detached again: another may be showing the same track.
*/
export function useAttachedTrack(track: LocalVideoTrack | null): {
videoRef: RefObject<HTMLVideoElement | null>;
mirrored: boolean;
} {
const videoRef = useRef<HTMLVideoElement | null>(null);
useEffect(() => {
const element = videoRef.current;
if (!track || !element) return;
track.attach(element);
return (): void => {
track.detach(element);
};
}, [track]);
return {
videoRef,
mirrored:
track !== null && facingModeFromLocalTrack(track).facingMode === "user",
};
}
+1 -1
View File
@@ -188,7 +188,7 @@ export function getBasicCallViewModelEnvironment(
vm, vm,
muteStates, muteStates,
mediaDevices, mediaDevices,
constant(false), constant({ supported: false, processor: undefined }),
"reactionId", "reactionId",
{ showControls: true, header: HeaderStyle.Standard }, { showControls: true, header: HeaderStyle.Standard },
); );