Compare commits

...
Author SHA1 Message Date
fkwpandClaude Opus 5.5 b74e63c1c0 Show the user their camera at the top of the camera menu
- The camera menu leads with the user's own camera, with the effect in
  force, full width at sixteen by nine: a background is judged by how it
  looks rather than by its thumbnail.
- It shows the pipeline's own track again, not a second capture. Each
  provider keeps the camera its pipeline is synced to, before joining and
  after, and hands it out with its state, so two calls on a page keep
  their own. It is reported again only when it changes, which keeps the
  preview from blanking on every change of state.
- It and the pre-join preview share useAttachedTrack, which detaches only
  its own element: the pre-join preview detached every element showing
  its track, the menu's among them.
- Paid for out of the list's share of the call, so a menu with it is no
  taller: at the top where the list keeps its floor after paying, else
  first among the options, scrolling away with them. Its size comes from
  the tokens the stylesheets draw it with, so it is placed once.
- While the first effect is prepared the preview shows the wait; with the
  camera off it says so, and an effect can still be chosen.
- The video is rounded itself as well as clipped by its frame: Firefox
  composites it on a layer of its own, which the frame's rounded clip
  didn't reach at 100% zoom. Confirmed by hand, as headless Firefox draws
  it rounded either way.
- Every camera story shows the preview, with a stand-in camera drawn to a
  stream. End to end on Chromium and Firefox, the lobby's menu shows the
  camera and then the chosen picture.

Rebased onto main, where the call's camera is the MatrixRTC SDK's track
rather than LiveKit's, and the Publisher that synced it is gone:
- The synced camera is a ShownCamera, which LiveKit's preview track and
  the SDK's call track both fill; useAttachedTrack and SelfPreview take
  that.
- CallViewModel exposes camera$, and the call view reports it with
  reportCamera, the same reporting the lobby's sync uses. An end-to-end
  test checks the call's menu shows the call's camera, then the picture.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-11-06 10:58:46 +01:00
23 changed files with 755 additions and 58 deletions
+60
View File
@@ -106,6 +106,66 @@ async function drawsPerSecond(page: Page): Promise<number> {
});
}
test.describe("the camera menu's preview", () => {
test("shows the camera with the effect in force", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(page, "Preview user", "Preview in the menu");
const camera = await cameraColour(page.locator("video").first());
await page.getByRole("button", { name: "Camera", exact: true }).click();
const preview = page.getByRole("menu").locator("video");
await expect(preview).toBeVisible();
await expect
.poll(async () => distance(await averageColour(preview), camera), {
timeout: 20_000,
})
.toBeLessThan(30);
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
});
test("shows the call's own camera once joined", async ({ page }) => {
test.setTimeout(120_000);
await page.goto("/");
await SpaHelpers.createCall(
page,
"Call preview user",
"Preview in the call's menu",
true,
);
await expect(page.getByTestId("incall_videomute")).toBeEnabled({
timeout: 10_000,
});
const camera = await cameraColour(page.locator("video").first());
await page.getByRole("button", { name: "Camera", exact: true }).click();
const preview = page.getByRole("menu").locator("video");
await expect(preview).toBeVisible();
await expect
.poll(async () => distance(await averageColour(preview), camera), {
timeout: 20_000,
})
.toBeLessThan(30);
const tile = page.getByRole("menuitemradio", { name: "Background 1" });
await tile.click();
const picture = await averageColour(tile.locator("img"));
await expect
.poll(async () => distance(await averageColour(preview), picture), {
timeout: 60_000,
})
.toBeLessThan(distance(camera, picture) / 2);
});
});
test.describe("background effects section", () => {
test("section stays within the call area at every size", async ({ page }) => {
test.setTimeout(120_000);
+1
View File
@@ -141,6 +141,7 @@ export const Default: Story = {
backgroundImageRefusal: undefined,
addBackgroundImage: fn(),
removeBackgroundEffect: fn(),
cameraTrack: null,
hangup: fn(),
buttonSize: "lg",
showFooter: true,
+6
View File
@@ -33,7 +33,9 @@ import { useBehavior } from "../useBehavior";
import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel";
import { LayoutSwitch } from "../room/LayoutSwitch";
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
import { SelfPreview } from "./SelfPreview";
import { type UnusableReason } from "../tracks/backgroundImages";
import { type ShownCamera } from "../tracks/cameraTrack";
export interface BackgroundImageRefusal {
reason: UnusableReason | "not-kept";
@@ -92,6 +94,8 @@ export interface FooterState {
backgroundEffectSettling: boolean;
/** Why the last file offered couldn't be kept, if it couldn't. */
backgroundImageRefusal: BackgroundImageRefusal | undefined;
/** The camera to preview, with the effect in force; null while it is off. */
cameraTrack: ShownCamera | null;
showFooter: boolean;
/* This is needed for WindowMode = "flat" */
@@ -181,6 +185,7 @@ export const CallFooter: FC<FooterProps> = ({
const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$);
const addBackgroundImage = useBehavior(vm.addBackgroundImage$);
const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$);
const cameraTrack = useBehavior(vm.cameraTrack$);
const refusal = useBehavior(vm.backgroundImageRefusal$);
const refusalMessage = useMemo(() => {
switch (refusal?.reason) {
@@ -264,6 +269,7 @@ export const CallFooter: FC<FooterProps> = ({
backgroundEffectSettling={backgroundEffectSettling}
onAddBackgroundImage={addBackgroundImage}
onRemoveBackgroundEffect={removeBackgroundEffect}
selfPreview={<SelfPreview track={cameraTrack} />}
backgroundImageRefusal={refusalMessage}
backgroundEffectNotice={
backgroundEffectNotice === "unavailable"
+48 -9
View File
@@ -24,6 +24,9 @@ import {
createLobbyFooterViewModel,
} from "./CallFooterViewModel";
import { HeaderStyle } from "../UrlParams";
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
import { type ShownCamera, SyncedCameraTrack } from "../tracks/cameraTrack";
import { type MuteStates } from "../state/MuteStates";
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
import type * as BackgroundImages from "../tracks/backgroundImages";
@@ -109,6 +112,11 @@ const pipLayout: Layout = {
spotlight: {} as SpotlightTileViewModel,
};
const noPipeline = constant<BackgroundEffectsState>({
supported: false,
processor: undefined,
});
const twoMicsAndOneCamMediaDevices = mockMediaDevices({
audioInput: {
available$: constant(
@@ -140,7 +148,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
constant(false),
noPipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -183,7 +191,7 @@ describe("createCallFooterViewModel", () => {
select: vi.fn(),
},
}),
constant(false),
noPipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -202,7 +210,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(layout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
constant(false),
noPipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -225,7 +233,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
constant(false),
noPipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
@@ -271,7 +279,7 @@ describe("createCallFooterViewModel", () => {
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
constant(false),
noPipeline,
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
@@ -356,18 +364,22 @@ describe("createCallFooterViewModel", () => {
it("passes on the wait for the first effect", () => {
platformMock.mockReturnValue("desktop");
const settling$ = new BehaviorSubject(true);
const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
settling: true,
});
const vm = createLobbyFooterViewModel(
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
settling$,
pipeline$,
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
expect(vm.backgroundEffectSettling$.value).toBe(true);
settling$.next(false);
pipeline$.next({ ...pipeline$.value, settling: false });
expect(vm.backgroundEffectSettling$.value).toBe(false);
});
@@ -427,6 +439,33 @@ describe("createCallFooterViewModel", () => {
expect(store.remove).toHaveBeenCalledWith("spare");
});
it("previews the camera only while it is on", () => {
const camera = new SyncedCameraTrack();
const track = {} as ShownCamera;
camera.report(track);
const videoEnabled$ = new BehaviorSubject(true);
const muteStates = mockMuteStates();
const vm = createLobbyFooterViewModel(
testScope(),
{
...muteStates,
video: { ...muteStates.video, enabled$: videoEnabled$ },
} as unknown as MuteStates,
twoMicsAndOneCamMediaDevices,
constant({
supported: true,
processor: undefined,
cameraTrack: camera,
}),
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
expect(vm.cameraTrack$.value).toBe(track);
videoEnabled$.next(false);
expect(vm.cameraTrack$.value).toBeNull();
});
it("availability is the same before and during a call", () => {
for (const supported of [true, false]) {
sdkSupportMock.mockReturnValue(supported);
@@ -436,7 +475,7 @@ describe("createCallFooterViewModel", () => {
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
constant(false),
noPipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
+28 -10
View File
@@ -11,6 +11,7 @@ import {
distinctUntilChanged,
map,
type Observable,
of,
switchMap,
} from "rxjs";
import { supportsAudioOutputSelection } from "livekit-client";
@@ -25,6 +26,7 @@ import {
serializeEffect,
shippedBackgrounds,
} from "../tracks/backgroundEffectCatalog";
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
import {
type AddedBackground,
addedBackgrounds,
@@ -87,7 +89,8 @@ function buildDeviceBehaviors(
mediaDevices: MediaDevices,
/** return empty arrays for audioOptions and videoOptions*/
disableSwitcher$: Behavior<boolean>,
backgroundEffectSettling$: Observable<boolean>,
backgroundEffectsState$: Observable<BackgroundEffectsState>,
videoEnabled$: Behavior<boolean>,
): Pick<
ViewModel<FooterSnapshot>,
| "audioOptions$"
@@ -107,6 +110,7 @@ function buildDeviceBehaviors(
| "addBackgroundImage$"
| "backgroundImageRefusal$"
| "removeBackgroundEffect$"
| "cameraTrack$"
> {
const options$ = (
available$: Behavior<Map<string, MenuOptions["label"]>>,
@@ -186,7 +190,10 @@ function buildDeviceBehaviors(
),
),
backgroundEffectSettling$: scope.behavior(
backgroundEffectSettling$.pipe(distinctUntilChanged()),
backgroundEffectsState$.pipe(
map(({ settling }) => settling ?? false),
distinctUntilChanged(),
),
false,
),
// Kept and offered, not put on: that waits for the user to choose it.
@@ -208,6 +215,15 @@ function buildDeviceBehaviors(
),
),
backgroundImageRefusal$: refusal$,
cameraTrack$: scope.behavior(
combineLatest([
backgroundEffectsState$.pipe(
switchMap(({ cameraTrack }) => cameraTrack?.track$ ?? of(null)),
),
videoEnabled$,
]).pipe(map(([track, videoEnabled]) => (videoEnabled ? track : null))),
null,
),
// Never the one in force: taking it away would leave nothing chosen.
removeBackgroundEffect$: scope.behavior(
offered$.pipe(
@@ -237,8 +253,8 @@ function buildDeviceBehaviors(
* @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc.
* @param muteStates - Audio and video mute state + toggles.
* @param mediaDevices - Available and selected input devices.
* @param backgroundEffectSettling$ - Whether the first effect chosen is still
* being prepared.
* @param backgroundEffectsState$ - The background pipeline's state: whether effects
* run here, the wait for the first, and the camera it is synced to.
* @param reactionIdentifier - The local user's reaction identifier string, or
* undefined when reactions are not supported (hides the reaction button).
* @param options - `showControls`: whether the call controls should be shown.
@@ -249,7 +265,7 @@ export function createCallFooterViewModel(
callModel: CallViewModel,
muteStates: MuteStates,
mediaDevices: MediaDevices,
backgroundEffectSettling$: Observable<boolean>,
backgroundEffectsState$: Observable<BackgroundEffectsState>,
reactionIdentifier: string | undefined,
options: { showControls: boolean; header: HeaderStyle },
): ViewModel<FooterSnapshot> {
@@ -268,7 +284,8 @@ export function createCallFooterViewModel(
scope,
mediaDevices,
disableDeviceSwitcher$,
backgroundEffectSettling$,
backgroundEffectsState$,
muteStates.video.enabled$,
),
// candidat to move into the FooterViewModel
showFooter$: callModel.showFooter$,
@@ -331,8 +348,8 @@ export function createCallFooterViewModel(
* @param scope - ObservableScope that bounds the lifetime of derived behaviors.
* @param muteStates - Audio and video mute state + toggles.
* @param mediaDevices - Available and selected input devices.
* @param backgroundEffectSettling$ - Whether the first effect chosen is still
* being prepared.
* @param backgroundEffectsState$ - The background pipeline's state: whether effects
* run here, the wait for the first, and the camera it is synced to.
* @param openSettings - Callback to open the settings modal, or undefined.
* @param hangup - Callback to leave/cancel, or undefined (hides the button).
* @param showLogo - Whether to show the Element Call logo.
@@ -341,7 +358,7 @@ export function createLobbyFooterViewModel(
scope: ObservableScope,
muteStates: MuteStates,
mediaDevices: MediaDevices,
backgroundEffectSettling$: Observable<boolean>,
backgroundEffectsState$: Observable<BackgroundEffectsState>,
openSettings: (() => void) | undefined,
hangup: (() => void) | undefined,
showLogo: boolean,
@@ -389,7 +406,8 @@ export function createLobbyFooterViewModel(
scope,
mediaDevices,
constant(false),
backgroundEffectSettling$,
backgroundEffectsState$,
muteStates.video.enabled$,
),
};
}
@@ -97,3 +97,35 @@ Please see LICENSE in the repository root for full details.
.refusal {
margin: var(--cpd-space-2x) var(--cpd-space-4x) 0;
}
.selfPreview {
position: relative;
display: flex;
align-items: center;
justify-content: center;
flex-shrink: 0;
aspect-ratio: 16 / 9;
margin-block-end: var(--cpd-space-2x);
overflow: clip;
border-start-start-radius: var(--cpd-space-3x);
border-start-end-radius: var(--cpd-space-3x);
background: var(--cpd-color-bg-subtle-secondary);
color: var(--cpd-color-icon-secondary);
}
.selfPreview > video {
position: absolute;
inset: 0;
inline-size: 100%;
block-size: 100%;
object-fit: cover;
/* Rounded itself too: a video can be drawn on a layer of its own, which the
frame's rounded clip doesn't always reach. */
border-radius: inherit;
}
/* Up to the menu's frame, over the padding at its top. */
.selfPreviewPinned,
.deviceListLeadsWithPreview {
margin-block-start: calc(-1 * var(--cpd-space-5x));
}
@@ -12,6 +12,7 @@ import type { Meta, StoryObj } from "@storybook/react-vite";
import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton";
import styles from "./MediaMuteAndSwitchButton.module.css";
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
import { SelfPreview } from "./SelfPreview";
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
import meterStyles from "./MicrophoneLevelMeter.module.css";
import { MediaDevicesContext } from "../MediaDevicesContext";
@@ -60,6 +61,50 @@ const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => {
return <>{children}</>;
};
/** A camera for the preview: a figure against a room, drawn to a stream. */
const FakeCamera: FC = () => {
const [stream, setStream] = useState<MediaStream>();
useEffect(() => {
const canvas = document.createElement("canvas");
canvas.width = 640;
canvas.height = 360;
const context = canvas.getContext("2d")!;
const draw = (): void => {
context.fillStyle = "#6b7f8e";
context.fillRect(0, 0, 640, 360);
context.fillStyle = "#c9a27e";
context.beginPath();
context.arc(320, 150, 62, 0, 2 * Math.PI);
context.fill();
context.fillStyle = "#34495e";
context.beginPath();
context.ellipse(320, 380, 150, 140, 0, Math.PI, 2 * Math.PI);
context.fill();
};
draw();
// Redrawn, so the stream keeps delivering frames.
const timer = setInterval(draw, 200);
const captured = canvas.captureStream(10);
setStream(captured);
return (): void => {
clearInterval(timer);
captured.getTracks().forEach((track) => track.stop());
};
}, []);
return (
<video
data-testid="preview-camera"
autoPlay
muted
playsInline
ref={(video): void => {
if (video && stream && video.srcObject !== stream)
video.srcObject = stream;
}}
/>
);
};
/** Supplies a call-sized root. Without one the list is bounded by the whole Storybook frame. */
const WithACallArea: FC<{ children: ReactNode; blockSize?: number }> = ({
children,
@@ -171,8 +216,8 @@ export const VideoMute: Story = {
{ label: { type: "name", name: "Camera 1" }, id: "1" },
{ label: { type: "name", name: "Camera 2" }, id: "2" },
],
selectedOption: "1",
selfPreview: <SelfPreview track={null} />,
},
};
@@ -199,6 +244,8 @@ export const VideoUnmute: Story = {
selectedBackgroundEffect: "none",
onSelectBackgroundEffect: fn(),
onAddBackgroundImage: fn(),
// As the footer always gives the camera menu one.
selfPreview: <FakeCamera />,
},
};
@@ -344,6 +391,9 @@ export const BackgroundEffectsSettling: Story = {
checked: false,
}))
await expect(other).toHaveAttribute("aria-busy", "false");
await expect(menu.queryByTestId("preview-camera")).toBeNull();
const frames = document.body.querySelectorAll(`.${styles.selfPreview} svg`);
await expect(frames).toHaveLength(1);
},
};
@@ -501,6 +551,113 @@ export const RemovingWithLiveSelection: Story = {
},
};
/** Where the call leaves room, the camera holds the top while the options scroll. */
export const BackgroundEffectsWithPreview: Story = {
args: {
...VideoUnmute.args,
// Enough that the list fills its share, so the preview's cost counts.
options: Array.from({ length: 8 }, (_, i) => ({
label: { type: "name" as const, name: `Camera ${i + 1}` },
id: `${i + 1}`,
})),
},
decorators: [
(Story): JSX.Element => (
<WithACallArea blockSize={640}>
<Story />
</WithACallArea>
),
],
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const picture = await within(document.body).findByTestId("preview-camera");
const preview = picture.parentElement!;
const list = document.body.querySelector<HTMLElement>(
`.${styles.deviceList}`,
)!;
const frame = document.body
.querySelector("[role='menu']")!
.getBoundingClientRect();
const box = preview.getBoundingClientRect();
await expect(list.contains(preview)).toBe(false);
await expect(box.top - frame.top).toBeLessThanOrEqual(2);
await expect(frame.width - box.width).toBeLessThanOrEqual(2);
await expect(box.width / box.height).toBeCloseTo(16 / 9, 1);
await expect(box.bottom).toBeLessThanOrEqual(
list.getBoundingClientRect().top,
);
const listShare = Math.round(640 * 0.6);
// The list runs on through the menu's padding at its foot, which isn't
// part of its share.
const listEnd =
list.getBoundingClientRect().bottom -
Number.parseFloat(getComputedStyle(list).paddingBlockEnd);
await expect(listEnd - box.top).toBeLessThanOrEqual(listShare + 1);
await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight);
list.scrollTop = list.scrollHeight;
await expect(preview.getBoundingClientRect().top).toBe(box.top);
},
};
/** With the camera off the preview says so, and effects can still be chosen. */
export const BackgroundEffectsPreviewWithTheCameraOff: Story = {
args: {
...VideoUnmute.args,
enabled: false,
selfPreview: <SelfPreview track={null} />,
},
play: async ({ args, canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const menu = within(document.body);
await menu.findByRole("group", { name: "Background effects" });
const preview = document.body.querySelector(`.${styles.selfPreview}`)!;
await expect(preview.querySelector("video")).toBeNull();
await expect(preview.querySelector("svg")).not.toBeNull();
await userEvent.click(menu.getByRole("menuitemradio", { name: "Blur" }));
await expect(args.onSelectBackgroundEffect).toHaveBeenCalledWith("blur");
},
};
/** In a call too short to hold it, the camera leads the options and scrolls away. */
export const BackgroundEffectsPreviewScrollsWithTheList: Story = {
args: VideoUnmute.args,
decorators: [
(Story): JSX.Element => (
<WithACallArea blockSize={300}>
<Story />
</WithACallArea>
),
],
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
const picture = await within(document.body).findByTestId("preview-camera");
const preview = picture.parentElement!;
const list = document.body.querySelector<HTMLElement>(
`.${styles.deviceList}`,
)!;
await expect(list.contains(preview)).toBe(true);
const cameras = within(document.body).getByRole("group", {
name: "Camera",
});
await expect(
preview.compareDocumentPosition(cameras) &
Node.DOCUMENT_POSITION_FOLLOWING,
).toBeTruthy();
list.scrollTop = list.scrollHeight;
await expect(preview.getBoundingClientRect().bottom).toBeLessThan(
list.getBoundingClientRect().top,
);
},
};
/** In a short call the effects scroll into view with the list. */
export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = {
args: {
+57 -6
View File
@@ -14,10 +14,12 @@ import {
type FC,
useEffect,
type ReactElement,
type ReactNode,
} from "react";
import {
Alert,
Button,
InlineSpinner,
Menu,
MenuItem,
MenuScrollArea,
@@ -87,6 +89,8 @@ export interface MediaMuteAndSwitchButtonProps {
onRemoveBackgroundEffect?: (id: string) => void;
/** Why the last file offered couldn't be used; a new object each time. */
backgroundImageRefusal?: { text: string };
/** The user's own camera, shown at the top of the camera menu. */
selfPreview?: ReactNode;
/**
* For any toggle and option this method will be called.
* So toggles need to be implemented by listening here and setting the right toggle item to `enabled`
@@ -103,6 +107,10 @@ const LIST_SHARE_OF_CALL = 0.6;
/** Smallest device list height in px, so a short call still shows more than one device. */
const MIN_LIST_HEIGHT = 160;
/** The menu's width as its stylesheet sets it, to place the preview before
the menu is drawn. */
const MENU_WIDTH = 296;
/** Space kept between the menu and the call area's sides. */
const MENU_MARGIN = 16;
@@ -124,6 +132,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
onAddBackgroundImage,
onRemoveBackgroundEffect,
backgroundImageRefusal,
selfPreview,
onSelect,
}) => {
// Requested but not yet selected. Keyed by kind too, since Chrome uses
@@ -202,21 +211,59 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
// Measured on the call area: CSS can't size the portalled menu against it.
const rootElement = useRootElement();
const [listMaxHeight, setListMaxHeight] = useState<number>();
const [listShare, setListShare] = useState<number>();
const [previewBlock, setPreviewBlock] = useState(0);
useEffect(() => {
if (!menuOpen) return;
// From the tokens the stylesheets size the preview with, so that it is
// placed once rather than measured and moved.
const tokens = getComputedStyle(rootElement);
const px = (token: string): number =>
parseFloat(tokens.getPropertyValue(token)) || 0;
const below = px("--cpd-space-2x") + px("--cpd-space-1x");
// Followed, since a host can resize the call while the menu is open.
const subscription = observeElementSize$(rootElement)
.pipe(
map(({ height }) =>
Math.max(MIN_LIST_HEIGHT, Math.round(height * LIST_SHARE_OF_CALL)),
map(({ width, height }) => ({
height: Math.round(height * LIST_SHARE_OF_CALL),
width: Math.min(MENU_WIDTH, Math.round(width - 2 * MENU_MARGIN)),
})),
distinctUntilChanged(
(a, b) => a.height === b.height && a.width === b.width,
),
distinctUntilChanged(),
)
.subscribe(setListMaxHeight);
.subscribe(({ height, width }) => {
setListShare(height);
setPreviewBlock(Math.round((width * 9) / 16) + below);
});
return (): void => subscription.unsubscribe();
}, [menuOpen, rootElement]);
// Paid for out of the list's share, so a preview makes the menu no taller.
const hasPreview =
iconsAndLabels === "video" &&
selfPreview !== undefined &&
listShare !== undefined;
const previewPinned =
hasPreview && listShare - previewBlock >= MIN_LIST_HEIGHT;
const listMaxHeight =
listShare === undefined
? undefined
: Math.max(
MIN_LIST_HEIGHT,
listShare - (previewPinned ? previewBlock : 0),
);
const preview = hasPreview && (
<div
aria-hidden
className={classNames(styles.selfPreview, {
[styles.selfPreviewPinned]: previewPinned,
})}
>
{backgroundEffectSettling ? <InlineSpinner size={32} /> : selfPreview}
</div>
);
useEffect(() => {
if (menuOpen) devices.requestDeviceNames(); // No-op after the first call
}, [menuOpen, devices]);
@@ -407,9 +454,12 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
/>
}
>
{previewPinned && preview}
<MenuScrollArea
ref={trackFocusSource}
className={styles.deviceList}
className={classNames(styles.deviceList, {
[styles.deviceListLeadsWithPreview]: hasPreview && !previewPinned,
})}
data-testid="device-list"
style={
{
@@ -418,6 +468,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
} as CSSProperties
}
>
{hasPreview && !previewPinned && preview}
{audioMeter && speakerOptions && (
<MenuScrollSection title={t("settings.devices.speaker")}>
{deviceItems(
+10
View File
@@ -0,0 +1,10 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
.mirrored {
transform: scaleX(-1);
}
+36
View File
@@ -0,0 +1,36 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type FC } from "react";
import { VideoCallOffIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
import styles from "./SelfPreview.module.css";
import { useAttachedTrack } from "../useAttachedTrack";
import { type ShownCamera } from "../tracks/cameraTrack";
interface Props {
/** The camera as the pipeline has it, or none while the camera is off. */
track: ShownCamera | null;
}
/**
* The user's own camera with the effect in force: the pipeline's own track,
* shown a second time rather than captured again.
*/
export const SelfPreview: FC<Props> = ({ track }) => {
const { videoRef, mirrored } = useAttachedTrack(track);
if (!track) return <VideoCallOffIcon width={32} height={32} />;
return (
<video
ref={videoRef}
muted
playsInline
className={mirrored ? styles.mirrored : undefined}
/>
);
};
+12 -2
View File
@@ -110,7 +110,11 @@ import { useBehavior } from "../useBehavior.ts";
import { useValueBehavior } from "../useValueBehavior.ts";
import { Toast } from "../Toast.tsx";
import overlayStyles from "../Overlay.module.css";
import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext";
import {
reportCamera,
useBackgroundEffectsState$,
} from "../tracks/BackgroundEffectsContext.tsx";
import { sdkCamera } from "../tracks/cameraTrack.ts";
import { type Layout } from "../state/layout-types.ts";
import { ObservableScope } from "../state/ObservableScope.ts";
import { CallFooter, type FooterSnapshot } from "../components/CallFooter.tsx";
@@ -211,6 +215,12 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
reactionsReader.reactions$,
);
setVm(vm);
// The call's camera is the one the camera menu shows from here on.
reportCamera(
scope,
vm.camera$.pipe(map((camera) => (camera ? sdkCamera(camera) : null))),
backgroundEffectsState$,
);
vm.leave$.pipe(scope.bind()).subscribe(props.onLeft);
@@ -241,7 +251,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
vm,
props.muteStates,
mediaDevices,
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
backgroundEffectsState$,
`${props.client.getUserId()}:${props.client.getDeviceId()}`,
{ showControls: urlParams.showControls, header: urlParams.header },
);
+1
View File
@@ -28,6 +28,7 @@ const stable = vi.hoisted(() => ({
media: undefined as unknown,
}));
vi.mock("../tracks/BackgroundEffectsContext", () => ({
reportCamera: (): void => {},
useBackgroundEffectsState$: (): Behavior<BackgroundEffectsState> =>
(stable.processorState$ ??= constant({
supported: false,
+1
View File
@@ -45,6 +45,7 @@ const leaveToHome = vi.fn();
// One instance: the view keys an effect on it
const processorState$ = vi.hoisted(() => ({ current: undefined as unknown }));
vi.mock("../tracks/BackgroundEffectsContext", () => ({
reportCamera: (): void => {},
useBackgroundEffectsState$: (): Behavior<BackgroundEffectsState> =>
(processorState$.current ??= constant({
supported: false,
+21 -12
View File
@@ -31,7 +31,11 @@ import { LeaveToHomeLink } from "../button/LeaveToHomeLink";
import { useLeaveToHome } from "../LeaveToHomeContext";
import { useMediaDevices } from "../MediaDevicesContext";
import { ObservableScope } from "../state/ObservableScope";
import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext";
import {
reportCamera,
useBackgroundEffectsState$,
} from "../tracks/BackgroundEffectsContext";
import { sdkCamera } from "../tracks/cameraTrack";
import { useBehavior } from "../useBehavior";
import { CallFooter, type FooterSnapshot } from "../components/CallFooter";
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
@@ -127,18 +131,23 @@ export const LobbyView: FC<Props> = ({
const [mediaVm, setMediaVm] = useState<LobbyMediaViewModel | null>(null);
useEffect(() => {
const mediaScope = new ObservableScope();
setMediaVm(
createLobbyMediaViewModel(
mediaScope,
media,
muteStates,
devices,
captureSettings(),
mediaScope.behavior(
backgroundEffectsState$.pipe(map((state) => state.processor)),
),
const vm = createLobbyMediaViewModel(
mediaScope,
media,
muteStates,
devices,
captureSettings(),
mediaScope.behavior(
backgroundEffectsState$.pipe(map((state) => state.processor)),
),
);
// The lobby's camera is the one the camera menu shows until the call's
reportCamera(
mediaScope,
vm.videoTrack$.pipe(map((camera) => camera && sdkCamera(camera))),
backgroundEffectsState$,
);
setMediaVm(vm);
return (): void => {
mediaScope.end();
};
@@ -164,7 +173,7 @@ export const LobbyView: FC<Props> = ({
footerScope,
muteStates,
devices,
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
backgroundEffectsState$,
openSettings,
hangup,
// Logo and header are connected: only show the logo in SPA with header.
+9 -16
View File
@@ -5,9 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { useEffect, useMemo, useRef, type FC, type ReactNode } from "react";
import { useMemo, type FC, type ReactNode } from "react";
import useMeasure from "react-use-measure";
import { constant, type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import classNames from "classnames";
import { useTranslation } from "react-i18next";
@@ -15,9 +15,8 @@ import { TileAvatar } from "../tile/TileAvatar";
import styles from "./VideoPreview.module.css";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import videoPlaceholder from "../graphics/video-placeholder.gif";
import { useBehavior } from "../useBehavior";
const noFacingMode = constant(undefined);
import { useAttachedTrack } from "../useAttachedTrack";
import { sdkCamera } from "../tracks/cameraTrack";
export type MatrixInfo = {
userId: string;
@@ -46,15 +45,9 @@ export const VideoPreview: FC<Props> = ({
const { t } = useTranslation();
const [previewRef, previewBounds] = useMeasure();
const videoEl = useRef<HTMLVideoElement | null>(null);
useEffect(() => {
const element = videoEl.current;
if (!element || !videoTrack) return;
videoTrack.attach(element);
return (): void => videoTrack.detach(element);
}, [videoTrack]);
const facingMode = useBehavior(videoTrack?.facingMode$ ?? noFacingMode);
const { videoRef, mirrored } = useAttachedTrack(
videoTrack && sdkCamera(videoTrack),
);
const cameraIsStarting = useMemo(
() => videoEnabled && !videoTrack,
@@ -64,8 +57,8 @@ export const VideoPreview: FC<Props> = ({
return (
<div className={classNames(styles.preview)} ref={previewRef}>
<video
className={facingMode === "user" ? styles.mirror : undefined}
ref={videoEl}
className={mirrored ? styles.mirror : undefined}
ref={videoRef}
muted
playsInline
// There's no reason for this to be focusable
+6
View File
@@ -32,6 +32,7 @@ import {
parseEffect,
} from "../tracks/backgroundEffectCatalog";
import { type AddedBackground } from "../tracks/backgroundImages";
import { SyncedCameraTrack } from "../tracks/cameraTrack";
const blurRadius = 15;
@@ -40,6 +41,8 @@ export interface BackgroundEffectsState {
processor: undefined | VideoProcessor;
/** From the first effect chosen until a frame carrying it is drawn. */
settling?: boolean;
/** The camera this provider's pipeline is synced to. */
cameraTrack?: SyncedCameraTrack;
}
export interface BackgroundEffectsOptions {
@@ -76,6 +79,8 @@ export function createBackgroundEffects(
): BackgroundEffects {
// Wrapped once, so the call and the preview see one processor throughout.
const processor = fromLivekitProcessor(pipeline);
// The camera the pipeline is synced to.
const cameraTrack = new SyncedCameraTrack();
const choice$ = effect$.pipe(map(parseEffect));
const wanted$ = choice$.pipe(
map((effect) => effect.kind !== "none"),
@@ -117,6 +122,7 @@ export function createBackgroundEffects(
supported,
processor: enable ? processor : undefined,
settling: enable && !drewAFrame,
cameraTrack,
};
},
{ supported, processor: undefined },
+3
View File
@@ -363,6 +363,8 @@ export interface CallViewModel {
keyRotationSuppressed$: Behavior<boolean>;
/** The local member, once our membership has been seen in the room. */
localMember$: Behavior<LocalRTCMember | null>;
/** Our own camera track, while one is published. */
camera$: Behavior<LocalVideoMediaTrack | undefined>;
remoteMembers$: Behavior<RemoteRTCMember[]>;
/** The transports the client holds a connection to, for the developer panel. */
connectedTransports$: Behavior<TransportMetadata[]>;
@@ -1508,6 +1510,7 @@ export function createCallViewModel$(
leave,
toggleScreenSharing: toggleScreenSharing,
sharingScreen$: sharingScreen$,
camera$,
tapScreen: (): void => screenTap$.next(),
tapControls: (): void => controlsTap$.next(),
+82 -1
View File
@@ -8,9 +8,11 @@ Please see LICENSE in the repository root for full details.
import { act, createElement, type FC } from "react";
import { render } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { BehaviorSubject } from "rxjs";
import {
BackgroundEffectsProvider,
reportCamera,
useBackgroundEffects,
createAppBackgroundEffects,
} from "./BackgroundEffectsContext";
@@ -19,8 +21,10 @@ import {
type BackgroundEffectsState,
} from "../state/BackgroundEffects";
import { backgroundEffect } from "../settings/settings";
import { sdkCamera, type ShownCamera, SyncedCameraTrack } from "./cameraTrack";
import { type AddedBackground } from "./backgroundImages";
import { flushPromises, testScope } from "../utils/test";
import { constant } from "../state/Behavior";
import { flushPromises, mockVideoTrack, testScope } from "../utils/test";
const sdkSupportMock = vi.hoisted(() => vi.fn(() => true));
const pipelines = vi.hoisted(() => ({
@@ -59,6 +63,83 @@ vi.mock("./BackgroundEffectTransformer", () => ({
BackgroundEffectTransformer: vi.fn(),
}));
describe("the synced camera", () => {
it("is the track last synced, and cleared only while still its own", () => {
const synced = new SyncedCameraTrack();
const pipeline = constant<BackgroundEffectsState>({
supported: false,
processor: undefined,
cameraTrack: synced,
});
const lobby = sdkCamera(mockVideoTrack({ id: "lobby" }));
const call = sdkCamera(mockVideoTrack({ id: "call" }));
const lobbyCamera$ = new BehaviorSubject<ShownCamera | null>(lobby);
reportCamera(testScope(), lobbyCamera$, pipeline);
expect(synced.track$.value).toBe(lobby);
const callScope = testScope();
reportCamera(callScope, constant(call), pipeline);
expect(synced.track$.value).toBe(call);
lobbyCamera$.next(null);
expect(synced.track$.value).toBe(call);
callScope.end();
expect(synced.track$.value).toBeNull();
});
it("reports the camera once, however often the pipeline's state changes", () => {
const synced = new SyncedCameraTrack();
const report = vi.spyOn(synced, "report");
const state$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
cameraTrack: synced,
});
reportCamera(testScope(), constant(sdkCamera(mockVideoTrack({}))), state$);
state$.next({ ...state$.value, settling: true });
state$.next({ ...state$.value, settling: false });
expect(report).toHaveBeenCalledOnce();
});
it("is the SDK's camera once the call reports it", () => {
const synced = new SyncedCameraTrack();
const state$ = constant<BackgroundEffectsState>({
supported: true,
processor: undefined,
cameraTrack: synced,
});
const callCamera = mockVideoTrack({ facingMode$: constant("user") });
const callCamera$ = new BehaviorSubject<ShownCamera | null>(
sdkCamera(callCamera),
);
reportCamera(testScope(), callCamera$, state$);
expect(synced.track$.value).toBe(sdkCamera(callCamera));
expect(synced.track$.value?.facesUser()).toBe(true);
const element = document.createElement("video");
synced.track$.value!.attach(element);
expect(callCamera.attach).toHaveBeenCalledWith(element);
callCamera$.next(null);
expect(synced.track$.value).toBeNull();
});
it("keeps each call's camera to its own", () => {
const mine = createAppBackgroundEffects(testScope());
const theirs = createAppBackgroundEffects(testScope());
const first = sdkCamera(mockVideoTrack({ id: "first" }));
const second = sdkCamera(mockVideoTrack({ id: "second" }));
reportCamera(testScope(), constant(first), mine.state$);
reportCamera(testScope(), constant(second), theirs.state$);
expect(mine.state$.value.cameraTrack).not.toBe(
theirs.state$.value.cameraTrack,
);
expect(mine.state$.value.cameraTrack?.track$.value).toBe(first);
expect(theirs.state$.value.cameraTrack?.track$.value).toBe(second);
});
});
describe("BackgroundEffectsProvider", () => {
let seen: BackgroundEffectsState[];
const Surface: FC = () => {
+38
View File
@@ -5,6 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
combineLatest,
distinctUntilChanged,
EMPTY,
map,
Observable,
switchMap,
} from "rxjs";
import {
BackgroundProcessorWrapper,
supportsBackgroundProcessors,
@@ -13,6 +21,7 @@ import { createContext, type FC, type JSX, use } from "react";
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
import { type ShownCamera } from "./cameraTrack";
import { addedBackgrounds } from "./backgroundImages";
import { type Behavior } from "../state/Behavior";
import { type ObservableScope } from "../state/ObservableScope";
@@ -71,3 +80,32 @@ export const BackgroundEffectsProvider: FC<Props> = ({ effects, children }) => (
{children}
</BackgroundEffectsContext>
);
/**
* Reports the camera the pipeline is synced to while the scope lasts: the
* lobby's before joining, or the call's, each syncing its own.
*/
export function reportCamera(
scope: ObservableScope,
camera$: Observable<ShownCamera | null>,
backgroundEffectsState$: Behavior<BackgroundEffectsState>,
): void {
// Keyed on the camera alone: reporting again for every change of state would
// blank the preview each time.
combineLatest([
camera$,
backgroundEffectsState$.pipe(
map(({ cameraTrack }) => cameraTrack),
distinctUntilChanged(),
),
])
.pipe(
switchMap(([camera, cameraTrack]) =>
camera === null || cameraTrack === undefined
? EMPTY
: new Observable<never>(() => cameraTrack.report(camera)),
),
scope.bind(),
)
.subscribe();
}
+61
View File
@@ -0,0 +1,61 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject } from "rxjs";
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import { type Behavior } from "../state/Behavior";
/**
* A camera that can be shown in a video element: the MatrixRTC SDK's track,
* before joining and in the call alike.
*/
export interface ShownCamera {
attach(element: HTMLVideoElement): void;
/** Detaches this element only: another may be showing the same camera. */
detach(element: HTMLVideoElement): void;
/** Whether it faces the user, so that it is shown mirrored. */
facesUser(): boolean;
}
// One per track, so that a view showing it isn't attached again on every render.
const shown = new WeakMap<object, ShownCamera>();
function shownOnce(track: object, make: () => ShownCamera): ShownCamera {
let camera = shown.get(track);
if (camera === undefined) {
camera = make();
shown.set(track, camera);
}
return camera;
}
/** A camera as the MatrixRTC SDK has it. */
export function sdkCamera(track: LocalVideoMediaTrack): ShownCamera {
return shownOnce(track, () => ({
attach: (element): void => track.attach(element),
detach: (element): void => track.detach(element),
facesUser: (): boolean => track.facingMode$.value === "user",
}));
}
/**
* The camera track a background pipeline is synced to: the lobby's before
* joining, the call's after, none while the camera is off.
*/
export class SyncedCameraTrack {
private readonly subject$ = new BehaviorSubject<ShownCamera | null>(null);
public readonly track$: Behavior<ShownCamera | null> = this.subject$;
public report(track: ShownCamera): () => void {
this.subject$.next(track);
// Only if still this one, so a lobby leaving doesn't blank a call starting.
return (): void => {
if (this.subject$.value === track) this.subject$.next(null);
};
}
}
+53
View File
@@ -0,0 +1,53 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type FC } from "react";
import { render } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";
import { useAttachedTrack } from "./useAttachedTrack";
import { type ShownCamera } from "./tracks/cameraTrack";
function cameraTrack(facing: string): ShownCamera {
return {
attach: vi.fn(),
detach: vi.fn(),
facesUser: (): boolean => facing === "user",
};
}
const Video: FC<{ track: ShownCamera | null }> = ({ track }) => {
const { videoRef, mirrored } = useAttachedTrack(track);
return <video ref={videoRef} data-mirrored={mirrored} />;
};
describe("useAttachedTrack", () => {
it("detaches only its own element from a track shown twice", () => {
const track = cameraTrack("user");
const first = render(<Video track={track} />);
const second = render(<Video track={track} />);
const [a, b] = [first, second].map((r) =>
r.container.querySelector("video"),
);
expect(track.attach).toHaveBeenCalledWith(a);
expect(track.attach).toHaveBeenCalledWith(b);
first.unmount();
expect(track.detach).toHaveBeenCalledExactlyOnceWith(a);
});
it("mirrors a camera facing the user, and only that", () => {
const { container, rerender } = render(
<Video track={cameraTrack("user")} />,
);
expect(container.querySelector("video")?.dataset.mirrored).toBe("true");
rerender(<Video track={cameraTrack("environment")} />);
expect(container.querySelector("video")?.dataset.mirrored).toBe("false");
rerender(<Video track={null} />);
expect(container.querySelector("video")?.dataset.mirrored).toBe("false");
});
});
+31
View File
@@ -0,0 +1,31 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RefObject, useEffect, useRef } from "react";
import { type ShownCamera } from "./tracks/cameraTrack";
/**
* Shows a camera in the video element given the returned ref, and says
* whether to mirror it, as a camera facing the user is. Only that element is
* detached again: another may be showing the same camera.
*/
export function useAttachedTrack(camera: ShownCamera | null): {
videoRef: RefObject<HTMLVideoElement | null>;
mirrored: boolean;
} {
const videoRef = useRef<HTMLVideoElement | null>(null);
useEffect(() => {
const element = videoRef.current;
if (!camera || !element) return;
camera.attach(element);
return (): void => {
camera.detach(element);
};
}, [camera]);
return { videoRef, mirrored: camera?.facesUser() ?? false };
}
+1 -1
View File
@@ -217,7 +217,7 @@ export function getBasicCallViewModelEnvironment(
vm,
muteStates,
mediaDevices,
constant(false),
constant({ supported: false, processor: undefined }),
"reactionId",
{ showControls: true, header: HeaderStyle.Standard },
);