Compare commits

..
Author SHA1 Message Date
Johannes Marbach 1cdcb62791 Request capability to receive m.rtc.encryption_key events 2026-11-18 11:16:41 +01:00
32 changed files with 1275 additions and 1419 deletions
+9 -16
View File
@@ -10,13 +10,6 @@
- Owning a resource: a class taking the scope in its constructor — `MediaDevices`,
`MuteStates`, `TileStore`, `Publisher`, `Connection`. Both styles are current;
don't convert one to the other in passing.
- A factory's required parameters are what the thing is made of: no sensible
default. Several of them go in a props interface, so each can carry a doc
comment. Its trailing `options` bag is how the host wants it to behave: every
field optional, each defaulting to standalone behaviour, so `{}` works. A
value with no default is never an option; a test seam is never an option
either — mock the dependency it comes from. See `createCallViewModel$`, with
`CallViewModelProps` and `CallViewModelOptions`.
- `foo$` is an observable. `Behavior<T>`, an observable with a current value, is
the default for anything a view reads.
@@ -43,15 +36,15 @@ Element Call can be mounted several times inside a host's React tree, so it owns
neither window, URL, document nor router. Each seam defaults to the old standalone
and widget behaviour.
| Never | Use |
| --------------------------------------------- | ----------------------------------------------------------- |
| the `widget` global | `useHostBridge()` — `src/HostBridge.ts` |
| `getUrlParams()`, `window.location` | `useUrlParams()` |
| `useNavigate("/")`, `<Link to="/">` | `useLeaveToHome()`, `LeaveToHomeLink` |
| `document.body` | `useRootElement()` |
| `window.innerWidth/Height`, `useMediaQuery` | `useRootSizeMatches()` in views, `viewSize$` in view models |
| global `i18next` | the instance in `src/utils/i18n.ts` |
| config or analytics reading their environment | `Config.initWith()`, `PosthogAnalytics.configure()` |
| Never | Use |
| --------------------------------------------- | ------------------------------------------------------------- |
| the `widget` global | `useHostBridge()` — `src/HostBridge.ts` |
| `getUrlParams()`, `window.location` | `useUrlParams()` |
| `useNavigate("/")`, `<Link to="/">` | `useLeaveToHome()`, `LeaveToHomeLink` |
| `document.body` | `useRootElement()` |
| `window.innerWidth/Height`, `useMediaQuery` | `useRootSizeMatches()` in views, `windowSize$` in view models |
| global `i18next` | the instance in `src/utils/i18n.ts` |
| config or analytics reading their environment | `Config.initWith()`, `PosthogAnalytics.configure()` |
- View models take values as options, never `getUrlParams()`
(`callViewModelOptionsFromParams`, `CallViewModelOptions.hostBridge`).
+9 -11
View File
@@ -128,22 +128,20 @@ export async function createMatrixRTCSdk(
});
vm = createCallViewModel$(
scope,
{
rtcParticipation: participation,
matrixRoom: room,
mediaDevices,
muteStates,
encryptionSystem,
// The bundle owns its page, so the body is the space it has
viewSize$: scope.behavior(observeElementSize$(document.body)),
handsRaisedSubject$: of({}),
reactionsSubject$: of({}),
},
participation,
room,
mediaDevices,
muteStates,
{
...callViewModelOptionsFromParams(urlParams),
encryptionSystem,
capture,
hostBridge,
// The bundle owns its page, so the body is the space it has
windowSize$: scope.behavior(observeElementSize$(document.body)),
},
of({}),
of({}),
);
// Once the hang-up has gone through, unstick and ask the host to close us
vm.leave$
-10
View File
@@ -152,15 +152,6 @@ describe("createLocalMedia", () => {
});
});
// As the built-in camera of a Mac
it("takes a camera that doesn't say which way it faces to face the user", async () => {
const media = createLocalMedia(fakeGetUserMedia());
const [camera] = (await media.capture([{ source: "camera" }])) as [
LocalVideoMediaTrack,
];
expect(camera.facingMode$.value).toBe("user");
});
it("releases one device and stops the rest", async () => {
const media = createLocalMedia(fakeGetUserMedia());
const [microphone, camera] = await media.capture([
@@ -207,7 +198,6 @@ function fakeTrack(
const track = {
kind,
id: `${kind}-${nextId++}`,
label: "",
enabled: true,
stop: vi.fn(),
getSettings: () => settings,
+15 -4
View File
@@ -19,7 +19,6 @@ import {
type VideoProcessor,
} from "./media-api";
import { constant } from "./reactive/Behavior";
import { facingModeOf } from "./utils/facingMode";
import { inTurn } from "./utils/processorTurns";
type Source = "microphone" | "camera";
@@ -162,7 +161,9 @@ function trackConstraints(request: Request): MediaTrackConstraints {
class CapturedTrack {
public readonly mediaTrack: LocalAudioMediaTrack | LocalVideoMediaTrack;
private readonly muted$ = new BehaviorSubject(false);
private readonly facingMode$: BehaviorSubject<"user" | "environment">;
private readonly facingMode$: BehaviorSubject<
"user" | "environment" | undefined
>;
private readonly elements = new Set<HTMLMediaElement>();
private processor: HandedOverTrack["processor"];
/** What the processor reads frames from; a processor needs a playing element. */
@@ -305,9 +306,10 @@ class CapturedTrack {
kind: "video",
facingMode$: this.facingMode$,
switchFacingMode: async () => {
const facingMode = this.facingMode$.value;
if (facingMode === undefined) return;
await this.reacquire({
facingMode:
this.facingMode$.value === "user" ? "environment" : "user",
facingMode: facingMode === "user" ? "environment" : "user",
});
return this.track.getSettings().deviceId;
},
@@ -323,3 +325,12 @@ function play(element: HTMLMediaElement, track: MediaStreamTrack): void {
const playing = element.play() as Promise<void> | undefined;
playing?.catch(() => undefined);
}
function facingModeOf(
track: MediaStreamTrack,
): "user" | "environment" | undefined {
const { facingMode } = track.getSettings();
return facingMode === "user" || facingMode === "environment"
? facingMode
: undefined;
}
+2 -6
View File
@@ -151,16 +151,12 @@ export interface LocalAudioMediaTrack
extends AudioMediaTrack, LocalMediaTrack {}
export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack {
/**
* For mirroring. A camera that does not say which way it faces is taken to
* face the user, unless it is known to film the room, as a phone used as a
* webcam. Undefined while there is no camera track.
*/
/** For mirroring; undefined where the camera does not say which way it faces. */
facingMode$: Behavior<"user" | "environment" | undefined>;
/**
* Restarts the camera facing the other way, on devices with a front and a
* back camera, and resolves with the id of the device now in use. Does
* nothing while there is no camera track.
* nothing where the facing mode is unknown.
*/
switchFacingMode(): Promise<string | undefined>;
/** Background blur and the like; undefined removes the processor. */
@@ -18,7 +18,15 @@ import {
TrackEvent,
type TrackPublication,
} from "livekit-client";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
afterEach,
beforeEach,
describe,
expect,
it,
onTestFinished,
vi,
} from "vitest";
import { ObservableScope } from "../../reactive/ObservableScope";
import {
@@ -225,6 +233,10 @@ describe("createLivekitMediaTrack", () => {
])(
"leaves a processed camera alone unless its device changes: $deviceId",
async ({ deviceId, switches }) => {
vi.stubGlobal("MediaStreamTrack", class {});
onTestFinished(() => {
vi.unstubAllGlobals();
});
const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
source: Track.Source.Camera,
_mediaStreamTrack: {
@@ -258,41 +270,6 @@ describe("createLivekitMediaTrack", () => {
},
);
it("takes the facing mode of a processed camera from the camera", async () => {
const source = {
readyState: "live",
label: "Back Camera",
getSettings: () => ({ facingMode: "environment" }),
};
const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
source: Track.Source.Camera,
_mediaStreamTrack: source,
mediaStream: { getVideoTracks: () => [source] },
processor: {
processedTrack: { label: "", getSettings: () => ({}) },
},
restartTrack: vi.fn().mockResolvedValue(undefined),
}) as LocalVideoTrack;
const { participant, publication, room } = fakes({
track: camera as unknown as FakeTrack,
});
const track = createLocalLivekitMediaTrack(
scope,
Object.assign(participant, {
isLocal: true,
}) as unknown as LocalParticipant,
publication as unknown as TrackPublication,
room,
vi.fn(),
) as LocalVideoMediaTrack;
expect(track.facingMode$.value).toBe("environment");
await track.switchFacingMode();
expect(camera.restartTrack).toHaveBeenCalledExactlyOnceWith({
facingMode: "user",
});
});
// LiveKit stops a camera that is turned off; the processor is built when it
// is turned on again
it.each([
@@ -302,6 +279,10 @@ describe("createLivekitMediaTrack", () => {
"gives a processor to a camera that $state: $takes",
async ({ isMuted, takes }) => {
// jsdom has none; LiveKit's facing mode checks against it
vi.stubGlobal("MediaStreamTrack", class {});
onTestFinished(() => {
vi.unstubAllGlobals();
});
const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
isMuted,
_mediaStreamTrack: {
@@ -11,6 +11,7 @@ import {
roomEventSelector,
} from "@livekit/components-core";
import {
facingModeFromLocalTrack,
type LocalParticipant,
LocalTrack,
LocalVideoTrack,
@@ -49,7 +50,6 @@ import {
type VideoMediaTrack,
} from "../../media-api";
import { MatrixRTCError } from "../../errors";
import { facingModeOf } from "../../utils/facingMode";
import { toLivekitProcessor, type LivekitProcessor } from "./videoProcessor";
export const mediaSources: Record<Track.Source, MediaSource> = {
@@ -231,9 +231,10 @@ export function createLocalLivekitMediaTrack(
switchFacingMode: async () => {
const track = videoTrack();
if (!track) return;
const { facingMode } = facingModeFromLocalTrack(track);
if (facingMode !== "user" && facingMode !== "environment") return;
await track.restartTrack({
facingMode:
facingModeOf(sourceTrack(track)) === "user" ? "environment" : "user",
facingMode: facingMode === "user" ? "environment" : "user",
});
return track.mediaStreamTrack.getSettings().deviceId;
},
@@ -287,14 +288,6 @@ async function rtpStreamStats(
return undefined;
}
/**
* The camera's own track. While a processor runs LiveKit's media stream track
* is the processed one, which says nothing of the camera.
*/
function sourceTrack(track: LocalVideoTrack): MediaStreamTrack {
return track.mediaStream?.getVideoTracks()[0] ?? track.mediaStreamTrack;
}
function facingMode$(
scope: ObservableScope,
track$: Behavior<Track | undefined>,
@@ -305,7 +298,12 @@ function facingMode$(
if (!(track instanceof LocalVideoTrack)) return of(undefined);
return fromEvent(track, TrackEvent.Restarted).pipe(
startWith(null),
map(() => facingModeOf(sourceTrack(track))),
map(() => {
const { facingMode } = facingModeFromLocalTrack(track);
return facingMode === "user" || facingMode === "environment"
? facingMode
: undefined;
}),
);
}),
),
-43
View File
@@ -1,43 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it } from "vitest";
import { facingModeOf } from "./facingMode";
function camera(label: string, facingMode?: string): MediaStreamTrack {
return {
label,
getSettings: () => (facingMode === undefined ? {} : { facingMode }),
} as MediaStreamTrack;
}
describe("facingModeOf", () => {
it.each([
{ label: "Back Camera", facingMode: "environment", faces: "environment" },
{ label: "iPhone Camera", facingMode: "user", faces: "user" },
{ label: "FaceTime HD Camera", facingMode: undefined, faces: "user" },
{ label: "", facingMode: undefined, faces: "user" },
{
label: "Alice's iPhone Camera",
facingMode: undefined,
faces: "environment",
},
{ label: "iPad Camera", facingMode: undefined, faces: "environment" },
{
label: " OBS Virtual Camera ",
facingMode: undefined,
faces: "environment",
},
{ label: "Integrated Camera", facingMode: "left", faces: "user" },
])(
"takes $label ($facingMode) to face $faces",
({ label, facingMode, faces }) => {
expect(facingModeOf(camera(label, facingMode))).toBe(faces);
},
);
});
-23
View File
@@ -1,23 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* Which way a camera faces. Laptop and desktop cameras don't say and face the
* user; of those, a phone used as a webcam (Continuity Camera) films with its
* rear camera and OBS shows a scene, so neither is taken to face the user.
* LiveKit makes the same guess.
*/
export function facingModeOf(track: MediaStreamTrack): "user" | "environment" {
const { facingMode } = track.getSettings();
if (facingMode === "user" || facingMode === "environment") return facingMode;
const label = track.label.trim().toLowerCase();
return label === "obs virtual camera" ||
label.includes("iphone") ||
label.includes("ipad")
? "environment"
: "user";
}
+1 -1
View File
@@ -45,7 +45,7 @@ interface Props {
/** Offers to add a background of one's own. Omit to leave that tile out. */
onAdd?: () => void;
addLabel?: string;
/** Removes a removable one, unless it is the selected one. Omit to offer none. */
/** Removes a removable one, never the one in force. Omit to offer none. */
onRemove?: (id: string) => void;
removeLabel?: string;
}
+193 -68
View File
@@ -22,7 +22,7 @@ import { MediaDevices } from "../state/MediaDevices";
import { globalScope } from "../state/ObservableScope";
import { constant } from "../state/Behavior";
import { type LayoutMode } from "../state/LayoutSwitchViewModel";
import { type MediaMenuSnapshot } from "./MediaMenuViewModel";
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
// consts for tests
const reactionIdentifier = "@user:example.com:DEVICE";
@@ -35,57 +35,10 @@ const mediaDevices = new MediaDevices(globalScope, {
controlledAudioDevices: false,
});
function menu(
kind: "audio" | "video",
withDevices: boolean,
): MediaMenuSnapshot {
const device = kind === "audio" ? "Microphone" : "Camera";
return {
enabled: true,
busy: false,
toggle: fn(),
options: withDevices
? [
{ id: "1", label: { type: "name", name: `${device} 1` } },
{ id: "2", label: { type: "name", name: `${device} 2` } },
]
: [],
selected: "1",
select: fn(),
outputOptions:
withDevices && kind === "audio"
? [
{
id: "default",
label: { type: "default", name: "Built-in Output" },
},
]
: [],
selectedOutput: "default",
selectOutput: fn(),
backgroundEffect: "none",
backgroundEffects:
kind === "video"
? [
{ id: "none", kind: "none" },
{ id: "blur", kind: "blur" },
]
: [],
backgroundEffectNotice: undefined,
backgroundEffectSettling: false,
backgroundImageRefusal: undefined,
selectBackgroundEffect: fn(),
addBackgroundImage: undefined,
removeBackgroundEffect: undefined,
cameraTrack: null,
};
}
/**
* A wrapper component that is used for:
* - exposing the snapshot via props so the storybook documents the snapshot properties (basically unpack them form the vm)
* - constructing the layout switch and the menus' view models; the menus'
* own states are in the MediaMenuButton stories
* - constructing the layout switch view model
* - Add additional react context
* The paraeters are all params from the FooterSnapshot,
* the Snapshot of the vm, the wrapper will create a mocked vm from it and pass it to the CallFooter.
@@ -96,22 +49,15 @@ function CallFooterStoryWrapper({
children,
layout,
setLayout,
withDevices,
...vmSnapshot
}: Omit<FooterSnapshot, "layoutSwitchVm" | "audioMenuVm" | "cameraMenuVm"> & {
}: Omit<FooterSnapshot, "layoutSwitchVm"> & {
children?: false | JSX.Element | JSX.Element[] | undefined;
layout: LayoutMode | null;
setLayout: (value: LayoutMode) => void;
/** Whether the menus have devices to pick, or are plain mute buttons. */
withDevices: boolean;
}): ReactNode {
const audioMenuVm = useStaticViewModel(menu("audio", withDevices));
const cameraMenuVm = useStaticViewModel(menu("video", withDevices));
const vm = useStaticViewModel({
...vmSnapshot,
layoutSwitchVm: layout && { layout$: constant(layout), setLayout },
audioMenuVm,
cameraMenuVm,
});
return (
<MediaDevicesContext value={mediaDevices}>
@@ -156,6 +102,8 @@ const meta = {
},
toggleScreenSharing: fnArgType,
openSettings: fnArgType,
toggleAudio: fnArgType,
toggleVideo: fnArgType,
hangup: fnArgType,
},
} satisfies Meta<typeof CallFooterStoryWrapper>;
@@ -168,9 +116,32 @@ export const Default: Story = {
showLogo: false,
layout: "grid",
setLayout: fn(),
withDevices: false,
audioEnabled: true,
audioBusy: false,
videoEnabled: true,
videoBusy: false,
openSettings: fn(),
toggleAudio: fn(),
toggleVideo: fn(),
toggleScreenSharing: fn(),
selectBackgroundEffect: fn(),
backgroundEffect: "blur",
backgroundEffects: [
{ id: "none", kind: "none", removable: false },
{ id: "blur", kind: "blur", removable: false },
...shippedBackgrounds.map((background) => ({
id: `image:${background.id}`,
kind: "image" as const,
imageUrl: background.imagePath,
removable: false,
})),
],
backgroundEffectNotice: undefined,
backgroundEffectSettling: false,
backgroundImageRefusal: undefined,
addBackgroundImage: fn(),
removeBackgroundEffect: fn(),
cameraTrack: null,
hangup: fn(),
buttonSize: "lg",
showFooter: true,
@@ -183,17 +154,49 @@ export const Default: Story = {
reactionData: undefined,
debugTileLayout: false,
tileStoreGeneration: undefined,
audioOptions: [],
audioOutputOptions: [],
videoOptions: [],
selectedAudio: undefined,
selectedAudioOutput: undefined,
selectedVideo: undefined,
selectAudioButtonOption: undefined,
selectAudioOutputOption: undefined,
selectVideoButtonOption: undefined,
},
parameters: {
layout: "fullscreen",
},
};
export const StaysWhileAMenuIsOpen: Story = {
export const WithAudioAndVideoOptions: Story = {
...Default,
args: {
...Default.args,
withDevices: true,
audioEnabled: false,
videoEnabled: true,
audioOptions: [
{ label: { type: "name", name: "Microphone 1" }, id: "1" },
{ label: { type: "name", name: "Microphone 2" }, id: "2" },
],
audioOutputOptions: [
{ label: { type: "default", name: "Built-in Output" }, id: "default" },
{ label: { type: "name", name: "Headset" }, id: "2" },
],
videoOptions: [
{ label: { type: "name", name: "Camera 1" }, id: "1" },
{ label: { type: "name", name: "Camera 2" }, id: "2" },
],
selectedAudio: "2",
selectedAudioOutput: "default",
selectedVideo: "1",
},
};
export const StaysWhileAMenuIsOpen: Story = {
...Default,
args: {
...WithAudioAndVideoOptions.args,
// In a short window the footer overlays the call and hides itself.
asOverlay: true,
showFooter: true,
@@ -216,6 +219,25 @@ export const StaysWhileAMenuIsOpen: Story = {
},
};
export const AudioBusy: Story = {
...Default,
args: {
...Default.args,
audioEnabled: true,
audioBusy: true,
videoEnabled: true,
},
};
export const VideoBusy: Story = {
...Default,
args: {
...Default.args,
audioEnabled: true,
videoEnabled: true,
videoBusy: true,
},
};
export const WithLogo: Story = {
...Default,
args: {
@@ -224,7 +246,45 @@ export const WithLogo: Story = {
},
};
/** Switching to grid, and the footer's own buttons, act. */
export const AudioVideoEnabled: Story = {
...Default,
args: {
...Default.args,
audioEnabled: true,
videoEnabled: true,
},
play: async ({ args, canvasElement }) => {
const canvas = within(canvasElement);
const spotlightRadio = canvas.getByRole("radio", { name: "Spotlight" });
await userEvent.click(spotlightRadio);
await expect(args.setLayout).toHaveBeenCalledWith("spotlight");
const micButtonMute = canvas.getByRole("switch", {
name: "Mute microphone",
});
await userEvent.click(micButtonMute);
await expect(args.toggleAudio).toHaveBeenCalled();
const videoMuteButton = canvas.getByRole("switch", {
name: "Stop video",
});
await userEvent.click(videoMuteButton);
await expect(args.toggleVideo).toHaveBeenCalled();
const screenShare = canvas.getByRole("switch", {
name: "Share screen",
});
await userEvent.click(screenShare);
await expect(args.toggleScreenSharing).toHaveBeenCalled();
const endCall = canvas.getByRole("button", {
name: "End call",
});
await userEvent.click(endCall);
await expect(args.hangup).toHaveBeenCalled();
},
};
/** used to test switching to grid mode */
export const SpotlightMode: Story = {
...Default,
args: {
@@ -233,12 +293,10 @@ export const SpotlightMode: Story = {
},
play: async ({ args, canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(canvas.getByRole("radio", { name: "Grid" }));
const spotlightRadio = canvas.getByRole("radio", { name: "Grid" });
await userEvent.click(spotlightRadio);
await expect(args.setLayout).toHaveBeenCalledWith("grid");
await userEvent.click(canvas.getByRole("switch", { name: "Share screen" }));
await expect(args.toggleScreenSharing).toHaveBeenCalled();
await userEvent.click(canvas.getByRole("button", { name: "End call" }));
await expect(args.hangup).toHaveBeenCalled();
},
};
@@ -272,10 +330,34 @@ export const Pip: Story = {
buttonSize: "md",
layout: null,
},
play: async ({ canvasElement }) => {
play: async ({ args, canvasElement }) => {
const canvas = within(canvasElement);
await expect(
within(canvasElement).queryByRole("radio", { name: "Spotlight" }),
canvas.queryByRole("radio", { name: "Spotlight" }),
).not.toBeInTheDocument();
const micButtonMute = canvas.getByRole("switch", {
name: "Mute microphone",
});
await userEvent.click(micButtonMute);
await expect(args.toggleAudio).toHaveBeenCalled();
const videoMuteButton = canvas.getByRole("switch", {
name: "Stop video",
});
await userEvent.click(videoMuteButton);
await expect(args.toggleVideo).toHaveBeenCalled();
const screenShare = canvas.getByRole("switch", {
name: "Share screen",
});
await userEvent.click(screenShare);
await expect(args.toggleScreenSharing).toHaveBeenCalled();
const endCall = canvas.getByRole("button", {
name: "End call",
});
await userEvent.click(endCall);
await expect(args.hangup).toHaveBeenCalled();
},
};
export const NoControlsWithLogo: Story = {
@@ -296,6 +378,18 @@ export const DebugData: Story = {
},
};
export const UnavailableMediaDevices: Story = {
...Default,
args: {
...Default.args,
audioEnabled: false,
videoEnabled: false,
toggleAudio: undefined,
toggleVideo: undefined,
audioOutputSwitcher: undefined,
},
};
export const MobileLayout: Story = {
...Default,
args: {
@@ -326,6 +420,37 @@ export const Lobby: Story = {
},
};
export const LobbyMobile: Story = {
...Default,
args: {
...Default.args,
showLogo: false,
layout: null,
toggleScreenSharing: undefined,
},
globals: {
viewport: { value: "mobile2", isRotated: false },
},
parameters: {
...Default.parameters,
},
};
export const LobbyRecentButton: Story = {
...Default,
args: {
...Default.args,
children: <Link>Back To Recents</Link>,
showLogo: false,
layout: null,
toggleScreenSharing: undefined,
},
parameters: {
...Default.parameters,
},
};
export const LobbyRecentButtonMobile: Story = {
...Default,
args: {
+165 -20
View File
@@ -7,11 +7,14 @@ Please see LICENSE in the repository root for full details.
import { type FC, type JSX, type Ref, useMemo } from "react";
import classNames from "classnames";
import { useTranslation } from "react-i18next";
import LogoMark from "../icons/LogoMark.svg?react";
import LogoType from "../icons/LogoType.svg?react";
import {
EndCallButton,
MicButton,
VideoButton,
ShareScreenButton,
SettingsButton,
ReactionToggleButton,
@@ -20,13 +23,23 @@ import {
type ReactionData,
} from "../button";
import styles from "./CallFooter.module.css";
import {
MediaMuteAndSwitchButton,
type MenuOptions,
} from "./MediaMuteAndSwitchButton";
import { type Behavior } from "../state/Behavior";
import { type ViewModel } from "../state/ViewModel";
import { useBehavior } from "../useBehavior";
import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel";
import { LayoutSwitch } from "../room/LayoutSwitch";
import { type MediaMenuViewModel } from "./MediaMenuViewModel";
import { MediaMenuButton } from "./MediaMenuButton";
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
import { SelfPreview } from "./SelfPreview";
import { type UnusableReason } from "../tracks/backgroundImages";
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
export interface BackgroundImageRefusal {
reason: UnusableReason | "not-kept";
}
export interface AudioOutputSwitcher {
targetOutput: string;
@@ -48,6 +61,16 @@ export interface AudioOutputSwitcher {
*/
export type FooterSnapshot = FooterActions & FooterState;
export interface FooterActions {
/** Also controls if the audioMute button is disabled */
toggleAudio: (() => void) | undefined;
/** Also controls if the videoMute button is disabled */
toggleVideo: (() => void) | undefined;
/** Undefined where background effects can't be chosen. */
selectBackgroundEffect: ((id: string) => void) | undefined;
/** Undefined where no background can be added. */
addBackgroundImage: ((file: File) => void) | undefined;
/** Removes an added background by its option id; never the one in force. */
removeBackgroundEffect: ((id: string) => void) | undefined;
toggleScreenSharing: (() => void) | undefined;
/** Also controls if the settings button is visible */
openSettings: (() => void) | undefined;
@@ -57,8 +80,22 @@ export interface FooterActions {
// we do not use any ? optional properties so that the vm type is including all fields.
export interface FooterState {
audioMenuVm: MediaMenuViewModel;
cameraMenuVm: MediaMenuViewModel;
audioEnabled: boolean;
audioBusy: boolean;
videoEnabled: boolean;
videoBusy: boolean;
/** The background effect in force, as its option id. */
backgroundEffect: string;
/** Every effect on offer, in the order they are shown. */
backgroundEffects: BackgroundEffectGridItem[];
/** What the user is told about effects here, if anything. */
backgroundEffectNotice: "unavailable" | "slow" | undefined;
/** Whether the first effect chosen is still being prepared. */
backgroundEffectSettling: boolean;
/** Why the last file offered couldn't be kept, if it couldn't. */
backgroundImageRefusal: BackgroundImageRefusal | undefined;
/** The camera to preview, with the effect in force; null while it is off. */
cameraTrack: LocalVideoMediaTrack | null;
showFooter: boolean;
/* This is needed for WindowMode = "flat" */
@@ -85,6 +122,19 @@ export interface FooterState {
// debug stuff
debugTileLayout: boolean;
tileStoreGeneration: number | undefined;
/** Providing no options `[]` or `undefined` will imply that we dont have a audio fast switcher */
audioOptions: MenuOptions[];
/** Output (speaker) devices. */
audioOutputOptions: MenuOptions[];
/** Providing no options `[]` or `undefined` will imply that we dont have a audio fast switcher */
videoOptions: MenuOptions[];
selectedAudio: string | undefined;
selectedAudioOutput: string | undefined;
selectedVideo: string | undefined;
selectAudioButtonOption: ((deviceId: string) => void) | undefined;
selectAudioOutputOption: ((deviceId: string) => void) | undefined;
selectVideoButtonOption: ((option: string) => void) | undefined;
}
export interface FooterProps {
@@ -100,14 +150,19 @@ export const CallFooter: FC<FooterProps> = ({
children,
vm,
}) => {
const { t } = useTranslation();
const asOverlay = useBehavior(vm.asOverlay$);
const showFooter = useBehavior(vm.showFooter$);
const hideControls = useBehavior(vm.hideControls$);
const showModals = useBehavior(vm.showModals$);
const layoutSwitchVm = useBehavior(vm.layoutSwitchVm$);
const audioMenuVm = useBehavior(vm.audioMenuVm$);
const cameraMenuVm = useBehavior(vm.cameraMenuVm$);
const openSettings = useBehavior(vm.openSettings$);
const audioEnabled = useBehavior(vm.audioEnabled$);
const audioBusy = useBehavior(vm.audioBusy$);
const videoEnabled = useBehavior(vm.videoEnabled$);
const videoBusy = useBehavior(vm.videoBusy$);
const toggleAudio = useBehavior(vm.toggleAudio$);
const toggleVideo = useBehavior(vm.toggleVideo$);
const sharingScreen = useBehavior(vm.sharingScreen$);
const toggleScreenSharing = useBehavior(vm.toggleScreenSharing$);
const reactionIdentifier = useBehavior(vm.reactionIdentifier$);
@@ -115,6 +170,38 @@ export const CallFooter: FC<FooterProps> = ({
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
const hangup = useBehavior(vm.hangup$);
const debugTileLayout = useBehavior(vm.debugTileLayout$);
const videoOptions = useBehavior(vm.videoOptions$);
const selectedVideo = useBehavior(vm.selectedVideo$);
const audioOptions = useBehavior(vm.audioOptions$);
const selectedAudio = useBehavior(vm.selectedAudio$);
const selectAudioButtonOption = useBehavior(vm.selectAudioButtonOption$);
const audioOutputOptions = useBehavior(vm.audioOutputOptions$);
const selectedAudioOutput = useBehavior(vm.selectedAudioOutput$);
const selectAudioOutputOption = useBehavior(vm.selectAudioOutputOption$);
const selectVideoButtonOption = useBehavior(vm.selectVideoButtonOption$);
const backgroundEffect = useBehavior(vm.backgroundEffect$);
const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$);
const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$);
const addBackgroundImage = useBehavior(vm.addBackgroundImage$);
const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$);
const cameraTrack = useBehavior(vm.cameraTrack$);
const refusal = useBehavior(vm.backgroundImageRefusal$);
const refusalMessage = useMemo(() => {
switch (refusal?.reason) {
case undefined:
return undefined;
case "not-an-image":
return { text: t("error.background_not_an_image") };
case "animated":
return { text: t("error.background_animated") };
case "undecodable":
return { text: t("error.background_undecodable") };
case "not-kept":
return { text: t("error.background_not_kept") };
}
}, [refusal, t]);
const backgroundEffects = useBehavior(vm.backgroundEffects$);
const buttonSize = useBehavior(vm.buttonSize$);
const showLogo = useBehavior(vm.showLogo$);
@@ -134,20 +221,78 @@ export const CallFooter: FC<FooterProps> = ({
);
}
buttons.push(
<MediaMenuButton
key="audio"
kind="audio"
vm={audioMenuVm}
size={buttonSize}
/>,
<MediaMenuButton
key="video"
kind="video"
vm={cameraMenuVm}
size={buttonSize}
/>,
);
if ((audioOptions?.length ?? 0) > 0) {
buttons.push(
<MediaMuteAndSwitchButton
key="audio"
iconsAndLabels="audio"
enabled={audioEnabled ?? false}
busy={audioBusy ?? false}
onMuteClick={toggleAudio}
data-testid="incall_mute"
options={audioOptions}
selectedOption={selectedAudio}
onSelect={selectAudioButtonOption}
outputOptions={audioOutputOptions}
selectedOutputOption={selectedAudioOutput}
onSelectOutput={selectAudioOutputOption}
/>,
);
} else {
buttons.push(
<MicButton
size={buttonSize}
key="audio"
enabled={audioEnabled ?? false}
busy={audioBusy ?? false}
onClick={toggleAudio}
disabled={(audioBusy ?? false) || toggleAudio === undefined}
data-testid="incall_mute"
/>,
);
}
if ((videoOptions?.length ?? 0) > 0) {
buttons.push(
<MediaMuteAndSwitchButton
key="video"
iconsAndLabels="video"
enabled={videoEnabled ?? false}
busy={videoBusy ?? false}
onMuteClick={toggleVideo}
options={videoOptions}
selectedOption={selectedVideo}
onSelect={selectVideoButtonOption}
backgroundEffects={backgroundEffects}
selectedBackgroundEffect={backgroundEffect}
onSelectBackgroundEffect={selectBackgroundEffect}
backgroundEffectSettling={backgroundEffectSettling}
onAddBackgroundImage={addBackgroundImage}
onRemoveBackgroundEffect={removeBackgroundEffect}
selfPreview={<SelfPreview track={cameraTrack} />}
backgroundImageRefusal={refusalMessage}
backgroundEffectNotice={
backgroundEffectNotice === "unavailable"
? t("background_effects.unavailable")
: backgroundEffectNotice === "slow"
? t("background_effects.slow")
: undefined
}
/>,
);
} else {
buttons.push(
<VideoButton
size={buttonSize}
key="video"
enabled={videoEnabled ?? false}
busy={videoBusy ?? false}
onClick={toggleVideo}
disabled={(videoBusy ?? false) || toggleVideo === undefined}
data-testid="incall_videomute"
/>,
);
}
if (toggleScreenSharing !== undefined) {
buttons.push(
+446 -63
View File
@@ -5,15 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it, vi } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import { BehaviorSubject } from "rxjs";
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import {
flushPromises,
testScope,
mockMuteStates,
mockMediaDevices,
mockVideoTrack,
} from "../utils/test";
import { constant } from "../state/Behavior";
import { type Behavior, constant } from "../state/Behavior";
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
import type { Layout } from "../state/layout-types";
@@ -24,6 +27,15 @@ import {
createLobbyFooterViewModel,
} from "./CallFooterViewModel";
import { HeaderStyle } from "../UrlParams";
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
import type * as BackgroundImages from "../tracks/backgroundImages";
import {
type AddedBackground,
addedBackgrounds,
maxAddedBackgrounds,
UnusableImage,
} from "../tracks/backgroundImages";
const platformMock = vi.hoisted(() => vi.fn(() => "desktop"));
vi.mock("../Platform", () => ({
@@ -32,6 +44,35 @@ vi.mock("../Platform", () => ({
},
}));
// The SDK's own check needs WebGL and canvas APIs jsdom does not have.
const sdkSupportMock = vi.hoisted(() => vi.fn(() => false));
const modernRouteMock = vi.hoisted(() => vi.fn(() => true));
vi.mock("@livekit/track-processors", () => ({
supportsBackgroundProcessors: (): boolean => sdkSupportMock(),
supportsModernBackgroundProcessors: (): boolean => modernRouteMock(),
}));
const store = vi.hoisted(() => ({
add: vi.fn<(file: Blob) => Promise<string>>(),
remove: vi.fn<(id: string) => Promise<void>>(),
}));
vi.mock("../tracks/backgroundImages", async (original) => {
const { BehaviorSubject } = await import("rxjs");
return {
...(await original<typeof BackgroundImages>()),
addedBackgrounds: {
added$: new BehaviorSubject<AddedBackground[] | undefined>([]),
add: store.add,
remove: store.remove,
},
};
});
const outputSelectionMock = vi.hoisted(() => vi.fn(() => true));
vi.mock("livekit-client", () => ({
supportsAudioOutputSelection: (): boolean => outputSelectionMock(),
}));
/**
* Returns the minimum set of CallViewModel fields required by
* createCallFooterViewModel, with all other properties stubbed to
@@ -72,82 +113,424 @@ const pipLayout: Layout = {
spotlight: {} as SpotlightTileViewModel,
};
const pipeline = constant<BackgroundEffectsState>({
supported: true,
const noPipeline = constant<BackgroundEffectsState>({
supported: false,
processor: undefined,
});
const oneOfEach = {
available$: constant(
new Map<string, DeviceLabel>([["1", { type: "number", number: 1 }]]),
),
selected$: constant(undefined),
select: vi.fn(),
};
const devices = mockMediaDevices({
audioInput: oneOfEach,
videoInput: oneOfEach,
/** The provider's verdict, which the SDK's check stands for here. */
function pipelineAsTheSdkSays(): Behavior<BackgroundEffectsState> {
return constant({ supported: sdkSupportMock(), processor: undefined });
}
const twoMicsAndOneCamMediaDevices = mockMediaDevices({
audioInput: {
available$: constant(
new Map<string, DeviceLabel>([
["mic1", { type: "number", number: 1 }],
["mic2", { type: "name", name: "Microphone 2" }],
]),
),
selected$: constant(undefined),
select: vi.fn(),
},
videoInput: {
available$: constant(
new Map<string, DeviceLabel>([
["cam1", { type: "name", name: "Camera 1" }],
]),
),
selected$: constant(undefined),
select: vi.fn(),
},
});
describe("createCallFooterViewModel", () => {
it.each([
["desktop", "grid", true, gridLayout],
["ios", "grid", false, gridLayout],
["desktop", "pip", false, pipLayout],
] as const)(
"on %s in a %s layout, offers the device menus: %s",
(platform, _, offered, layout) => {
describe("selectAudioOutputOption", () => {
function buildFooterVm(): ReturnType<typeof createCallFooterViewModel> {
platformMock.mockReturnValue("desktop");
return createCallFooterViewModel(
testScope(),
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
noPipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
}
it("is withheld where the platform cannot route audio to a chosen device", () => {
outputSelectionMock.mockReturnValue(false);
expect(buildFooterVm().selectAudioOutputOption$.value).toBeUndefined();
});
it("is offered where the platform can route audio to a chosen device", () => {
outputSelectionMock.mockReturnValue(true);
expect(buildFooterVm().selectAudioOutputOption$.value).toBeDefined();
});
});
describe("audioOutputOptions", () => {
it("is an empty list, not absent, where the platform enumerates no outputs", () => {
platformMock.mockReturnValue("desktop");
outputSelectionMock.mockReturnValue(true);
const vm = createCallFooterViewModel(
testScope(),
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
mockMediaDevices({
audioInput: {
available$: constant(
new Map<string, DeviceLabel>([
["mic1", { type: "name", name: "Microphone 1" }],
]),
),
selected$: constant(undefined),
select: vi.fn(),
},
// As Safari: no outputs listed.
audioOutput: {
available$: constant(new Map<string, DeviceLabel>()),
selected$: constant(undefined),
select: vi.fn(),
},
}),
noPipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
// Empty, not undefined: undefined would hide the section.
expect(vm.audioOutputOptions$.value).toEqual([]);
});
});
describe("audioOptions and videoOptions", () => {
function checkEmptyFor(platform: string, layout: Layout): void {
platformMock.mockReturnValue(platform);
const vm = createCallFooterViewModel(
testScope(),
buildMinimalCallViewModel(layout),
mockMuteStates(),
devices,
pipeline,
twoMicsAndOneCamMediaDevices,
noPipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
const audio = vm.audioMenuVm$.value;
const camera = vm.cameraMenuVm$.value;
expect(audio.options$.value.length > 0).toBe(offered);
expect(camera.options$.value.length > 0).toBe(offered);
expect(camera.selectBackgroundEffect$.value !== undefined).toBe(offered);
},
);
it("previews the call's camera", () => {
const track = mockVideoTrack({});
const vm = createCallFooterViewModel(
testScope(),
{ ...buildMinimalCallViewModel(gridLayout), camera$: constant(track) },
mockMuteStates(),
devices,
pipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
expect(vm.cameraMenuVm$.value.cameraTrack$.value).toBe(track);
expect(vm.audioOptions$.value).toEqual([]);
expect(vm.videoOptions$.value).toEqual([]);
}
it("are both empty when the platform is iOS", () => {
checkEmptyFor("ios", gridLayout);
});
it("are both empty when the layout is pip", () => {
checkEmptyFor("desktop", pipLayout);
});
it("are populated when the platform is desktop and the layout is not PiP", () => {
platformMock.mockReturnValue("desktop");
const vm = createCallFooterViewModel(
testScope(),
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
noPipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
expect(vm.audioOptions$?.value).toEqual([
{
id: "mic1",
label: {
number: 1,
type: "number",
},
},
{
id: "mic2",
label: {
name: "Microphone 2",
type: "name",
},
},
]);
expect(vm.videoOptions$?.value).toEqual([
{
id: "cam1",
label: {
name: "Camera 1",
type: "name",
},
},
]);
});
});
});
describe("createLobbyFooterViewModel", () => {
it("offers the device menus even on a phone, previewing the lobby's camera", () => {
platformMock.mockReturnValue("ios");
const track = mockVideoTrack({});
const vm = createLobbyFooterViewModel(
testScope(),
mockMuteStates(),
devices,
pipeline,
constant(track),
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
const camera = vm.cameraMenuVm$.value;
expect(vm.audioMenuVm$.value.options$.value).not.toEqual([]);
expect(camera.options$.value).not.toEqual([]);
expect(camera.selectBackgroundEffect$.value).toBeDefined();
expect(camera.cameraTrack$.value).toBe(track);
describe("background effects", () => {
afterEach(() => backgroundEffectSetting.setValue("none"));
// The lobby, because in a call the whole device switcher is already
// withheld on a phone, whatever the verdict says.
function lobbyOn(
platform: string,
): ReturnType<typeof createLobbyFooterViewModel> {
platformMock.mockReturnValue(platform);
return createLobbyFooterViewModel(
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
pipelineAsTheSdkSays(),
/* camera$ */ constant(null),
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
}
it("offers them on a phone whose browser can run them", () => {
sdkSupportMock.mockReturnValue(true);
expect(lobbyOn("ios").selectBackgroundEffect$.value).toBeDefined();
});
it("offers them on a desktop whose browser can run them", () => {
sdkSupportMock.mockReturnValue(true);
expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeDefined();
});
it("offers nothing where the browser itself cannot run them", () => {
sdkSupportMock.mockReturnValue(false);
expect(lobbyOn("desktop").selectBackgroundEffect$.value).toBeUndefined();
expect(lobbyOn("desktop").addBackgroundImage$.value).toBeUndefined();
});
it("puts no effect in force where the browser cannot run them", () => {
backgroundEffectSetting.setValue("blur");
sdkSupportMock.mockReturnValue(false);
expect(lobbyOn("desktop").backgroundEffect$.value).toBe("none");
sdkSupportMock.mockReturnValue(true);
expect(lobbyOn("desktop").backgroundEffect$.value).toBe("blur");
});
it("stores the effect chosen", () => {
sdkSupportMock.mockReturnValue(true);
const vm = lobbyOn("desktop");
vm.selectBackgroundEffect$.value?.("image:arc");
expect(vm.backgroundEffect$.value).toBe("image:arc");
// What isn't an effect on offer is stored as none.
vm.selectBackgroundEffect$.value?.("image:gone");
expect(backgroundEffectSetting.getValue()).toBe("none");
});
it("says they run slowly where only the slower route exists", () => {
sdkSupportMock.mockReturnValue(true);
modernRouteMock.mockReturnValue(false);
const slow = lobbyOn("desktop");
expect(slow.backgroundEffectNotice$.value).toBe("slow");
expect(slow.selectBackgroundEffect$.value).toBeDefined();
modernRouteMock.mockReturnValue(true);
expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBeUndefined();
});
it("offers every effect in order, the added ones removable", () => {
sdkSupportMock.mockReturnValue(true);
const added$ = addedBackgrounds.added$ as BehaviorSubject<
AddedBackground[] | undefined
>;
added$.next([{ id: "mine", url: "blob:mine" }]);
expect(lobbyOn("desktop").backgroundEffects$.value).toEqual([
{ id: "none", kind: "none", removable: false },
{ id: "blur", kind: "blur", removable: false },
...shippedBackgrounds.map((background) => ({
id: `image:${background.id}`,
kind: "image",
imageUrl: background.imagePath,
removable: false,
})),
{
id: "added:mine",
kind: "image",
imageUrl: "blob:mine",
removable: true,
},
]);
added$.next([]);
});
it("says they are unavailable where they cannot be chosen", () => {
sdkSupportMock.mockReturnValue(false);
expect(lobbyOn("desktop").backgroundEffectNotice$.value).toBe(
"unavailable",
);
});
it("passes on the wait for the first effect", () => {
platformMock.mockReturnValue("desktop");
const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
wait: "settling",
});
const vm = createLobbyFooterViewModel(
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
pipeline$,
/* camera$ */ constant(null),
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
expect(vm.backgroundEffectSettling$.value).toBe(true);
pipeline$.next({ ...pipeline$.value, wait: undefined });
expect(vm.backgroundEffectSettling$.value).toBe(false);
});
it("offers nothing once the pipeline fails to build, keeping the choice", () => {
platformMock.mockReturnValue("desktop");
backgroundEffectSetting.setValue("blur");
const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
});
const vm = createLobbyFooterViewModel(
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
pipeline$,
/* camera$ */ constant(null),
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
expect(vm.backgroundEffect$.value).toBe("blur");
pipeline$.next({ supported: false, processor: undefined });
expect(vm.selectBackgroundEffect$.value).toBeUndefined();
expect(vm.addBackgroundImage$.value).toBeUndefined();
expect(vm.backgroundEffect$.value).toBe("none");
expect(backgroundEffectSetting.getValue()).toBe("blur");
});
it("adding is unavailable at the limit", () => {
sdkSupportMock.mockReturnValue(true);
const added$ = addedBackgrounds.added$ as BehaviorSubject<
AddedBackground[] | undefined
>;
const vm = lobbyOn("desktop");
expect(vm.addBackgroundImage$.value).toBeDefined();
added$.next(
Array.from({ length: maxAddedBackgrounds }, (_, i) => ({
id: `${i}`,
url: `blob:${i}`,
})),
);
expect(vm.addBackgroundImage$.value).toBeUndefined();
added$.next(added$.value!.slice(1));
expect(vm.addBackgroundImage$.value).toBeDefined();
added$.next([]);
});
it("says why each refused file was refused", async () => {
sdkSupportMock.mockReturnValue(true);
const vm = lobbyOn("desktop");
store.add.mockRejectedValue(new UnusableImage("animated"));
vm.addBackgroundImage$.value!(new File(["x"], "a.gif"));
await flushPromises();
const first = vm.backgroundImageRefusal$.value;
expect(first).toEqual({ reason: "animated" });
// The same again is a refusal of its own, so it is shown again.
vm.addBackgroundImage$.value!(new File(["x"], "a.gif"));
await flushPromises();
expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "animated" });
expect(vm.backgroundImageRefusal$.value).not.toBe(first);
store.add.mockRejectedValue(new Error("quota"));
vm.addBackgroundImage$.value!(new File(["x"], "b.png"));
await flushPromises();
expect(vm.backgroundImageRefusal$.value).toEqual({ reason: "not-kept" });
store.add.mockResolvedValue("kept");
vm.addBackgroundImage$.value!(new File(["x"], "c.png"));
await flushPromises();
expect(vm.backgroundImageRefusal$.value).toBeUndefined();
});
it("never removes the background in force", () => {
sdkSupportMock.mockReturnValue(true);
store.remove.mockResolvedValue();
const vm = lobbyOn("desktop");
backgroundEffectSetting.setValue("added:worn");
vm.removeBackgroundEffect$.value!("added:worn");
expect(store.remove).not.toHaveBeenCalled();
vm.removeBackgroundEffect$.value!("added:spare");
expect(store.remove).toHaveBeenCalledWith("spare");
});
it("previews the lobby's camera while there is one", () => {
const track = mockVideoTrack({});
const camera$ = new BehaviorSubject<LocalVideoMediaTrack | null>(track);
const vm = createLobbyFooterViewModel(
testScope(),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
noPipeline,
camera$,
/* openSettings */ undefined,
/* hangup */ undefined,
/* showLogo */ false,
);
expect(vm.cameraTrack$.value).toBe(track);
// The lobby releases the camera when it is turned off
camera$.next(null);
expect(vm.cameraTrack$.value).toBeNull();
});
it("previews the call's camera only while it is not muted", () => {
const muted$ = new BehaviorSubject(false);
const track = mockVideoTrack({ muted$ });
const vm = createCallFooterViewModel(
testScope(),
{ ...buildMinimalCallViewModel(gridLayout), camera$: constant(track) },
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
noPipeline,
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
expect(vm.cameraTrack$.value).toBe(track);
muted$.next(true);
expect(vm.cameraTrack$.value).toBeNull();
muted$.next(false);
expect(vm.cameraTrack$.value).toBe(track);
});
it("availability is the same before and during a call", () => {
for (const supported of [true, false]) {
sdkSupportMock.mockReturnValue(supported);
const lobby = lobbyOn("desktop");
const inCall = createCallFooterViewModel(
testScope(),
buildMinimalCallViewModel(gridLayout),
mockMuteStates(),
twoMicsAndOneCamMediaDevices,
pipelineAsTheSdkSays(),
/* reactionIdentifier */ undefined,
{ showControls: true, header: HeaderStyle.Standard },
);
const offeredInLobby =
lobby.selectBackgroundEffect$.value !== undefined;
expect(inCall.selectBackgroundEffect$.value !== undefined).toBe(
offeredInLobby,
);
expect(offeredInLobby).toBe(supported);
}
});
});
});
+305 -54
View File
@@ -5,24 +5,253 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { combineLatest, map, type Observable } from "rxjs";
import {
BehaviorSubject,
combineLatest,
distinctUntilChanged,
map,
type Observable,
of,
switchMap,
} from "rxjs";
import { supportsAudioOutputSelection } from "livekit-client";
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import { supportsModernBackgroundProcessors } from "@livekit/track-processors";
import { logger } from "matrix-js-sdk/lib/logger";
import {
parseEffect,
serializeEffect,
shippedBackgrounds,
} from "../tracks/backgroundEffectCatalog";
import {
type AddedBackground,
addedBackgrounds,
maxAddedBackgrounds,
UnusableImage,
} from "../tracks/backgroundImages";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
import { type MenuOptions } from "./MediaMuteAndSwitchButton";
import { type MediaDevices } from "../state/MediaDevices";
import { debugTileLayout as debugTileLayoutSetting } from "../settings/settings";
import {
backgroundEffect as backgroundEffectSetting,
debugTileLayout as debugTileLayoutSetting,
} from "../settings/settings";
import { type Behavior, constant } from "../state/Behavior";
import type { ObservableScope } from "../state/ObservableScope";
import { type MuteStates } from "../state/MuteStates";
import { createStaticViewModel, type ViewModel } from "../state/ViewModel";
import { HeaderStyle } from "../UrlParams";
import { platform } from "../Platform";
import { type FooterSnapshot } from "./CallFooter";
import {
createAudioMenuViewModel,
createCameraMenuViewModel,
} from "./MediaMenuViewModel";
import { type BackgroundImageRefusal, type FooterSnapshot } from "./CallFooter";
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
/**
* Shared helper: maps MuteStates into the audio/video enabled + toggle behaviors
* needed by FooterSnapshot.
*/
function buildMuteBehaviors(
scope: ObservableScope,
muteStates: MuteStates,
): Pick<
ViewModel<FooterSnapshot>,
| "audioEnabled$"
| "audioBusy$"
| "toggleAudio$"
| "videoEnabled$"
| "videoBusy$"
| "toggleVideo$"
> {
return {
audioEnabled$: muteStates.audio.enabled$,
audioBusy$: muteStates.audio.syncing$,
toggleAudio$: scope.behavior(
muteStates.audio.toggle$.pipe(map((t) => t ?? undefined)),
),
videoEnabled$: muteStates.video.enabled$,
videoBusy$: muteStates.video.syncing$,
toggleVideo$: scope.behavior(
muteStates.video.toggle$.pipe(map((t) => t ?? undefined)),
),
};
}
/**
* Shared helper: maps MediaDevices into the audio/video device-list behaviors
* needed by FooterSnapshot (options, selection, callbacks, background effect).
*/
function buildDeviceBehaviors(
scope: ObservableScope,
mediaDevices: MediaDevices,
/** return empty arrays for audioOptions and videoOptions*/
disableSwitcher$: Behavior<boolean>,
backgroundEffectsState$: Observable<BackgroundEffectsState>,
/** The camera as the pipeline has it, for the menu's preview. */
camera$: Observable<LocalVideoMediaTrack | null | undefined>,
): Pick<
ViewModel<FooterSnapshot>,
| "audioOptions$"
| "selectedAudio$"
| "selectAudioButtonOption$"
| "audioOutputOptions$"
| "selectedAudioOutput$"
| "selectAudioOutputOption$"
| "videoOptions$"
| "selectedVideo$"
| "selectVideoButtonOption$"
| "backgroundEffect$"
| "selectBackgroundEffect$"
| "backgroundEffects$"
| "backgroundEffectNotice$"
| "backgroundEffectSettling$"
| "addBackgroundImage$"
| "backgroundImageRefusal$"
| "removeBackgroundEffect$"
| "cameraTrack$"
> {
const options$ = (
available$: Behavior<Map<string, MenuOptions["label"]>>,
): Observable<MenuOptions[]> =>
disableSwitcher$.pipe(
switchMap((disable) =>
disable
? constant([] as MenuOptions[])
: available$.pipe(
map((available) =>
[...available.entries()].map(([id, label]) => ({ id, label })),
),
),
),
);
// Also false once the pipeline has failed to build this session.
const supported$ = backgroundEffectsState$.pipe(
map(({ supported }) => supported === true),
distinctUntilChanged(),
);
const slow = !supportsModernBackgroundProcessors();
const offered$ = combineLatest([disableSwitcher$, supported$]).pipe(
map(([switcherDisabled, supported]) => !switcherDisabled && supported),
);
// A new object for each refusal, so the same one twice is shown twice.
const refusal$ = new BehaviorSubject<BackgroundImageRefusal | undefined>(
undefined,
);
return {
audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)),
selectedAudio$: scope.behavior(
mediaDevices.audioInput.selected$.pipe(map((s) => s?.id)),
),
selectAudioButtonOption$: constant(mediaDevices.audioInput.select),
audioOutputOptions$: scope.behavior(
options$(mediaDevices.audioOutput.available$),
),
selectedAudioOutput$: scope.behavior(
mediaDevices.audioOutput.selected$.pipe(map((s) => s?.id)),
),
// Withheld where the platform can't route audio to a chosen device, which
// disables the speaker section.
selectAudioOutputOption$: constant(
supportsAudioOutputSelection()
? mediaDevices.audioOutput.select
: undefined,
),
videoOptions$: scope.behavior(options$(mediaDevices.videoInput.available$)),
selectedVideo$: scope.behavior(
mediaDevices.videoInput.selected$.pipe(map((s) => s?.id)),
),
selectVideoButtonOption$: constant(mediaDevices.videoInput.select),
backgroundEffect$: scope.behavior(
combineLatest([backgroundEffectSetting.value$, supported$]).pipe(
map(([raw, supported]) =>
supported ? serializeEffect(parseEffect(raw)) : "none",
),
),
),
selectBackgroundEffect$: scope.behavior(
offered$.pipe(
map((offered) =>
offered
? (id: string): void =>
backgroundEffectSetting.setValue(
serializeEffect(parseEffect(id)),
)
: undefined,
),
),
),
backgroundEffects$: scope.behavior(
addedBackgrounds.added$.pipe(
map((added) => backgroundEffectChoices(added ?? [])),
),
),
backgroundEffectNotice$: scope.behavior(
offered$.pipe(
map((offered) =>
!offered ? "unavailable" : slow ? "slow" : undefined,
),
),
),
backgroundEffectSettling$: scope.behavior(
backgroundEffectsState$.pipe(
map(({ wait }) => wait !== undefined),
distinctUntilChanged(),
),
false,
),
// Kept and offered, not put on: that waits for the user to choose it.
addBackgroundImage$: scope.behavior(
combineLatest([offered$, addedBackgrounds.added$]).pipe(
map(([offered, added]) =>
offered && (added?.length ?? 0) < maxAddedBackgrounds
? (file: File): void => {
refusal$.next(undefined);
addedBackgrounds.add(file).catch((e) => {
logger.warn("Could not keep that background", e);
refusal$.next({
reason: e instanceof UnusableImage ? e.reason : "not-kept",
});
});
}
: undefined,
),
),
),
backgroundImageRefusal$: refusal$,
// None while muted: the call mutes its track rather than releasing it.
cameraTrack$: scope.behavior(
camera$.pipe(
switchMap((camera) =>
camera
? camera.muted$.pipe(map((muted) => (muted ? null : camera)))
: of(null),
),
),
null,
),
// Never the one in force: taking it away would leave nothing chosen.
removeBackgroundEffect$: scope.behavior(
offered$.pipe(
map((offered) =>
offered
? (optionId: string): void => {
const effect = parseEffect(optionId);
if (effect.kind !== "added") return;
if (backgroundEffectSetting.getValue() === optionId) return;
addedBackgrounds
.remove(effect.id)
.catch((e) =>
logger.warn("Could not remove that background", e),
);
}
: undefined,
),
),
),
};
}
/**
* Creates the ViewModel for the CallFooter.
@@ -57,23 +286,13 @@ export function createCallFooterViewModel(
isPip$.pipe(map((isPip) => isPip || platform !== "desktop")),
);
return {
audioMenuVm$: constant(
createAudioMenuViewModel(
scope,
muteStates,
mediaDevices,
disableDeviceSwitcher$,
),
),
cameraMenuVm$: constant(
createCameraMenuViewModel(
scope,
muteStates,
mediaDevices,
disableDeviceSwitcher$,
backgroundEffectsState$,
callModel.camera$,
),
...buildMuteBehaviors(scope, muteStates),
...buildDeviceBehaviors(
scope,
mediaDevices,
disableDeviceSwitcher$,
backgroundEffectsState$,
callModel.camera$,
),
// candidat to move into the FooterViewModel
showFooter$: callModel.showFooter$,
@@ -153,40 +372,72 @@ export function createLobbyFooterViewModel(
hangup: (() => void) | undefined,
showLogo: boolean,
): ViewModel<FooterSnapshot> {
return createStaticViewModel({
// we can safly skip any props that we do not need.
// The view model will then have less keys.
// But as soon as we call `useViewModel` and convert back to a snapshot the missing props will
// be correcty matching the snapshot type.
showLogo,
hideControls: false,
asOverlay: false,
showModals: true,
buttonSize: "lg",
openSettings,
hangup,
debugTileLayout: false,
showFooter: true,
toggleScreenSharing: undefined,
layoutSwitchVm: null,
sharingScreen: false,
audioOutputSwitcher: undefined,
reactionIdentifier: undefined,
reactionData: undefined,
tileStoreGeneration: undefined,
audioMenuVm: createAudioMenuViewModel(
return {
...createStaticViewModel({
// we can safly skip any props that we do not need.
// The view model will then have less keys.
// But as soon as we call `useViewModel` and convert back to a snapshot the missing props will
// be correcty matching the snapshot type.
showLogo,
hideControls: false,
asOverlay: false,
showModals: true,
buttonSize: "lg",
openSettings,
hangup,
debugTileLayout: false,
showFooter: true,
toggleAudio: undefined,
toggleVideo: undefined,
toggleScreenSharing: undefined,
audioEnabled: undefined,
audioBusy: false,
videoEnabled: undefined,
videoBusy: false,
layoutSwitchVm: null,
sharingScreen: false,
audioOutputSwitcher: undefined,
reactionIdentifier: undefined,
reactionData: undefined,
tileStoreGeneration: undefined,
audioOptions: undefined,
audioOutputOptions: undefined,
videoOptions: undefined,
selectedAudio: undefined,
selectedAudioOutput: undefined,
selectedVideo: undefined,
selectAudioButtonOption: undefined,
selectAudioOutputOption: undefined,
selectVideoButtonOption: undefined,
}),
...buildMuteBehaviors(scope, muteStates),
...buildDeviceBehaviors(
scope,
muteStates,
mediaDevices,
constant(false),
),
cameraMenuVm: createCameraMenuViewModel(
scope,
muteStates,
mediaDevices,
constant(false),
backgroundEffectsState$,
camera$,
),
});
};
}
function backgroundEffectChoices(
added: AddedBackground[],
): BackgroundEffectGridItem[] {
return [
{ id: serializeEffect({ kind: "none" }), kind: "none", removable: false },
{ id: serializeEffect({ kind: "blur" }), kind: "blur", removable: false },
...shippedBackgrounds.map((background) => ({
id: serializeEffect({ kind: "shipped", id: background.id }),
kind: "image" as const,
imageUrl: background.imagePath,
removable: false,
})),
...added.map((background) => ({
id: serializeEffect({ kind: "added", id: background.id }),
kind: "image" as const,
imageUrl: background.url,
removable: true,
})),
];
}
-193
View File
@@ -1,193 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, fn, userEvent, within } from "storybook/test";
import { type JSX, type ReactNode } from "react";
import type { Meta, StoryObj } from "@storybook/react-vite";
import { MediaMenuButton } from "./MediaMenuButton";
import { type MediaMenuSnapshot } from "./MediaMenuViewModel";
import { useStaticViewModel } from "../state/ViewModel";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { MediaDevices } from "../state/MediaDevices";
import { globalScope } from "../state/ObservableScope";
const mediaDevices = new MediaDevices(globalScope, {
controlledAudioDevices: false,
});
/** Builds the view model from the story's args, so each field is a control. */
function MediaMenuButtonStoryWrapper({
kind,
...snapshot
}: MediaMenuSnapshot & { kind: "audio" | "video" }): ReactNode {
const vm = useStaticViewModel(snapshot);
return <MediaMenuButton kind={kind} vm={vm} size="lg" />;
}
const meta = {
component: MediaMenuButtonStoryWrapper,
decorators: [
(Story): JSX.Element => (
<MediaDevicesContext value={mediaDevices}>
<Story />
</MediaDevicesContext>
),
],
} satisfies Meta<typeof MediaMenuButtonStoryWrapper>;
export default meta;
type Story = StoryObj<typeof meta>;
const empty = {
outputOptions: [],
selectedOutput: undefined,
selectOutput: undefined,
backgroundEffect: undefined,
backgroundEffects: [],
backgroundEffectNotice: undefined,
backgroundEffectSettling: false,
backgroundImageRefusal: undefined,
selectBackgroundEffect: undefined,
addBackgroundImage: undefined,
removeBackgroundEffect: undefined,
cameraTrack: null,
} satisfies Partial<MediaMenuSnapshot>;
export const Microphone: Story = {
args: {
...empty,
kind: "audio",
enabled: true,
busy: false,
toggle: fn(),
options: [
{ id: "mic1", label: { type: "name", name: "Microphone 1" } },
{ id: "mic2", label: { type: "name", name: "Microphone 2" } },
],
selected: "mic1",
select: fn(),
outputOptions: [
{ id: "default", label: { type: "default", name: "Built-in Output" } },
],
selectedOutput: "default",
selectOutput: fn(),
},
play: async ({ args, canvasElement }) => {
await userEvent.click(
within(canvasElement).getByRole("switch", { name: "Mute microphone" }),
);
await expect(args.toggle).toHaveBeenCalled();
},
};
export const Camera: Story = {
args: {
...empty,
kind: "video",
enabled: true,
busy: false,
toggle: fn(),
options: [{ id: "cam1", label: { type: "name", name: "Camera 1" } }],
selected: "cam1",
select: fn(),
backgroundEffect: "none",
backgroundEffects: [
{ id: "none", kind: "none" },
{ id: "blur", kind: "blur" },
],
selectBackgroundEffect: fn(),
},
play: async ({ args, canvasElement }) => {
await userEvent.click(
within(canvasElement).getByRole("switch", { name: "Stop video" }),
);
await expect(args.toggle).toHaveBeenCalled();
},
};
export const CameraBusy: Story = {
args: { ...Camera.args, busy: true },
};
/** Opens the effects, for the stories that check what they say. */
async function openEffects(canvasElement: HTMLElement): Promise<HTMLElement> {
await userEvent.click(
within(canvasElement).getByRole("button", { name: "Camera" }),
);
return within(document.body).findByRole("group", {
name: "Background effects",
});
}
/** The view model says why; the view says it in words. */
export const CameraEffectsUnavailable: Story = {
args: {
...Camera.args,
selectBackgroundEffect: undefined,
backgroundEffectNotice: "unavailable",
},
play: async ({ canvasElement }) => {
const section = await openEffects(canvasElement);
await expect(
within(section).getByRole("group"),
).toHaveAccessibleDescription(
"Background effects are not supported on this platform.",
);
},
};
export const CameraEffectsSlow: Story = {
args: {
...Camera.args,
backgroundEffectNotice: "slow",
},
play: async ({ canvasElement }) => {
const section = await openEffects(canvasElement);
await expect(
within(section).getByRole("group"),
).toHaveAccessibleDescription(
"Background effects run slowly on this platform, which may cause your video to stutter.",
);
},
};
export const CameraFileRefused: Story = {
args: {
...Camera.args,
addBackgroundImage: fn(),
backgroundImageRefusal: { reason: "not-an-image", dismiss: fn() },
},
play: async ({ canvasElement }) => {
await openEffects(canvasElement);
await expect(
await within(document.body).findByText(
"That file is not a supported image",
),
).toBeInTheDocument();
},
};
/** No devices to pick and none to mute: a plain, disabled mute button. */
export const WithoutDevices: Story = {
args: {
...Camera.args,
enabled: false,
toggle: undefined,
options: [],
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await expect(
canvas.queryByRole("button", { name: "Camera" }),
).not.toBeInTheDocument();
await expect(canvas.getByTestId("incall_videomute")).toHaveAttribute(
"aria-disabled",
"true",
);
},
};
-124
View File
@@ -1,124 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type FC, useMemo } from "react";
import { useTranslation } from "react-i18next";
import { MicButton, VideoButton } from "../button";
import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton";
import { type MediaMenuViewModel } from "./MediaMenuViewModel";
import { SelfPreview } from "./SelfPreview";
import { useBehavior } from "../useBehavior";
interface Props {
vm: MediaMenuViewModel;
kind: "audio" | "video";
size: "md" | "lg";
}
/**
* The microphone's or the camera's mute button: with a menu to pick the
* device while there are devices to offer, a plain mute button otherwise.
*/
export const MediaMenuButton: FC<Props> = ({ vm, kind, size }) => {
const { t } = useTranslation();
const enabled = useBehavior(vm.enabled$);
const busy = useBehavior(vm.busy$);
const toggle = useBehavior(vm.toggle$);
const options = useBehavior(vm.options$);
const selected = useBehavior(vm.selected$);
const select = useBehavior(vm.select$);
const outputOptions = useBehavior(vm.outputOptions$);
const selectedOutput = useBehavior(vm.selectedOutput$);
const selectOutput = useBehavior(vm.selectOutput$);
const backgroundEffect = useBehavior(vm.backgroundEffect$);
const backgroundEffects = useBehavior(vm.backgroundEffects$);
const notice = useBehavior(vm.backgroundEffectNotice$);
const settling = useBehavior(vm.backgroundEffectSettling$);
const refusal = useBehavior(vm.backgroundImageRefusal$);
const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
const addBackgroundImage = useBehavior(vm.addBackgroundImage$);
const removeBackgroundEffect = useBehavior(vm.removeBackgroundEffect$);
const cameraTrack = useBehavior(vm.cameraTrack$);
const testId = kind === "audio" ? "incall_mute" : "incall_videomute";
const noticeText = useMemo(() => {
switch (notice) {
case undefined:
return undefined;
case "unavailable":
return t("background_effects.unavailable");
case "slow":
return t("background_effects.slow");
}
}, [notice, t]);
const refusalText = useMemo(() => {
switch (refusal?.reason) {
case undefined:
return undefined;
case "not-an-image":
return t("error.background_not_an_image");
case "animated":
return t("error.background_animated");
case "undecodable":
return t("error.background_undecodable");
case "not-kept":
return t("error.background_not_kept");
}
}, [refusal, t]);
if (options.length === 0) {
const PlainButton = kind === "audio" ? MicButton : VideoButton;
return (
<PlainButton
size={size}
enabled={enabled}
busy={busy}
onClick={toggle}
disabled={busy || toggle === undefined}
data-testid={testId}
/>
);
}
return kind === "audio" ? (
<MediaMuteAndSwitchButton
iconsAndLabels="audio"
enabled={enabled}
busy={busy}
onMuteClick={toggle}
data-testid={testId}
options={options}
selectedOption={selected}
onSelect={select}
outputOptions={outputOptions}
selectedOutputOption={selectedOutput}
onSelectOutput={selectOutput}
/>
) : (
<MediaMuteAndSwitchButton
iconsAndLabels="video"
enabled={enabled}
busy={busy}
onMuteClick={toggle}
options={options}
selectedOption={selected}
onSelect={select}
backgroundEffects={backgroundEffects}
selectedBackgroundEffect={backgroundEffect}
onSelectBackgroundEffect={selectBackgroundEffect}
backgroundEffectSettling={settling}
onAddBackgroundImage={addBackgroundImage}
onRemoveBackgroundEffect={removeBackgroundEffect}
selfPreview={<SelfPreview track={cameraTrack} />}
backgroundImageRefusal={refusalText}
onDismissBackgroundImageRefusal={refusal?.dismiss}
backgroundEffectNotice={noticeText}
/>
);
};
-294
View File
@@ -1,294 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, describe, expect, it, vi } from "vitest";
import { BehaviorSubject, type Observable } from "rxjs";
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import {
flushPromises,
testScope,
mockMuteStates,
mockMediaDevices,
mockVideoTrack,
} from "../utils/test";
import { constant } from "../state/Behavior";
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
import type { DeviceLabel } from "../state/MediaDevices";
import {
createAudioMenuViewModel,
createCameraMenuViewModel,
type MediaMenuViewModel,
} from "./MediaMenuViewModel";
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
import { shippedBackgrounds } from "../tracks/backgroundEffectCatalog";
import type * as BackgroundImages from "../tracks/backgroundImages";
import {
type AddedBackground,
addedBackgrounds,
maxAddedBackgrounds,
UnusableImage,
} from "../tracks/backgroundImages";
const modernRouteMock = vi.hoisted(() => vi.fn(() => true));
vi.mock("@livekit/track-processors", () => ({
supportsModernBackgroundProcessors: (): boolean => modernRouteMock(),
}));
const outputSelectionMock = vi.hoisted(() => vi.fn(() => true));
vi.mock("livekit-client", () => ({
supportsAudioOutputSelection: (): boolean => outputSelectionMock(),
}));
const store = vi.hoisted(() => ({
add: vi.fn<(file: Blob) => Promise<string>>(),
remove: vi.fn<(id: string) => Promise<void>>(),
}));
vi.mock("../tracks/backgroundImages", async (original) => {
const { BehaviorSubject } = await import("rxjs");
return {
...(await original<typeof BackgroundImages>()),
addedBackgrounds: {
added$: new BehaviorSubject<AddedBackground[] | undefined>([]),
add: store.add,
remove: store.remove,
},
};
});
const added$ = addedBackgrounds.added$ as BehaviorSubject<
AddedBackground[] | undefined
>;
const devices = mockMediaDevices({
audioInput: {
available$: constant(
new Map<string, DeviceLabel>([
["mic1", { type: "number", number: 1 }],
["mic2", { type: "name", name: "Microphone 2" }],
]),
),
selected$: constant(undefined),
select: vi.fn(),
},
// As Safari: no outputs listed.
audioOutput: {
available$: constant(new Map<string, DeviceLabel>()),
selected$: constant(undefined),
select: vi.fn(),
},
});
function audioMenu(disableSwitcher = false): MediaMenuViewModel {
return createAudioMenuViewModel(
testScope(),
mockMuteStates(),
devices,
constant(disableSwitcher),
);
}
function cameraMenu({
supported = true,
disableSwitcher = false,
pipeline$ = constant({ supported, processor: undefined }),
camera$ = constant(null),
}: {
supported?: boolean;
disableSwitcher?: boolean;
pipeline$?: Observable<BackgroundEffectsState>;
camera$?: Observable<LocalVideoMediaTrack | null>;
} = {}): MediaMenuViewModel {
return createCameraMenuViewModel(
testScope(),
mockMuteStates(),
devices,
constant(disableSwitcher),
pipeline$,
camera$,
);
}
describe("createAudioMenuViewModel", () => {
it("lists the devices, unless the switcher is withheld", () => {
expect(audioMenu().options$.value).toEqual([
{ id: "mic1", label: { type: "number", number: 1 } },
{ id: "mic2", label: { type: "name", name: "Microphone 2" } },
]);
expect(audioMenu(true).options$.value).toEqual([]);
});
it("lists no outputs as an empty list, which keeps the section", () => {
expect(audioMenu().outputOptions$.value).toEqual([]);
});
it.each([true, false])(
"offers to pick an output only where the platform can route to it (%s)",
(canRoute) => {
outputSelectionMock.mockReturnValue(canRoute);
expect(audioMenu().selectOutput$.value !== undefined).toBe(canRoute);
},
);
});
describe("createCameraMenuViewModel", () => {
afterEach(() => {
backgroundEffectSetting.setValue("none");
added$.next([]);
});
it("offers effects where they run, saying where they run slowly", () => {
modernRouteMock.mockReturnValue(true);
const vm = cameraMenu();
expect(vm.selectBackgroundEffect$.value).toBeDefined();
expect(vm.addBackgroundImage$.value).toBeDefined();
expect(vm.backgroundEffectNotice$.value).toBeUndefined();
modernRouteMock.mockReturnValue(false);
const slow = cameraMenu();
expect(slow.selectBackgroundEffect$.value).toBeDefined();
expect(slow.backgroundEffectNotice$.value).toBe("slow");
modernRouteMock.mockReturnValue(true);
});
it("offers nothing while the switcher is withheld", () => {
const vm = cameraMenu({ disableSwitcher: true });
expect(vm.options$.value).toEqual([]);
expect(vm.selectBackgroundEffect$.value).toBeUndefined();
expect(vm.backgroundEffectNotice$.value).toBe("unavailable");
});
it("offers nothing once the pipeline fails to build, keeping the choice", () => {
backgroundEffectSetting.setValue("blur");
const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
});
const vm = cameraMenu({ pipeline$ });
expect(vm.backgroundEffect$.value).toBe("blur");
pipeline$.next({ supported: false, processor: undefined });
expect(vm.selectBackgroundEffect$.value).toBeUndefined();
expect(vm.addBackgroundImage$.value).toBeUndefined();
expect(vm.removeBackgroundEffect$.value).toBeUndefined();
expect(vm.backgroundEffectNotice$.value).toBe("unavailable");
expect(vm.backgroundEffect$.value).toBe("none");
expect(backgroundEffectSetting.getValue()).toBe("blur");
});
it("saves the effect chosen, and none for one not on offer", () => {
const vm = cameraMenu();
vm.selectBackgroundEffect$.value!("image:arc");
expect(vm.backgroundEffect$.value).toBe("image:arc");
vm.selectBackgroundEffect$.value!("image:gone");
expect(backgroundEffectSetting.getValue()).toBe("none");
});
it("offers every effect in order, the added ones removable", () => {
added$.next([{ id: "mine", url: "blob:mine" }]);
expect(cameraMenu().backgroundEffects$.value).toEqual([
{ id: "none", kind: "none", removable: false },
{ id: "blur", kind: "blur", removable: false },
...shippedBackgrounds.map((background) => ({
id: `image:${background.id}`,
kind: "image",
imageUrl: background.imagePath,
removable: false,
})),
{
id: "added:mine",
kind: "image",
imageUrl: "blob:mine",
removable: true,
},
]);
});
it("passes on the wait for the first effect", () => {
const pipeline$ = new BehaviorSubject<BackgroundEffectsState>({
supported: true,
processor: undefined,
wait: "settling",
});
const vm = cameraMenu({ pipeline$ });
expect(vm.backgroundEffectSettling$.value).toBe(true);
pipeline$.next({ ...pipeline$.value, wait: undefined });
expect(vm.backgroundEffectSettling$.value).toBe(false);
});
it("offers adding only below the limit", () => {
const vm = cameraMenu();
added$.next(
Array.from({ length: maxAddedBackgrounds }, (_, i) => ({
id: `${i}`,
url: `blob:${i}`,
})),
);
expect(vm.addBackgroundImage$.value).toBeUndefined();
added$.next(added$.value!.slice(1));
expect(vm.addBackgroundImage$.value).toBeDefined();
});
it("says why each refused file was refused", async () => {
const vm = cameraMenu();
store.add.mockRejectedValue(new UnusableImage("animated"));
vm.addBackgroundImage$.value!(new File(["x"], "a.gif"));
await flushPromises();
expect(vm.backgroundImageRefusal$.value?.reason).toBe("animated");
store.add.mockRejectedValue(new Error("quota"));
vm.addBackgroundImage$.value!(new File(["x"], "b.png"));
await flushPromises();
expect(vm.backgroundImageRefusal$.value?.reason).toBe("not-kept");
store.add.mockResolvedValue("kept");
vm.addBackgroundImage$.value!(new File(["x"], "c.png"));
await flushPromises();
expect(vm.backgroundImageRefusal$.value).toBeUndefined();
});
it("drops a refusal once dismissed, and only that one", async () => {
const vm = cameraMenu();
store.add.mockRejectedValue(new UnusableImage("animated"));
vm.addBackgroundImage$.value!(new File(["x"], "a.gif"));
await flushPromises();
const first = vm.backgroundImageRefusal$.value!;
first.dismiss();
expect(vm.backgroundImageRefusal$.value).toBeUndefined();
// The same again is a refusal of its own, which the first can't dismiss.
vm.addBackgroundImage$.value!(new File(["x"], "a.gif"));
await flushPromises();
expect(vm.backgroundImageRefusal$.value?.reason).toBe("animated");
first.dismiss();
expect(vm.backgroundImageRefusal$.value?.reason).toBe("animated");
});
it("never removes the selected background", () => {
store.remove.mockResolvedValue();
const vm = cameraMenu();
backgroundEffectSetting.setValue("added:worn");
vm.removeBackgroundEffect$.value!("added:worn");
expect(store.remove).not.toHaveBeenCalled();
vm.removeBackgroundEffect$.value!("added:spare");
expect(store.remove).toHaveBeenCalledWith("spare");
});
it("previews the camera only while there is one and it is not muted", () => {
const muted$ = new BehaviorSubject(false);
const track = mockVideoTrack({ muted$ });
const camera$ = new BehaviorSubject<LocalVideoMediaTrack | null>(track);
const vm = cameraMenu({ camera$ });
expect(vm.cameraTrack$.value).toBe(track);
muted$.next(true);
expect(vm.cameraTrack$.value).toBeNull();
muted$.next(false);
expect(vm.cameraTrack$.value).toBe(track);
camera$.next(null);
expect(vm.cameraTrack$.value).toBeNull();
});
});
-324
View File
@@ -1,324 +0,0 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
BehaviorSubject,
combineLatest,
distinctUntilChanged,
map,
type Observable,
of,
switchMap,
} from "rxjs";
import { supportsAudioOutputSelection } from "livekit-client";
import { type LocalVideoMediaTrack } from "@element-hq/matrixrtc-sdk";
import { supportsModernBackgroundProcessors } from "@livekit/track-processors";
import { logger } from "matrix-js-sdk/lib/logger";
import {
parseEffect,
serializeEffect,
shippedBackgrounds,
} from "../tracks/backgroundEffectCatalog";
import {
type AddedBackground,
addedBackgrounds,
maxAddedBackgrounds,
type UnusableReason,
UnusableImage,
} from "../tracks/backgroundImages";
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
import { type MenuOptions } from "./MediaMuteAndSwitchButton";
import { type MediaDevices } from "../state/MediaDevices";
import { backgroundEffect as backgroundEffectSetting } from "../settings/settings";
import { type Behavior, constant } from "../state/Behavior";
import type { ObservableScope } from "../state/ObservableScope";
import { type MuteStates } from "../state/MuteStates";
import { createStaticViewModel, type ViewModel } from "../state/ViewModel";
import { type BackgroundEffectGridItem } from "./BackgroundEffectGrid";
export interface BackgroundImageRefusal {
reason: UnusableReason | "not-kept";
/** Done with it: once read, or once the menu showing it closes. */
dismiss: () => void;
}
/**
* A mute button with a menu to pick the device, shared by the microphone and
* the camera. Fields that belong to the other device are left empty.
*/
export interface MediaMenuSnapshot {
/** Whether the device is on. (muted/unmuted state) */
enabled: boolean;
busy: boolean;
/** Turns the device on or off. Undefined while it can't be toggled. (disable interaction state) */
toggle: (() => void) | undefined;
options: MenuOptions[];
selected: string | undefined;
select: ((id: string) => void) | undefined;
/** Output (speaker) devices. Microphone only. */
outputOptions: MenuOptions[];
selectedOutput: string | undefined;
/** Undefined if the output device is not configurable. */
selectOutput: ((id: string) => void) | undefined;
/** The selected background effect, as its option id. Camera only. */
backgroundEffect: string | undefined;
/** Every effect on offer, in the order they are shown. */
backgroundEffects: BackgroundEffectGridItem[];
/** What the user is told about effects here, if anything. */
backgroundEffectNotice: "unavailable" | "slow" | undefined;
/** Whether the first effect chosen is still being prepared. */
backgroundEffectSettling: boolean;
/** Why the last file offered couldn't be kept, if it couldn't. */
backgroundImageRefusal: BackgroundImageRefusal | undefined;
/** Undefined where background effects can't be chosen. */
selectBackgroundEffect: ((id: string) => void) | undefined;
/** Undefined where no background can be added. */
addBackgroundImage: ((file: File) => void) | undefined;
/** Removes an added background by its option id, unless it is the selected one. */
removeBackgroundEffect: ((id: string) => void) | undefined;
/** The camera to preview, with the selected effect; null while it is off. */
cameraTrack: LocalVideoMediaTrack | null;
}
export type MediaMenuViewModel = ViewModel<MediaMenuSnapshot>;
const noBackgroundEffects = {
backgroundEffect: undefined,
backgroundEffects: [],
backgroundEffectNotice: undefined,
backgroundEffectSettling: false,
backgroundImageRefusal: undefined,
selectBackgroundEffect: undefined,
addBackgroundImage: undefined,
removeBackgroundEffect: undefined,
cameraTrack: null,
} satisfies Partial<MediaMenuSnapshot>;
function options$(
disableSwitcher$: Behavior<boolean>,
available$: Behavior<Map<string, MenuOptions["label"]>>,
): Observable<MenuOptions[]> {
return disableSwitcher$.pipe(
switchMap((disable) =>
disable
? constant([] as MenuOptions[])
: available$.pipe(
map((available) =>
[...available.entries()].map(([id, label]) => ({ id, label })),
),
),
),
);
}
/**
* The microphone's mute button and menu, with the speaker choice.
*
* @param disableSwitcher$ - True to leave out the menu.
*/
export function createAudioMenuViewModel(
scope: ObservableScope,
muteStates: MuteStates,
mediaDevices: MediaDevices,
disableSwitcher$: Behavior<boolean>,
): MediaMenuViewModel {
const { audio } = muteStates;
return {
...createStaticViewModel(noBackgroundEffects),
enabled$: audio.enabled$,
busy$: audio.syncing$,
toggle$: scope.behavior(audio.toggle$.pipe(map((t) => t ?? undefined))),
options$: scope.behavior(
options$(disableSwitcher$, mediaDevices.audioInput.available$),
),
selected$: scope.behavior(
mediaDevices.audioInput.selected$.pipe(map((s) => s?.id)),
),
select$: constant(mediaDevices.audioInput.select),
outputOptions$: scope.behavior(
options$(disableSwitcher$, mediaDevices.audioOutput.available$),
),
selectedOutput$: scope.behavior(
mediaDevices.audioOutput.selected$.pipe(map((s) => s?.id)),
),
selectOutput$: constant(
supportsAudioOutputSelection()
? mediaDevices.audioOutput.select
: undefined,
),
};
}
/**
* The camera's mute button and menu, with the background effects and a
* preview of the camera.
*
* @param disableSwitcher$ - True to leave out the menu.
* @param backgroundEffectsState$ - The background pipeline's state: whether
* effects run here, and the wait for the first.
* @param camera$ - The camera as the pipeline has it, for the preview.
*/
export function createCameraMenuViewModel(
scope: ObservableScope,
muteStates: MuteStates,
mediaDevices: MediaDevices,
disableSwitcher$: Behavior<boolean>,
backgroundEffectsState$: Observable<BackgroundEffectsState>,
camera$: Observable<LocalVideoMediaTrack | null | undefined>,
): MediaMenuViewModel {
const { video } = muteStates;
// Also false once the pipeline has failed to build this session.
const supported$ = backgroundEffectsState$.pipe(
map(({ supported }) => supported === true),
distinctUntilChanged(),
);
const slow = !supportsModernBackgroundProcessors();
const offered$ = combineLatest([disableSwitcher$, supported$]).pipe(
map(([switcherDisabled, supported]) => !switcherDisabled && supported),
);
const refusal$ = new BehaviorSubject<BackgroundImageRefusal | undefined>(
undefined,
);
const refuse = (reason: BackgroundImageRefusal["reason"]): void => {
const refusal: BackgroundImageRefusal = {
reason,
// A later refusal is not this one's to dismiss.
dismiss: () => {
if (refusal$.value === refusal) refusal$.next(undefined);
},
};
refusal$.next(refusal);
};
return {
...createStaticViewModel({
outputOptions: [],
selectedOutput: undefined,
selectOutput: undefined,
}),
enabled$: video.enabled$,
busy$: video.syncing$,
toggle$: scope.behavior(video.toggle$.pipe(map((t) => t ?? undefined))),
options$: scope.behavior(
options$(disableSwitcher$, mediaDevices.videoInput.available$),
),
selected$: scope.behavior(
mediaDevices.videoInput.selected$.pipe(map((s) => s?.id)),
),
select$: constant(mediaDevices.videoInput.select),
backgroundEffect$: scope.behavior(
combineLatest([backgroundEffectSetting.value$, supported$]).pipe(
map(([raw, supported]) =>
supported ? serializeEffect(parseEffect(raw)) : "none",
),
),
),
selectBackgroundEffect$: scope.behavior(
offered$.pipe(
map((offered) =>
offered
? (id: string): void =>
backgroundEffectSetting.setValue(
serializeEffect(parseEffect(id)),
)
: undefined,
),
),
),
backgroundEffects$: scope.behavior(
addedBackgrounds.added$.pipe(
map((added) => backgroundEffectChoices(added ?? [])),
),
),
backgroundEffectNotice$: scope.behavior(
offered$.pipe(
map((offered) =>
!offered ? "unavailable" : slow ? "slow" : undefined,
),
),
),
backgroundEffectSettling$: scope.behavior(
backgroundEffectsState$.pipe(
map(({ wait }) => wait !== undefined),
distinctUntilChanged(),
),
false,
),
// Kept and offered, not put on: that waits for the user to choose it.
addBackgroundImage$: scope.behavior(
combineLatest([offered$, addedBackgrounds.added$]).pipe(
map(([offered, added]) =>
offered && (added?.length ?? 0) < maxAddedBackgrounds
? (file: File): void => {
refusal$.next(undefined);
addedBackgrounds.add(file).catch((e) => {
logger.warn("Could not keep that background", e);
refuse(e instanceof UnusableImage ? e.reason : "not-kept");
});
}
: undefined,
),
),
),
backgroundImageRefusal$: refusal$,
// Muting stops the camera, and so its light.
// The track outlives the mute, so its mute state is checked.
cameraTrack$: scope.behavior(
camera$.pipe(
switchMap((camera) =>
camera
? camera.muted$.pipe(map((muted) => (muted ? null : camera)))
: of(null),
),
),
null,
),
removeBackgroundEffect$: scope.behavior(
offered$.pipe(
map((offered) =>
offered
? (optionId: string): void => {
const effect = parseEffect(optionId);
if (effect.kind !== "added") return;
// The selected effect can't be removed: that would leave no
// effect selected.
if (backgroundEffectSetting.getValue() === optionId) return;
addedBackgrounds
.remove(effect.id)
.catch((e) =>
logger.warn("Could not remove that background", e),
);
}
: undefined,
),
),
),
};
}
function backgroundEffectChoices(
added: AddedBackground[],
): BackgroundEffectGridItem[] {
return [
{ id: serializeEffect({ kind: "none" }), kind: "none", removable: false },
{ id: serializeEffect({ kind: "blur" }), kind: "blur", removable: false },
...shippedBackgrounds.map((background) => ({
id: serializeEffect({ kind: "shipped", id: background.id }),
kind: "image" as const,
imageUrl: background.imagePath,
removable: false,
})),
...added.map((background) => ({
id: serializeEffect({ kind: "added", id: background.id }),
kind: "image" as const,
imageUrl: background.url,
removable: true,
})),
];
}
@@ -404,7 +404,7 @@ export const BackgroundImageRefused: Story = {
selectedBackgroundEffect: "blur",
backgroundEffectNotice:
"Background effects run slowly on this platform, which may cause your video to stutter.",
backgroundImageRefusal: "That file is not a supported image",
backgroundImageRefusal: { text: "That file is not a supported image" },
},
decorators: [
(Story): JSX.Element => (
@@ -440,7 +440,7 @@ function swatch(colour: string): string {
)}`;
}
/** One's own backgrounds are removable, unless one is the selected one. */
/** One's own backgrounds are removable, never the one in force. */
export const AddedBackgroundsCanBeRemoved: Story = {
args: {
...VideoUnmute.args,
@@ -437,7 +437,7 @@ describe("MediaMuteAndSwitchButton", () => {
}
});
test("a refusal is dismissed when closed, or with the menu", async () => {
test("a refusal can be dismissed and does not outlive the menu", async () => {
const user = userEvent.setup();
// jsdom lays nothing out, so has nothing to scroll.
const scroll = Element.prototype.scrollIntoView;
@@ -445,8 +445,7 @@ describe("MediaMuteAndSwitchButton", () => {
onTestFinished(() => {
Element.prototype.scrollIntoView = scroll;
});
const onDismiss = vi.fn();
const menu = (refusal?: string): JSX.Element =>
const menu = (refusal?: { text: string }): JSX.Element =>
withProviders(
<MediaMuteAndSwitchButton
iconsAndLabels="video"
@@ -458,22 +457,28 @@ describe("MediaMuteAndSwitchButton", () => {
onSelectBackgroundEffect={vi.fn()}
onAddBackgroundImage={vi.fn()}
backgroundImageRefusal={refusal}
onDismissBackgroundImageRefusal={onDismiss}
/>,
);
const { rerender, getByRole } = render(menu());
await user.click(getByRole("button", { name: "Camera" }));
rerender(menu("That file is not a supported image"));
rerender(menu({ text: "That file is not a supported image" }));
expect(
screen.getByText("That file is not a supported image"),
).toBeVisible();
screen.getByRole("menuitemradio", { name: "Blur", checked: true });
await user.click(screen.getByRole("button", { name: "Close" }));
expect(onDismiss).toHaveBeenCalledTimes(1);
expect(screen.queryByText("That file is not a supported image")).toBeNull();
// The next refusal is shown, even if it says the same.
rerender(menu({ text: "That file is not a supported image" }));
expect(
screen.getByText("That file is not a supported image"),
).toBeVisible();
await user.keyboard("{Escape}");
expect(onDismiss).toHaveBeenCalledTimes(2);
await user.click(getByRole("button", { name: "Camera" }));
expect(screen.queryByText("That file is not a supported image")).toBeNull();
});
test("offers the background effects in a phone's drawer", async () => {
+16 -17
View File
@@ -85,12 +85,10 @@ export interface MediaMuteAndSwitchButtonProps {
backgroundEffectSettling?: boolean;
/** Called with the file chosen from the add tile. Omit to leave it out. */
onAddBackgroundImage?: (file: File) => void;
/** Removes a removable effect, unless it is the selected one. */
/** Removes a removable effect, never the one in force. */
onRemoveBackgroundEffect?: (id: string) => void;
/** Why the last file offered couldn't be used, if it couldn't. */
backgroundImageRefusal?: string;
/** Called as the refusal is closed, or the menu showing it. */
onDismissBackgroundImageRefusal?: () => void;
/** Why the last file offered couldn't be used; a new object each time. */
backgroundImageRefusal?: { text: string };
/** The user's own camera, shown at the top of the camera menu. */
selfPreview?: ReactNode;
/**
@@ -137,7 +135,6 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
onAddBackgroundImage,
onRemoveBackgroundEffect,
backgroundImageRefusal,
onDismissBackgroundImageRefusal,
selfPreview,
onSelect,
}) => {
@@ -152,18 +149,20 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
// elsewhere; it is held open until the picker is done.
const choosingFile = useRef(false);
const fileInput = useRef<HTMLInputElement | null>(null);
// A refusal is about the attempt that caused it: shown until dismissed or
// until the menu closes, and again for the next one.
const [seenRefusal, setSeenRefusal] = useState<object>();
const refusal =
backgroundImageRefusal !== seenRefusal ? backgroundImageRefusal : undefined;
const onOpenChange = useCallback(
(open: boolean): void => {
if (!open && choosingFile.current) return;
setMenuOpen(open);
if (!open) {
// Drop a request that never arrived.
setPlannedSelection(null);
// A refusal is about the attempt that caused it.
onDismissBackgroundImageRefusal?.();
}
// Drop a request that never arrived.
if (!open) setPlannedSelection(null);
if (!open) setSeenRefusal(backgroundImageRefusal);
},
[onDismissBackgroundImageRefusal],
[backgroundImageRefusal],
);
// Scrolled to as it appears, as the list may be scrolled away from it.
const scrollIntoView = useCallback((element: HTMLElement | null): void => {
@@ -513,7 +512,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
removeLabel={t("action.remove")}
/>
{/* In the list, so the menu grows no taller for it. */}
{backgroundImageRefusal !== undefined && (
{refusal !== undefined && (
<div
ref={scrollIntoView}
role="none"
@@ -521,12 +520,12 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
>
<Alert
type="critical"
title={backgroundImageRefusal}
onClose={onDismissBackgroundImageRefusal}
title={refusal.text}
onClose={(): void => setSeenRefusal(refusal)}
/>
</div>
)}
{backgroundImageRefusal === undefined &&
{refusal === undefined &&
backgroundEffectNotice !== undefined && (
<div id={noticeId} role="none" className={styles.notice}>
<InfoIcon width={20} height={20} />
+8 -10
View File
@@ -186,18 +186,13 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
const vm = createCallViewModel$(
scope,
{
rtcParticipation,
matrixRoom: props.matrixRoom,
mediaDevices,
muteStates: props.muteStates,
encryptionSystem: props.e2eeSystem,
viewSize$: scope.behavior(observeElementSize$(rootElement)),
handsRaisedSubject$: reactionsReader.raisedHands$,
reactionsSubject$: reactionsReader.reactions$,
},
rtcParticipation,
props.matrixRoom,
mediaDevices,
props.muteStates,
{
...callViewModelOptionsFromParams(urlParams),
encryptionSystem: props.e2eeSystem,
capture,
backgroundEffectsState$,
hostBridge,
@@ -207,7 +202,10 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
scope,
props.rtcSession,
),
windowSize$: scope.behavior(observeElementSize$(rootElement)),
},
reactionsReader.raisedHands$,
reactionsReader.reactions$,
);
setVm(vm);
+4 -4
View File
@@ -75,7 +75,7 @@ describe("the pipeline's state", () => {
const effects = createBackgroundEffects(testScope(), {
supported: true,
effect$: behavior(effect, { n: "none", b: "blur" }),
clearEffect: vi.fn(),
setEffect: vi.fn(),
added$: new BehaviorSubject<AddedBackground[] | undefined>([]),
canSegment: () => cold(answer, { y: true, n: false }),
pipeline: fakePipeline().pipeline,
@@ -110,7 +110,7 @@ describe("the pipeline's state", () => {
describe("background effects", () => {
let effect$: BehaviorSubject<string>;
let added$: BehaviorSubject<AddedBackground[] | undefined>;
let clearEffect: () => void;
let setEffect: (raw: string) => void;
let fake: ReturnType<typeof fakePipeline>;
function build(
@@ -119,7 +119,7 @@ describe("background effects", () => {
return createBackgroundEffects(testScope(), {
supported: true,
effect$,
clearEffect,
setEffect,
added$,
canSegment: () => of(true),
pipeline: fake.pipeline,
@@ -135,7 +135,7 @@ describe("background effects", () => {
beforeEach(() => {
effect$ = new BehaviorSubject("none");
added$ = new BehaviorSubject<AddedBackground[] | undefined>(undefined);
clearEffect = vi.fn(() => effect$.next("none"));
setEffect = vi.fn((raw: string) => effect$.next(raw));
fake = fakePipeline();
});
+5 -4
View File
@@ -33,6 +33,7 @@ import { type Behavior } from "./Behavior";
import { type ObservableScope } from "./ObservableScope";
import {
type BackgroundEffect,
type EffectId,
imagePathFor,
parseEffect,
} from "../tracks/backgroundEffectCatalog";
@@ -56,8 +57,8 @@ export interface BackgroundEffectsOptions {
supported: boolean;
/** The effect chosen, as the setting stores it. */
effect$: Behavior<string>;
/** Resets the choice to no effect, for when the chosen one can't run here. */
clearEffect: () => void;
/** Stores a choice, to forget one that can't be honoured. */
setEffect: (id: EffectId) => void;
/** The backgrounds the device keeps, undefined until it has said. */
added$: Behavior<AddedBackground[] | undefined>;
/** Builds a segmenter on trial, and answers whether it could. */
@@ -79,7 +80,7 @@ export function createBackgroundEffects(
{
supported,
effect$,
clearEffect,
setEffect,
added$,
canSegment,
pipeline,
@@ -106,7 +107,7 @@ export function createBackgroundEffects(
added !== undefined &&
!added.some(({ id }) => id === effect.id))
)
clearEffect();
setEffect("none");
});
// Asked once an effect is first wanted. A browser update may fix what
+10 -5
View File
@@ -36,6 +36,7 @@ import {
mockRtcMembership,
testScope,
} from "../../utils/test.ts";
import { E2eeType } from "../../e2ee/e2eeType.ts";
import {
alice,
aliceId,
@@ -505,7 +506,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
videoEnabled: new Map([
[localParticipant, behavior(videoInputMarbles, yesNo)],
]),
viewSize$: constant({ width: 380, height: 700 }), // Mobile phone in portrait
windowSize$: constant({ width: 380, height: 700 }), // Mobile phone in portrait
},
(vm) => {
schedule(tapScreenInputMarbles, { a: () => vm.tapScreen() });
@@ -547,7 +548,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
// Both Alice and Bob are with us in the room
roomMembers: [local, alice, bob],
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
viewSize$: constant({ width: 380, height: 700 }), // Mobile phone
windowSize$: constant({ width: 380, height: 700 }), // Mobile phone
},
(vm) => {
// Uses one-on-one mobile layout
@@ -594,7 +595,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
sharingScreen: new Map([
[aliceParticipant, behavior(aliceSharingInputMarbles, yesNo)],
]),
viewSize$: constant({ width: 700, height: 380 }), // Mobile phone in landscape
windowSize$: constant({ width: 700, height: 380 }), // Mobile phone in landscape
},
(vm) => {
expectObservable(summarizeLayout$(vm.layout$)).toBe(
@@ -782,13 +783,13 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
test("layout reacts to window size", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const viewSizeInputMarbles = "abc";
const windowSizeInputMarbles = "abc";
const expectedLayoutMarbles = " abc";
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant]),
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
viewSize$: behavior(viewSizeInputMarbles, {
windowSize$: behavior(windowSizeInputMarbles, {
a: { width: 380, height: 700 }, // Start very narrow, like a phone
b: { width: 1000, height: 800 }, // Go to normal desktop window size
c: { width: 200, height: 180 }, // Go to PiP size
@@ -1465,6 +1466,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
},
{
autoLeaveWhenOthersLeft: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
@@ -1482,6 +1484,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
},
{
autoLeaveWhenOthersLeft: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
@@ -1499,6 +1502,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
},
{
autoLeaveWhenOthersLeft: false,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
@@ -1528,6 +1532,7 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
},
{
autoLeaveWhenOthersLeft: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
+25 -47
View File
@@ -185,39 +185,8 @@ export function initialPublishRequests(
return requests;
}
/**
* What the call is made of: the room, the participation, the devices, the
* things with no sensible default. How the host wants it to behave is in
* {@link CallViewModelOptions} instead.
*/
export interface CallViewModelProps {
rtcParticipation: RTCParticipation;
/** A call is permanently tied to a single Matrix room. */
matrixRoom: MatrixRoom;
mediaDevices: MediaDevices;
muteStates: MuteStates;
encryptionSystem: EncryptionSystem;
/**
* The size of the space the call is drawn in: the page when Element Call
* owns it, or the container a host mounted it in when it is a component.
* The layout — whether the call is shown full size, flat, narrow or as a
* picture-in-picture — follows this rather than the size of the window, so
* that a component shrunk by its host adapts even though the window has not
* changed.
*/
viewSize$: Behavior<{ width: number; height: number }>;
handsRaisedSubject$: Observable<Record<string, RaisedHandInfo>>;
reactionsSubject$: Observable<Record<string, ReactionInfo>>;
}
/**
* How the host wants the call to behave. Every field is optional and defaults
* to what standalone Element Call does, so `{}` is a working standalone call.
* What the call is made of is in {@link CallViewModelProps} instead. A test
* that wants to replace behaviour mocks the dependency it comes from rather
* than adding an option here.
*/
export interface CallViewModelOptions {
encryptionSystem: EncryptionSystem;
/**
* How our tracks are captured and encoded; the browser's and LiveKit's
* defaults otherwise.
@@ -253,6 +222,17 @@ export interface CallViewModelOptions {
* Defaults to none sent.
*/
sentCallNotification$?: Observable<CallNotificationWrapper | null>;
/**
* The size of the space the call is drawn in: the page when Element Call
* owns it, or the container a host mounted it in when it is a component.
* The layout — whether the call is shown full size, flat, narrow or as a
* picture-in-picture — follows this rather than the size of the window, so
* that a component shrunk by its host adapts even though the window has not
* changed.
*/
windowSize$: Behavior<{ width: number; height: number }>;
/** Optional behavior overriding for the screensharing, for testing */
toggleScreensharing?: () => void;
}
/**
@@ -489,17 +469,14 @@ export interface CallViewModel {
*/
export function createCallViewModel$(
scope: ObservableScope,
{
rtcParticipation,
matrixRoom,
mediaDevices,
muteStates,
encryptionSystem,
viewSize$,
handsRaisedSubject$,
reactionsSubject$,
}: CallViewModelProps,
options: CallViewModelOptions = {},
rtcParticipation: RTCParticipation,
// A call is permanently tied to a single Matrix room
matrixRoom: MatrixRoom,
mediaDevices: MediaDevices,
muteStates: MuteStates,
options: CallViewModelOptions,
handsRaisedSubject$: Observable<Record<string, RaisedHandInfo>>,
reactionsSubject$: Observable<Record<string, ReactionInfo>>,
): CallViewModel {
const logger = rootLogger.getChild("[CallViewModel]");
const client = matrixRoom.client;
@@ -725,7 +702,7 @@ export function createCallViewModel$(
userId: member.userId,
rtcBackendIdentity: member.rtcBackendIdentity,
member,
encryptionSystem,
encryptionSystem: options.encryptionSystem,
transport$: scope.behavior(
member.transport$.pipe(map((transport) => transport?.raw)),
),
@@ -939,7 +916,7 @@ export function createCallViewModel$(
// A guess at what the window's mode should be based on the size and shape of
// the space we have to draw in.
const naturalWindowMode$ = scope.behavior<WindowMode>(
viewSize$.pipe(
options.windowSize$.pipe(
map(({ width, height }) => {
if (height <= 400 && width <= 340) return "pip";
// Our layouts for flat windows are better at adapting to a small width
@@ -1523,7 +1500,8 @@ export function createCallViewModel$(
* the platform cannot capture a screen, or the host hides the button.
*/
const toggleScreenSharing =
hideScreensharing || !("getDisplayMedia" in (navigator.mediaDevices ?? {}))
options.toggleScreensharing ??
(hideScreensharing || !("getDisplayMedia" in (navigator.mediaDevices ?? {}))
? null
: (): void => {
const track = screenShare$.value;
@@ -1544,7 +1522,7 @@ export function createCallViewModel$(
e instanceof Error ? e : new Error(String(e)),
);
});
};
});
// Tell the host and the analytics about the user's joins and leaves. The
// participation is joined by the time the view model is built on it
@@ -81,7 +81,7 @@ export interface CallViewModelInputs {
sharingScreen: Map<Participant, Behavior<boolean>>;
mediaDevices: MediaDevices;
initialSyncState: SyncState;
viewSize$: Behavior<{ width: number; height: number }>;
windowSize$: Behavior<{ width: number; height: number }>;
}
export const localParticipant: Participant = { identity: "" };
@@ -106,7 +106,7 @@ export function withCallViewModel(mode: MatrixRTCMode) {
sharingScreen = new Map(),
mediaDevices = mockMediaDevices({}),
initialSyncState = SyncState.Syncing,
viewSize$ = constant({ width: 1000, height: 800 }),
windowSize$ = constant({ width: 1000, height: 800 }),
}: Partial<CallViewModelInputs> = {},
continuation: (
vm: CallViewModel,
@@ -262,24 +262,22 @@ export function withCallViewModel(mode: MatrixRTCMode) {
const vm = createCallViewModel$(
scope,
rtcParticipation,
room,
mediaDevices,
muteStates,
{
rtcParticipation,
matrixRoom: room,
mediaDevices,
muteStates,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
viewSize$,
handsRaisedSubject$: raisedHands$,
reactionsSubject$: reactions$,
},
{
autoLeaveWhenOthersLeft: false,
sentCallNotification$: createSentCallNotification$(
scope,
rtcSession.asMockedSession(),
),
windowSize$,
...options,
},
raisedHands$,
reactions$,
);
void mode;
+2
View File
@@ -17,6 +17,7 @@ import {
type HostRequest,
nullHostBridge,
} from "../HostBridge.ts";
import { E2eeType } from "../e2ee/e2eeType.ts";
import { MatrixRTCMode } from "../config/ConfigOptions.ts";
it.each([[MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0]])(
@@ -40,6 +41,7 @@ it.each([[MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0]])(
hangUp$.next({ data: {}, reply });
},
{
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
hostBridge,
},
);
+1 -1
View File
@@ -57,7 +57,7 @@ export function createAppBackgroundEffects(
return createBackgroundEffects(scope, {
supported: supportsBackgroundProcessors(),
effect$: backgroundEffectSetting.value$,
clearEffect: () => backgroundEffectSetting.setValue("none"),
setEffect: backgroundEffectSetting.setValue,
added$: addedBackgrounds.added$,
canSegment,
pipeline: new BackgroundProcessorWrapper(transformer, "background-effect"),
+7 -9
View File
@@ -196,23 +196,21 @@ export function getBasicCallViewModelEnvironment(
});
const vm = createCallViewModel$(
scope,
rtcParticipation,
matrixRoom,
mediaDevices,
muteStates,
{
rtcParticipation,
matrixRoom,
mediaDevices,
muteStates,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
viewSize$: constant({ width: 1000, height: 800 }),
handsRaisedSubject$: handRaisedSubject$,
reactionsSubject$,
},
{
sentCallNotification$: createSentCallNotification$(
scope,
rtcSession.asMockedSession(),
),
windowSize$: constant({ width: 1000, height: 800 }),
...callViewModelOptions,
},
handRaisedSubject$,
reactionsSubject$,
);
const footerVm = createCallFooterViewModel(
testScope(),
+1
View File
@@ -100,6 +100,7 @@ describe("widget", () => {
EventType.CallSDPStreamMetadataChangedPrefix,
EventType.CallReplaces,
EventType.CallEncryptionKeysPrefix,
"m.rtc.encryption_key",
];
expect(createRoomWidgetClientSpy.mock.calls[0][1]).toStrictEqual({
+1
View File
@@ -162,6 +162,7 @@ export const initializeWidget = (
EventType.CallSDPStreamMetadataChangedPrefix,
EventType.CallReplaces,
EventType.CallEncryptionKeysPrefix,
"m.rtc.encryption_key",
];
const client = createRoomWidgetClient(