diff --git a/locales/en/app.json b/locales/en/app.json index f3d568bb8..e31c7d50a 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -163,6 +163,12 @@ "login_auth_links_prompt": "Not registered yet?", "login_subheading": "To continue to Element", "login_title": "Login", + "microphone_level": { + "label": "Microphone level", + "no_device": "No microphone found. Connect one, then try again.", + "permission_denied": "Microphone access is blocked. Allow it, then try again.", + "value": "{{level}} of {{max}}" + }, "microphone_off": "Microphone off", "microphone_on": "Microphone on", "mute_microphone_button_label": "Mute microphone", @@ -215,11 +221,14 @@ "activating": "Activating…", "camera": "Camera", "camera_numbered": "Camera {{n}}", + "camera_source": "Camera Source", "change_device_button": "Change audio device", "default": "Default", "default_named": "Default <2>({{name}})", + "default_named_plain": "Default ({{name}})", "handset": "Handset", "loudspeaker": "Loudspeaker", + "mic_source": "Mic Source", "microphone": "Microphone", "microphone_numbered": "Microphone {{n}}", "speaker": "Speaker", diff --git a/playwright/audio-menu.spec.ts b/playwright/audio-menu.spec.ts new file mode 100644 index 000000000..99385c890 --- /dev/null +++ b/playwright/audio-menu.spec.ts @@ -0,0 +1,229 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test, type Locator, type Page } from "@playwright/test"; + +import { SpaHelpers } from "./spa-helpers.ts"; +import { installFakeDevices } from "./utils/fake-devices.ts"; + +test.describe("the quick audio menu", () => { + test("lists speakers and microphones with a live level meter", async ({ + page, + }) => { + await installFakeDevices(page, { microphones: 3, speakers: 3 }); + await joinACall(page, "Menu user", "Audio menu"); + await openAudioMenu(page); + + await expect(page.getByRole("group", { name: "Speaker" })).toBeVisible(); + await expect(page.getByRole("group", { name: "Microphone" })).toBeVisible(); + // By name, not count: the browser adds its own fake output and a Default. + for (const n of [1, 2, 3]) + await expect( + page + .getByRole("group", { name: "Speaker" }) + .getByRole("menuitemradio", { name: `Fake Speaker ${n}` }), + ).toBeVisible(); + + await expect( + page.getByRole("menuitemradio", { checked: true }), + ).toHaveCount(2); + const meter = page.getByRole("meter", { name: "Microphone level" }); + await expect(meter).toBeVisible(); + await expect(meter).toHaveAttribute("aria-valuenow", /\d+/); + await expect(meter).toHaveAttribute("aria-valuetext", /\d+ of \d+/); + + // Both browsers here can route audio; the platform that can't is a unit check. + await expect( + page + .getByRole("group", { name: "Speaker" }) + .getByRole("menuitemradio") + .first(), + ).toHaveAttribute("aria-disabled", "false"); + }); + + test("moves the microphone and the speaker without disturbing the call", async ({ + browser, + }) => { + // Two browsers and a real call. + test.slow(); + const hostContext = await browser.newContext({ reducedMotion: "reduce" }); + const host = await hostContext.newPage(); + await installFakeDevices(host, { microphones: 3, speakers: 3 }); + await joinACall(host, "Host", "Device switch"); + + const inviteLink = await SpaHelpers.getCallInviteLink(host); + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guest = await guestContext.newPage(); + await SpaHelpers.joinCallFromInviteLink(guest, inviteLink, "Guest"); + await SpaHelpers.expectVideoTilesCount(guest, 2); + + await openAudioMenu(host); + await selectDevice(host, "Microphone", "Fake Microphone 2"); + await openAudioMenu(host); + await selectDevice(host, "Speaker", "Fake Speaker 2"); + + // Not a rejoin: neither side drops, and the guest still has both tiles. + await expect( + host.getByRole("dialog", { name: "Reconnecting…" }), + ).not.toBeVisible(); + await expect( + guest.getByRole("dialog", { name: "Reconnecting…" }), + ).not.toBeVisible(); + await SpaHelpers.expectVideoTilesCount(guest, 2); + await expect(guest.getByText("Waiting for media...")).not.toBeVisible(); + + await hostContext.close(); + await guestContext.close(); + }); + + test("keeps the meter moving while muted, and sends nothing", async ({ + browser, + }) => { + // Two browsers and a real call. + test.slow(); + const hostContext = await browser.newContext({ reducedMotion: "reduce" }); + const host = await hostContext.newPage(); + await installFakeDevices(host); + await joinACall(host, "Muted host", "Muted meter"); + + const inviteLink = await SpaHelpers.getCallInviteLink(host); + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guest = await guestContext.newPage(); + await SpaHelpers.joinCallFromInviteLink(guest, inviteLink, "Listener"); + await SpaHelpers.expectVideoTilesCount(guest, 2); + + const mute = host.getByTestId("incall_mute"); + await mute.click(); + await expect(mute).toHaveAttribute("aria-checked", "false"); + await openAudioMenu(host); + + const meter = host.getByRole("meter", { name: "Microphone level" }); + await expect(meter).toBeVisible(); + // By test id: the modal menu hides the rest of the call from the a11y tree. + await expect(mute).toHaveAttribute("aria-checked", "false"); + await expect(mute).toBeVisible(); + // And the guest is shown the mute. + await expect( + guest.getByTestId("videoTile").filter({ hasText: "Muted host" }), + ).toBeVisible(); + + await hostContext.close(); + await guestContext.close(); + }); + + test("keeps the meter at the foot of the list while it scrolls", async ({ + page, + }) => { + await installFakeDevices(page, { microphones: 20, speakers: 4 }); + await joinACall(page, "Scroller", "Long device list"); + await openAudioMenu(page); + + const meter = page.getByRole("meter", { name: "Microphone level" }); + await expect(meter).toBeVisible(); + + // Scrolled so the microphones start at the top: only there does a pinned + // meter differ from one that is simply last. + const list = page.locator("[role='menu'] div[role='none']").first(); + await list.evaluate((element) => { + const group = element.querySelector("[role='group'][aria-label*='icro']"); + element.scrollTop += + group!.getBoundingClientRect().top - + element.getBoundingClientRect().top; + }); + + await expect(meter).toBeInViewport(); + await expectPinnedInside(meter, list); + await expect( + page.getByRole("menuitemradio", { name: "Fake Microphone 20" }), + ).toBeVisible(); + }); + + test("shows the focus ring only when the keyboard moved the focus", async ({ + page, + browserName, + }) => { + test.skip( + browserName === "firefox", + "Headless Firefox does not deliver synthetic key presses reliably; see reconnect.spec.ts", + ); + await installFakeDevices(page); + await joinACall(page, "Keyboard user", "Focus ring"); + await openAudioMenu(page); + + const first = page.getByRole("menuitemradio").first(); + // Opened by pointer: no ring, though Radix has moved focus into the menu. + await expect.poll(async () => outlineWidth(first)).toBe(0); + + await page.keyboard.press("ArrowDown"); + const focused = page.locator("[role='menuitemradio']:focus"); + await expect.poll(async () => outlineWidth(focused)).toBeGreaterThan(0); + + // And the pointer takes it away again. + await first.hover(); + await expect.poll(async () => outlineWidth(focused)).toBe(0); + }); +}); + +/** Creates a call and joins it, leaving the page in the call. */ +async function joinACall( + page: Page, + userName: string, + callName: string, +): Promise { + await page.goto("/"); + await SpaHelpers.createCall(page, userName, callName, true); + await expect(page.getByTestId("name_tag")).toContainText(userName); + // The media controls stay disabled until devices have enumerated. + await expect(page.getByTestId("incall_mute")).toBeEnabled({ + timeout: 10_000, + }); +} + +async function openAudioMenu(page: Page): Promise { + await page.getByRole("button", { name: "Microphone" }).click(); + await expect(page.getByRole("menu")).toBeVisible(); +} + +async function selectDevice( + page: Page, + section: "Speaker" | "Microphone", + name: string, +): Promise { + const item = page + .getByRole("group", { name: section }) + .getByRole("menuitemradio", { name }); + await item.click(); + // Selecting doesn't close the menu, so dismiss it. + await page.keyboard.press("Escape"); + await expect(page.getByRole("menu")).not.toBeVisible(); +} + +/** Asserts the meter sits within the scrollport and inside the menu's frame. */ +async function expectPinnedInside( + meter: Locator, + list: Locator, +): Promise { + const meterBox = (await meter.boundingBox())!; + const listBox = (await list.boundingBox())!; + const frame = (await meter.page().getByRole("menu").boundingBox())!; + + expect(meterBox.y + meterBox.height).toBeLessThanOrEqual( + listBox.y + listBox.height + 1, + ); + expect(meterBox.y).toBeGreaterThanOrEqual(listBox.y - 1); + expect(meterBox.x).toBeGreaterThan(frame.x); + expect(meterBox.x + meterBox.width).toBeLessThan(frame.x + frame.width); +} + +/** Painted outline width, in px. */ +async function outlineWidth(item: Locator): Promise { + if ((await item.count()) === 0) return 0; + return item.first().evaluate((element) => { + const { outlineStyle, outlineWidth } = getComputedStyle(element); + return outlineStyle === "none" ? 0 : Number.parseFloat(outlineWidth) || 0; + }); +} diff --git a/playwright/component/audio-menu.spec.ts b/playwright/component/audio-menu.spec.ts new file mode 100644 index 000000000..f4d811598 --- /dev/null +++ b/playwright/component/audio-menu.spec.ts @@ -0,0 +1,135 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Locator, type Page, test } from "@playwright/test"; + +import { createUserAndRoom, resizeContainer, startHarness } from "./harness.ts"; +import { installFakeDevices } from "../utils/fake-devices.ts"; + +/** + * The device menu with Element Call as a component in a host page, where the + * portalled menu must be sized against the call, not the window. Driven from + * the lobby: it builds the same menu, without two connections' worth of flake. + */ + +// Sign-in, crypto setup and sync happen twice before anything shows. +test.describe.configure({ timeout: 180_000 }); + +test("sizes the device list against the call, not the window", async ({ + page, +}) => { + await installFakeDevices(page, { microphones: 20, speakers: 4 }); + const { username, roomId } = await createUserAndRoom("menusize"); + const panes = await startHarness(page, username, roomId); + const pane = panes.first(); + + // A short call in a much taller page. + const container = pane.getByTestId("call-container"); + await resizeContainer(container, { width: 900, height: 400 }); + await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({ + timeout: 60_000, + }); + + const list = await openDeviceList(page, pane); + const callHeight = (await container.boundingBox())!.height; + const windowHeight = page.viewportSize()!.height; + const listHeight = (await list.boundingBox())!.height; + + expect(callHeight).toBeLessThan(windowHeight * 0.75); + expect(listHeight).toBeLessThanOrEqual(callHeight); + expect(listHeight).toBeLessThan(windowHeight * 0.6); +}); + +test("follows the call area when the host resizes it", async ({ page }) => { + await installFakeDevices(page, { microphones: 20, speakers: 4 }); + const { username, roomId } = await createUserAndRoom("menuresize"); + const panes = await startHarness(page, username, roomId); + const pane = panes.first(); + + const container = pane.getByTestId("call-container"); + await resizeContainer(container, { width: 900, height: 360 }); + await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({ + timeout: 60_000, + }); + + const list = await openDeviceList(page, pane); + const whenShort = (await list.boundingBox())!.height; + + // The host grows the call while the menu is open. + await resizeContainer(container, { width: 900, height: 700 }); + await expect + .poll(async () => (await list.boundingBox())!.height) + .toBeGreaterThan(whenShort); +}); + +test("keeps every device reachable in a small container", async ({ page }) => { + await installFakeDevices(page, { microphones: 20, speakers: 4 }); + const { username, roomId } = await createUserAndRoom("menureach"); + const panes = await startHarness(page, username, roomId); + const pane = panes.first(); + + const container = pane.getByTestId("call-container"); + await resizeContainer(container, { width: 400, height: 360 }); + await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({ + timeout: 60_000, + }); + + const list = await openDeviceList(page, pane); + // More devices than fit, so the list must scroll. + expect(await list.evaluate((el) => el.scrollHeight > el.clientHeight)).toBe( + true, + ); + + const last = page.getByRole("menuitemradio", { name: "Fake Microphone 20" }); + await last.scrollIntoViewIfNeeded(); + await expect(last).toBeInViewport(); + // Reachable means clickable: the menu may be drawn outside the call area. + await last.click(); + await expect(last).toHaveAttribute("aria-checked", "true"); +}); + +test("tracks the focus source of its own call, not the page", async ({ + page, +}) => { + await installFakeDevices(page); + const { username, roomId } = await createUserAndRoom("menufocus"); + const panes = await startHarness(page, username, roomId); + const pane = panes.first(); + const other = panes.nth(1); + + await expect(pane.getByTestId("lobby_joinCall")).toBeVisible({ + timeout: 60_000, + }); + await openDeviceList(page, pane); + const menu = page.getByRole("menu"); + + // Asserted on the attribute: the component build's scoped stylesheet doesn't + // reach the portalled menu, so its ring can't be read here. + await expect(menu).toHaveAttribute("data-focus-source", "pointer"); + + // A key pressed in the other call says nothing about this menu. + await other.evaluate((element) => + element.dispatchEvent( + new KeyboardEvent("keydown", { key: "ArrowDown", bubbles: true }), + ), + ); + await expect(menu).toHaveAttribute("data-focus-source", "pointer"); + + await page.keyboard.press("ArrowDown"); + await expect(menu).toHaveAttribute("data-focus-source", "keyboard"); +}); + +/** Opens a component's microphone menu and returns its device list. */ +async function openDeviceList(page: Page, pane: Locator): Promise { + await pane + .getByRole("button", { name: "Microphone" }) + .click({ timeout: 60_000 }); + await expect(page.getByRole("menu")).toBeVisible(); + const list = page.locator("[role='menu'] div[role='none']").first(); + await expect(list).toBeVisible(); + return list; +} diff --git a/playwright/utils/fake-devices.ts b/playwright/utils/fake-devices.ts new file mode 100644 index 000000000..e67f053bd --- /dev/null +++ b/playwright/utils/fake-devices.ts @@ -0,0 +1,85 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type Page } from "@playwright/test"; + +/** + * Adds synthetic devices beside the browser's one fake microphone and speaker, + * so the menu has a choice to show. They share one capture and `setSinkId` is + * accepted but not honoured: routing still needs hardware and a manual check. + * Call before the page navigates. + */ +export async function installFakeDevices( + page: Page, + { microphones = 2, speakers = 2 } = {}, +): Promise { + await page.addInitScript( + ({ microphones, speakers }) => { + const synthetic = ( + kind: MediaDeviceKind, + count: number, + name: string, + ): MediaDeviceInfo[] => + Array.from({ length: count }, (_, i) => { + const info = { + deviceId: `${kind}-${i + 1}`, + groupId: `${kind}-group-${i + 1}`, + kind, + label: `${name} ${i + 1}`, + }; + return { ...info, toJSON: () => info } as MediaDeviceInfo; + }); + const ids = new Set( + [ + ...synthetic("audioinput", microphones, ""), + ...synthetic("audiooutput", speakers, ""), + ].map((d) => d.deviceId), + ); + + const devices = navigator.mediaDevices; + const enumerate = devices.enumerateDevices.bind(devices); + devices.enumerateDevices = async (): Promise => [ + ...(await enumerate()), + ...synthetic("audioinput", microphones, "Fake Microphone"), + ...synthetic("audiooutput", speakers, "Fake Speaker"), + ]; + + // Our ids name no hardware, so drop the exact-device constraint. + const getUserMedia = devices.getUserMedia.bind(devices); + devices.getUserMedia = async ( + constraints?: MediaStreamConstraints, + ): Promise => { + const audio = constraints?.audio; + if (typeof audio === "object") { + const requested = audio.deviceId; + const id = + typeof requested === "object" && requested !== null + ? ((requested as ConstrainDOMStringParameters).exact as string) + : (requested as string | undefined); + if (id !== undefined && ids.has(id)) + return getUserMedia({ ...constraints, audio: true }); + } + return getUserMedia(constraints); + }; + + // Accept routing to our ids on both media elements and the AudioContext, + // which the app also routes. + for (const proto of [ + HTMLMediaElement.prototype, + AudioContext.prototype, + ]) { + const sink = proto as { setSinkId?: (id: string) => Promise }; + const setSinkId = sink.setSinkId; + if (setSinkId === undefined) continue; + sink.setSinkId = async function (id: string): Promise { + if (!ids.has(id)) await setSinkId.call(this, id); + }; + } + }, + { microphones, speakers }, + ); +} diff --git a/src/components/CallFooter.module.css b/src/components/CallFooter.module.css index e006a5f7d..01c1ac901 100644 --- a/src/components/CallFooter.module.css +++ b/src/components/CallFooter.module.css @@ -43,7 +43,10 @@ Please see LICENSE in the repository root for full details. inset-inline: 0; } -.footer.overlay:has(:focus-visible) { +/* Keep the footer while it is in use: focus inside it, or a menu opened from + it. The menu is portalled out, so the trigger's aria-expanded tracks it. */ +.footer.overlay:has(:focus-visible), +.footer.overlay:has([aria-expanded="true"]) { opacity: 1; pointer-events: initial; } diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index 3c3f46074..96e010d85 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -13,6 +13,7 @@ import { Link } from "@vector-im/compound-web"; import type { Meta, StoryObj } from "@storybook/react-vite"; import { CallFooter, type FooterSnapshot } from "./CallFooter"; import inCallViewStyles from "../room/InCallView.module.css"; +import styles from "./CallFooter.module.css"; import { useStaticViewModel } from "../state/ViewModel"; import { ReactionsSenderContext } from "../reactions/useReactionsSender"; import { type ReactionOption } from "../reactions"; @@ -137,10 +138,13 @@ export const Default: Story = { debugTileLayout: false, tileStoreGeneration: undefined, audioOptions: [], + audioOutputOptions: [], videoOptions: [], selectedAudio: undefined, + selectedAudioOutput: undefined, selectedVideo: undefined, selectAudioButtonOption: undefined, + selectAudioOutputOption: undefined, selectVideoButtonOption: undefined, }, parameters: { @@ -158,15 +162,46 @@ export const WithAudioAndVideoOptions: Story = { { label: { type: "name", name: "Microphone 1" }, id: "1" }, { label: { type: "name", name: "Microphone 2" }, id: "2" }, ], + audioOutputOptions: [ + { label: { type: "default", name: "Built-in Output" }, id: "default" }, + { label: { type: "name", name: "Headset" }, id: "2" }, + ], videoOptions: [ { label: { type: "name", name: "Camera 1" }, id: "1" }, { label: { type: "name", name: "Camera 2" }, id: "2" }, ], selectedAudio: "2", + selectedAudioOutput: "default", selectedVideo: "1", }, }; +export const StaysWhileAMenuIsOpen: Story = { + ...Default, + args: { + ...WithAudioAndVideoOptions.args, + // In a short window the footer overlays the call and hides itself. + asOverlay: true, + showFooter: true, + }, + play: async ({ canvasElement }): Promise => { + const footer = canvasElement.querySelector( + '[data-testid="footer-container"]', + )!; + await userEvent.click( + within(canvasElement).getByRole("button", { name: "Microphone" }), + ); + await expect(document.body.querySelector('[role="menu"]')).not.toBeNull(); + + // The call hides the footer with this class. + footer.classList.add(styles.hidden); + + // Read after the 0.15s fade, not mid-transition. + await new Promise((settled) => setTimeout(settled, 400)); + await expect(getComputedStyle(footer).opacity).toBe("1"); + }, +}; + export const AudioBusy: Story = { ...Default, args: { diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index 4f79f236c..6fdf13dda 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -99,11 +99,15 @@ export interface FooterState { /** Providing no options `[]` or `undefined` will imply that we dont have a audio fast switcher */ audioOptions: MenuOptions[]; + /** Output (speaker) devices. */ + audioOutputOptions: MenuOptions[]; /** Providing no options `[]` or `undefined` will imply that we dont have a audio fast switcher */ videoOptions: MenuOptions[]; selectedAudio: string | undefined; + selectedAudioOutput: string | undefined; selectedVideo: string | undefined; selectAudioButtonOption: ((deviceId: string) => void) | undefined; + selectAudioOutputOption: ((deviceId: string) => void) | undefined; selectVideoButtonOption: ((option: string) => void) | undefined; } @@ -143,6 +147,9 @@ export const CallFooter: FC = ({ const audioOptions = useBehavior(vm.audioOptions$); const selectedAudio = useBehavior(vm.selectedAudio$); const selectAudioButtonOption = useBehavior(vm.selectAudioButtonOption$); + const audioOutputOptions = useBehavior(vm.audioOutputOptions$); + const selectedAudioOutput = useBehavior(vm.selectedAudioOutput$); + const selectAudioOutputOption = useBehavior(vm.selectAudioOutputOption$); const selectVideoButtonOption = useBehavior(vm.selectVideoButtonOption$); const toggleBlur = useBehavior(vm.toggleBlur$); const videoBlurEnabled = useBehavior(vm.videoBlurEnabled$); @@ -168,7 +175,6 @@ export const CallFooter: FC = ({ if ((audioOptions?.length ?? 0) > 0) { buttons.push( = ({ options={audioOptions} selectedOption={selectedAudio} onSelect={selectAudioButtonOption} + outputOptions={audioOutputOptions} + selectedOutputOption={selectedAudioOutput} + onSelectOutput={selectAudioOutputOption} />, ); } else { @@ -197,7 +206,6 @@ export const CallFooter: FC = ({ if ((videoOptions?.length ?? 0) > 0) { buttons.push( ({ supportsBackgroundProcessors: (): boolean => false, })); +const outputSelectionMock = vi.hoisted(() => vi.fn(() => true)); +vi.mock("livekit-client", () => ({ + supportsAudioOutputSelection: (): boolean => outputSelectionMock(), +})); + /** * Returns the minimum set of CallViewModel fields required by * createCallFooterViewModel, with all other properties stubbed to @@ -96,6 +101,65 @@ const twoMicsAndOneCamMediaDevices = mockMediaDevices({ }); describe("createCallFooterViewModel", () => { + describe("selectAudioOutputOption", () => { + function buildFooterVm(): ReturnType { + platformMock.mockReturnValue("desktop"); + return createCallFooterViewModel( + testScope(), + buildMinimalCallViewModel(gridLayout), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + /* reactionIdentifier */ undefined, + { showControls: true, header: HeaderStyle.Standard }, + ); + } + + it("is withheld where the platform cannot route audio to a chosen device", () => { + outputSelectionMock.mockReturnValue(false); + expect(buildFooterVm().selectAudioOutputOption$.value).toBeUndefined(); + }); + + it("is offered where the platform can route audio to a chosen device", () => { + outputSelectionMock.mockReturnValue(true); + expect(buildFooterVm().selectAudioOutputOption$.value).toBeDefined(); + }); + }); + + describe("audioOutputOptions", () => { + it("is an empty list, not absent, where the platform enumerates no outputs", () => { + platformMock.mockReturnValue("desktop"); + outputSelectionMock.mockReturnValue(true); + + const vm = createCallFooterViewModel( + testScope(), + buildMinimalCallViewModel(gridLayout), + mockMuteStates(), + mockMediaDevices({ + audioInput: { + available$: constant( + new Map([ + ["mic1", { type: "name", name: "Microphone 1" }], + ]), + ), + selected$: constant(undefined), + select: vi.fn(), + }, + // As Safari: no outputs listed. + audioOutput: { + available$: constant(new Map()), + selected$: constant(undefined), + select: vi.fn(), + }, + }), + /* reactionIdentifier */ undefined, + { showControls: true, header: HeaderStyle.Standard }, + ); + + // Empty, not undefined: undefined would hide the section. + expect(vm.audioOutputOptions$.value).toEqual([]); + }); + }); + describe("audioOptions and videoOptions", () => { function checkEmptyFor(platform: string, layout: Layout): void { platformMock.mockReturnValue(platform); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index a2ca6c88e..50efb42f9 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -5,8 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { combineLatest, map, switchMap } from "rxjs"; +import { combineLatest, map, type Observable, switchMap } from "rxjs"; import { supportsBackgroundProcessors } from "@livekit/track-processors"; +import { supportsAudioOutputSelection } from "livekit-client"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { type MenuOptions } from "./MediaMuteAndSwitchButton"; @@ -67,49 +68,50 @@ function buildDeviceBehaviors( | "audioOptions$" | "selectedAudio$" | "selectAudioButtonOption$" + | "audioOutputOptions$" + | "selectedAudioOutput$" + | "selectAudioOutputOption$" | "videoOptions$" | "selectedVideo$" | "selectVideoButtonOption$" | "toggleBlur$" | "videoBlurEnabled$" > { - return { - audioOptions$: scope.behavior( - disableSwitcher$.pipe( - switchMap((disable) => - disable - ? constant([] as MenuOptions[]) - : mediaDevices.audioInput.available$.pipe( - map((available) => - [...available.entries()].map(([id, label]) => ({ - id, - label, - })), - ), + const options$ = ( + available$: Behavior>, + ): Observable => + disableSwitcher$.pipe( + switchMap((disable) => + disable + ? constant([] as MenuOptions[]) + : available$.pipe( + map((available) => + [...available.entries()].map(([id, label]) => ({ id, label })), ), - ), + ), ), - ), + ); + + return { + audioOptions$: scope.behavior(options$(mediaDevices.audioInput.available$)), selectedAudio$: scope.behavior( mediaDevices.audioInput.selected$.pipe(map((s) => s?.id)), ), selectAudioButtonOption$: constant(mediaDevices.audioInput.select), - videoOptions$: scope.behavior( - disableSwitcher$.pipe( - switchMap((disable) => - disable - ? constant([] as MenuOptions[]) - : mediaDevices.videoInput.available$.pipe( - map((available) => - [...available.entries()].map(([id, label]) => ({ - id, - label, - })), - ), - ), - ), - ), + audioOutputOptions$: scope.behavior( + options$(mediaDevices.audioOutput.available$), ), + selectedAudioOutput$: scope.behavior( + mediaDevices.audioOutput.selected$.pipe(map((s) => s?.id)), + ), + // Withheld where the platform can't route audio to a chosen device, which + // disables the speaker section. + selectAudioOutputOption$: constant( + supportsAudioOutputSelection() + ? mediaDevices.audioOutput.select + : undefined, + ), + videoOptions$: scope.behavior(options$(mediaDevices.videoInput.available$)), selectedVideo$: scope.behavior( mediaDevices.videoInput.selected$.pipe(map((s) => s?.id)), ), @@ -263,10 +265,13 @@ export function createLobbyFooterViewModel( reactionData: undefined, tileStoreGeneration: undefined, audioOptions: undefined, + audioOutputOptions: undefined, videoOptions: undefined, selectedAudio: undefined, + selectedAudioOutput: undefined, selectedVideo: undefined, selectAudioButtonOption: undefined, + selectAudioOutputOption: undefined, selectVideoButtonOption: undefined, }), ...buildMuteBehaviors(scope, muteStates), diff --git a/src/components/MediaMuteAndSwitchButton.module.css b/src/components/MediaMuteAndSwitchButton.module.css index e5bba2383..1d5242689 100644 --- a/src/components/MediaMuteAndSwitchButton.module.css +++ b/src/components/MediaMuteAndSwitchButton.module.css @@ -35,3 +35,60 @@ Please see LICENSE in the repository root for full details. transform: rotate(360deg); } } + +.menu { + display: flex; + flex-direction: column; +} + +/* Only the device lists scroll, bounded by the measured call area: the menu is + portalled out of the root, so CSS can't size it against the call. */ +.deviceList { + overflow-y: auto; + min-block-size: 0; + max-block-size: var(--device-list-max-height); + /* Keeps a row reached by keyboard clear of the sticky heading and meter. */ + scroll-padding-block: var(--device-list-scroll-padding-start, 0) + var(--device-list-scroll-padding-end, 0); +} + +/* Compound's heading rule heads each section; the design adds space below it. */ +.menu h3 { + margin-block-end: var(--cpd-space-5x); +} + +/* And more above a heading that follows another section. */ +.deviceList [role="group"] + [role="group"] h3 { + margin-block-start: var(--cpd-space-7x); +} + +/* Sticky within its own section. Opaque and inset a border width so it doesn't + paint over the menu's outline. */ +.sectionHeading { + position: sticky; + inset-block-start: 0; + background: var(--cpd-color-bg-canvas-default); + margin-inline: var(--cpd-border-width-1); + margin-block-start: var(--cpd-border-width-1); +} + +/* Sticky at the foot of the microphone section; opaque and inset as above. */ +.stickyMeter { + position: sticky; + inset-block-end: 0; + background: var(--cpd-color-bg-canvas-default); + margin-inline: var(--cpd-border-width-1); + margin-block-end: var(--cpd-border-width-1); +} + +/* Radix focuses whatever the pointer is over, so the browser's ring would + follow the mouse. Replaced by one shown only for keyboard focus. */ +.menu [role^="menuitem"]:focus, +.menu [role^="menuitem"]:focus-visible { + outline: none; +} + +.menu[data-focus-source="keyboard"] [role^="menuitem"]:focus { + outline: var(--cpd-border-width-2) solid var(--cpd-color-border-focused); + outline-offset: calc(-1 * var(--cpd-border-width-2)); +} diff --git a/src/components/MediaMuteAndSwitchButton.stories.tsx b/src/components/MediaMuteAndSwitchButton.stories.tsx index 21def4007..d4965892f 100644 --- a/src/components/MediaMuteAndSwitchButton.stories.tsx +++ b/src/components/MediaMuteAndSwitchButton.stories.tsx @@ -5,12 +5,15 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { fn, userEvent, within, expect } from "storybook/test"; -import { type JSX } from "react"; +import { fn, userEvent, waitFor, within, expect } from "storybook/test"; +import { useEffect, useState, type FC, type JSX, type ReactNode } from "react"; import type { Meta, StoryObj } from "@storybook/react-vite"; import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton"; +import styles from "./MediaMuteAndSwitchButton.module.css"; +import meterStyles from "./MicrophoneLevelMeter.module.css"; import { MediaDevicesContext } from "../MediaDevicesContext"; +import { RootElementProvider } from "../RootElementContext"; import { MediaDevices } from "../state/MediaDevices"; import { globalScope } from "../state/ObservableScope"; @@ -18,12 +21,74 @@ const mediaDevices = new MediaDevices(globalScope, { controlledAudioDevices: false, }); +/** Supplies a microphone, a wavering tone: a story's invented device ids match no hardware. */ +const WithAMicrophone: FC<{ children: ReactNode }> = ({ children }) => { + useEffect(() => { + const context = new AudioContext(); + const microphone = context.createMediaStreamDestination(); + const tone = context.createOscillator(); + const loudness = context.createGain(); + // Wavers between a third and two thirds of full scale, so it reads as live. + const swing = context.createOscillator(); + const depth = context.createGain(); + loudness.gain.value = 0.25; + depth.gain.value = 0.2; + swing.frequency.value = 0.6; + tone.frequency.value = 220; + swing.connect(depth).connect(loudness.gain); + tone.connect(loudness).connect(microphone); + tone.start(); + swing.start(); + + const devices = navigator.mediaDevices; + const openedForReal = devices.getUserMedia.bind(devices); + const opened = Promise.resolve(microphone.stream); + // A fresh clone each time, so a caller stopping its tracks doesn't end the next. + devices.getUserMedia = async (): Promise => + (await opened).clone(); + + return (): void => { + devices.getUserMedia = openedForReal; + tone.stop(); + swing.stop(); + void context.close(); + }; + }, []); + + return <>{children}; +}; + +/** Supplies a call-sized root. Without one the list is bounded by the whole Storybook frame. */ +const WithACallArea: FC<{ children: ReactNode }> = ({ children }) => { + const [callArea, setCallArea] = useState(null); + return ( +
+ {callArea !== null && ( + {children} + )} +
+ ); +}; + const meta = { component: MediaMuteAndSwitchButton, decorators: [ (Story): JSX.Element => ( - + + + + + ), ], @@ -34,7 +99,6 @@ type Story = StoryObj; export const Default: Story = { args: { - title: "SomeMenu", iconsAndLabels: "audio", enabled: true, options: [ @@ -42,6 +106,13 @@ export const Default: Story = { { label: { type: "name", name: "Option 2" }, id: "2" }, ], selectedOption: "1", + // The footer always passes an output list to the audio menu. + outputOptions: [ + { label: { type: "default", name: "Built-in Output" }, id: "default" }, + { label: { type: "name", name: "Headset" }, id: "spk2" }, + ], + selectedOutputOption: "default", + onSelectOutput: fn(), onMuteClick: fn(), onSelect: fn(), }, @@ -50,7 +121,6 @@ export const Default: Story = { export const AudioMute: Story = { args: { ...Default.args, - title: "Microphone", iconsAndLabels: "audio", enabled: false, options: [ @@ -73,21 +143,19 @@ export const AudioMute: Story = { export const AudioUnmute: Story = { args: { - title: "Microphone", + ...Default.args, iconsAndLabels: "audio", enabled: true, options: [ { label: { type: "name", name: "Microphone 1" }, id: "1" }, { label: { type: "name", name: "Microphone 2" }, id: "2" }, ], - selectedOption: "2", }, }; export const VideoMute: Story = { args: { - title: "Camera", iconsAndLabels: "video", enabled: false, options: [ @@ -101,7 +169,6 @@ export const VideoMute: Story = { export const VideoUnmute: Story = { args: { - title: "Camera", iconsAndLabels: "video", enabled: true, options: [ @@ -113,3 +180,479 @@ export const VideoUnmute: Story = { selectedOption: "2", }, }; + +export const SpeakerAndMicrophoneSections: Story = { + args: { + ...Default.args, + iconsAndLabels: "audio", + enabled: true, + options: [ + { label: { type: "name", name: "Microphone 1" }, id: "mic1" }, + { label: { type: "name", name: "Microphone 2" }, id: "mic2" }, + ], + selectedOption: "mic1", + outputOptions: [ + { label: { type: "default", name: "Built-in Output" }, id: "default" }, + { label: { type: "name", name: "Headset" }, id: "spk2" }, + ], + selectedOutputOption: "default", + onSelectOutput: fn(), + }, + play: async ({ args, canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Microphone" })); + + const headset = await within(document.body).findByRole("menuitemradio", { + name: "Headset", + }); + await userEvent.click(headset); + await expect(args.onSelectOutput).toHaveBeenCalledWith("spk2"); + + // A few devices fit, so the list doesn't scroll. + const list = document.body.querySelector( + `.${styles.deviceList}`, + )!; + await expect(list.scrollHeight).toBe(list.clientHeight); + + // Each section is headed by its own edge-to-edge rule, with no separator. + const menu = document.body.querySelector("[role='menu']")!; + await expect( + document.body.querySelectorAll("[role='separator']"), + ).toHaveLength(0); + const headings = document.body.querySelectorAll( + `.${styles.sectionHeading}`, + ); + await expect(headings).toHaveLength(2); + const frame = menu.getBoundingClientRect(); + for (const heading of headings) { + const rule = heading.querySelector("h3")!; + await expect( + Number.parseFloat(getComputedStyle(rule).borderBottomWidth), + ).toBeGreaterThan(0); + // Edge to edge, within the frame. + const box = rule.getBoundingClientRect(); + await expect(box.left - frame.left).toBeLessThanOrEqual(2); + await expect(frame.right - box.right).toBeLessThanOrEqual(2); + } + + // The first device sits further below the rule than from the menu's edge. + // Asserted as a relationship, not pixels: the asymmetry is the design. + const control = document.body.querySelector("input[type='radio']")!; + const ruleBottom = headings[0] + .querySelector("h3")! + .getBoundingClientRect().bottom; + const box = control.getBoundingClientRect(); + await expect(box.top - ruleBottom).toBeGreaterThan(box.left - frame.left); + + // And a section stands further from the one above than from its own first + // device. + const groups = document.body.querySelectorAll("[role='group']"); + const speakers = groups[0].querySelectorAll("input[type='radio']"); + const lastSpeaker = speakers[speakers.length - 1].getBoundingClientRect(); + const nextHeading = groups[1]!.querySelector("h3")!.getBoundingClientRect(); + await expect(nextHeading.top - lastSpeaker.bottom).toBeGreaterThan( + box.top - ruleBottom, + ); + }, +}; + +export const OutputCannotBeChosen: Story = { + args: { + ...Default.args, + iconsAndLabels: "audio", + enabled: true, + options: [ + { label: { type: "name", name: "Microphone 1" }, id: "mic1" }, + { label: { type: "name", name: "Microphone 2" }, id: "mic2" }, + ], + selectedOption: "mic1", + outputOptions: [ + { label: { type: "name", name: "Speakers" }, id: "spk1" }, + { label: { type: "name", name: "Headset" }, id: "spk2" }, + ], + selectedOutputOption: "spk1", + // No callback: the speakers are listed, but none can be picked. + onSelectOutput: undefined, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Microphone" })); + + const speakers = await within(document.body).findByRole("menuitemradio", { + name: "Speakers", + }); + await expect(speakers).toHaveAttribute("aria-disabled", "true"); + }, +}; + +export const OnlyOneDevice: Story = { + args: { + ...Default.args, + iconsAndLabels: "audio", + enabled: true, + options: [{ label: { type: "name", name: "Microphone 1" }, id: "mic1" }], + selectedOption: "mic1", + outputOptions: [{ label: { type: "name", name: "Speakers" }, id: "spk1" }], + selectedOutputOption: "spk1", + onSelectOutput: fn(), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Microphone" })); + + // Shown, disabled, rather than hidden. + const only = await within(document.body).findByRole("menuitemradio", { + name: "Microphone 1", + }); + await expect(only).toHaveAttribute("aria-disabled", "true"); + }, +}; + +/** A requested device hasn't arrived: nothing in either section can be picked. */ +export const SelectionSettling: Story = { + args: { + ...Default.args, + iconsAndLabels: "audio", + enabled: true, + options: [ + { label: { type: "name", name: "Microphone 1" }, id: "mic1" }, + { label: { type: "name", name: "Microphone 2" }, id: "mic2" }, + ], + selectedOption: "mic1", + outputOptions: [ + { label: { type: "name", name: "Speakers" }, id: "spk1" }, + { label: { type: "name", name: "Headset" }, id: "spk2" }, + ], + selectedOutputOption: "spk1", + onSelectOutput: fn(), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Microphone" })); + const menu = within(document.body); + + // selectedOption never changes, so the request stays in flight. + await userEvent.click( + await menu.findByRole("menuitemradio", { name: "Microphone 2" }), + ); + + await expect(await menu.findByLabelText("Activating…")).toBeVisible(); + for (const item of menu.getAllByRole("menuitemradio")) + await expect(item).toHaveAttribute("aria-disabled", "true"); + }, +}; + +/** + * The ring shows for keyboard focus only. Asserted on the painted outline: + * asserting data-focus-source would still pass with the CSS rule deleted. + */ +export const KeyboardFocusRing: Story = { + args: { + ...Default.args, + iconsAndLabels: "audio", + enabled: true, + options: [ + { label: { type: "name", name: "Microphone 1" }, id: "mic1" }, + { label: { type: "name", name: "Microphone 2" }, id: "mic2" }, + ], + selectedOption: "mic1", + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Microphone" })); + const menu = within(document.body); + const first = await menu.findByRole("menuitemradio", { + name: "Microphone 1", + }); + + // Opened by pointer: no ring, even though Radix has moved focus. + await expect(outlineWidth(first)).toBe(0); + + await userEvent.keyboard("{ArrowDown}"); + const focused = document.activeElement as HTMLElement; + await expect(focused).toHaveRole("menuitemradio"); + await expect(outlineWidth(focused)).toBeGreaterThan(0); + + // And the pointer takes it away again. + await userEvent.hover(first); + await expect(outlineWidth(document.activeElement as HTMLElement)).toBe(0); + }, +}; + +/** The list scrolls; the meter stays at the foot of the microphone section. */ +export const ManyDevices: Story = { + args: { + ...Default.args, + iconsAndLabels: "audio", + enabled: true, + options: Array.from({ length: 20 }, (_, i) => ({ + label: { type: "name" as const, name: `Microphone ${i + 1}` }, + id: `mic${i + 1}`, + })), + selectedOption: "mic1", + outputOptions: Array.from({ length: 6 }, (_, i) => ({ + label: { type: "name" as const, name: `Speaker ${i + 1}` }, + id: `spk${i + 1}`, + })), + selectedOutputOption: "spk1", + onSelectOutput: fn(), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Microphone" })); + const menu = within(document.body); + + // The sticky wrapper rather than the meter: without a fake microphone (as on + // WebKit) the meter shows a message instead of a level. + const list = document.body.querySelector( + `.${styles.deviceList}`, + )!; + const sticky = await waitFor(() => { + const element = document.body.querySelector( + `.${styles.stickyMeter}`, + ); + if (element === null) throw new Error("the meter has not rendered yet"); + return element; + }); + await expect(list.scrollHeight).toBeGreaterThan(list.clientHeight); + + // Scrolled so the microphone section starts at the top: only there does a + // pinned meter differ from one that is simply last. + const group = await menu.findByRole("group", { name: "Microphone" }); + list.scrollTop += + group.getBoundingClientRect().top - list.getBoundingClientRect().top; + await expect(list.scrollTop + list.clientHeight).toBeLessThan( + list.scrollHeight, + ); + + const scrollport = list.getBoundingClientRect(); + const pinned = sticky.getBoundingClientRect(); + await expect(pinned.bottom).toBeLessThanOrEqual(scrollport.bottom + 1); + await expect(pinned.top).toBeGreaterThanOrEqual(scrollport.top - 1); + + // The whole menu is on screen, not bounded by the document. + const frame = document.body + .querySelector("[role='menu']")! + .getBoundingClientRect(); + await expect(frame.top).toBeGreaterThanOrEqual(0); + await expect(frame.bottom).toBeLessThanOrEqual(window.innerHeight + 1); + + // The meter is the menu's one opaque part, so it must stay inside the frame. + await expect(pinned.left).toBeGreaterThan(frame.left); + await expect(pinned.right).toBeLessThan(frame.right); + }, +}; + +/** Painted outline width, in px. */ +function outlineWidth(element: HTMLElement): number { + const { outlineStyle, outlineWidth } = getComputedStyle(element); + if (outlineStyle === "none") return 0; + return Number.parseFloat(outlineWidth) || 0; +} + +/** Safari lists no outputs: a disabled, selected Default stands in. */ +export const OutputNotEnumerated: Story = { + args: { + ...Default.args, + iconsAndLabels: "audio", + enabled: true, + options: [ + { label: { type: "name", name: "Microphone 1" }, id: "mic1" }, + { label: { type: "name", name: "Microphone 2" }, id: "mic2" }, + ], + selectedOption: "mic1", + outputOptions: [], + selectedOutputOption: undefined, + onSelectOutput: undefined, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Microphone" })); + + const speakers = await within(document.body).findByRole("menuitemradio", { + name: "Default", + }); + await expect(speakers).toHaveAttribute("aria-disabled", "true"); + }, +}; + +/** The meter's icon shares the radio controls' centre line; only a real browser lays this out. */ +export const MeterAlignsWithTheDeviceRows: Story = { + args: { + ...Default.args, + iconsAndLabels: "audio", + enabled: true, + options: [ + { label: { type: "name", name: "Microphone 1" }, id: "mic1" }, + { label: { type: "name", name: "Microphone 2" }, id: "mic2" }, + ], + selectedOption: "mic1", + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Microphone" })); + + const menu = document.body.querySelector("[role='menu']")!; + const radio = menu.querySelector("input[type='radio']")!; + const icon = await waitFor(() => { + const element = document.body.querySelector(`.${meterStyles.icon}`); + if (element === null) throw new Error("the meter has not rendered yet"); + return element; + }); + + // A pixel of slack, for subpixel layout. + await expect(Math.abs(centre(icon) - centre(radio))).toBeLessThanOrEqual(1); + }, +}; + +/** Inline-axis centre of an element. */ +function centre(element: Element): number { + const box = element.getBoundingClientRect(); + return box.left + box.width / 2; +} + +/** Every row the keyboard reaches is fully visible, not under a heading or the meter. */ +export const KeyboardReachesEveryDevice: Story = { + args: { + ...Default.args, + iconsAndLabels: "audio", + enabled: true, + // Enough to scroll both ways, so either end can hide a row. + options: Array.from({ length: 20 }, (_, i) => ({ + label: { type: "name" as const, name: `Microphone ${i + 1}` }, + id: `mic${i + 1}`, + })), + selectedOption: "mic1", + outputOptions: Array.from({ length: 4 }, (_, i) => ({ + label: { type: "name" as const, name: `Speaker ${i + 1}` }, + id: `spk${i + 1}`, + })), + selectedOutputOption: "spk1", + onSelectOutput: fn(), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Microphone" })); + + const meter = await waitFor(() => { + const element = document.body.querySelector( + `.${styles.stickyMeter}`, + ); + if (element === null) throw new Error("the meter has not rendered yet"); + return element; + }); + + // Drawn over the list: the sticky headings and the meter. + const overlays = [ + meter, + ...document.body.querySelectorAll( + `.${styles.sectionHeading}`, + ), + ]; + const items = within(document.body).getAllByRole("menuitemradio"); + + // Down to the last device and back up. + for (const key of ["{ArrowDown}", "{ArrowUp}"]) + for (let i = 0; i < items.length; i++) { + await userEvent.keyboard(key); + const focused = document.activeElement as HTMLElement; + await expect(focused).toHaveRole("menuitemradio"); + // Checked as overlap: a heading only covers rows while its section is on + // screen. + await expect(overlapping(focused, overlays)).toBeLessThanOrEqual(1); + } + }, +}; + +/** A section's heading stays at the top while the section is in view, and leaves with it. */ +export const HeadingsStayWhileScrolling: Story = { + args: { + ...Default.args, + iconsAndLabels: "audio", + enabled: true, + options: Array.from({ length: 20 }, (_, i) => ({ + label: { type: "name" as const, name: `Microphone ${i + 1}` }, + id: `mic${i + 1}`, + })), + selectedOption: "mic1", + outputOptions: Array.from({ length: 4 }, (_, i) => ({ + label: { type: "name" as const, name: `Speaker ${i + 1}` }, + id: `spk${i + 1}`, + })), + selectedOutputOption: "spk1", + onSelectOutput: fn(), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Microphone" })); + const menu = within(document.body); + + const list = document.body.querySelector( + `.${styles.deviceList}`, + )!; + const group = await menu.findByRole("group", { name: "Microphone" }); + // By class: the heading is aria-hidden, and its group carries the name. + const heading = group.querySelector( + `.${styles.sectionHeading}`, + )!; + + // Far enough that the heading is only on screen if it is stuck there. + list.scrollTop += + group.getBoundingClientRect().top - list.getBoundingClientRect().top + 80; + + const scrollport = list.getBoundingClientRect(); + await expect(heading.getBoundingClientRect().top).toBeLessThanOrEqual( + scrollport.top + 2, + ); + await expect(heading.getBoundingClientRect().bottom).toBeGreaterThan( + scrollport.top, + ); + // Clear of the menu's frame. + const frame = document.body + .querySelector("[role='menu']")! + .getBoundingClientRect(); + await expect(heading.getBoundingClientRect().left).toBeGreaterThan( + frame.left, + ); + }, +}; + +/** How far the most overlapping of `overlays` covers `element`, in px. */ +function overlapping(element: Element, overlays: Element[]): number { + const box = element.getBoundingClientRect(); + return overlays.reduce((worst, overlay) => { + const over = overlay.getBoundingClientRect(); + const shared = + Math.min(box.bottom, over.bottom) - Math.max(box.top, over.top); + return Math.max(worst, shared); + }, 0); +} + +/** The blur toggle gets the keyboard ring too: it is the menu's child, not the list's. */ +export const FocusRingCoversTheBlurToggle: Story = { + args: { + ...VideoUnmute.args, + iconsAndLabels: "video", + videoBlurEnabled: false, + videoBlurToggleClick: fn(), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Camera" })); + const toggle = await within(document.body).findByRole("menuitemcheckbox", { + name: /Blur background/, + }); + + // Arrow down past the cameras to the toggle. + for (let i = 0; i < 6 && document.activeElement !== toggle; i++) + await userEvent.keyboard("{ArrowDown}"); + await expect(document.activeElement).toBe(toggle); + // The same ring the device rows get. + await expect(outlineWidth(toggle)).toBeGreaterThan(0); + + // And the pointer takes it away, with the toggle still focused. + await userEvent.hover(toggle); + await expect(document.activeElement).toBe(toggle); + await expect(outlineWidth(toggle)).toBe(0); + }, +}; diff --git a/src/components/MediaMuteAndSwitchButton.test.tsx b/src/components/MediaMuteAndSwitchButton.test.tsx index fcc9f0b4e..9fd67dfdb 100644 --- a/src/components/MediaMuteAndSwitchButton.test.tsx +++ b/src/components/MediaMuteAndSwitchButton.test.tsx @@ -5,40 +5,61 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, test, vi } from "vitest"; -import { act, render, screen, type RenderResult } from "@testing-library/react"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { axe } from "vitest-axe"; +import { + act, + render, + screen, + within, + type RenderResult, +} from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import { type JSX, useState, type ReactNode } from "react"; +import { Profiler, type JSX, useState, type ReactNode } from "react"; import { TooltipProvider } from "@vector-im/compound-web"; -import { MediaMuteAndSwitchButton } from "./MediaMuteAndSwitchButton"; +import { + MediaMuteAndSwitchButton, + type MenuOptions, +} from "./MediaMuteAndSwitchButton"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { type MediaDevices } from "../state/MediaDevices"; +import { restoreAudioCapture, stubAudioCapture } from "../utils/test"; interface RenderOptions { requestDeviceNames: () => void; } -function renderComponent( +function withProviders( component: ReactNode, { requestDeviceNames = (): void => {} }: Partial = {}, -): RenderResult { - return render( +): JSX.Element { + return ( {component} - , + ); } +function renderComponent( + component: ReactNode, + options: Partial = {}, +): RenderResult { + return render(withProviders(component, options)); +} + describe("MediaMuteAndSwitchButton", () => { + // Only one test stubs the capture; don't let it leak into the rest. + afterEach(restoreAudioCapture); + test("renders", () => { const { container } = renderComponent( - + , ); expect(container).toMatchSnapshot(); @@ -50,11 +71,7 @@ describe("MediaMuteAndSwitchButton", () => { enabled: boolean, ): RenderResult => { return renderComponent( - , + , ); }; const renderAudioEndabled = renderLabels("audio", true); @@ -81,7 +98,6 @@ describe("MediaMuteAndSwitchButton", () => { const onMute = vi.fn(); const { getByRole } = renderComponent( { const onMute = vi.fn(); const { getByRole } = renderComponent( { const onMute = vi.fn(); const { getByRole } = renderComponent( { const user = userEvent.setup(); const requestDeviceNames = vi.fn(); renderComponent( - , + , { requestDeviceNames }, ); @@ -157,7 +167,6 @@ describe("MediaMuteAndSwitchButton", () => { renderComponent( <> { selectedOption="mic1" /> { const onSelect = vi.fn(); const { getByRole } = renderComponent( { const onSelect = vi.fn(); const { getByRole } = renderComponent( { const [selectedOption, setSelectedOption] = useState("mic1"); return ( { const onVideoBlurToggle = vi.fn(); const { getByRole } = renderComponent( { expect(onVideoBlurToggle).toHaveBeenCalled(); }); - test("renders check icon to mark the selected menu item", async () => { + test("marks the selected menu item as checked", async () => { const user = userEvent.setup(); const { getByRole } = renderComponent( { />, ); - // open menu await user.click(getByRole("button", { name: "Microphone" })); - // The selected item (mic2) renders both an IconOptions SVG and a CheckIcon SVG - const mic1Item = screen.getByRole("menuitemradio", { - name: "Microphone 2", - }); - expect(mic1Item.querySelectorAll("svg").length).toBe(2); + screen.getByRole("menuitemradio", { name: "Microphone 2", checked: true }); + screen.getByRole("menuitemradio", { name: "Microphone 1", checked: false }); + }); - // The unselected item (mic1) only renders its IconOptions SVG - const mic2Item = screen.getByRole("menuitemradio", { + test("disables every device while a selection is settling", async () => { + const user = userEvent.setup(); + const { promise, resolve } = Promise.withResolvers(); + function Wrapper(): JSX.Element { + const [selectedOption, setSelectedOption] = useState("mic1"); + return ( + { + void promise.then(() => setSelectedOption(id)); + }} + outputOptions={[ + { label: { type: "name", name: "Speakers" }, id: "spk1" }, + { label: { type: "name", name: "Headset" }, id: "spk2" }, + ]} + selectedOutputOption="spk1" + onSelectOutput={vi.fn()} + /> + ); + } + + const { getByRole } = renderComponent(); + await user.click(getByRole("button", { name: "Microphone" })); + await user.click( + screen.getByRole("menuitemradio", { name: "Microphone 2" }), + ); + + // In flight: nothing else can be picked, in either section. + for (const name of ["Microphone 1", "Speakers", "Headset"]) { + expect(screen.getByRole("menuitemradio", { name })).toHaveAttribute( + "aria-disabled", + "true", + ); + } + + await act(async () => { + resolve(); + await promise; + }); + + // Settled: selectable again. + expect( + screen.getByRole("menuitemradio", { name: "Microphone 1" }), + ).not.toHaveAttribute("aria-disabled", "true"); + expect( + screen.getByRole("menuitemradio", { name: "Headset" }), + ).not.toHaveAttribute("aria-disabled", "true"); + }); + + test("lets go of a device switch that never arrives", async () => { + const user = userEvent.setup(); + // onSelect never reports back, as when a device is removed mid-switch. + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Microphone" })); + await user.click( + screen.getByRole("menuitemradio", { name: "Microphone 2" }), + ); + await user.keyboard("{Escape}"); + await user.click(getByRole("button", { name: "Microphone" })); + + for (const name of ["Microphone 1", "Speakers", "Headset"]) { + expect(screen.getByRole("menuitemradio", { name })).not.toHaveAttribute( + "aria-disabled", + "true", + ); + } + }); + + test("lets go of a device switch whose device is unplugged", async () => { + const user = userEvent.setup(); + const mics: MenuOptions[] = [ + { label: { type: "name", name: "Microphone 1" }, id: "mic1" }, + { label: { type: "name", name: "Microphone 2" }, id: "mic2" }, + ]; + const menu = (options: MenuOptions[]): JSX.Element => ( + + ); + + const { getByRole, rerender } = renderComponent(menu(mics)); + await user.click(getByRole("button", { name: "Microphone" })); + await user.click( + screen.getByRole("menuitemradio", { name: "Microphone 2" }), + ); + + // The second microphone is unplugged before the switch lands. + rerender(withProviders(menu(mics.slice(0, 1)))); + + // Selectable again without closing the menu. + expect( + screen.getByRole("menuitemradio", { name: "Headset" }), + ).not.toHaveAttribute("aria-disabled", "true"); + }); + + test("moves the level without re-rendering anything", async () => { + // The level is drawn into the DOM, so a moving level commits nothing. + const capture = stubAudioCapture(); + const user = userEvent.setup(); + let commits = 0; + + const { getByRole } = renderComponent( + { + commits++; + }} + > + + , + ); + + await user.click(getByRole("button", { name: "Microphone" })); + capture.grant(); + await vi.waitFor(() => expect(capture.contexts).toHaveLength(1)); + const meter = await screen.findByRole("meter"); + // The capture's arrival is one render: the idle level swapped for its own. + await act(async () => {}); + const settled = commits; + + // The meter smooths by elapsed time, so hand-driven frames need a clock. + let elapsed = performance.now(); + const clock = vi + .spyOn(performance, "now") + .mockImplementation(() => (elapsed += 16)); + + // One frame per task, as a browser delivers them: one act() would batch them. + for (let step = 1; step <= 8; step++) { + capture.speak(step / 8); + await act(async () => { + capture.drawFrames(1); + await Promise.resolve(); + }); + } + clock.mockRestore(); + + expect(meter.getAttribute("aria-valuenow")).not.toBe("0"); + expect(commits - settled).toBe(0); + }); + + test("camera menu uses the same selection pattern and keeps the blur toggle", async () => { + const user = userEvent.setup(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Camera" })); + + screen.getByRole("menuitemradio", { name: "Camera 1", checked: true }); + screen.getByRole("menuitemradio", { name: "Camera 2", checked: false }); + expect( + screen.getByRole("menuitemcheckbox", { name: "Blur background" }), + ).toBeInTheDocument(); + }); + + test("marks focus as keyboard-driven only when the keyboard moved it", async () => { + const user = userEvent.setup(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Microphone" })); + const list = screen + .getByRole("menuitemradio", { name: "Microphone 1" }) + .closest("[data-focus-source]"); + + // The menu focuses whatever the pointer is over, so focus alone says nothing. + expect(list).toHaveAttribute("data-focus-source", "pointer"); + + await user.keyboard("{ArrowDown}"); + expect(list).toHaveAttribute("data-focus-source", "keyboard"); + + await user.pointer({ + target: screen.getByRole("menuitemradio", { name: "Microphone 2" }), + coords: { clientX: 10, clientY: 10 }, + }); + expect(list).toHaveAttribute("data-focus-source", "pointer"); + }); + + test("marks the selected device with the accent fill", async () => { + const user = userEvent.setup(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Microphone" })); + + const selected = screen + .getByRole("menuitemradio", { name: "Microphone 2" }) + .querySelector("input[type=radio]"); + expect(selected).toBeChecked(); + // readOnly would paint the selected radio muted. + expect(selected).not.toHaveAttribute("readonly"); + }); + + test("the open menu has no accessibility violations", async () => { + const user = userEvent.setup(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Microphone" })); + + const menu = document.querySelector('[role="menu"]'); + expect(await axe(menu as HTMLElement)).toHaveNoViolations(); + }); + + test("puts the speaker section above the microphone section", async () => { + const user = userEvent.setup(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Microphone" })); + + const speakers = screen.getByRole("menuitemradio", { name: "Speakers" }); + const microphone = screen.getByRole("menuitemradio", { name: "Microphone 1", }); - expect(mic2Item.querySelectorAll("svg").length).toBe(1); + expect( + speakers.compareDocumentPosition(microphone) & + Node.DOCUMENT_POSITION_FOLLOWING, + ).toBeTruthy(); + }); + + test("lists speaker and microphone sections", async () => { + const user = userEvent.setup(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Microphone" })); + + screen.getByRole("menuitemradio", { + name: "Default (Built-in Output)", + checked: true, + }); + screen.getByRole("menuitemradio", { name: "Headset", checked: false }); + screen.getByRole("menuitemradio", { name: "Microphone 1", checked: true }); + screen.getByRole("menuitemradio", { name: "Microphone 2", checked: false }); + }); + + test("calls the output select callback on speaker click", async () => { + const user = userEvent.setup(); + const onSelectOutput = vi.fn(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Microphone" })); + await user.click(screen.getByRole("menuitemradio", { name: "Headset" })); + + expect(onSelectOutput).toHaveBeenCalledWith("spk2"); + }); + + test("shows a single device entry disabled", async () => { + const user = userEvent.setup(); + const onSelect = vi.fn(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Microphone" })); + + // Shown, but not selectable. + const only = screen.getByRole("menuitemradio", { name: "Microphone 1" }); + expect(only).toHaveAttribute("aria-disabled", "true"); + }); + + test("shows a default speaker where the platform lists none", async () => { + const user = userEvent.setup(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Microphone" })); + + const speakers = screen + .getAllByRole("group") + .find((group) => group.getAttribute("aria-label") === "Speaker")!; + const entries = within(speakers).getAllByRole("menuitemradio"); + expect(entries).toHaveLength(1); + expect(entries[0]).toHaveAccessibleName("Default"); + expect(entries[0]).toHaveAttribute("aria-disabled", "true"); + expect(entries[0]).toHaveAttribute("aria-checked", "true"); + expect( + within(entries[0]).getByRole("radio", { hidden: true }), + ).toBeChecked(); + }); + + test("shows the speaker section disabled when output selection is unsupported", async () => { + const user = userEvent.setup(); + const { getByRole } = renderComponent( + , + ); + + await user.click(getByRole("button", { name: "Microphone" })); + + expect( + screen.getByRole("menuitemradio", { name: "Speakers" }), + ).toHaveAttribute("aria-disabled", "true"); + expect( + screen.getByRole("menuitemradio", { name: "Headset" }), + ).toHaveAttribute("aria-disabled", "true"); + expect( + screen.getByRole("menuitemradio", { name: "Microphone 2" }), + ).not.toHaveAttribute("aria-disabled", "true"); }); }); diff --git a/src/components/MediaMuteAndSwitchButton.tsx b/src/components/MediaMuteAndSwitchButton.tsx index 3e344dd3a..25342f001 100644 --- a/src/components/MediaMuteAndSwitchButton.tsx +++ b/src/components/MediaMuteAndSwitchButton.tsx @@ -5,37 +5,48 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type ComponentType, useState, type FC, useEffect } from "react"; +import { + useCallback, + useState, + type CSSProperties, + type FC, + useEffect, + type ReactElement, +} from "react"; import { Button, Menu, MenuItem, + MenuTitle, + RadioInput, ToggleMenuItem, } from "@vector-im/compound-web"; import { - CheckIcon, ChevronUpIcon, ChevronDownIcon, - MicOnIcon, SpinnerIcon, - VideoCallIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import classNames from "classnames"; import { useTranslation } from "react-i18next"; +import { distinctUntilChanged, map } from "rxjs"; import styles from "./MediaMuteAndSwitchButton.module.css"; import { MicButton, VideoButton } from "../button"; -import { type DeviceLabel } from "../state/MediaDevices"; +import { + type AudioOutputDeviceLabel, + type DeviceLabel, +} from "../state/MediaDevices"; import { useMediaDevices } from "../MediaDevicesContext"; +import { useRootElement } from "../RootElementContext"; +import { observeElementSize$ } from "../utils/elementSize"; +import { LiveMicrophoneLevelMeter } from "./MicrophoneLevelMeter"; export interface MenuOptions { - label: DeviceLabel; + label: DeviceLabel | AudioOutputDeviceLabel; id: string; } export interface MediaMuteAndSwitchButtonProps { - /** The title used in the Switcher modal. */ - title: string; /** If the Mute button is enabled */ enabled?: boolean; /** Callback if the mute button is clicked */ @@ -47,6 +58,12 @@ export interface MediaMuteAndSwitchButtonProps { options?: MenuOptions[]; /** The option that will currently be rendered as the selected option */ selectedOption?: string; + /** Output (speaker) devices. Audio menu only. */ + outputOptions?: MenuOptions[]; + /** The output option currently rendered as selected */ + selectedOutputOption?: string; + /** Picks an output device. Undefined disables the speaker section. */ + onSelectOutput?: (id: string) => void; videoBlurToggleClick?: () => void; videoBlurEnabled?: boolean; /** @@ -58,90 +75,240 @@ export interface MediaMuteAndSwitchButtonProps { const BLUR_ID = "blur"; +/** Id of the placeholder "Default" row, shown when the platform lists no outputs. */ +const DEFAULT_OUTPUT_ID = "default"; + +/** Largest share of the call area's height the device list may take. */ +const LIST_SHARE_OF_CALL = 0.6; + +/** Smallest device list height in px, so a short call still shows more than one device. */ +const MIN_LIST_HEIGHT = 160; + export const MediaMuteAndSwitchButton: FC = ({ - title, enabled, busy, onMuteClick, iconsAndLabels, options, selectedOption, + outputOptions, + selectedOutputOption, + onSelectOutput, videoBlurEnabled, videoBlurToggleClick, onSelect, }) => { - const [plannedSelection, setPlannedSelection] = useState(null); + // Requested but not yet selected. Keyed by kind too, since Chrome uses + // "default" for both an input and an output. + const [plannedSelection, setPlannedSelection] = useState<{ + kind: "input" | "output"; + id: string; + } | null>(null); const [menuOpen, setMenuOpen] = useState(false); + const onOpenChange = useCallback((open: boolean): void => { + setMenuOpen(open); + // Drop a request that never arrived. + if (!open) setPlannedSelection(null); + }, []); const isBusy = busy ?? false; const { t } = useTranslation(); const devices = useMediaDevices(); + /** + * Records on the menu whether the keyboard or the pointer moved focus, for + * the focus ring. `:focus-visible` can't tell: Radix focuses whatever the + * pointer is over. Listened for on the menu, not the document, so a second + * Element Call on the page doesn't answer for this one. + */ + const trackFocusSource = useCallback( + (list: HTMLDivElement | null): (() => void) | undefined => { + const menu = list?.closest('[role="menu"]'); + if (menu === null || menu === undefined) return; + const record = (source: "keyboard" | "pointer"): void => { + menu.dataset.focusSource = source; + }; + record("pointer"); + const usedKeyboard = (): void => record("keyboard"); + const usedPointer = (): void => record("pointer"); + menu.addEventListener("keydown", usedKeyboard, true); + menu.addEventListener("pointermove", usedPointer, true); + return (): void => { + menu.removeEventListener("keydown", usedKeyboard, true); + menu.removeEventListener("pointermove", usedPointer, true); + }; + }, + [], + ); + + // Measured on the call area: CSS can't size the portalled menu against it. + const rootElement = useRootElement(); + const [listMaxHeight, setListMaxHeight] = useState(); + useEffect(() => { + if (!menuOpen) return; + // Followed, since a host can resize the call while the menu is open. + const subscription = observeElementSize$(rootElement) + .pipe( + map(({ height }) => + Math.max(MIN_LIST_HEIGHT, Math.round(height * LIST_SHARE_OF_CALL)), + ), + distinctUntilChanged(), + ) + .subscribe(setListMaxHeight); + return (): void => subscription.unsubscribe(); + }, [menuOpen, rootElement]); + + // Kept clear at the list's foot, so a row reached by keyboard isn't under + // the meter. + const [meterHeight, meter] = useMeasuredHeight(); + + // Likewise at its head, for the sticky headings. + const [headingHeight, heading] = useMeasuredHeight(); + useEffect(() => { if (menuOpen) devices.requestDeviceNames(); // No-op after the first call }, [menuOpen, devices]); - let button; - let toggles: { label: string; enabled: boolean; id: string }[] = []; - switch (iconsAndLabels) { - case "video": - button = ( - { - onMuteClick?.(); - e.preventDefault(); - e.stopPropagation(); - }} - disabled={isBusy || onMuteClick === undefined} - data-testid="incall_videomute" - /> - ); - if (videoBlurToggleClick !== undefined) { - toggles = [ + const MuteButton = iconsAndLabels === "audio" ? MicButton : VideoButton; + const button = ( + { + onMuteClick?.(); + e.preventDefault(); + e.stopPropagation(); + }} + disabled={isBusy || onMuteClick === undefined} + data-testid={ + iconsAndLabels === "audio" ? "incall_mute" : "incall_videomute" + } + /> + ); + + const toggles = + iconsAndLabels === "video" && videoBlurToggleClick !== undefined + ? [ { label: t("action.blur_background"), enabled: videoBlurEnabled ?? false, id: BLUR_ID, }, - ]; - } - break; - case "audio": - button = ( - { - onMuteClick?.(); - e.preventDefault(); - e.stopPropagation(); - }} - disabled={isBusy || onMuteClick === undefined} - data-testid="incall_mute" - /> - ); - break; - } + ] + : []; - let IconOptions: ComponentType> | undefined; let optionsButtonLabel: string; + let menuTitle: string; let numberedLabel: (number: number) => string; switch (iconsAndLabels) { case "video": - IconOptions = VideoCallIcon; optionsButtonLabel = t("settings.devices.camera"); + menuTitle = t("settings.devices.camera_source"); numberedLabel = (n): string => t("settings.devices.camera_numbered", { n }); break; case "audio": - IconOptions = MicOnIcon; optionsButtonLabel = t("settings.devices.microphone"); + menuTitle = t("settings.devices.mic_source"); numberedLabel = (n): string => t("settings.devices.microphone_numbered", { n }); break; } + const labelText = ( + label: MenuOptions["label"], + numbered: (n: number) => string, + ): string => { + switch (label.type) { + case "name": + return label.name; + case "number": + return numbered(label.number); + case "default": + return label.name === null + ? t("settings.devices.default") + : t("settings.devices.default_named_plain", { name: label.name }); + case "speaker": + return t("settings.devices.loudspeaker"); + case "earpiece": + return t("settings.devices.handset"); + } + }; + + // Nothing can be picked while a requested device is on offer but not yet + // selected, so a second request can't overtake it. + const plannedOutput = plannedSelection?.kind === "output"; + const selectedOfPlannedKind = plannedOutput + ? selectedOutputOption + : selectedOption; + const offeredOfPlannedKind = plannedOutput ? outputOptions : options; + const settling = + plannedSelection !== null && + plannedSelection.id !== selectedOfPlannedKind && + offeredOfPlannedKind?.some(({ id }) => id === plannedSelection.id) === true; + + // Safari lists no outputs: show a disabled, selected Default rather than an + // empty section. + const noOutputsListed = outputOptions?.length === 0; + const speakerOptions: MenuOptions[] | undefined = noOutputsListed + ? [{ id: DEFAULT_OUTPUT_ID, label: { type: "default", name: null } }] + : outputOptions; + const selectedSpeaker = noOutputsListed + ? DEFAULT_OUTPUT_ID + : selectedOutputOption; + + const deviceItems = ( + kind: "input" | "output", + items: MenuOptions[] | undefined, + selected: string | undefined, + select: ((id: string) => void) | undefined, + numbered: (n: number) => string, + ): ReactElement[] => { + const list = items ?? []; + // Disabled rather than hidden, so the menu keeps its shape. + const disabled = select === undefined || list.length <= 1 || settling; + return list.map(({ label, id }) => ( + + {}} + /> + + } + onSelect={(e) => { + e.preventDefault(); + if (id === selected) return; + setPlannedSelection({ kind, id }); + select?.(id); + }} + key={id} + role="menuitemradio" + aria-checked={selected === id} + > + {selected !== id && + plannedSelection?.kind === kind && + plannedSelection.id === id && ( + + )} + + )); + }; + return (
= ({ {/* The mute button lives inside */} {button} = ({ /> } > - {options?.map(({ label, id }) => { - let labelText: string; - switch (label.type) { - case "name": - labelText = label.name; - break; - case "number": - labelText = numberedLabel(label.number); - break; +
- ) - } - onSelect={(e) => { - e.preventDefault(); - if (id === selectedOption) return; - setPlannedSelection(id); - onSelect?.(id); - }} - key={id} - role="menuitemradio" - aria-checked={selectedOption === id} - > - {selectedOption === id && ( - + {iconsAndLabels === "audio" && speakerOptions && ( + <> + {/* A menu may only contain items, separators and groups, so each + heading is a hidden part of a named group. */} +
+
+ +
+ {deviceItems( + "output", + speakerOptions, + selectedSpeaker, + onSelectOutput, + (n) => t("settings.devices.speaker_numbered", { n }), + )} +
+ + )} +
+
+ +
+ {/* Apart from the heading, so the sticky meter can't ride over it. */} +
+ {deviceItems( + "input", + options, + selectedOption, + onSelect, + numberedLabel, + )} + {iconsAndLabels === "audio" && ( + )} - {selectedOption !== id && plannedSelection === id && ( - - )} - - ); - })} - {(toggles?.length ?? 0) > 0 &&
} - {toggles?.map((toggle) => ( +
+
+
+ {toggles.length > 0 &&
} + {toggles.map((toggle) => ( { videoBlurToggleClick?.(); e.preventDefault(); }} - checked={toggle.enabled ?? false} + checked={toggle.enabled} key={toggle.id} /> ))} @@ -239,3 +415,25 @@ export const MediaMuteAndSwitchButton: FC = ({
); }; + +/** Follows an element's height. */ +function useMeasuredHeight(): [ + number | undefined, + (element: HTMLElement | null) => (() => void) | undefined, +] { + const [height, setHeight] = useState(); + const ref = useCallback( + (element: HTMLElement | null): (() => void) | undefined => { + if (element === null) return; + const subscription = observeElementSize$(element) + .pipe( + map((size) => size.height), + distinctUntilChanged(), + ) + .subscribe(setHeight); + return (): void => subscription.unsubscribe(); + }, + [], + ); + return [height, ref]; +} diff --git a/src/components/MicrophoneLevelMeter.module.css b/src/components/MicrophoneLevelMeter.module.css new file mode 100644 index 000000000..83e440c55 --- /dev/null +++ b/src/components/MicrophoneLevelMeter.module.css @@ -0,0 +1,65 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +.meter { + /* A device row's icon column, so the bars start where the names do. */ + --meter-icon-size: 24px; + + /* Compound's radio size, whose centre line the icon shares. */ + --device-control-size: 20px; + + display: flex; + align-items: center; + gap: var(--cpd-space-3x); + padding-block: var(--cpd-space-2x); + /* Centres the icon on the radio column: less the row's inset border and half + the icon's extra width. Checked in the MeterAlignsWithTheDeviceRows story. */ + padding-inline-start: calc( + var(--cpd-space-4x) - var(--cpd-border-width-1) - + (var(--meter-icon-size) - var(--device-control-size)) / 2 + ); + padding-inline-end: calc(var(--cpd-space-4x) * 2 + var(--cpd-space-2x)); +} + +.icon { + color: var(--cpd-color-icon-secondary); + flex-shrink: 0; + inline-size: var(--meter-icon-size); + block-size: var(--meter-icon-size); +} + +.segments { + /* Bars keep one size and the width decides how many; the component reads the + sizes back from here. */ + flex: 1; + /* So the bars never set the menu's width. */ + contain: inline-size; + min-inline-size: 0; + display: flex; + align-items: center; + /* Half again a bar's width, per the design. */ + gap: var(--cpd-space-1-5x); +} + +.segment { + flex: none; + inline-size: var(--cpd-space-1x); + block-size: var(--cpd-space-4x); + border-radius: var(--cpd-space-1x); + background: var(--cpd-color-bg-subtle-primary); +} + +/* The count of lit bars carries the level as well as their colour. */ +.segmentLit { + background: var(--cpd-color-bg-accent-rest); +} + +.message { + color: var(--cpd-color-text-secondary); + /* A paragraph's margin would lift the text off centre. */ + margin-block: 0; +} diff --git a/src/components/MicrophoneLevelMeter.stories.tsx b/src/components/MicrophoneLevelMeter.stories.tsx new file mode 100644 index 000000000..933fa1866 --- /dev/null +++ b/src/components/MicrophoneLevelMeter.stories.tsx @@ -0,0 +1,161 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, within } from "storybook/test"; +import { type JSX } from "react"; + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { + MicrophoneLevelMeter, + type MicrophoneLevelMeterProps, +} from "./MicrophoneLevelMeter"; +import styles from "./MicrophoneLevelMeter.module.css"; +import { LEVEL_SCALE } from "../state/MicrophoneLevel"; +import { constant } from "../state/Behavior"; + +/** Roughly the menu's width. It only decides how many bars fit. */ +const STORY_WIDTH = 256; + +const meta = { + component: MicrophoneLevelMeter, + decorators: [ + (Story): JSX.Element => ( +
+ +
+ ), + ], + argTypes: { + state: { + description: + "What the selected microphone can say about itself: a level, or a reason there is none.", + }, + }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +/** A quiet room: hiss below the noise floor lights nothing. */ +export const Silent: Story = { + args: { state: { type: "level", level$: constant(0) } }, + play: async ({ canvasElement }) => { + await expect(litSegments(canvasElement)).toBe(0); + }, +}; + +export const QuietSpeech: Story = { + args: { state: { type: "level", level$: constant(5) } }, +}; + +export const NormalSpeech: Story = { + args: { state: { type: "level", level$: constant(12) } }, + play: async ({ canvasElement }) => { + // A floor, not a count: the count follows from the design's bar and gap sizes. + await expect( + canvasElement.getElementsByClassName(styles.segment).length, + ).toBeGreaterThanOrEqual(15); + }, +}; + +export const LoudSpeech: Story = { + args: { state: { type: "level", level$: constant(LEVEL_SCALE) } }, +}; + +/** The three volumes differ in how many bars are lit, not only in colour. */ +export const VolumesAreDistinguishable: Story = { + args: { state: { type: "level", level$: constant(5) } }, + play: async ({ canvasElement, mount }) => { + const lit: number[] = []; + for (const level of [5, 12, LEVEL_SCALE]) { + await mount( + , + ); + lit.push(litSegments(canvasElement)); + await expect(within(canvasElement).getByRole("meter")).toHaveAttribute( + "aria-valuenow", + String(level), + ); + } + await expect(new Set(lit).size).toBe(lit.length); + await expect(lit).toEqual([...lit].sort((a, b) => a - b)); + }, +}; + +/** Permission refused: a message with a next action, not a still meter. */ +export const PermissionDenied: Story = { + args: { state: { type: "permission-denied" } }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.queryByRole("meter")).toBeNull(); + await expect( + canvas.getByText(/Microphone access is blocked/), + ).toBeVisible(); + }, +}; + +/** No input device, told apart from a refusal. */ +export const NoDevice: Story = { + args: { state: { type: "no-device" } }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.queryByRole("meter")).toBeNull(); + await expect(canvas.getByText(/No microphone found/)).toBeVisible(); + + // The icon sits on the middle of the text, however many lines it runs to. + const middle = (element: Element): number => { + const box = element.getBoundingClientRect(); + return box.top + box.height / 2; + }; + const icon = canvasElement.getElementsByClassName(styles.icon)[0]; + const words = canvasElement.getElementsByClassName(styles.message)[0]; + await expect(Math.abs(middle(icon) - middle(words))).toBeLessThanOrEqual(1); + }, +}; + +/** The same meter at two widths: the bars keep their size and only their count changes. */ +export const ShapeStaysTheSameAtAnyWidth: Story = { + args: { state: { type: "level", level$: constant(12) } }, + play: async ({ mount, args }) => { + const narrow = await measureAt(mount, args, 180); + const wide = await measureAt(mount, args, 400); + + await expect(narrow.bar).toBe(wide.bar); + await expect(narrow.gap).toBe(wide.gap); + await expect(narrow.count).toBeLessThan(wide.count); + // The bars are still separate, not one run of colour. + await expect(narrow.gap).toBeGreaterThan(0); + }, +}; + +/** How many bars are lit. */ +function litSegments(canvasElement: HTMLElement): number { + return canvasElement.getElementsByClassName(styles.segmentLit).length; +} + +/** Renders the meter at one width and reports its bars' shape. */ +async function measureAt( + mount: (ui: JSX.Element) => Promise, + args: MicrophoneLevelMeterProps, + width: number, +): Promise<{ count: number; bar: number; gap: number }> { + await mount( +
+ +
, + ); + const bars = Array.from(document.body.getElementsByClassName(styles.segment)); + const first = bars[0].getBoundingClientRect(); + const second = bars[1].getBoundingClientRect(); + return { + count: bars.length, + bar: first.width, + gap: second.left - first.right, + }; +} diff --git a/src/components/MicrophoneLevelMeter.test.tsx b/src/components/MicrophoneLevelMeter.test.tsx new file mode 100644 index 000000000..171f208c1 --- /dev/null +++ b/src/components/MicrophoneLevelMeter.test.tsx @@ -0,0 +1,42 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, test } from "vitest"; +import { render, screen } from "@testing-library/react"; + +import { MicrophoneLevelMeter } from "./MicrophoneLevelMeter"; +import { LEVEL_SCALE } from "../state/MicrophoneLevel"; +import { constant } from "../state/Behavior"; + +describe("MicrophoneLevelMeter", () => { + test("announces the level rather than relying on hue", () => { + render( + , + ); + + const meter = screen.getByRole("meter", { name: "Microphone level" }); + expect(meter).toHaveAttribute("aria-valuenow", "6"); + expect(meter).toHaveAttribute("aria-valuemax", String(LEVEL_SCALE)); + expect(meter).toHaveAttribute("aria-valuetext", `6 of ${LEVEL_SCALE}`); + }); + + test("shows distinct messages for denied permission and no input device", () => { + const denied = render( + , + ); + expect( + denied.getByText(/Microphone access is blocked/), + ).toBeInTheDocument(); + expect(denied.queryByRole("meter")).toBeNull(); + + const missing = render( + , + ); + expect(missing.getByText(/No microphone found/)).toBeInTheDocument(); + expect(missing.queryByRole("meter")).toBeNull(); + }); +}); diff --git a/src/components/MicrophoneLevelMeter.tsx b/src/components/MicrophoneLevelMeter.tsx new file mode 100644 index 000000000..6b21491a4 --- /dev/null +++ b/src/components/MicrophoneLevelMeter.tsx @@ -0,0 +1,146 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + useCallback, + useLayoutEffect, + useRef, + useState, + type FC, + type Ref, +} from "react"; +import { Text } from "@vector-im/compound-web"; +import { MicOnIcon } from "@vector-im/compound-design-tokens/assets/web/icons"; +import classNames from "classnames"; +import { useTranslation } from "react-i18next"; + +import { distinctUntilChanged, map } from "rxjs"; + +import styles from "./MicrophoneLevelMeter.module.css"; +import { LEVEL_SCALE, type MicrophoneState } from "../state/MicrophoneLevel"; +import { observeElementSize$ } from "../utils/elementSize"; +import { useMicrophoneLevel } from "./useMicrophoneLevel"; + +export interface LiveMicrophoneLevelMeterProps { + /** The microphone to listen to. */ + deviceId: string | undefined; + /** Whether to hold a capture at all. */ + active: boolean; + className?: string; + ref?: Ref; +} + +/** + * The meter, wired to a live capture. Its own component so a changing level + * re-renders the meter, not its parent. + */ +export const LiveMicrophoneLevelMeter: FC = ({ + deviceId, + active, + className, + ref, +}) => { + const state = useMicrophoneLevel(deviceId, active); + return ; +}; + +export interface MicrophoneLevelMeterProps { + state: MicrophoneState; + className?: string; + ref?: Ref; +} + +/** The input level of a microphone, or why there is none. */ +export const MicrophoneLevelMeter: FC = ({ + state, + className, + ref, +}) => { + const { t } = useTranslation(); + // Bars keep one size, so their count follows the width. Starts full so the + // first paint, and jsdom, draw a whole meter. + const [barCount, setBarCount] = useState(LEVEL_SCALE); + const segments = useRef(null); + const track = useCallback( + (element: HTMLDivElement | null): (() => void) | undefined => { + if (element === null) return; + segments.current = element; + const subscription = observeElementSize$(element) + .pipe( + map(({ width }) => barsThatFit(element, width)), + distinctUntilChanged(), + ) + .subscribe(setBarCount); + return (): void => { + subscription.unsubscribe(); + segments.current = null; + }; + }, + [], + ); + + // Drawn straight into the DOM: the level changes many times a second, and + // re-rendering for each change is what this avoids. + useLayoutEffect(() => { + const element = segments.current; + if (state.type !== "level" || element === null) return; + const subscription = state.level$.subscribe((level) => { + element.setAttribute("aria-valuenow", String(level)); + element.setAttribute( + "aria-valuetext", + t("microphone_level.value", { level, max: LEVEL_SCALE }), + ); + // The level is a share of the scale, not a bar count. + const lit = Math.round((level / LEVEL_SCALE) * barCount); + Array.from(element.children).forEach((bar, i) => + bar.classList.toggle(styles.segmentLit, i < lit), + ); + }); + return (): void => subscription.unsubscribe(); + }, [state, barCount, t]); + + return ( +
+ + {state.type !== "level" ? ( + + {state.type === "permission-denied" + ? t("microphone_level.permission_denied") + : t("microphone_level.no_device")} + + ) : ( +
+ {Array.from({ length: barCount }, (_, i) => ( + + ))} +
+ )} +
+ ); +}; + +/** How many bars fit across `width`, measured so the stylesheet alone sets their size. */ +function barsThatFit(track: HTMLElement, width: number): number { + const gap = Number.parseFloat(getComputedStyle(track).columnGap); + const bar = track.firstElementChild?.getBoundingClientRect().width ?? 0; + // No layout (jsdom): keep the full count. + if (!(bar > 0) || !(gap >= 0)) return LEVEL_SCALE; + return Math.max(1, Math.floor((width + gap) / (bar + gap))); +} diff --git a/src/components/useMicrophoneLevel.test.tsx b/src/components/useMicrophoneLevel.test.tsx new file mode 100644 index 000000000..47b33b996 --- /dev/null +++ b/src/components/useMicrophoneLevel.test.tsx @@ -0,0 +1,41 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, describe, expect, test } from "vitest"; +import { renderHook } from "@testing-library/react"; + +import { useMicrophoneLevel } from "./useMicrophoneLevel"; +import { restoreAudioCapture, stubAudioCapture } from "../utils/test"; + +// Capture and release are covered in MicrophoneLevel.test.ts; this covers the bridge. +describe("useMicrophoneLevel", () => { + afterEach(restoreAudioCapture); + + test("holds no capture while the meter is not shown", () => { + const capture = stubAudioCapture(); + + renderHook(() => useMicrophoneLevel("mic1", false)); + + expect(capture.getUserMedia).not.toHaveBeenCalled(); + }); + + test("starts from nothing rather than the previous device's level", () => { + const capture = stubAudioCapture(); + + const { result, rerender } = renderHook( + ({ id }: { id: string }) => useMicrophoneLevel(id, true), + { initialProps: { id: "mic1" } }, + ); + capture.grant(); + + rerender({ id: "mic2" }); + + // No level carried over from the previous device. + const state = result.current; + expect(state.type === "level" ? state.level$.value : state.type).toBe(0); + }); +}); diff --git a/src/components/useMicrophoneLevel.ts b/src/components/useMicrophoneLevel.ts new file mode 100644 index 000000000..a787a23d5 --- /dev/null +++ b/src/components/useMicrophoneLevel.ts @@ -0,0 +1,34 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { useEffect, useState } from "react"; + +import { + type MicrophoneState, + observeMicrophoneState$, +} from "../state/MicrophoneLevel"; +import { constant } from "../state/Behavior"; + +const IDLE: MicrophoneState = { type: "level", level$: constant(0) }; + +/** The live level of a microphone, captured only while `active`. */ +export function useMicrophoneLevel( + deviceId: string | undefined, + active: boolean, +): MicrophoneState { + const [state, setState] = useState(IDLE); + + useEffect(() => { + if (!active) return; + // Idle first, so a new device doesn't start from the previous level. + setState(IDLE); + const subscription = observeMicrophoneState$(deviceId).subscribe(setState); + return (): void => subscription.unsubscribe(); + }, [deviceId, active]); + + return state; +} diff --git a/src/room/LobbyView.test.tsx b/src/room/LobbyView.test.tsx index 7f03f2d29..aedd9fdd5 100644 --- a/src/room/LobbyView.test.tsx +++ b/src/room/LobbyView.test.tsx @@ -5,8 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, it, vi } from "vitest"; -import { render } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { of } from "rxjs"; import { LeaveToHomeProvider } from "../LeaveToHomeContext"; import { TooltipProvider } from "@vector-im/compound-web"; import { type MatrixClient } from "matrix-js-sdk"; @@ -18,7 +20,13 @@ import { import { LobbyView } from "./LobbyView"; import { E2eeType } from "../e2ee/e2eeType"; -import { mockMediaDevices, mockMuteStates } from "../utils/test"; +import { + mockMediaDevices, + mockMuteStates, + restoreAudioCapture, + stubAudioCapture, +} from "../utils/test"; +import { type MediaDevices } from "../state/MediaDevices"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; @@ -77,9 +85,10 @@ function renderLobbyView( props: Partial[0]> = {}, withAppBar = false, platform = "android", + devices: Partial = {}, ): ReturnType { platformMock.mockReturnValue(platform); - const mediaDevices = mockMediaDevices({}); + const mediaDevices = mockMediaDevices(devices); const muteStates = mockMuteStates(); const hideHeader = withAppBar ? true : false; const lobbyView = ( @@ -178,3 +187,30 @@ describe("LobbyView", () => { expect(await axe(container)).toHaveNoViolations(); }); }); + +describe("LobbyView microphone level", () => { + afterEach(restoreAudioCapture); + + it("shows the microphone level meter", async () => { + const capture = stubAudioCapture(); + capture.grant(); + const user = userEvent.setup(); + const { getByRole } = renderLobbyView({}, false, "desktop", { + requestDeviceNames: (): void => {}, + audioInput: { + available$: of( + new Map([["mic1", { type: "name", name: "Microphone 1" }]]), + ), + selected$: of({ id: "mic1" }), + select: (): void => {}, + }, + } as unknown as Partial); + + // Pre-join reaches the meter through the same chevron as a call. + await user.click(getByRole("button", { name: "Microphone" })); + + expect( + await screen.findByRole("meter", { name: "Microphone level" }), + ).toBeInTheDocument(); + }); +}); diff --git a/src/state/MediaDevices.test.ts b/src/state/MediaDevices.test.ts index f65b024e2..69c23765c 100644 --- a/src/state/MediaDevices.test.ts +++ b/src/state/MediaDevices.test.ts @@ -5,8 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, test, vi } from "vitest"; -import { of } from "rxjs"; +import { afterEach, describe, expect, test, vi } from "vitest"; const getPlatform = vi.hoisted(() => vi.fn(() => "desktop")); vi.mock("../Platform", () => ({ @@ -15,9 +14,22 @@ vi.mock("../Platform", () => ({ }, isFirefox: (): boolean => false, })); -vi.mock("@livekit/components-core", () => ({ - createMediaDeviceObserver: () => of([]), -})); +const observers = vi.hoisted( + () => new Map void }>(), +); +vi.mock("@livekit/components-core", async () => { + const { BehaviorSubject: Subject } = await import("rxjs"); + return { + createMediaDeviceObserver: (kind: string) => { + let observer = observers.get(kind); + if (observer === undefined) { + observer = new Subject([]); + observers.set(kind, observer); + } + return observer; + }, + }; +}); import { AudioOutput, MediaDevices } from "./MediaDevices"; import { AndroidControlledAudioOutput } from "./AndroidControlledAudioOutput"; @@ -57,3 +69,125 @@ describe("MediaDevices audio output", () => { expect(devices.audioOutput).toBeInstanceOf(IOSControlledAudioOutput); }); }); + +function device(deviceId: string, label: string, groupId = deviceId): object { + return { deviceId, label, groupId, kind: "audioinput" }; +} + +/** Replaces the devices of one kind, as the browser reports them. */ +function setDevices(kind: string, devices: object[]): void { + const observer = observers.get(kind); + if (observer === undefined) throw new Error(`nothing observing ${kind}`); + observer.next(devices); +} + +function newMediaDevices(): MediaDevices { + return new MediaDevices(new ObservableScope(), { + controlledAudioDevices: false, + }); +} + +describe("MediaDevices selection", () => { + afterEach(() => { + localStorage.clear(); + for (const kind of observers.keys()) setDevices(kind, []); + }); + + test("persists the selected device across sessions", () => { + const devices = newMediaDevices(); + setDevices("audioinput", [ + device("mic1", "Microphone 1"), + device("mic2", "Microphone 2"), + ]); + + devices.audioInput.select("mic2"); + + // A later call reads the same stored preference. + expect(newMediaDevices().audioInput.selected$.value?.id).toBe("mic2"); + }); + + test("updates the available devices when hardware changes", () => { + const devices = newMediaDevices(); + setDevices("audioinput", [device("mic1", "Microphone 1")]); + expect([...devices.audioInput.available$.value.keys()]).toEqual(["mic1"]); + + setDevices("audioinput", [ + device("mic1", "Microphone 1"), + device("mic2", "Headset"), + ]); + expect([...devices.audioInput.available$.value.keys()]).toEqual([ + "mic1", + "mic2", + ]); + + setDevices("audioinput", [device("mic1", "Microphone 1")]); + expect([...devices.audioInput.available$.value.keys()]).toEqual(["mic1"]); + }); + + test("falls back to the default device when the selected device disappears", () => { + const devices = newMediaDevices(); + setDevices("audioinput", [ + device("mic1", "Microphone 1"), + device("mic2", "Headset"), + ]); + devices.audioInput.select("mic2"); + expect(devices.audioInput.selected$.value?.id).toBe("mic2"); + + setDevices("audioinput", [device("mic1", "Microphone 1")]); + + expect(devices.audioInput.selected$.value?.id).toBe("mic1"); + }); + + test("falls back when the remembered device is absent", () => { + const devices = newMediaDevices(); + setDevices("audioinput", [device("mic1", "Microphone 1")]); + devices.audioInput.select("a-device-from-last-time"); + + expect(devices.audioInput.selected$.value?.id).toBe("mic1"); + }); + + test("falls back to numbered labels when labels are unavailable", () => { + const devices = newMediaDevices(); + // Names are withheld until permission is granted. + setDevices("audioinput", [device("mic1", ""), device("mic2", "")]); + + expect([...devices.audioInput.available$.value.values()]).toEqual([ + { type: "number", number: 1 }, + { type: "number", number: 2 }, + ]); + }); + + test("lists Default as a distinct entry", () => { + const devices = newMediaDevices(); + setDevices("audiooutput", [device("spk1", "Speakers")]); + + const available = devices.audioOutput.available$.value; + // Default is its own entry, unnamed: which device it resolves to isn't knowable. + expect(available.get("spk1")).toEqual({ type: "name", name: "Speakers" }); + expect(available.get("")).toEqual({ type: "default", name: null }); + }); + + test("selecting one device kind leaves the others unchanged", () => { + const devices = newMediaDevices(); + setDevices("audioinput", [ + device("mic1", "Microphone 1"), + device("mic2", "Headset"), + ]); + setDevices("audiooutput", [ + device("spk1", "Speakers"), + device("spk2", "Headset"), + ]); + setDevices("videoinput", [device("cam1", "Camera 1")]); + + devices.audioOutput.select("spk2"); + const audioInputBefore = devices.audioInput.selected$.value?.id; + const videoInputBefore = devices.videoInput.selected$.value?.id; + + devices.audioInput.select("mic2"); + + expect(devices.audioOutput.selected$.value?.id).toBe("spk2"); + expect(devices.videoInput.selected$.value?.id).toBe(videoInputBefore); + expect(audioInputBefore).not.toBe("mic2"); + expect(devices.audioInput.selected$.value?.id).toBe("mic2"); + }); +}); diff --git a/src/state/MicrophoneLevel.test.ts b/src/state/MicrophoneLevel.test.ts new file mode 100644 index 000000000..e8f684845 --- /dev/null +++ b/src/state/MicrophoneLevel.test.ts @@ -0,0 +1,183 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, describe, expect, test, vi } from "vitest"; + +import { + ATTACK_MS, + LEVEL_SCALE, + observeMicrophoneState$, + RELEASE_MS, + segmentsForVolume, + smoothVolume, +} from "./MicrophoneLevel"; +import { restoreAudioCapture, stubAudioCapture } from "../utils/test"; + +describe("segmentsForVolume", () => { + test("shows nothing for silence", () => { + expect(segmentsForVolume(0)).toBe(0); + }); + + test("shows nothing for the hiss of a quiet room", () => { + expect(segmentsForVolume(0.005)).toBe(0); + expect(segmentsForVolume(0.015)).toBe(0); + }); + + test("distinguishes quiet, normal and loud speech", () => { + const quiet = segmentsForVolume(0.06); + const normal = segmentsForVolume(0.2); + const loud = segmentsForVolume(0.8); + + expect(quiet).toBeGreaterThan(0); + expect(normal).toBeGreaterThan(quiet); + expect(loud).toBeGreaterThan(normal); + }); + + test("moves the meter visibly for normal speech", () => { + // Ordinary speech reaches the middle of the meter. + expect(segmentsForVolume(0.2)).toBeGreaterThanOrEqual(LEVEL_SCALE / 4); + }); + + test("never exceeds the meter", () => { + expect(segmentsForVolume(1)).toBe(LEVEL_SCALE); + expect(segmentsForVolume(4)).toBe(LEVEL_SCALE); + }); + + test("treats a missing reading as silence", () => { + expect(segmentsForVolume(NaN)).toBe(0); + expect(segmentsForVolume(-1)).toBe(0); + }); +}); + +describe("smoothVolume", () => { + test("rises faster than it falls", () => { + const rise = smoothVolume(0, 1, 50); + const fall = 1 - smoothVolume(1, 0, 50); + + expect(rise).toBeGreaterThan(fall); + }); + + test("registers a syllable as it starts", () => { + // Most of the way within one attack time constant. + expect(smoothVolume(0, 1, ATTACK_MS)).toBeGreaterThan(0.6); + }); + + test("rides over the gaps between words", () => { + // A short pause doesn't collapse the meter... + expect(smoothVolume(1, 0, 30)).toBeGreaterThan(0.7); + // ...but a real silence brings it down. + expect(smoothVolume(1, 0, RELEASE_MS * 3)).toBeLessThan(0.1); + }); + + test("behaves the same whatever the frame rate", () => { + const oneStep = smoothVolume(0, 1, 32); + let twoSteps = smoothVolume(0, 1, 16); + twoSteps = smoothVolume(twoSteps, 1, 16); + + expect(twoSteps).toBeCloseTo(oneStep, 5); + }); + + test("holds still when no time has passed", () => { + expect(smoothVolume(0.5, 1, 0)).toBe(0.5); + }); +}); + +describe("observeMicrophoneState$", () => { + afterEach(restoreAudioCapture); + + test("releases a capture the browser grants after nobody is watching", async () => { + const capture = stubAudioCapture(); + + const subscription = observeMicrophoneState$("mic1").subscribe(); + // The menu closes before the browser hands the microphone over. + subscription.unsubscribe(); + capture.grant(); + await vi.waitFor(() => expect(capture.track.stop).toHaveBeenCalled()); + }); + + test("releases the capture and the audio context when the subscription ends", async () => { + const capture = stubAudioCapture(); + + const subscription = observeMicrophoneState$("mic1").subscribe(); + capture.grant(); + await vi.waitFor(() => expect(capture.contexts).toHaveLength(1)); + + subscription.unsubscribe(); + + expect(capture.track.stop).toHaveBeenCalled(); + expect(capture.contexts[0].close).toHaveBeenCalled(); + }); + + test("gives the microphone back when the audio graph fails to build", async () => { + const capture = stubAudioCapture(); + // Fails only after getUserMedia has granted the device. + vi.stubGlobal( + "AudioContext", + class { + public constructor() { + throw new Error("no audio context for you"); + } + }, + ); + + const seen: string[] = []; + const subscription = observeMicrophoneState$("mic1").subscribe((state) => + seen.push(state.type), + ); + capture.grant(); + + await vi.waitFor(() => expect(seen).toContain("no-device")); + expect(capture.track.stop).toHaveBeenCalled(); + + subscription.unsubscribe(); + }); + + test("tells denied permission and a missing device apart", async () => { + for (const [name, expected] of [ + ["NotAllowedError", "permission-denied"], + ["NotFoundError", "no-device"], + ] as const) { + const capture = stubAudioCapture(); + capture.getUserMedia.mockRejectedValue(named(new Error(name), name)); + + const seen: string[] = []; + const subscription = observeMicrophoneState$("mic1").subscribe((state) => + seen.push(state.type), + ); + await vi.waitFor(() => expect(seen).toContain(expected)); + subscription.unsubscribe(); + restoreAudioCapture(); + } + }); + + test("says nothing on a frame that did not change the level", async () => { + const capture = stubAudioCapture(); + + const states: string[] = []; + let levels = 0; + const subscription = observeMicrophoneState$("mic1").subscribe((state) => { + states.push(state.type); + if (state.type === "level") state.level$.subscribe(() => levels++); + }); + capture.grant(); + await vi.waitFor(() => expect(levels).toBe(1)); + + // Read every frame, but a steady signal emits once, and the state itself + // arrives only once. + capture.drawFrames(20); + expect(levels).toBe(1); + expect(states).toEqual(["level"]); + + subscription.unsubscribe(); + }); +}); + +/** An error carrying the browser's `name`. */ +function named(error: Error, name: string): Error { + error.name = name; + return error; +} diff --git a/src/state/MicrophoneLevel.ts b/src/state/MicrophoneLevel.ts new file mode 100644 index 000000000..390f2b3f8 --- /dev/null +++ b/src/state/MicrophoneLevel.ts @@ -0,0 +1,143 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { BehaviorSubject, Observable } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { type Behavior } from "./Behavior"; + +/** What the microphone picks up, or why it can't be read. */ +export type MicrophoneState = + // A Behavior, so a changing level can be drawn without re-rendering. + | { type: "level"; level$: Behavior } + | { type: "permission-denied" } + | { type: "no-device" }; + +/** Scale a level is reported on. Fixed rather than the bar count, so a level reads the same at any width. */ +export const LEVEL_SCALE = 24; + +/** + * The level of a microphone, captured while subscribed. Its own capture rather + * than the call's track, because pre-join freezes that track to the device + * selected at mount. + */ +export function observeMicrophoneState$( + deviceId: string | undefined, +): Observable { + return new Observable((subscriber) => { + let stream: MediaStream | undefined; + let context: AudioContext | undefined; + let frame: number | undefined; + let level: BehaviorSubject | undefined; + + // Idempotent: teardown and start can both call it. + const release = (): void => { + if (frame !== undefined) cancelAnimationFrame(frame); + stream?.getTracks().forEach((track) => track.stop()); + void context?.close(); + frame = undefined; + stream = undefined; + context = undefined; + level?.complete(); + level = undefined; + }; + + const start = async (): Promise => { + stream = await navigator.mediaDevices.getUserMedia({ + audio: + deviceId === undefined ? true : { deviceId: { exact: deviceId } }, + }); + // Unsubscribed while the permission prompt was open. + if (subscriber.closed) return release(); + + context = new AudioContext(); + // Starts suspended outside a user gesture, which would read as silence. + if (context.state === "suspended") await context.resume(); + if (subscriber.closed) return release(); + + const analyser = context.createAnalyser(); + analyser.fftSize = 1024; + context.createMediaStreamSource(stream).connect(analyser); + const samples = new Uint8Array(analyser.fftSize); + let displayed = 0; + let previousFrame = performance.now(); + const current = new BehaviorSubject(0); + level = current; + subscriber.next({ type: "level", level$: current }); + + const read = (): void => { + analyser.getByteTimeDomainData(samples); + // RMS: perceived loudness rather than the peak. + let sum = 0; + for (const sample of samples) { + const centred = (sample - 128) / 128; + sum += centred * centred; + } + const now = performance.now(); + displayed = smoothVolume( + displayed, + Math.sqrt(sum / samples.length), + now - previousFrame, + ); + previousFrame = now; + // Frames that don't move the quantised level say nothing. + const next = segmentsForVolume(displayed); + if (next !== current.value) current.next(next); + frame = requestAnimationFrame(read); + }; + read(); + }; + + start().catch((e: unknown) => { + // Building the graph can fail after the device was granted. + release(); + subscriber.next(stateForFailure(e)); + }); + + return release; + }); +} + +/** What a failure to open the microphone means for the person using it. */ +function stateForFailure(e: unknown): MicrophoneState { + const name = e instanceof Error ? e.name : ""; + if (name === "NotAllowedError" || name === "SecurityError") + return { type: "permission-denied" }; + if (name === "NotFoundError" || name === "OverconstrainedError") + return { type: "no-device" }; + logger.error("Could not read the microphone level", e); + return { type: "no-device" }; +} + +/** Below this counts as silence, so a quiet room's hiss doesn't light the first bars. */ +const NOISE_FLOOR = 0.02; + +/** Quantises a 0..1 volume onto {@link LEVEL_SCALE}. */ +export function segmentsForVolume(volume: number): number { + if (!Number.isFinite(volume) || volume <= NOISE_FLOOR) return 0; + // Square root, so ordinary speech reaches the middle of the scale. + const aboveFloor = (Math.min(volume, 1) - NOISE_FLOOR) / (1 - NOISE_FLOOR); + return Math.min(LEVEL_SCALE, Math.ceil(Math.sqrt(aboveFloor) * LEVEL_SCALE)); +} + +/** Rise time constant: short, so a syllable registers as it starts. */ +export const ATTACK_MS = 50; + +/** Fall time constant: longer, so the gaps between words don't flicker. */ +export const RELEASE_MS = 120; + +/** Eases towards a reading, by elapsed time so the frame rate doesn't matter. */ +export function smoothVolume( + displayed: number, + reading: number, + elapsedMs: number, +): number { + if (elapsedMs <= 0) return displayed; + const timeConstant = reading > displayed ? ATTACK_MS : RELEASE_MS; + const towards = 1 - Math.exp(-elapsedMs / timeConstant); + return displayed + (reading - displayed) * towards; +} diff --git a/src/utils/test.ts b/src/utils/test.ts index 49ae0a3f1..405e5947b 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -8,6 +8,7 @@ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; import { expect, + type Mock, type MockedObject, type MockInstance, onTestFinished, @@ -590,3 +591,98 @@ export class MockConnection extends Connection { public async start(): Promise {} public async stop(): Promise {} } + +export interface StubbedCapture { + getUserMedia: Mock; + /** Grants the microphone, as the browser does once permission is given. */ + grant: () => void; + track: { stop: Mock }; + contexts: { close: Mock }[]; + /** Runs the pending animation frames, in order. */ + drawFrames: (count: number) => void; + /** Sets the microphone's loudness from the next frame, 0 to 1. */ + speak: (amplitude: number) => void; +} + +/** + * Stubs the capture and Web Audio APIs, with the grant and the animation + * frames driven by the test. Undo with {@link restoreAudioCapture}. + */ +export function stubAudioCapture(): StubbedCapture { + const track = { stop: vi.fn() }; + const contexts: { close: Mock }[] = []; + let grant = (): void => {}; + const granted = new Promise((resolve) => { + grant = (): void => + resolve({ getTracks: () => [track] } as unknown as MediaStream); + }); + + const frames: FrameRequestCallback[] = []; + let amplitude = 0; + vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { + frames.push(callback); + return frames.length; + }); + vi.stubGlobal("cancelAnimationFrame", vi.fn()); + vi.stubGlobal( + "AudioContext", + class { + public readonly state = "running"; + public readonly close = vi.fn(); + public constructor() { + contexts.push(this); + } + public createAnalyser(): object { + return { + fftSize: 1024, + getByteTimeDomainData: (samples: Uint8Array): void => { + // Silence is the midpoint of the range; a zeroed buffer reads as full scale. + if (amplitude <= 0) { + samples.fill(128); + return; + } + const peak = Math.round(Math.min(1, amplitude) * 127); + for (let i = 0; i < samples.length; i++) + samples[i] = 128 + (i % 2 ? peak : -peak); + }, + }; + } + public createMediaStreamSource(): object { + return { connect: (): void => {} }; + } + }, + ); + // Only this property: replacing navigator loses getters like userAgent. + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: { getUserMedia: vi.fn().mockReturnValue(granted) }, + }); + + return { + getUserMedia: navigator.mediaDevices.getUserMedia as unknown as Mock, + grant: () => grant(), + track, + contexts, + drawFrames: (count): void => { + for (let i = 0; i < count; i++) frames.shift()?.(i); + }, + speak: (next): void => { + amplitude = next; + }, + }; +} + +const realMediaDevices = Object.getOwnPropertyDescriptor( + navigator, + "mediaDevices", +); + +/** Undoes {@link stubAudioCapture}. */ +export function restoreAudioCapture(): void { + vi.unstubAllGlobals(); + if (realMediaDevices === undefined) { + Reflect.deleteProperty(navigator, "mediaDevices"); + } else { + Object.defineProperty(navigator, "mediaDevices", realMediaDevices); + } +}