From 9a68de193f5f3c686213fe13173fbe4aeb9cd628 Mon Sep 17 00:00:00 2001 From: fkwp Date: Wed, 16 Sep 2026 14:52:59 +0200 Subject: [PATCH] Show a live microphone level in the audio menu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Draw the input level beneath the microphone list as 32 discrete segments, so the level reads by count and not by hue alone. - Expose it through role="meter" with aria-valuenow and aria-valuetext, so a screen reader announces it. - Give denied permission and no input device their own message and next action, rather than a flat meter that reads as silence. - Read the level only while the menu is open, so nothing holds a second capture of the microphone for the length of a call. - Read the device the picker has selected, so the meter re-points as soon as the selection changes. - Ignore anything below a noise floor: a quiet room is never digitally silent, and its hiss otherwise lights the first segments permanently, which reads as "it can hear me" when nobody is speaking. - Quantise loudness with a square root above that floor: amplitude puts ordinary speech near the bottom of the range, where a linear meter barely moves. - Withhold the output select callback where the platform cannot route audio to a chosen device, which is what renders the speaker section disabled. Safari and most Firefox builds land here. - Track the in-flight device by kind as well as id: Chrome names both an input and an output "default", so the spinner appeared on the wrong row. - Disable every device in the menu while a selection is settling, so a second request cannot overtake the first. The meter opens its own short-lived capture rather than tapping the call's audio track. That is what lets it follow the picker instantly, and it avoids the pre-join track, which is deliberately frozen to the device selected when the screen mounted. The cost is a second capture of the same device while the menu is open. See the spec's notes sidecar. Spec: FEATURES_SPEC/2026-09_Quick_Audio_Menu.md — AC1, AC6, AC7, AC13, AC14, AC15, AC17, AC18, AC21, AC22 --- locales/en/app.json | 6 + src/components/CallFooterViewModel.test.ts | 30 +++++ src/components/CallFooterViewModel.tsx | 9 +- .../MediaMuteAndSwitchButton.test.tsx | 59 +++++++++- src/components/MediaMuteAndSwitchButton.tsx | 54 +++++++-- .../MicrophoneLevelMeter.module.css | 46 ++++++++ src/components/MicrophoneLevelMeter.test.tsx | 40 +++++++ src/components/MicrophoneLevelMeter.tsx | 70 ++++++++++++ src/components/useMicrophoneLevel.ts | 105 ++++++++++++++++++ src/state/MicrophoneLevel.test.ts | 49 ++++++++ src/state/MicrophoneLevel.ts | 52 +++++++++ 11 files changed, 506 insertions(+), 14 deletions(-) create mode 100644 src/components/MicrophoneLevelMeter.module.css create mode 100644 src/components/MicrophoneLevelMeter.test.tsx create mode 100644 src/components/MicrophoneLevelMeter.tsx create mode 100644 src/components/useMicrophoneLevel.ts create mode 100644 src/state/MicrophoneLevel.test.ts create mode 100644 src/state/MicrophoneLevel.ts diff --git a/locales/en/app.json b/locales/en/app.json index 919aa607e..a852e6505 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -163,6 +163,12 @@ "login_auth_links_prompt": "Not registered yet?", "login_subheading": "To continue to Element", "login_title": "Login", + "microphone_level": { + "label": "Microphone level", + "no_device": "No microphone found. Connect one, then try again.", + "permission_denied": "Element Call cannot use your microphone. Allow access in your browser settings.", + "value": "{{level}} of {{max}}" + }, "microphone_off": "Microphone off", "microphone_on": "Microphone on", "mute_microphone_button_label": "Mute microphone", diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index 9e73393be..82d92d395 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -30,6 +30,11 @@ vi.mock("@livekit/track-processors", () => ({ supportsBackgroundProcessors: (): boolean => false, })); +const outputSelectionMock = vi.hoisted(() => vi.fn(() => true)); +vi.mock("livekit-client", () => ({ + supportsAudioOutputSelection: (): boolean => outputSelectionMock(), +})); + /** * Returns the minimum set of CallViewModel fields required by * createCallFooterViewModel, with all other properties stubbed to @@ -96,6 +101,31 @@ const twoMicsAndOneCamMediaDevices = mockMediaDevices({ }); describe("createCallFooterViewModel", () => { + describe("selectAudioOutputOption", () => { + function buildFooterVm(): ReturnType { + platformMock.mockReturnValue("desktop"); + return createCallFooterViewModel( + testScope(), + buildMinimalCallViewModel(gridLayout), + mockMuteStates(), + twoMicsAndOneCamMediaDevices, + /* reactionIdentifier */ undefined, + { showControls: true, header: HeaderStyle.Standard }, + ); + } + + it("is withheld where the platform cannot route audio to a chosen device", () => { + outputSelectionMock.mockReturnValue(false); + // Undefined is what renders the speaker section disabled. + expect(buildFooterVm().selectAudioOutputOption$.value).toBeUndefined(); + }); + + it("is offered where the platform can route audio to a chosen device", () => { + outputSelectionMock.mockReturnValue(true); + expect(buildFooterVm().selectAudioOutputOption$.value).toBeDefined(); + }); + }); + describe("audioOptions and videoOptions", () => { function checkEmptyFor(platform: string, layout: Layout): void { platformMock.mockReturnValue(platform); diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index 08f5a6cd2..305b2a434 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { combineLatest, map, type Observable, switchMap } from "rxjs"; import { supportsBackgroundProcessors } from "@livekit/track-processors"; +import { supportsAudioOutputSelection } from "livekit-client"; import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { type MenuOptions } from "./MediaMuteAndSwitchButton"; @@ -103,7 +104,13 @@ function buildDeviceBehaviors( selectedAudioOutput$: scope.behavior( mediaDevices.audioOutput.selected$.pipe(map((s) => s?.id)), ), - selectAudioOutputOption$: constant(mediaDevices.audioOutput.select), + // Safari and most Firefox builds cannot route audio to a chosen device at + // all. Withholding the callback is what renders the section disabled. + selectAudioOutputOption$: constant( + supportsAudioOutputSelection() + ? mediaDevices.audioOutput.select + : undefined, + ), videoOptions$: scope.behavior(options$(mediaDevices.videoInput.available$)), selectedVideo$: scope.behavior( mediaDevices.videoInput.selected$.pipe(map((s) => s?.id)), diff --git a/src/components/MediaMuteAndSwitchButton.test.tsx b/src/components/MediaMuteAndSwitchButton.test.tsx index d7edacff1..09546c619 100644 --- a/src/components/MediaMuteAndSwitchButton.test.tsx +++ b/src/components/MediaMuteAndSwitchButton.test.tsx @@ -348,6 +348,63 @@ describe("MediaMuteAndSwitchButton", () => { screen.getByRole("menuitemradio", { name: "Microphone 1", checked: false }); }); + test("disables every device while a selection is settling", async () => { + const user = userEvent.setup(); + const { promise, resolve } = Promise.withResolvers(); + function Wrapper(): JSX.Element { + const [selectedOption, setSelectedOption] = useState("mic1"); + return ( + { + void promise.then(() => setSelectedOption(id)); + }} + outputOptions={[ + { label: { type: "name", name: "Speakers" }, id: "spk1" }, + { label: { type: "name", name: "Headset" }, id: "spk2" }, + ]} + selectedOutputOption="spk1" + onSelectOutput={vi.fn()} + /> + ); + } + + const { getByRole } = renderComponent(); + await user.click(getByRole("button", { name: "Microphone" })); + await user.click( + screen.getByRole("menuitemradio", { name: "Microphone 2" }), + ); + + // In flight: nothing else can be picked, in either section, so a second + // request cannot overtake the first. + for (const name of ["Microphone 1", "Speakers", "Headset"]) { + expect(screen.getByRole("menuitemradio", { name })).toHaveAttribute( + "aria-disabled", + "true", + ); + } + + await act(async () => { + resolve(); + await promise; + }); + + // Settled: choosable again. + expect( + screen.getByRole("menuitemradio", { name: "Microphone 1" }), + ).not.toHaveAttribute("aria-disabled", "true"); + expect( + screen.getByRole("menuitemradio", { name: "Headset" }), + ).not.toHaveAttribute("aria-disabled", "true"); + }); + test("lists speaker and microphone sections", async () => { const user = userEvent.setup(); const { getByRole } = renderComponent( @@ -435,7 +492,7 @@ describe("MediaMuteAndSwitchButton", () => { expect(only).toHaveAttribute("aria-disabled", "true"); }); - test("shows the speaker section disabled when no output can be chosen", async () => { + test("shows the speaker section disabled when output selection is unsupported", async () => { const user = userEvent.setup(); const { getByRole } = renderComponent( = ({ videoBlurToggleClick, onSelect, }) => { - const [plannedSelection, setPlannedSelection] = useState(null); + // Which device we have asked for but not yet been given. Carries the kind as + // well as the id, because an input and an output can share an id: "default" + // names both on Chrome. + const [plannedSelection, setPlannedSelection] = useState<{ + kind: "input" | "output"; + id: string; + } | null>(null); const [menuOpen, setMenuOpen] = useState(false); const isBusy = busy ?? false; const { t } = useTranslation(); const devices = useMediaDevices(); + // Only while the menu is open, so nothing holds a second capture of the + // microphone for the length of a call. + const microphoneState = useMicrophoneLevel( + selectedOption, + menuOpen && iconsAndLabels === "audio", + ); useEffect(() => { if (menuOpen) devices.requestDeviceNames(); // No-op after the first call @@ -184,7 +198,17 @@ export const MediaMuteAndSwitchButton: FC = ({ } }; + // A device we asked for that has not arrived yet. Until it does, nothing in + // the menu can be picked, so a second request cannot overtake the first. + const settling = + plannedSelection !== null && + plannedSelection.id !== + (plannedSelection.kind === "output" + ? selectedOutputOption + : selectedOption); + const deviceItems = ( + kind: "input" | "output", items: MenuOptions[] | undefined, selected: string | undefined, select: ((id: string) => void) | undefined, @@ -194,7 +218,7 @@ export const MediaMuteAndSwitchButton: FC = ({ // Shown but not choosable when nothing can be picked here, or when there is // only one device. The entry stays visible so the menu keeps the same shape // on every platform. - const disabled = select === undefined || list.length <= 1; + const disabled = select === undefined || list.length <= 1 || settling; return list.map(({ label, id }) => ( = ({ onSelect={(e) => { e.preventDefault(); if (id === selected) return; - setPlannedSelection(id); + setPlannedSelection({ kind, id }); select?.(id); }} key={id} role="menuitemradio" aria-checked={selected === id} > - {selected !== id && plannedSelection === id && ( - - )} + {selected !== id && + plannedSelection?.kind === kind && + plannedSelection.id === id && ( + + )} )); }; @@ -271,6 +297,7 @@ export const MediaMuteAndSwitchButton: FC = ({ <> {deviceItems( + "output", outputOptions, selectedOutputOption, onSelectOutput, @@ -280,7 +307,10 @@ export const MediaMuteAndSwitchButton: FC = ({ )} - {deviceItems(options, selectedOption, onSelect, numberedLabel)} + {deviceItems("input", options, selectedOption, onSelect, numberedLabel)} + {iconsAndLabels === "audio" && ( + + )} {(toggles?.length ?? 0) > 0 &&
} {toggles?.map((toggle) => ( { + test("announces the level rather than relying on hue", () => { + render(); + + const meter = screen.getByRole("meter", { name: "Microphone level" }); + expect(meter).toHaveAttribute("aria-valuenow", "6"); + expect(meter).toHaveAttribute("aria-valuemax", String(METER_SEGMENTS)); + expect(meter).toHaveAttribute("aria-valuetext", `6 of ${METER_SEGMENTS}`); + }); + + test("shows distinct messages for denied permission and no input device", () => { + const denied = render( + , + ); + // A next action, not a flat meter that reads as silence. + expect( + denied.getByText(/Allow access in your browser settings/), + ).toBeInTheDocument(); + expect(denied.queryByRole("meter")).toBeNull(); + + const missing = render( + , + ); + expect(missing.getByText(/No microphone found/)).toBeInTheDocument(); + expect(missing.queryByRole("meter")).toBeNull(); + }); +}); diff --git a/src/components/MicrophoneLevelMeter.tsx b/src/components/MicrophoneLevelMeter.tsx new file mode 100644 index 000000000..f393685f4 --- /dev/null +++ b/src/components/MicrophoneLevelMeter.tsx @@ -0,0 +1,70 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type FC } from "react"; +import { Text } from "@vector-im/compound-web"; +import { MicOnIcon } from "@vector-im/compound-design-tokens/assets/web/icons"; +import classNames from "classnames"; +import { useTranslation } from "react-i18next"; + +import styles from "./MicrophoneLevelMeter.module.css"; +import { METER_SEGMENTS, type MicrophoneState } from "../state/MicrophoneLevel"; + +interface Props { + state: MicrophoneState; +} + +/** + * The live input level of the selected microphone, shown beneath it. + * + * It says whether the microphone is picking anything up, which is not the same + * as whether the user is being heard: it keeps moving while muted, and the mute + * control is what says nothing is transmitted. + */ +export const MicrophoneLevelMeter: FC = ({ state }) => { + const { t } = useTranslation(); + + if (state.type !== "level") + return ( +
+ + + {state.type === "permission-denied" + ? t("microphone_level.permission_denied") + : t("microphone_level.no_device")} + +
+ ); + + return ( +
+ +
+ {Array.from({ length: METER_SEGMENTS }, (_, i) => ( + + ))} +
+
+ ); +}; diff --git a/src/components/useMicrophoneLevel.ts b/src/components/useMicrophoneLevel.ts new file mode 100644 index 000000000..edf56af31 --- /dev/null +++ b/src/components/useMicrophoneLevel.ts @@ -0,0 +1,105 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { useEffect, useState } from "react"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { + type MicrophoneState, + segmentsForVolume, +} from "../state/MicrophoneLevel"; + +/** + * Reads the live input level of a microphone, while `active`. + * + * Capture is scoped to the caller being on screen: the meter only exists while + * the menu that shows it is open, so nothing holds a second capture of the + * device for the length of a call. + * + * The level says whether the microphone is picking anything up, which is not + * the same as whether the user is being heard. It keeps moving while muted, and + * the mute control is what says nothing is transmitted. + */ +export function useMicrophoneLevel( + deviceId: string | undefined, + active: boolean, +): MicrophoneState { + const [state, setState] = useState({ + type: "level", + level: 0, + }); + + useEffect(() => { + if (!active) return; + + let stopped = false; + let stream: MediaStream | undefined; + let context: AudioContext | undefined; + let frame: number | undefined; + + const stop = (): void => { + stopped = true; + if (frame !== undefined) cancelAnimationFrame(frame); + stream?.getTracks().forEach((track) => track.stop()); + void context?.close(); + }; + + const start = async (): Promise => { + stream = await navigator.mediaDevices.getUserMedia({ + audio: + deviceId === undefined ? true : { deviceId: { exact: deviceId } }, + }); + if (stopped) return; + + context = new AudioContext(); + // Chrome starts the context suspended unless it was created during a + // gesture; opening the menu is one, but resume explicitly so the meter + // cannot silently sit at zero. + if (context.state === "suspended") await context.resume(); + if (stopped) return; + const analyser = context.createAnalyser(); + analyser.fftSize = 1024; + context.createMediaStreamSource(stream).connect(analyser); + const samples = new Uint8Array(analyser.fftSize); + + const read = (): void => { + analyser.getByteTimeDomainData(samples); + // Root mean square of the waveform around its centre, which is the + // loudness a listener perceives rather than the tallest spike. + let sum = 0; + for (const sample of samples) { + const centred = (sample - 128) / 128; + sum += centred * centred; + } + const level = segmentsForVolume(Math.sqrt(sum / samples.length)); + setState((current) => + current.type === "level" && current.level === level + ? current + : { type: "level", level }, + ); + frame = requestAnimationFrame(read); + }; + read(); + }; + + start().catch((e: unknown) => { + const name = e instanceof Error ? e.name : ""; + if (name === "NotAllowedError" || name === "SecurityError") { + setState({ type: "permission-denied" }); + } else if (name === "NotFoundError" || name === "OverconstrainedError") { + setState({ type: "no-device" }); + } else { + logger.error("Could not read the microphone level", e); + setState({ type: "no-device" }); + } + }); + + return stop; + }, [deviceId, active]); + + return state; +} diff --git a/src/state/MicrophoneLevel.test.ts b/src/state/MicrophoneLevel.test.ts new file mode 100644 index 000000000..3dfc872fe --- /dev/null +++ b/src/state/MicrophoneLevel.test.ts @@ -0,0 +1,49 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, test } from "vitest"; + +import { METER_SEGMENTS, segmentsForVolume } from "./MicrophoneLevel"; + +describe("segmentsForVolume", () => { + test("shows nothing for silence", () => { + expect(segmentsForVolume(0)).toBe(0); + }); + + test("shows nothing for the hiss of a quiet room", () => { + // Without a noise floor these light the first segments permanently, which + // reads as "it can hear me" when nobody is speaking. + expect(segmentsForVolume(0.005)).toBe(0); + expect(segmentsForVolume(0.015)).toBe(0); + }); + + test("distinguishes quiet, normal and loud speech", () => { + const quiet = segmentsForVolume(0.06); + const normal = segmentsForVolume(0.2); + const loud = segmentsForVolume(0.8); + + expect(quiet).toBeGreaterThan(0); + expect(normal).toBeGreaterThan(quiet); + expect(loud).toBeGreaterThan(normal); + }); + + test("moves the meter visibly for normal speech", () => { + // Ordinary speech should reach the middle of the meter, not scrape along + // the floor: a meter that barely moves reads as a broken microphone. + expect(segmentsForVolume(0.2)).toBeGreaterThanOrEqual(METER_SEGMENTS / 4); + }); + + test("never exceeds the meter", () => { + expect(segmentsForVolume(1)).toBe(METER_SEGMENTS); + expect(segmentsForVolume(4)).toBe(METER_SEGMENTS); + }); + + test("treats a missing reading as silence", () => { + expect(segmentsForVolume(NaN)).toBe(0); + expect(segmentsForVolume(-1)).toBe(0); + }); +}); diff --git a/src/state/MicrophoneLevel.ts b/src/state/MicrophoneLevel.ts new file mode 100644 index 000000000..94521c227 --- /dev/null +++ b/src/state/MicrophoneLevel.ts @@ -0,0 +1,52 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * What the microphone selector can say about the input, beyond its level. + * + * Silence and a broken microphone look identical on a meter, so the states a + * user has to act on are named rather than drawn as a flat bar. + */ +export type MicrophoneState = + | { type: "level"; level: number } + | { type: "permission-denied" } + | { type: "no-device" }; + +/** + * Steps the meter is drawn in. The level is quantised to these. + * + * Enough of them that they sit close together across the width of the menu: + * the bars keep a fixed size, so too few leaves visible gaps between them. + */ +export const METER_SEGMENTS = 32; + +/** + * Loudness below which the microphone is treated as picking up nothing. + * + * A quiet room is never digitally silent, and without a floor that hiss lights + * the first segments permanently — which reads as "it can hear me" when nobody + * is speaking. + */ +const NOISE_FLOOR = 0.02; + +/** + * Quantises a 0..1 volume to a whole number of meter segments. + * + * Exported for the tests: the mapping from loudness to segments is the part + * that decides whether quiet, normal and loud speech look different. + */ +export function segmentsForVolume(volume: number): number { + if (!Number.isFinite(volume) || volume <= NOISE_FLOOR) return 0; + // Volume arrives as amplitude, where speech occupies a small part of the top + // of the range. A square root spreads that out, so ordinary speech moves the + // meter through its middle rather than barely leaving the floor. + const aboveFloor = (Math.min(volume, 1) - NOISE_FLOOR) / (1 - NOISE_FLOOR); + return Math.min( + METER_SEGMENTS, + Math.ceil(Math.sqrt(aboveFloor) * METER_SEGMENTS), + ); +}