[Feature] Quick audio menu (#4275)

* Give the microphone level its own observable
* Add a microphone level meter
* Show speakers and microphones in the quick audio menu
* Wire speaker selection through the call footer
* Add stories for the meter and the device menu
* Add end-to-end specs for the quick audio menu
* Keep the footer while a menu opened from it is open
* Use the shared audio capture stub in the lobby test
---------

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

src/state/MediaDevices.test.ts

@@ -5,8 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { describe, expect, test, vi } from "vitest";
-import { of } from "rxjs";
+import { afterEach, describe, expect, test, vi } from "vitest";
 
 const getPlatform = vi.hoisted(() => vi.fn(() => "desktop"));
 vi.mock("../Platform", () => ({
@@ -15,9 +14,22 @@ vi.mock("../Platform", () => ({
   },
   isFirefox: (): boolean => false,
 }));
-vi.mock("@livekit/components-core", () => ({
-  createMediaDeviceObserver: () => of([]),
-}));
+const observers = vi.hoisted(
+  () => new Map<string, { next: (devices: unknown[]) => void }>(),
+);
+vi.mock("@livekit/components-core", async () => {
+  const { BehaviorSubject: Subject } = await import("rxjs");
+  return {
+    createMediaDeviceObserver: (kind: string) => {
+      let observer = observers.get(kind);
+      if (observer === undefined) {
+        observer = new Subject<unknown[]>([]);
+        observers.set(kind, observer);
+      }
+      return observer;
+    },
+  };
+});
 
 import { AudioOutput, MediaDevices } from "./MediaDevices";
 import { AndroidControlledAudioOutput } from "./AndroidControlledAudioOutput";
@@ -57,3 +69,125 @@ describe("MediaDevices audio output", () => {
     expect(devices.audioOutput).toBeInstanceOf(IOSControlledAudioOutput);
   });
 });
+
+function device(deviceId: string, label: string, groupId = deviceId): object {
+  return { deviceId, label, groupId, kind: "audioinput" };
+}
+
+/** Replaces the devices of one kind, as the browser reports them. */
+function setDevices(kind: string, devices: object[]): void {
+  const observer = observers.get(kind);
+  if (observer === undefined) throw new Error(`nothing observing ${kind}`);
+  observer.next(devices);
+}
+
+function newMediaDevices(): MediaDevices {
+  return new MediaDevices(new ObservableScope(), {
+    controlledAudioDevices: false,
+  });
+}
+
+describe("MediaDevices selection", () => {
+  afterEach(() => {
+    localStorage.clear();
+    for (const kind of observers.keys()) setDevices(kind, []);
+  });
+
+  test("persists the selected device across sessions", () => {
+    const devices = newMediaDevices();
+    setDevices("audioinput", [
+      device("mic1", "Microphone 1"),
+      device("mic2", "Microphone 2"),
+    ]);
+
+    devices.audioInput.select("mic2");
+
+    // A later call reads the same stored preference.
+    expect(newMediaDevices().audioInput.selected$.value?.id).toBe("mic2");
+  });
+
+  test("updates the available devices when hardware changes", () => {
+    const devices = newMediaDevices();
+    setDevices("audioinput", [device("mic1", "Microphone 1")]);
+    expect([...devices.audioInput.available$.value.keys()]).toEqual(["mic1"]);
+
+    setDevices("audioinput", [
+      device("mic1", "Microphone 1"),
+      device("mic2", "Headset"),
+    ]);
+    expect([...devices.audioInput.available$.value.keys()]).toEqual([
+      "mic1",
+      "mic2",
+    ]);
+
+    setDevices("audioinput", [device("mic1", "Microphone 1")]);
+    expect([...devices.audioInput.available$.value.keys()]).toEqual(["mic1"]);
+  });
+
+  test("falls back to the default device when the selected device disappears", () => {
+    const devices = newMediaDevices();
+    setDevices("audioinput", [
+      device("mic1", "Microphone 1"),
+      device("mic2", "Headset"),
+    ]);
+    devices.audioInput.select("mic2");
+    expect(devices.audioInput.selected$.value?.id).toBe("mic2");
+
+    setDevices("audioinput", [device("mic1", "Microphone 1")]);
+
+    expect(devices.audioInput.selected$.value?.id).toBe("mic1");
+  });
+
+  test("falls back when the remembered device is absent", () => {
+    const devices = newMediaDevices();
+    setDevices("audioinput", [device("mic1", "Microphone 1")]);
+    devices.audioInput.select("a-device-from-last-time");
+
+    expect(devices.audioInput.selected$.value?.id).toBe("mic1");
+  });
+
+  test("falls back to numbered labels when labels are unavailable", () => {
+    const devices = newMediaDevices();
+    // Names are withheld until permission is granted.
+    setDevices("audioinput", [device("mic1", ""), device("mic2", "")]);
+
+    expect([...devices.audioInput.available$.value.values()]).toEqual([
+      { type: "number", number: 1 },
+      { type: "number", number: 2 },
+    ]);
+  });
+
+  test("lists Default as a distinct entry", () => {
+    const devices = newMediaDevices();
+    setDevices("audiooutput", [device("spk1", "Speakers")]);
+
+    const available = devices.audioOutput.available$.value;
+    // Default is its own entry, unnamed: which device it resolves to isn't knowable.
+    expect(available.get("spk1")).toEqual({ type: "name", name: "Speakers" });
+    expect(available.get("")).toEqual({ type: "default", name: null });
+  });
+
+  test("selecting one device kind leaves the others unchanged", () => {
+    const devices = newMediaDevices();
+    setDevices("audioinput", [
+      device("mic1", "Microphone 1"),
+      device("mic2", "Headset"),
+    ]);
+    setDevices("audiooutput", [
+      device("spk1", "Speakers"),
+      device("spk2", "Headset"),
+    ]);
+    setDevices("videoinput", [device("cam1", "Camera 1")]);
+
+    devices.audioOutput.select("spk2");
+    const audioInputBefore = devices.audioInput.selected$.value?.id;
+    const videoInputBefore = devices.videoInput.selected$.value?.id;
+
+    devices.audioInput.select("mic2");
+
+    expect(devices.audioOutput.selected$.value?.id).toBe("spk2");
+    expect(devices.videoInput.selected$.value?.id).toBe(videoInputBefore);
+    expect(audioInputBefore).not.toBe("mic2");
+    expect(devices.audioInput.selected$.value?.id).toBe("mic2");
+  });
+});

src/state/MicrophoneLevel.test.ts

@@ -0,0 +1,183 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { afterEach, describe, expect, test, vi } from "vitest";
+
+import {
+  ATTACK_MS,
+  LEVEL_SCALE,
+  observeMicrophoneState$,
+  RELEASE_MS,
+  segmentsForVolume,
+  smoothVolume,
+} from "./MicrophoneLevel";
+import { restoreAudioCapture, stubAudioCapture } from "../utils/test";
+
+describe("segmentsForVolume", () => {
+  test("shows nothing for silence", () => {
+    expect(segmentsForVolume(0)).toBe(0);
+  });
+
+  test("shows nothing for the hiss of a quiet room", () => {
+    expect(segmentsForVolume(0.005)).toBe(0);
+    expect(segmentsForVolume(0.015)).toBe(0);
+  });
+
+  test("distinguishes quiet, normal and loud speech", () => {
+    const quiet = segmentsForVolume(0.06);
+    const normal = segmentsForVolume(0.2);
+    const loud = segmentsForVolume(0.8);
+
+    expect(quiet).toBeGreaterThan(0);
+    expect(normal).toBeGreaterThan(quiet);
+    expect(loud).toBeGreaterThan(normal);
+  });
+
+  test("moves the meter visibly for normal speech", () => {
+    // Ordinary speech reaches the middle of the meter.
+    expect(segmentsForVolume(0.2)).toBeGreaterThanOrEqual(LEVEL_SCALE / 4);
+  });
+
+  test("never exceeds the meter", () => {
+    expect(segmentsForVolume(1)).toBe(LEVEL_SCALE);
+    expect(segmentsForVolume(4)).toBe(LEVEL_SCALE);
+  });
+
+  test("treats a missing reading as silence", () => {
+    expect(segmentsForVolume(NaN)).toBe(0);
+    expect(segmentsForVolume(-1)).toBe(0);
+  });
+});
+
+describe("smoothVolume", () => {
+  test("rises faster than it falls", () => {
+    const rise = smoothVolume(0, 1, 50);
+    const fall = 1 - smoothVolume(1, 0, 50);
+
+    expect(rise).toBeGreaterThan(fall);
+  });
+
+  test("registers a syllable as it starts", () => {
+    // Most of the way within one attack time constant.
+    expect(smoothVolume(0, 1, ATTACK_MS)).toBeGreaterThan(0.6);
+  });
+
+  test("rides over the gaps between words", () => {
+    // A short pause doesn't collapse the meter...
+    expect(smoothVolume(1, 0, 30)).toBeGreaterThan(0.7);
+    // ...but a real silence brings it down.
+    expect(smoothVolume(1, 0, RELEASE_MS * 3)).toBeLessThan(0.1);
+  });
+
+  test("behaves the same whatever the frame rate", () => {
+    const oneStep = smoothVolume(0, 1, 32);
+    let twoSteps = smoothVolume(0, 1, 16);
+    twoSteps = smoothVolume(twoSteps, 1, 16);
+
+    expect(twoSteps).toBeCloseTo(oneStep, 5);
+  });
+
+  test("holds still when no time has passed", () => {
+    expect(smoothVolume(0.5, 1, 0)).toBe(0.5);
+  });
+});
+
+describe("observeMicrophoneState$", () => {
+  afterEach(restoreAudioCapture);
+
+  test("releases a capture the browser grants after nobody is watching", async () => {
+    const capture = stubAudioCapture();
+
+    const subscription = observeMicrophoneState$("mic1").subscribe();
+    // The menu closes before the browser hands the microphone over.
+    subscription.unsubscribe();
+    capture.grant();
+    await vi.waitFor(() => expect(capture.track.stop).toHaveBeenCalled());
+  });
+
+  test("releases the capture and the audio context when the subscription ends", async () => {
+    const capture = stubAudioCapture();
+
+    const subscription = observeMicrophoneState$("mic1").subscribe();
+    capture.grant();
+    await vi.waitFor(() => expect(capture.contexts).toHaveLength(1));
+
+    subscription.unsubscribe();
+
+    expect(capture.track.stop).toHaveBeenCalled();
+    expect(capture.contexts[0].close).toHaveBeenCalled();
+  });
+
+  test("gives the microphone back when the audio graph fails to build", async () => {
+    const capture = stubAudioCapture();
+    // Fails only after getUserMedia has granted the device.
+    vi.stubGlobal(
+      "AudioContext",
+      class {
+        public constructor() {
+          throw new Error("no audio context for you");
+        }
+      },
+    );
+
+    const seen: string[] = [];
+    const subscription = observeMicrophoneState$("mic1").subscribe((state) =>
+      seen.push(state.type),
+    );
+    capture.grant();
+
+    await vi.waitFor(() => expect(seen).toContain("no-device"));
+    expect(capture.track.stop).toHaveBeenCalled();
+
+    subscription.unsubscribe();
+  });
+
+  test("tells denied permission and a missing device apart", async () => {
+    for (const [name, expected] of [
+      ["NotAllowedError", "permission-denied"],
+      ["NotFoundError", "no-device"],
+    ] as const) {
+      const capture = stubAudioCapture();
+      capture.getUserMedia.mockRejectedValue(named(new Error(name), name));
+
+      const seen: string[] = [];
+      const subscription = observeMicrophoneState$("mic1").subscribe((state) =>
+        seen.push(state.type),
+      );
+      await vi.waitFor(() => expect(seen).toContain(expected));
+      subscription.unsubscribe();
+      restoreAudioCapture();
+    }
+  });
+
+  test("says nothing on a frame that did not change the level", async () => {
+    const capture = stubAudioCapture();
+
+    const states: string[] = [];
+    let levels = 0;
+    const subscription = observeMicrophoneState$("mic1").subscribe((state) => {
+      states.push(state.type);
+      if (state.type === "level") state.level$.subscribe(() => levels++);
+    });
+    capture.grant();
+    await vi.waitFor(() => expect(levels).toBe(1));
+
+    // Read every frame, but a steady signal emits once, and the state itself
+    // arrives only once.
+    capture.drawFrames(20);
+    expect(levels).toBe(1);
+    expect(states).toEqual(["level"]);
+
+    subscription.unsubscribe();
+  });
+});
+
+/** An error carrying the browser's `name`. */
+function named(error: Error, name: string): Error {
+  error.name = name;
+  return error;
+}

src/state/MicrophoneLevel.ts

@@ -0,0 +1,143 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { BehaviorSubject, Observable } from "rxjs";
+import { logger } from "matrix-js-sdk/lib/logger";
+
+import { type Behavior } from "./Behavior";
+
+/** What the microphone picks up, or why it can't be read. */
+export type MicrophoneState =
+  // A Behavior, so a changing level can be drawn without re-rendering.
+  | { type: "level"; level$: Behavior<number> }
+  | { type: "permission-denied" }
+  | { type: "no-device" };
+
+/** Scale a level is reported on. Fixed rather than the bar count, so a level reads the same at any width. */
+export const LEVEL_SCALE = 24;
+
+/**
+ * The level of a microphone, captured while subscribed. Its own capture rather
+ * than the call's track, because pre-join freezes that track to the device
+ * selected at mount.
+ */
+export function observeMicrophoneState$(
+  deviceId: string | undefined,
+): Observable<MicrophoneState> {
+  return new Observable<MicrophoneState>((subscriber) => {
+    let stream: MediaStream | undefined;
+    let context: AudioContext | undefined;
+    let frame: number | undefined;
+    let level: BehaviorSubject<number> | undefined;
+
+    // Idempotent: teardown and start can both call it.
+    const release = (): void => {
+      if (frame !== undefined) cancelAnimationFrame(frame);
+      stream?.getTracks().forEach((track) => track.stop());
+      void context?.close();
+      frame = undefined;
+      stream = undefined;
+      context = undefined;
+      level?.complete();
+      level = undefined;
+    };
+
+    const start = async (): Promise<void> => {
+      stream = await navigator.mediaDevices.getUserMedia({
+        audio:
+          deviceId === undefined ? true : { deviceId: { exact: deviceId } },
+      });
+      // Unsubscribed while the permission prompt was open.
+      if (subscriber.closed) return release();
+
+      context = new AudioContext();
+      // Starts suspended outside a user gesture, which would read as silence.
+      if (context.state === "suspended") await context.resume();
+      if (subscriber.closed) return release();
+
+      const analyser = context.createAnalyser();
+      analyser.fftSize = 1024;
+      context.createMediaStreamSource(stream).connect(analyser);
+      const samples = new Uint8Array(analyser.fftSize);
+      let displayed = 0;
+      let previousFrame = performance.now();
+      const current = new BehaviorSubject(0);
+      level = current;
+      subscriber.next({ type: "level", level$: current });
+
+      const read = (): void => {
+        analyser.getByteTimeDomainData(samples);
+        // RMS: perceived loudness rather than the peak.
+        let sum = 0;
+        for (const sample of samples) {
+          const centred = (sample - 128) / 128;
+          sum += centred * centred;
+        }
+        const now = performance.now();
+        displayed = smoothVolume(
+          displayed,
+          Math.sqrt(sum / samples.length),
+          now - previousFrame,
+        );
+        previousFrame = now;
+        // Frames that don't move the quantised level say nothing.
+        const next = segmentsForVolume(displayed);
+        if (next !== current.value) current.next(next);
+        frame = requestAnimationFrame(read);
+      };
+      read();
+    };
+
+    start().catch((e: unknown) => {
+      // Building the graph can fail after the device was granted.
+      release();
+      subscriber.next(stateForFailure(e));
+    });
+
+    return release;
+  });
+}
+
+/** What a failure to open the microphone means for the person using it. */
+function stateForFailure(e: unknown): MicrophoneState {
+  const name = e instanceof Error ? e.name : "";
+  if (name === "NotAllowedError" || name === "SecurityError")
+    return { type: "permission-denied" };
+  if (name === "NotFoundError" || name === "OverconstrainedError")
+    return { type: "no-device" };
+  logger.error("Could not read the microphone level", e);
+  return { type: "no-device" };
+}
+
+/** Below this counts as silence, so a quiet room's hiss doesn't light the first bars. */
+const NOISE_FLOOR = 0.02;
+
+/** Quantises a 0..1 volume onto {@link LEVEL_SCALE}. */
+export function segmentsForVolume(volume: number): number {
+  if (!Number.isFinite(volume) || volume <= NOISE_FLOOR) return 0;
+  // Square root, so ordinary speech reaches the middle of the scale.
+  const aboveFloor = (Math.min(volume, 1) - NOISE_FLOOR) / (1 - NOISE_FLOOR);
+  return Math.min(LEVEL_SCALE, Math.ceil(Math.sqrt(aboveFloor) * LEVEL_SCALE));
+}
+
+/** Rise time constant: short, so a syllable registers as it starts. */
+export const ATTACK_MS = 50;
+
+/** Fall time constant: longer, so the gaps between words don't flicker. */
+export const RELEASE_MS = 120;
+
+/** Eases towards a reading, by elapsed time so the frame rate doesn't matter. */
+export function smoothVolume(
+  displayed: number,
+  reading: number,
+  elapsedMs: number,
+): number {
+  if (elapsedMs <= 0) return displayed;
+  const timeConstant = reading > displayed ? ATTACK_MS : RELEASE_MS;
+  const towards = 1 - Math.exp(-elapsedMs / timeConstant);
+  return displayed + (reading - displayed) * towards;
+}