diff --git a/src/components/useMicrophoneLevel.test.tsx b/src/components/useMicrophoneLevel.test.tsx new file mode 100644 index 000000000..1258bfeea --- /dev/null +++ b/src/components/useMicrophoneLevel.test.tsx @@ -0,0 +1,42 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, describe, expect, test } from "vitest"; +import { renderHook } from "@testing-library/react"; + +import { useMicrophoneLevel } from "./useMicrophoneLevel"; +import { restoreAudioCapture, stubAudioCapture } from "../utils/test"; + +// The capture itself, and releasing it, belong to observeMicrophoneState$ and +// are covered in src/state/MicrophoneLevel.test.ts. What is left here is the +// bridging: when the hook watches, and what it reports before it has an answer. +describe("useMicrophoneLevel", () => { + afterEach(restoreAudioCapture); + + test("holds no capture while the meter is not shown", () => { + const capture = stubAudioCapture(); + + renderHook(() => useMicrophoneLevel("mic1", false)); + + expect(capture.getUserMedia).not.toHaveBeenCalled(); + }); + + test("starts from nothing rather than the previous device's level", () => { + const capture = stubAudioCapture(); + + const { result, rerender } = renderHook( + ({ id }: { id: string }) => useMicrophoneLevel(id, true), + { initialProps: { id: "mic1" } }, + ); + capture.grant(); + + rerender({ id: "mic2" }); + + // No level carried over: the meter reads the new device or nothing at all. + expect(result.current).toEqual({ type: "level", level: 0 }); + }); +}); diff --git a/src/components/useMicrophoneLevel.ts b/src/components/useMicrophoneLevel.ts index 1ea5cff0e..6e295d3e4 100644 --- a/src/components/useMicrophoneLevel.ts +++ b/src/components/useMicrophoneLevel.ts @@ -6,109 +6,35 @@ Please see LICENSE in the repository root for full details. */ import { useEffect, useState } from "react"; -import { logger } from "matrix-js-sdk/lib/logger"; import { type MicrophoneState, - segmentsForVolume, - smoothVolume, + observeMicrophoneState$, } from "../state/MicrophoneLevel"; +const IDLE: MicrophoneState = { type: "level", level: 0 }; + /** * Reads the live input level of a microphone, while `active`. * - * Capture is scoped to the caller being on screen: the meter only exists while - * the menu that shows it is open, so nothing holds a second capture of the - * device for the length of a call. - * - * The level says whether the microphone is picking anything up, which is not - * the same as whether the user is being heard. It keeps moving while muted, and - * the mute control is what says nothing is transmitted. + * A bridge and nothing else: the capture, its lifetime and the maths belong to + * {@link observeMicrophoneState$}. Scoped to `active` so the device is held + * only while whatever shows the meter is on screen, rather than for the length + * of a call. */ export function useMicrophoneLevel( deviceId: string | undefined, active: boolean, ): MicrophoneState { - const [state, setState] = useState({ - type: "level", - level: 0, - }); + const [state, setState] = useState(IDLE); useEffect(() => { if (!active) return; - - let stopped = false; - let stream: MediaStream | undefined; - let context: AudioContext | undefined; - let frame: number | undefined; - - const stop = (): void => { - stopped = true; - if (frame !== undefined) cancelAnimationFrame(frame); - stream?.getTracks().forEach((track) => track.stop()); - void context?.close(); - }; - - const start = async (): Promise => { - stream = await navigator.mediaDevices.getUserMedia({ - audio: - deviceId === undefined ? true : { deviceId: { exact: deviceId } }, - }); - if (stopped) return; - - context = new AudioContext(); - // Chrome starts the context suspended unless it was created during a - // gesture; opening the menu is one, but resume explicitly so the meter - // cannot silently sit at zero. - if (context.state === "suspended") await context.resume(); - if (stopped) return; - const analyser = context.createAnalyser(); - analyser.fftSize = 1024; - context.createMediaStreamSource(stream).connect(analyser); - const samples = new Uint8Array(analyser.fftSize); - let displayed = 0; - let previousFrame = performance.now(); - - const read = (): void => { - analyser.getByteTimeDomainData(samples); - // Root mean square of the waveform around its centre, which is the - // loudness a listener perceives rather than the tallest spike. - let sum = 0; - for (const sample of samples) { - const centred = (sample - 128) / 128; - sum += centred * centred; - } - const now = performance.now(); - displayed = smoothVolume( - displayed, - Math.sqrt(sum / samples.length), - now - previousFrame, - ); - previousFrame = now; - const level = segmentsForVolume(displayed); - setState((current) => - current.type === "level" && current.level === level - ? current - : { type: "level", level }, - ); - frame = requestAnimationFrame(read); - }; - read(); - }; - - start().catch((e: unknown) => { - const name = e instanceof Error ? e.name : ""; - if (name === "NotAllowedError" || name === "SecurityError") { - setState({ type: "permission-denied" }); - } else if (name === "NotFoundError" || name === "OverconstrainedError") { - setState({ type: "no-device" }); - } else { - logger.error("Could not read the microphone level", e); - setState({ type: "no-device" }); - } - }); - - return stop; + // Idle first, so a new device starts from nothing rather than from the + // level the previous one was reading. + setState(IDLE); + const subscription = observeMicrophoneState$(deviceId).subscribe(setState); + return (): void => subscription.unsubscribe(); }, [deviceId, active]); return state; diff --git a/src/state/MicrophoneLevel.test.ts b/src/state/MicrophoneLevel.test.ts index 8ce3544b8..38e9cf604 100644 --- a/src/state/MicrophoneLevel.test.ts +++ b/src/state/MicrophoneLevel.test.ts @@ -5,15 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { ATTACK_MS, METER_SEGMENTS, + observeMicrophoneState$, RELEASE_MS, segmentsForVolume, smoothVolume, } from "./MicrophoneLevel"; +import { restoreAudioCapture, stubAudioCapture } from "../utils/test"; describe("segmentsForVolume", () => { test("shows nothing for silence", () => { @@ -88,3 +90,73 @@ describe("smoothVolume", () => { expect(smoothVolume(0.5, 1, 0)).toBe(0.5); }); }); + +describe("observeMicrophoneState$", () => { + afterEach(restoreAudioCapture); + + test("releases a capture the browser grants after nobody is watching", async () => { + const capture = stubAudioCapture(); + + const subscription = observeMicrophoneState$("mic1").subscribe(); + // The user gives up on the permission prompt and closes the menu, and only + // then does the browser hand the microphone over. + subscription.unsubscribe(); + capture.grant(); + await vi.waitFor(() => expect(capture.track.stop).toHaveBeenCalled()); + }); + + test("releases the capture and the audio context when the subscription ends", async () => { + const capture = stubAudioCapture(); + + const subscription = observeMicrophoneState$("mic1").subscribe(); + capture.grant(); + await vi.waitFor(() => expect(capture.contexts).toHaveLength(1)); + + subscription.unsubscribe(); + + expect(capture.track.stop).toHaveBeenCalled(); + expect(capture.contexts[0].close).toHaveBeenCalled(); + }); + + test("tells denied permission and a missing device apart", async () => { + for (const [name, expected] of [ + ["NotAllowedError", "permission-denied"], + ["NotFoundError", "no-device"], + ] as const) { + const capture = stubAudioCapture(); + capture.getUserMedia.mockRejectedValue(named(new Error(name), name)); + + const seen: string[] = []; + const subscription = observeMicrophoneState$("mic1").subscribe((state) => + seen.push(state.type), + ); + await vi.waitFor(() => expect(seen).toContain(expected)); + subscription.unsubscribe(); + restoreAudioCapture(); + } + }); + + test("says nothing on a frame that did not change the level", async () => { + const capture = stubAudioCapture(); + + let emissions = 0; + const subscription = observeMicrophoneState$("mic1").subscribe( + () => emissions++, + ); + capture.grant(); + await vi.waitFor(() => expect(emissions).toBe(1)); + + // The analyser is read every animation frame, but the meter has only + // METER_SEGMENTS steps: a steady signal must not redraw the meter. + capture.drawFrames(20); + expect(emissions).toBe(1); + + subscription.unsubscribe(); + }); +}); + +/** An error with the `name` the browser would give it, not just a message. */ +function named(error: Error, name: string): Error { + error.name = name; + return error; +} diff --git a/src/state/MicrophoneLevel.ts b/src/state/MicrophoneLevel.ts index 8a391cbb2..ace564da4 100644 --- a/src/state/MicrophoneLevel.ts +++ b/src/state/MicrophoneLevel.ts @@ -5,6 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +import { distinctUntilChanged, Observable } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + /** * What the microphone selector can say about the input, beyond its level. * @@ -24,6 +27,112 @@ export type MicrophoneState = */ export const METER_SEGMENTS = 24; +/** + * Observes what a microphone is picking up, as the meter should show it. + * + * Subscribing opens the device and unsubscribing releases it, so it is held for + * exactly as long as something is watching — the menu, and not the call. + * + * Opens a capture of its own rather than reading the track the call already + * holds. That is a shortcut, recorded as S1 in the feature spec: the pre-join + * screen freezes its audio track to the device selected when it mounted, so a + * meter fed from that track could not follow the picker. + * + * The level says whether the microphone is picking anything up, which is not + * the same as whether the user is being heard: it keeps reading while muted, + * and the mute control is what says nothing is transmitted. + */ +export function observeMicrophoneState$( + deviceId: string | undefined, +): Observable { + return new Observable((subscriber) => { + let stream: MediaStream | undefined; + let context: AudioContext | undefined; + let frame: number | undefined; + + // Safe to call more than once: unsubscribing runs it, and `start` runs it + // again for anything the browser handed over after that. + const release = (): void => { + if (frame !== undefined) cancelAnimationFrame(frame); + stream?.getTracks().forEach((track) => track.stop()); + void context?.close(); + frame = undefined; + stream = undefined; + context = undefined; + }; + + const start = async (): Promise => { + stream = await navigator.mediaDevices.getUserMedia({ + audio: + deviceId === undefined ? true : { deviceId: { exact: deviceId } }, + }); + // A permission prompt outlives the subscription that asked for it, so by + // now nobody may be watching — and `release` ran while `stream` was still + // undefined. Nothing will call it again, so release here or the device + // stays held, with the indicator lit and no meter on screen. + if (subscriber.closed) return release(); + + context = new AudioContext(); + // Chrome starts the context suspended unless it was created during a + // gesture; opening the menu is one, but resume explicitly so the meter + // cannot silently sit at zero. + if (context.state === "suspended") await context.resume(); + if (subscriber.closed) return release(); + + const analyser = context.createAnalyser(); + analyser.fftSize = 1024; + context.createMediaStreamSource(stream).connect(analyser); + const samples = new Uint8Array(analyser.fftSize); + let displayed = 0; + let previousFrame = performance.now(); + + const read = (): void => { + analyser.getByteTimeDomainData(samples); + // Root mean square of the waveform around its centre, which is the + // loudness a listener perceives rather than the tallest spike. + let sum = 0; + for (const sample of samples) { + const centred = (sample - 128) / 128; + sum += centred * centred; + } + const now = performance.now(); + displayed = smoothVolume( + displayed, + Math.sqrt(sum / samples.length), + now - previousFrame, + ); + previousFrame = now; + subscriber.next({ type: "level", level: segmentsForVolume(displayed) }); + frame = requestAnimationFrame(read); + }; + read(); + }; + + start().catch((e: unknown) => subscriber.next(stateForFailure(e))); + + return release; + }).pipe( + // Read every animation frame, but quantised to a whole number of segments, + // so most frames say nothing new and should not reach React. + distinctUntilChanged( + (a, b) => + a.type === b.type && + (a.type !== "level" || b.type !== "level" || a.level === b.level), + ), + ); +} + +/** What a failure to open the microphone means for the person using it. */ +function stateForFailure(e: unknown): MicrophoneState { + const name = e instanceof Error ? e.name : ""; + if (name === "NotAllowedError" || name === "SecurityError") + return { type: "permission-denied" }; + if (name === "NotFoundError" || name === "OverconstrainedError") + return { type: "no-device" }; + logger.error("Could not read the microphone level", e); + return { type: "no-device" }; +} + /** * Loudness below which the microphone is treated as picking up nothing. * diff --git a/src/utils/test.ts b/src/utils/test.ts index 49ae0a3f1..d65cd3607 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -8,6 +8,7 @@ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; import { expect, + type Mock, type MockedObject, type MockInstance, onTestFinished, @@ -590,3 +591,92 @@ export class MockConnection extends Connection { public async start(): Promise {} public async stop(): Promise {} } + +export interface StubbedCapture { + getUserMedia: Mock; + /** Hands the microphone over, as the browser does once permission is given. */ + grant: () => void; + track: { stop: Mock }; + contexts: { close: Mock }[]; + /** Runs the animation frames the level meter reads on, in order. */ + drawFrames: (count: number) => void; +} + +/** + * Stubs just enough of the capture and Web Audio APIs for a microphone level to + * be read, with the grant held back so a test decides when — or whether — it + * lands, and with animation frames driven by hand rather than by a clock. + * + * Call {@link restoreAudioCapture} afterwards, or every later test in the run + * inherits the stub. + */ +export function stubAudioCapture(): StubbedCapture { + const track = { stop: vi.fn() }; + const contexts: { close: Mock }[] = []; + let grant = (): void => {}; + const granted = new Promise((resolve) => { + grant = (): void => + resolve({ getTracks: () => [track] } as unknown as MediaStream); + }); + + const frames: FrameRequestCallback[] = []; + vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { + frames.push(callback); + return frames.length; + }); + vi.stubGlobal("cancelAnimationFrame", vi.fn()); + vi.stubGlobal( + "AudioContext", + class { + public readonly state = "running"; + public readonly close = vi.fn(); + public constructor() { + contexts.push(this); + } + public createAnalyser(): object { + return { + fftSize: 1024, + // Digital silence, which is the midpoint of the range and not zero: + // a buffer left at zero reads as a full-scale waveform. + getByteTimeDomainData: (samples: Uint8Array): void => { + samples.fill(128); + }, + }; + } + public createMediaStreamSource(): object { + return { connect: (): void => {} }; + } + }, + ); + // Only this property: replacing navigator wholesale drops the getters on its + // prototype, such as userAgent. + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: { getUserMedia: vi.fn().mockReturnValue(granted) }, + }); + + return { + getUserMedia: navigator.mediaDevices.getUserMedia as unknown as Mock, + grant: () => grant(), + track, + contexts, + drawFrames: (count): void => { + for (let i = 0; i < count; i++) frames.shift()?.(i); + }, + }; +} + +const realMediaDevices = Object.getOwnPropertyDescriptor( + navigator, + "mediaDevices", +); + +/** Undoes {@link stubAudioCapture}. */ +export function restoreAudioCapture(): void { + vi.unstubAllGlobals(); + if (realMediaDevices === undefined) { + Reflect.deleteProperty(navigator, "mediaDevices"); + } else { + Object.defineProperty(navigator, "mediaDevices", realMediaDevices); + } +}