diff --git a/src/state/MicrophoneLevel.test.ts b/src/state/MicrophoneLevel.test.ts new file mode 100644 index 000000000..79c849782 --- /dev/null +++ b/src/state/MicrophoneLevel.test.ts @@ -0,0 +1,189 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, describe, expect, test, vi } from "vitest"; + +import { + ATTACK_MS, + LEVEL_SCALE, + observeMicrophoneState$, + RELEASE_MS, + segmentsForVolume, + smoothVolume, +} from "./MicrophoneLevel"; +import { restoreAudioCapture, stubAudioCapture } from "../utils/test"; + +describe("segmentsForVolume", () => { + test("shows nothing for silence", () => { + expect(segmentsForVolume(0)).toBe(0); + }); + + test("shows nothing for the hiss of a quiet room", () => { + // Without a noise floor these light the first segments permanently, which + // reads as "it can hear me" when nobody is speaking. + expect(segmentsForVolume(0.005)).toBe(0); + expect(segmentsForVolume(0.015)).toBe(0); + }); + + test("distinguishes quiet, normal and loud speech", () => { + const quiet = segmentsForVolume(0.06); + const normal = segmentsForVolume(0.2); + const loud = segmentsForVolume(0.8); + + expect(quiet).toBeGreaterThan(0); + expect(normal).toBeGreaterThan(quiet); + expect(loud).toBeGreaterThan(normal); + }); + + test("moves the meter visibly for normal speech", () => { + // Ordinary speech should reach the middle of the meter, not scrape along + // the floor: a meter that barely moves reads as a broken microphone. + expect(segmentsForVolume(0.2)).toBeGreaterThanOrEqual(LEVEL_SCALE / 4); + }); + + test("never exceeds the meter", () => { + expect(segmentsForVolume(1)).toBe(LEVEL_SCALE); + expect(segmentsForVolume(4)).toBe(LEVEL_SCALE); + }); + + test("treats a missing reading as silence", () => { + expect(segmentsForVolume(NaN)).toBe(0); + expect(segmentsForVolume(-1)).toBe(0); + }); +}); + +describe("smoothVolume", () => { + test("rises faster than it falls", () => { + const rise = smoothVolume(0, 1, 50); + const fall = 1 - smoothVolume(1, 0, 50); + + expect(rise).toBeGreaterThan(fall); + }); + + test("registers a syllable as it starts", () => { + // Most of the way there within one attack time constant, so speech does + // not lag the speaker. + expect(smoothVolume(0, 1, ATTACK_MS)).toBeGreaterThan(0.6); + }); + + test("rides over the gaps between words", () => { + // A pause of a few tens of milliseconds should not collapse the meter, or + // it flickers rather than reading as a level. + expect(smoothVolume(1, 0, 30)).toBeGreaterThan(0.7); + // A real silence still brings it down. + expect(smoothVolume(1, 0, RELEASE_MS * 3)).toBeLessThan(0.1); + }); + + test("behaves the same whatever the frame rate", () => { + const oneStep = smoothVolume(0, 1, 32); + let twoSteps = smoothVolume(0, 1, 16); + twoSteps = smoothVolume(twoSteps, 1, 16); + + expect(twoSteps).toBeCloseTo(oneStep, 5); + }); + + test("holds still when no time has passed", () => { + expect(smoothVolume(0.5, 1, 0)).toBe(0.5); + }); +}); + +describe("observeMicrophoneState$", () => { + afterEach(restoreAudioCapture); + + test("releases a capture the browser grants after nobody is watching", async () => { + const capture = stubAudioCapture(); + + const subscription = observeMicrophoneState$("mic1").subscribe(); + // The user gives up on the permission prompt and closes the menu, and only + // then does the browser hand the microphone over. + subscription.unsubscribe(); + capture.grant(); + await vi.waitFor(() => expect(capture.track.stop).toHaveBeenCalled()); + }); + + test("releases the capture and the audio context when the subscription ends", async () => { + const capture = stubAudioCapture(); + + const subscription = observeMicrophoneState$("mic1").subscribe(); + capture.grant(); + await vi.waitFor(() => expect(capture.contexts).toHaveLength(1)); + + subscription.unsubscribe(); + + expect(capture.track.stop).toHaveBeenCalled(); + expect(capture.contexts[0].close).toHaveBeenCalled(); + }); + + test("gives the microphone back when the audio graph fails to build", async () => { + const capture = stubAudioCapture(); + // The graph fails only after getUserMedia has handed the device over, so + // there is a live capture to lose. + vi.stubGlobal( + "AudioContext", + class { + public constructor() { + throw new Error("no audio context for you"); + } + }, + ); + + const seen: string[] = []; + const subscription = observeMicrophoneState$("mic1").subscribe((state) => + seen.push(state.type), + ); + capture.grant(); + + await vi.waitFor(() => expect(seen).toContain("no-device")); + // Without this the microphone stays open, and its in-use light on, behind + // a meter that says it is unavailable. + expect(capture.track.stop).toHaveBeenCalled(); + + subscription.unsubscribe(); + }); + + test("tells denied permission and a missing device apart", async () => { + for (const [name, expected] of [ + ["NotAllowedError", "permission-denied"], + ["NotFoundError", "no-device"], + ] as const) { + const capture = stubAudioCapture(); + capture.getUserMedia.mockRejectedValue(named(new Error(name), name)); + + const seen: string[] = []; + const subscription = observeMicrophoneState$("mic1").subscribe((state) => + seen.push(state.type), + ); + await vi.waitFor(() => expect(seen).toContain(expected)); + subscription.unsubscribe(); + restoreAudioCapture(); + } + }); + + test("says nothing on a frame that did not change the level", async () => { + const capture = stubAudioCapture(); + + let emissions = 0; + const subscription = observeMicrophoneState$("mic1").subscribe( + () => emissions++, + ); + capture.grant(); + await vi.waitFor(() => expect(emissions).toBe(1)); + + // The analyser is read every animation frame, but the meter has only + // LEVEL_SCALE steps: a steady signal must not redraw the meter. + capture.drawFrames(20); + expect(emissions).toBe(1); + + subscription.unsubscribe(); + }); +}); + +/** An error with the `name` the browser would give it, not just a message. */ +function named(error: Error, name: string): Error { + error.name = name; + return error; +} diff --git a/src/state/MicrophoneLevel.ts b/src/state/MicrophoneLevel.ts new file mode 100644 index 000000000..43f57ec29 --- /dev/null +++ b/src/state/MicrophoneLevel.ts @@ -0,0 +1,186 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { distinctUntilChanged, Observable } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +/** + * What the microphone selector can say about the input, beyond its level. + * + * Silence and a broken microphone look identical on a meter, so the states a + * user has to act on are named rather than drawn as a flat bar. + */ +export type MicrophoneState = + | { type: "level"; level: number } + | { type: "permission-denied" } + | { type: "no-device" }; + +/** + * The scale a level is reported on: 0 means silence, this means full scale. + * + * Fixed, and deliberately not the number of bars drawn — that follows the + * width available. A scale that moved with the width would announce the same + * loudness as different numbers in different places, and would make this layer + * depend on how wide something is drawn. + */ +export const LEVEL_SCALE = 24; + +/** + * Observes what a microphone is picking up, as the meter should show it. + * + * - Held for exactly as long as something is watching: subscribing opens the + * device, unsubscribing releases it. The menu, not the call. + * - Its own capture rather than the track the call holds, by design. The + * pre-join screen freezes that track to the device selected when it mounted, + * so a meter fed from it could not follow the picker. + * - Says whether the microphone hears anything, not whether anyone hears the + * user: it keeps reading while muted, and the mute control carries that. + */ +export function observeMicrophoneState$( + deviceId: string | undefined, +): Observable { + return new Observable((subscriber) => { + let stream: MediaStream | undefined; + let context: AudioContext | undefined; + let frame: number | undefined; + + // Safe to call more than once: unsubscribing runs it, and `start` runs it + // again for anything the browser handed over after that. + const release = (): void => { + if (frame !== undefined) cancelAnimationFrame(frame); + stream?.getTracks().forEach((track) => track.stop()); + void context?.close(); + frame = undefined; + stream = undefined; + context = undefined; + }; + + const start = async (): Promise => { + stream = await navigator.mediaDevices.getUserMedia({ + audio: + deviceId === undefined ? true : { deviceId: { exact: deviceId } }, + }); + // A permission prompt outlives the subscription that asked for it, so by + // now nobody may be watching — and `release` ran while `stream` was still + // undefined. Nothing will call it again, so release here or the device + // stays held, with the indicator lit and no meter on screen. + if (subscriber.closed) return release(); + + context = new AudioContext(); + // Chrome starts the context suspended unless it was created during a + // gesture; opening the menu is one, but resume explicitly so the meter + // cannot silently sit at zero. + if (context.state === "suspended") await context.resume(); + if (subscriber.closed) return release(); + + const analyser = context.createAnalyser(); + analyser.fftSize = 1024; + context.createMediaStreamSource(stream).connect(analyser); + const samples = new Uint8Array(analyser.fftSize); + let displayed = 0; + let previousFrame = performance.now(); + + const read = (): void => { + analyser.getByteTimeDomainData(samples); + // Root mean square of the waveform around its centre, which is the + // loudness a listener perceives rather than the tallest spike. + let sum = 0; + for (const sample of samples) { + const centred = (sample - 128) / 128; + sum += centred * centred; + } + const now = performance.now(); + displayed = smoothVolume( + displayed, + Math.sqrt(sum / samples.length), + now - previousFrame, + ); + previousFrame = now; + subscriber.next({ type: "level", level: segmentsForVolume(displayed) }); + frame = requestAnimationFrame(read); + }; + read(); + }; + + start().catch((e: unknown) => { + // Building the graph can fail after getUserMedia has already resolved, + // and the capture would then outlive its own failure: the microphone + // open and its in-use light on, behind a meter reporting it as + // unavailable. Releasing is its own step, which teardown and this path + // both take. + release(); + subscriber.next(stateForFailure(e)); + }); + + return release; + }).pipe( + // Read every animation frame, but quantised to a whole number of segments, + // so most frames say nothing new and should not reach React. + distinctUntilChanged( + (a, b) => + a.type === b.type && + (a.type !== "level" || b.type !== "level" || a.level === b.level), + ), + ); +} + +/** What a failure to open the microphone means for the person using it. */ +function stateForFailure(e: unknown): MicrophoneState { + const name = e instanceof Error ? e.name : ""; + if (name === "NotAllowedError" || name === "SecurityError") + return { type: "permission-denied" }; + if (name === "NotFoundError" || name === "OverconstrainedError") + return { type: "no-device" }; + logger.error("Could not read the microphone level", e); + return { type: "no-device" }; +} + +/** + * Loudness below which the microphone counts as hearing nothing. A quiet room + * is never digitally silent, and without a floor that hiss lights the first + * bars permanently — which reads as "it can hear me" when nobody is speaking. + */ +const NOISE_FLOOR = 0.02; + +/** + * Quantises a 0..1 volume onto {@link LEVEL_SCALE}. Exported for the tests: + * this mapping is what decides whether quiet, normal and loud look different. + */ +export function segmentsForVolume(volume: number): number { + if (!Number.isFinite(volume) || volume <= NOISE_FLOOR) return 0; + // Volume arrives as amplitude, where speech occupies a small part of the top + // of the range. A square root spreads that out, so ordinary speech moves the + // meter through its middle rather than barely leaving the floor. + const aboveFloor = (Math.min(volume, 1) - NOISE_FLOOR) / (1 - NOISE_FLOOR); + return Math.min(LEVEL_SCALE, Math.ceil(Math.sqrt(aboveFloor) * LEVEL_SCALE)); +} + +/** Time constant for a rise. Short, so a syllable registers as it starts. */ +export const ATTACK_MS = 50; + +/** + * Time constant for a fall. Longer than the attack: speech is full of gaps a + * few tens of milliseconds long, and tracking them exactly would flicker. + */ +export const RELEASE_MS = 120; + +/** + * Moves a displayed level towards a new reading, fast up and slowly down. + * + * In elapsed time rather than frames, so it behaves the same at 60Hz and + * 120Hz, and does not jump when a frame is dropped. + */ +export function smoothVolume( + displayed: number, + reading: number, + elapsedMs: number, +): number { + if (elapsedMs <= 0) return displayed; + const timeConstant = reading > displayed ? ATTACK_MS : RELEASE_MS; + const towards = 1 - Math.exp(-elapsedMs / timeConstant); + return displayed + (reading - displayed) * towards; +} diff --git a/src/utils/test.ts b/src/utils/test.ts index 49ae0a3f1..d8964d6ac 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -8,6 +8,7 @@ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; import { expect, + type Mock, type MockedObject, type MockInstance, onTestFinished, @@ -590,3 +591,107 @@ export class MockConnection extends Connection { public async start(): Promise {} public async stop(): Promise {} } + +export interface StubbedCapture { + getUserMedia: Mock; + /** Hands the microphone over, as the browser does once permission is given. */ + grant: () => void; + track: { stop: Mock }; + contexts: { close: Mock }[]; + /** Runs the animation frames the level meter reads on, in order. */ + drawFrames: (count: number) => void; + /** + * Sets how loud the microphone is, 0 for silence and 1 for full scale. + * Takes effect on the next frame drawn. + */ + speak: (amplitude: number) => void; +} + +/** + * Stubs just enough of the capture and Web Audio APIs for a microphone level to + * be read, with the grant held back so a test decides when — or whether — it + * lands, and with animation frames driven by hand rather than by a clock. + * + * Call {@link restoreAudioCapture} afterwards, or every later test in the run + * inherits the stub. + */ +export function stubAudioCapture(): StubbedCapture { + const track = { stop: vi.fn() }; + const contexts: { close: Mock }[] = []; + let grant = (): void => {}; + const granted = new Promise((resolve) => { + grant = (): void => + resolve({ getTracks: () => [track] } as unknown as MediaStream); + }); + + const frames: FrameRequestCallback[] = []; + let amplitude = 0; + vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { + frames.push(callback); + return frames.length; + }); + vi.stubGlobal("cancelAnimationFrame", vi.fn()); + vi.stubGlobal( + "AudioContext", + class { + public readonly state = "running"; + public readonly close = vi.fn(); + public constructor() { + contexts.push(this); + } + public createAnalyser(): object { + return { + fftSize: 1024, + getByteTimeDomainData: (samples: Uint8Array): void => { + // Digital silence is the midpoint of the range and not zero: a + // buffer left at zero reads as a full-scale waveform. + if (amplitude <= 0) { + samples.fill(128); + return; + } + const peak = Math.round(Math.min(1, amplitude) * 127); + for (let i = 0; i < samples.length; i++) + samples[i] = 128 + (i % 2 ? peak : -peak); + }, + }; + } + public createMediaStreamSource(): object { + return { connect: (): void => {} }; + } + }, + ); + // Only this property: replacing navigator wholesale drops the getters on its + // prototype, such as userAgent. + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: { getUserMedia: vi.fn().mockReturnValue(granted) }, + }); + + return { + getUserMedia: navigator.mediaDevices.getUserMedia as unknown as Mock, + grant: () => grant(), + track, + contexts, + drawFrames: (count): void => { + for (let i = 0; i < count; i++) frames.shift()?.(i); + }, + speak: (next): void => { + amplitude = next; + }, + }; +} + +const realMediaDevices = Object.getOwnPropertyDescriptor( + navigator, + "mediaDevices", +); + +/** Undoes {@link stubAudioCapture}. */ +export function restoreAudioCapture(): void { + vi.unstubAllGlobals(); + if (realMediaDevices === undefined) { + Reflect.deleteProperty(navigator, "mediaDevices"); + } else { + Object.defineProperty(navigator, "mediaDevices", realMediaDevices); + } +}