mirror of
https://github.com/vector-im/element-call.git
synced 2026-09-22 22:29:30 +00:00
Give the microphone level its own observable
- Opens a short-lived capture of a named device and reports a smoothed level. - Owns the capture, the audio context and the frame loop, and releases all three idempotently, including when cleanup lands before getUserMedia has resolved. - Reports denied permission and a missing device as states of their own, so a caller never has to read silence as an error. - Gives the microphone back when the audio graph fails to build. That can fail after getUserMedia has resolved, and the capture would otherwise outlive its own failure: the device open and its in-use light on, behind a meter saying it is unavailable. - Shares the capture stub through the test utilities, silence and a signal alike, so the meter's tests and its stories drive the same microphone. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
fkwp
@@ -0,0 +1,189 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
||||
|
||||
import {
|
||||
ATTACK_MS,
|
||||
LEVEL_SCALE,
|
||||
observeMicrophoneState$,
|
||||
RELEASE_MS,
|
||||
segmentsForVolume,
|
||||
smoothVolume,
|
||||
} from "./MicrophoneLevel";
|
||||
import { restoreAudioCapture, stubAudioCapture } from "../utils/test";
|
||||
|
||||
describe("segmentsForVolume", () => {
|
||||
test("shows nothing for silence", () => {
|
||||
expect(segmentsForVolume(0)).toBe(0);
|
||||
});
|
||||
|
||||
test("shows nothing for the hiss of a quiet room", () => {
|
||||
// Without a noise floor these light the first segments permanently, which
|
||||
// reads as "it can hear me" when nobody is speaking.
|
||||
expect(segmentsForVolume(0.005)).toBe(0);
|
||||
expect(segmentsForVolume(0.015)).toBe(0);
|
||||
});
|
||||
|
||||
test("distinguishes quiet, normal and loud speech", () => {
|
||||
const quiet = segmentsForVolume(0.06);
|
||||
const normal = segmentsForVolume(0.2);
|
||||
const loud = segmentsForVolume(0.8);
|
||||
|
||||
expect(quiet).toBeGreaterThan(0);
|
||||
expect(normal).toBeGreaterThan(quiet);
|
||||
expect(loud).toBeGreaterThan(normal);
|
||||
});
|
||||
|
||||
test("moves the meter visibly for normal speech", () => {
|
||||
// Ordinary speech should reach the middle of the meter, not scrape along
|
||||
// the floor: a meter that barely moves reads as a broken microphone.
|
||||
expect(segmentsForVolume(0.2)).toBeGreaterThanOrEqual(LEVEL_SCALE / 4);
|
||||
});
|
||||
|
||||
test("never exceeds the meter", () => {
|
||||
expect(segmentsForVolume(1)).toBe(LEVEL_SCALE);
|
||||
expect(segmentsForVolume(4)).toBe(LEVEL_SCALE);
|
||||
});
|
||||
|
||||
test("treats a missing reading as silence", () => {
|
||||
expect(segmentsForVolume(NaN)).toBe(0);
|
||||
expect(segmentsForVolume(-1)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("smoothVolume", () => {
|
||||
test("rises faster than it falls", () => {
|
||||
const rise = smoothVolume(0, 1, 50);
|
||||
const fall = 1 - smoothVolume(1, 0, 50);
|
||||
|
||||
expect(rise).toBeGreaterThan(fall);
|
||||
});
|
||||
|
||||
test("registers a syllable as it starts", () => {
|
||||
// Most of the way there within one attack time constant, so speech does
|
||||
// not lag the speaker.
|
||||
expect(smoothVolume(0, 1, ATTACK_MS)).toBeGreaterThan(0.6);
|
||||
});
|
||||
|
||||
test("rides over the gaps between words", () => {
|
||||
// A pause of a few tens of milliseconds should not collapse the meter, or
|
||||
// it flickers rather than reading as a level.
|
||||
expect(smoothVolume(1, 0, 30)).toBeGreaterThan(0.7);
|
||||
// A real silence still brings it down.
|
||||
expect(smoothVolume(1, 0, RELEASE_MS * 3)).toBeLessThan(0.1);
|
||||
});
|
||||
|
||||
test("behaves the same whatever the frame rate", () => {
|
||||
const oneStep = smoothVolume(0, 1, 32);
|
||||
let twoSteps = smoothVolume(0, 1, 16);
|
||||
twoSteps = smoothVolume(twoSteps, 1, 16);
|
||||
|
||||
expect(twoSteps).toBeCloseTo(oneStep, 5);
|
||||
});
|
||||
|
||||
test("holds still when no time has passed", () => {
|
||||
expect(smoothVolume(0.5, 1, 0)).toBe(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("observeMicrophoneState$", () => {
|
||||
afterEach(restoreAudioCapture);
|
||||
|
||||
test("releases a capture the browser grants after nobody is watching", async () => {
|
||||
const capture = stubAudioCapture();
|
||||
|
||||
const subscription = observeMicrophoneState$("mic1").subscribe();
|
||||
// The user gives up on the permission prompt and closes the menu, and only
|
||||
// then does the browser hand the microphone over.
|
||||
subscription.unsubscribe();
|
||||
capture.grant();
|
||||
await vi.waitFor(() => expect(capture.track.stop).toHaveBeenCalled());
|
||||
});
|
||||
|
||||
test("releases the capture and the audio context when the subscription ends", async () => {
|
||||
const capture = stubAudioCapture();
|
||||
|
||||
const subscription = observeMicrophoneState$("mic1").subscribe();
|
||||
capture.grant();
|
||||
await vi.waitFor(() => expect(capture.contexts).toHaveLength(1));
|
||||
|
||||
subscription.unsubscribe();
|
||||
|
||||
expect(capture.track.stop).toHaveBeenCalled();
|
||||
expect(capture.contexts[0].close).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("gives the microphone back when the audio graph fails to build", async () => {
|
||||
const capture = stubAudioCapture();
|
||||
// The graph fails only after getUserMedia has handed the device over, so
|
||||
// there is a live capture to lose.
|
||||
vi.stubGlobal(
|
||||
"AudioContext",
|
||||
class {
|
||||
public constructor() {
|
||||
throw new Error("no audio context for you");
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
const seen: string[] = [];
|
||||
const subscription = observeMicrophoneState$("mic1").subscribe((state) =>
|
||||
seen.push(state.type),
|
||||
);
|
||||
capture.grant();
|
||||
|
||||
await vi.waitFor(() => expect(seen).toContain("no-device"));
|
||||
// Without this the microphone stays open, and its in-use light on, behind
|
||||
// a meter that says it is unavailable.
|
||||
expect(capture.track.stop).toHaveBeenCalled();
|
||||
|
||||
subscription.unsubscribe();
|
||||
});
|
||||
|
||||
test("tells denied permission and a missing device apart", async () => {
|
||||
for (const [name, expected] of [
|
||||
["NotAllowedError", "permission-denied"],
|
||||
["NotFoundError", "no-device"],
|
||||
] as const) {
|
||||
const capture = stubAudioCapture();
|
||||
capture.getUserMedia.mockRejectedValue(named(new Error(name), name));
|
||||
|
||||
const seen: string[] = [];
|
||||
const subscription = observeMicrophoneState$("mic1").subscribe((state) =>
|
||||
seen.push(state.type),
|
||||
);
|
||||
await vi.waitFor(() => expect(seen).toContain(expected));
|
||||
subscription.unsubscribe();
|
||||
restoreAudioCapture();
|
||||
}
|
||||
});
|
||||
|
||||
test("says nothing on a frame that did not change the level", async () => {
|
||||
const capture = stubAudioCapture();
|
||||
|
||||
let emissions = 0;
|
||||
const subscription = observeMicrophoneState$("mic1").subscribe(
|
||||
() => emissions++,
|
||||
);
|
||||
capture.grant();
|
||||
await vi.waitFor(() => expect(emissions).toBe(1));
|
||||
|
||||
// The analyser is read every animation frame, but the meter has only
|
||||
// LEVEL_SCALE steps: a steady signal must not redraw the meter.
|
||||
capture.drawFrames(20);
|
||||
expect(emissions).toBe(1);
|
||||
|
||||
subscription.unsubscribe();
|
||||
});
|
||||
});
|
||||
|
||||
/** An error with the `name` the browser would give it, not just a message. */
|
||||
function named(error: Error, name: string): Error {
|
||||
error.name = name;
|
||||
return error;
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { distinctUntilChanged, Observable } from "rxjs";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
/**
|
||||
* What the microphone selector can say about the input, beyond its level.
|
||||
*
|
||||
* Silence and a broken microphone look identical on a meter, so the states a
|
||||
* user has to act on are named rather than drawn as a flat bar.
|
||||
*/
|
||||
export type MicrophoneState =
|
||||
| { type: "level"; level: number }
|
||||
| { type: "permission-denied" }
|
||||
| { type: "no-device" };
|
||||
|
||||
/**
|
||||
* The scale a level is reported on: 0 means silence, this means full scale.
|
||||
*
|
||||
* Fixed, and deliberately not the number of bars drawn — that follows the
|
||||
* width available. A scale that moved with the width would announce the same
|
||||
* loudness as different numbers in different places, and would make this layer
|
||||
* depend on how wide something is drawn.
|
||||
*/
|
||||
export const LEVEL_SCALE = 24;
|
||||
|
||||
/**
|
||||
* Observes what a microphone is picking up, as the meter should show it.
|
||||
*
|
||||
* - Held for exactly as long as something is watching: subscribing opens the
|
||||
* device, unsubscribing releases it. The menu, not the call.
|
||||
* - Its own capture rather than the track the call holds, by design. The
|
||||
* pre-join screen freezes that track to the device selected when it mounted,
|
||||
* so a meter fed from it could not follow the picker.
|
||||
* - Says whether the microphone hears anything, not whether anyone hears the
|
||||
* user: it keeps reading while muted, and the mute control carries that.
|
||||
*/
|
||||
export function observeMicrophoneState$(
|
||||
deviceId: string | undefined,
|
||||
): Observable<MicrophoneState> {
|
||||
return new Observable<MicrophoneState>((subscriber) => {
|
||||
let stream: MediaStream | undefined;
|
||||
let context: AudioContext | undefined;
|
||||
let frame: number | undefined;
|
||||
|
||||
// Safe to call more than once: unsubscribing runs it, and `start` runs it
|
||||
// again for anything the browser handed over after that.
|
||||
const release = (): void => {
|
||||
if (frame !== undefined) cancelAnimationFrame(frame);
|
||||
stream?.getTracks().forEach((track) => track.stop());
|
||||
void context?.close();
|
||||
frame = undefined;
|
||||
stream = undefined;
|
||||
context = undefined;
|
||||
};
|
||||
|
||||
const start = async (): Promise<void> => {
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio:
|
||||
deviceId === undefined ? true : { deviceId: { exact: deviceId } },
|
||||
});
|
||||
// A permission prompt outlives the subscription that asked for it, so by
|
||||
// now nobody may be watching — and `release` ran while `stream` was still
|
||||
// undefined. Nothing will call it again, so release here or the device
|
||||
// stays held, with the indicator lit and no meter on screen.
|
||||
if (subscriber.closed) return release();
|
||||
|
||||
context = new AudioContext();
|
||||
// Chrome starts the context suspended unless it was created during a
|
||||
// gesture; opening the menu is one, but resume explicitly so the meter
|
||||
// cannot silently sit at zero.
|
||||
if (context.state === "suspended") await context.resume();
|
||||
if (subscriber.closed) return release();
|
||||
|
||||
const analyser = context.createAnalyser();
|
||||
analyser.fftSize = 1024;
|
||||
context.createMediaStreamSource(stream).connect(analyser);
|
||||
const samples = new Uint8Array(analyser.fftSize);
|
||||
let displayed = 0;
|
||||
let previousFrame = performance.now();
|
||||
|
||||
const read = (): void => {
|
||||
analyser.getByteTimeDomainData(samples);
|
||||
// Root mean square of the waveform around its centre, which is the
|
||||
// loudness a listener perceives rather than the tallest spike.
|
||||
let sum = 0;
|
||||
for (const sample of samples) {
|
||||
const centred = (sample - 128) / 128;
|
||||
sum += centred * centred;
|
||||
}
|
||||
const now = performance.now();
|
||||
displayed = smoothVolume(
|
||||
displayed,
|
||||
Math.sqrt(sum / samples.length),
|
||||
now - previousFrame,
|
||||
);
|
||||
previousFrame = now;
|
||||
subscriber.next({ type: "level", level: segmentsForVolume(displayed) });
|
||||
frame = requestAnimationFrame(read);
|
||||
};
|
||||
read();
|
||||
};
|
||||
|
||||
start().catch((e: unknown) => {
|
||||
// Building the graph can fail after getUserMedia has already resolved,
|
||||
// and the capture would then outlive its own failure: the microphone
|
||||
// open and its in-use light on, behind a meter reporting it as
|
||||
// unavailable. Releasing is its own step, which teardown and this path
|
||||
// both take.
|
||||
release();
|
||||
subscriber.next(stateForFailure(e));
|
||||
});
|
||||
|
||||
return release;
|
||||
}).pipe(
|
||||
// Read every animation frame, but quantised to a whole number of segments,
|
||||
// so most frames say nothing new and should not reach React.
|
||||
distinctUntilChanged(
|
||||
(a, b) =>
|
||||
a.type === b.type &&
|
||||
(a.type !== "level" || b.type !== "level" || a.level === b.level),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** What a failure to open the microphone means for the person using it. */
|
||||
function stateForFailure(e: unknown): MicrophoneState {
|
||||
const name = e instanceof Error ? e.name : "";
|
||||
if (name === "NotAllowedError" || name === "SecurityError")
|
||||
return { type: "permission-denied" };
|
||||
if (name === "NotFoundError" || name === "OverconstrainedError")
|
||||
return { type: "no-device" };
|
||||
logger.error("Could not read the microphone level", e);
|
||||
return { type: "no-device" };
|
||||
}
|
||||
|
||||
/**
|
||||
* Loudness below which the microphone counts as hearing nothing. A quiet room
|
||||
* is never digitally silent, and without a floor that hiss lights the first
|
||||
* bars permanently — which reads as "it can hear me" when nobody is speaking.
|
||||
*/
|
||||
const NOISE_FLOOR = 0.02;
|
||||
|
||||
/**
|
||||
* Quantises a 0..1 volume onto {@link LEVEL_SCALE}. Exported for the tests:
|
||||
* this mapping is what decides whether quiet, normal and loud look different.
|
||||
*/
|
||||
export function segmentsForVolume(volume: number): number {
|
||||
if (!Number.isFinite(volume) || volume <= NOISE_FLOOR) return 0;
|
||||
// Volume arrives as amplitude, where speech occupies a small part of the top
|
||||
// of the range. A square root spreads that out, so ordinary speech moves the
|
||||
// meter through its middle rather than barely leaving the floor.
|
||||
const aboveFloor = (Math.min(volume, 1) - NOISE_FLOOR) / (1 - NOISE_FLOOR);
|
||||
return Math.min(LEVEL_SCALE, Math.ceil(Math.sqrt(aboveFloor) * LEVEL_SCALE));
|
||||
}
|
||||
|
||||
/** Time constant for a rise. Short, so a syllable registers as it starts. */
|
||||
export const ATTACK_MS = 50;
|
||||
|
||||
/**
|
||||
* Time constant for a fall. Longer than the attack: speech is full of gaps a
|
||||
* few tens of milliseconds long, and tracking them exactly would flicker.
|
||||
*/
|
||||
export const RELEASE_MS = 120;
|
||||
|
||||
/**
|
||||
* Moves a displayed level towards a new reading, fast up and slowly down.
|
||||
*
|
||||
* In elapsed time rather than frames, so it behaves the same at 60Hz and
|
||||
* 120Hz, and does not jump when a frame is dropped.
|
||||
*/
|
||||
export function smoothVolume(
|
||||
displayed: number,
|
||||
reading: number,
|
||||
elapsedMs: number,
|
||||
): number {
|
||||
if (elapsedMs <= 0) return displayed;
|
||||
const timeConstant = reading > displayed ? ATTACK_MS : RELEASE_MS;
|
||||
const towards = 1 - Math.exp(-elapsedMs / timeConstant);
|
||||
return displayed + (reading - displayed) * towards;
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import { map, type Observable, of, type SchedulerLike } from "rxjs";
|
||||
import { type RunHelpers, TestScheduler } from "rxjs/testing";
|
||||
import {
|
||||
expect,
|
||||
type Mock,
|
||||
type MockedObject,
|
||||
type MockInstance,
|
||||
onTestFinished,
|
||||
@@ -590,3 +591,107 @@ export class MockConnection extends Connection {
|
||||
public async start(): Promise<void> {}
|
||||
public async stop(): Promise<void> {}
|
||||
}
|
||||
|
||||
export interface StubbedCapture {
|
||||
getUserMedia: Mock;
|
||||
/** Hands the microphone over, as the browser does once permission is given. */
|
||||
grant: () => void;
|
||||
track: { stop: Mock };
|
||||
contexts: { close: Mock }[];
|
||||
/** Runs the animation frames the level meter reads on, in order. */
|
||||
drawFrames: (count: number) => void;
|
||||
/**
|
||||
* Sets how loud the microphone is, 0 for silence and 1 for full scale.
|
||||
* Takes effect on the next frame drawn.
|
||||
*/
|
||||
speak: (amplitude: number) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stubs just enough of the capture and Web Audio APIs for a microphone level to
|
||||
* be read, with the grant held back so a test decides when — or whether — it
|
||||
* lands, and with animation frames driven by hand rather than by a clock.
|
||||
*
|
||||
* Call {@link restoreAudioCapture} afterwards, or every later test in the run
|
||||
* inherits the stub.
|
||||
*/
|
||||
export function stubAudioCapture(): StubbedCapture {
|
||||
const track = { stop: vi.fn() };
|
||||
const contexts: { close: Mock }[] = [];
|
||||
let grant = (): void => {};
|
||||
const granted = new Promise<MediaStream>((resolve) => {
|
||||
grant = (): void =>
|
||||
resolve({ getTracks: () => [track] } as unknown as MediaStream);
|
||||
});
|
||||
|
||||
const frames: FrameRequestCallback[] = [];
|
||||
let amplitude = 0;
|
||||
vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => {
|
||||
frames.push(callback);
|
||||
return frames.length;
|
||||
});
|
||||
vi.stubGlobal("cancelAnimationFrame", vi.fn());
|
||||
vi.stubGlobal(
|
||||
"AudioContext",
|
||||
class {
|
||||
public readonly state = "running";
|
||||
public readonly close = vi.fn();
|
||||
public constructor() {
|
||||
contexts.push(this);
|
||||
}
|
||||
public createAnalyser(): object {
|
||||
return {
|
||||
fftSize: 1024,
|
||||
getByteTimeDomainData: (samples: Uint8Array): void => {
|
||||
// Digital silence is the midpoint of the range and not zero: a
|
||||
// buffer left at zero reads as a full-scale waveform.
|
||||
if (amplitude <= 0) {
|
||||
samples.fill(128);
|
||||
return;
|
||||
}
|
||||
const peak = Math.round(Math.min(1, amplitude) * 127);
|
||||
for (let i = 0; i < samples.length; i++)
|
||||
samples[i] = 128 + (i % 2 ? peak : -peak);
|
||||
},
|
||||
};
|
||||
}
|
||||
public createMediaStreamSource(): object {
|
||||
return { connect: (): void => {} };
|
||||
}
|
||||
},
|
||||
);
|
||||
// Only this property: replacing navigator wholesale drops the getters on its
|
||||
// prototype, such as userAgent.
|
||||
Object.defineProperty(navigator, "mediaDevices", {
|
||||
configurable: true,
|
||||
value: { getUserMedia: vi.fn().mockReturnValue(granted) },
|
||||
});
|
||||
|
||||
return {
|
||||
getUserMedia: navigator.mediaDevices.getUserMedia as unknown as Mock,
|
||||
grant: () => grant(),
|
||||
track,
|
||||
contexts,
|
||||
drawFrames: (count): void => {
|
||||
for (let i = 0; i < count; i++) frames.shift()?.(i);
|
||||
},
|
||||
speak: (next): void => {
|
||||
amplitude = next;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const realMediaDevices = Object.getOwnPropertyDescriptor(
|
||||
navigator,
|
||||
"mediaDevices",
|
||||
);
|
||||
|
||||
/** Undoes {@link stubAudioCapture}. */
|
||||
export function restoreAudioCapture(): void {
|
||||
vi.unstubAllGlobals();
|
||||
if (realMediaDevices === undefined) {
|
||||
Reflect.deleteProperty(navigator, "mediaDevices");
|
||||
} else {
|
||||
Object.defineProperty(navigator, "mediaDevices", realMediaDevices);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user