mirror of
https://github.com/vector-im/element-call.git
synced 2026-09-22 22:29:30 +00:00
Own the microphone level in the state layer
- The capture, the analyser loop and the loudness maths were in a component hook. Call logic belongs in the state layer; components render. - observeMicrophoneState$ opens the device when something subscribes and releases it when nothing does. The hook is left as a bridge, no domain logic. - Fixes a leak. A permission prompt outlives the menu that opened it, so the browser could hand over a microphone after cleanup had already run — with the variable that cleanup would have released still unset. The device stayed held, indicator lit, no meter on screen. - Expressed now as the subscriber being closed, which the observable can always answer. - The hook had no tests at all, which is why this survived. Four now, and the capture stub they need is in the shared factories rather than hand-rolled. - One counts emissions across twenty animation frames and expects one: the redraw cost AGENTS.md asks continuous drawing to account for.
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, test } from "vitest";
|
||||
import { renderHook } from "@testing-library/react";
|
||||
|
||||
import { useMicrophoneLevel } from "./useMicrophoneLevel";
|
||||
import { restoreAudioCapture, stubAudioCapture } from "../utils/test";
|
||||
|
||||
// The capture itself, and releasing it, belong to observeMicrophoneState$ and
|
||||
// are covered in src/state/MicrophoneLevel.test.ts. What is left here is the
|
||||
// bridging: when the hook watches, and what it reports before it has an answer.
|
||||
describe("useMicrophoneLevel", () => {
|
||||
afterEach(restoreAudioCapture);
|
||||
|
||||
test("holds no capture while the meter is not shown", () => {
|
||||
const capture = stubAudioCapture();
|
||||
|
||||
renderHook(() => useMicrophoneLevel("mic1", false));
|
||||
|
||||
expect(capture.getUserMedia).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("starts from nothing rather than the previous device's level", () => {
|
||||
const capture = stubAudioCapture();
|
||||
|
||||
const { result, rerender } = renderHook(
|
||||
({ id }: { id: string }) => useMicrophoneLevel(id, true),
|
||||
{ initialProps: { id: "mic1" } },
|
||||
);
|
||||
capture.grant();
|
||||
|
||||
rerender({ id: "mic2" });
|
||||
|
||||
// No level carried over: the meter reads the new device or nothing at all.
|
||||
expect(result.current).toEqual({ type: "level", level: 0 });
|
||||
});
|
||||
});
|
||||
@@ -6,109 +6,35 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import {
|
||||
type MicrophoneState,
|
||||
segmentsForVolume,
|
||||
smoothVolume,
|
||||
observeMicrophoneState$,
|
||||
} from "../state/MicrophoneLevel";
|
||||
|
||||
const IDLE: MicrophoneState = { type: "level", level: 0 };
|
||||
|
||||
/**
|
||||
* Reads the live input level of a microphone, while `active`.
|
||||
*
|
||||
* Capture is scoped to the caller being on screen: the meter only exists while
|
||||
* the menu that shows it is open, so nothing holds a second capture of the
|
||||
* device for the length of a call.
|
||||
*
|
||||
* The level says whether the microphone is picking anything up, which is not
|
||||
* the same as whether the user is being heard. It keeps moving while muted, and
|
||||
* the mute control is what says nothing is transmitted.
|
||||
* A bridge and nothing else: the capture, its lifetime and the maths belong to
|
||||
* {@link observeMicrophoneState$}. Scoped to `active` so the device is held
|
||||
* only while whatever shows the meter is on screen, rather than for the length
|
||||
* of a call.
|
||||
*/
|
||||
export function useMicrophoneLevel(
|
||||
deviceId: string | undefined,
|
||||
active: boolean,
|
||||
): MicrophoneState {
|
||||
const [state, setState] = useState<MicrophoneState>({
|
||||
type: "level",
|
||||
level: 0,
|
||||
});
|
||||
const [state, setState] = useState<MicrophoneState>(IDLE);
|
||||
|
||||
useEffect(() => {
|
||||
if (!active) return;
|
||||
|
||||
let stopped = false;
|
||||
let stream: MediaStream | undefined;
|
||||
let context: AudioContext | undefined;
|
||||
let frame: number | undefined;
|
||||
|
||||
const stop = (): void => {
|
||||
stopped = true;
|
||||
if (frame !== undefined) cancelAnimationFrame(frame);
|
||||
stream?.getTracks().forEach((track) => track.stop());
|
||||
void context?.close();
|
||||
};
|
||||
|
||||
const start = async (): Promise<void> => {
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio:
|
||||
deviceId === undefined ? true : { deviceId: { exact: deviceId } },
|
||||
});
|
||||
if (stopped) return;
|
||||
|
||||
context = new AudioContext();
|
||||
// Chrome starts the context suspended unless it was created during a
|
||||
// gesture; opening the menu is one, but resume explicitly so the meter
|
||||
// cannot silently sit at zero.
|
||||
if (context.state === "suspended") await context.resume();
|
||||
if (stopped) return;
|
||||
const analyser = context.createAnalyser();
|
||||
analyser.fftSize = 1024;
|
||||
context.createMediaStreamSource(stream).connect(analyser);
|
||||
const samples = new Uint8Array(analyser.fftSize);
|
||||
let displayed = 0;
|
||||
let previousFrame = performance.now();
|
||||
|
||||
const read = (): void => {
|
||||
analyser.getByteTimeDomainData(samples);
|
||||
// Root mean square of the waveform around its centre, which is the
|
||||
// loudness a listener perceives rather than the tallest spike.
|
||||
let sum = 0;
|
||||
for (const sample of samples) {
|
||||
const centred = (sample - 128) / 128;
|
||||
sum += centred * centred;
|
||||
}
|
||||
const now = performance.now();
|
||||
displayed = smoothVolume(
|
||||
displayed,
|
||||
Math.sqrt(sum / samples.length),
|
||||
now - previousFrame,
|
||||
);
|
||||
previousFrame = now;
|
||||
const level = segmentsForVolume(displayed);
|
||||
setState((current) =>
|
||||
current.type === "level" && current.level === level
|
||||
? current
|
||||
: { type: "level", level },
|
||||
);
|
||||
frame = requestAnimationFrame(read);
|
||||
};
|
||||
read();
|
||||
};
|
||||
|
||||
start().catch((e: unknown) => {
|
||||
const name = e instanceof Error ? e.name : "";
|
||||
if (name === "NotAllowedError" || name === "SecurityError") {
|
||||
setState({ type: "permission-denied" });
|
||||
} else if (name === "NotFoundError" || name === "OverconstrainedError") {
|
||||
setState({ type: "no-device" });
|
||||
} else {
|
||||
logger.error("Could not read the microphone level", e);
|
||||
setState({ type: "no-device" });
|
||||
}
|
||||
});
|
||||
|
||||
return stop;
|
||||
// Idle first, so a new device starts from nothing rather than from the
|
||||
// level the previous one was reading.
|
||||
setState(IDLE);
|
||||
const subscription = observeMicrophoneState$(deviceId).subscribe(setState);
|
||||
return (): void => subscription.unsubscribe();
|
||||
}, [deviceId, active]);
|
||||
|
||||
return state;
|
||||
|
||||
@@ -5,15 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
||||
|
||||
import {
|
||||
ATTACK_MS,
|
||||
METER_SEGMENTS,
|
||||
observeMicrophoneState$,
|
||||
RELEASE_MS,
|
||||
segmentsForVolume,
|
||||
smoothVolume,
|
||||
} from "./MicrophoneLevel";
|
||||
import { restoreAudioCapture, stubAudioCapture } from "../utils/test";
|
||||
|
||||
describe("segmentsForVolume", () => {
|
||||
test("shows nothing for silence", () => {
|
||||
@@ -88,3 +90,73 @@ describe("smoothVolume", () => {
|
||||
expect(smoothVolume(0.5, 1, 0)).toBe(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("observeMicrophoneState$", () => {
|
||||
afterEach(restoreAudioCapture);
|
||||
|
||||
test("releases a capture the browser grants after nobody is watching", async () => {
|
||||
const capture = stubAudioCapture();
|
||||
|
||||
const subscription = observeMicrophoneState$("mic1").subscribe();
|
||||
// The user gives up on the permission prompt and closes the menu, and only
|
||||
// then does the browser hand the microphone over.
|
||||
subscription.unsubscribe();
|
||||
capture.grant();
|
||||
await vi.waitFor(() => expect(capture.track.stop).toHaveBeenCalled());
|
||||
});
|
||||
|
||||
test("releases the capture and the audio context when the subscription ends", async () => {
|
||||
const capture = stubAudioCapture();
|
||||
|
||||
const subscription = observeMicrophoneState$("mic1").subscribe();
|
||||
capture.grant();
|
||||
await vi.waitFor(() => expect(capture.contexts).toHaveLength(1));
|
||||
|
||||
subscription.unsubscribe();
|
||||
|
||||
expect(capture.track.stop).toHaveBeenCalled();
|
||||
expect(capture.contexts[0].close).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("tells denied permission and a missing device apart", async () => {
|
||||
for (const [name, expected] of [
|
||||
["NotAllowedError", "permission-denied"],
|
||||
["NotFoundError", "no-device"],
|
||||
] as const) {
|
||||
const capture = stubAudioCapture();
|
||||
capture.getUserMedia.mockRejectedValue(named(new Error(name), name));
|
||||
|
||||
const seen: string[] = [];
|
||||
const subscription = observeMicrophoneState$("mic1").subscribe((state) =>
|
||||
seen.push(state.type),
|
||||
);
|
||||
await vi.waitFor(() => expect(seen).toContain(expected));
|
||||
subscription.unsubscribe();
|
||||
restoreAudioCapture();
|
||||
}
|
||||
});
|
||||
|
||||
test("says nothing on a frame that did not change the level", async () => {
|
||||
const capture = stubAudioCapture();
|
||||
|
||||
let emissions = 0;
|
||||
const subscription = observeMicrophoneState$("mic1").subscribe(
|
||||
() => emissions++,
|
||||
);
|
||||
capture.grant();
|
||||
await vi.waitFor(() => expect(emissions).toBe(1));
|
||||
|
||||
// The analyser is read every animation frame, but the meter has only
|
||||
// METER_SEGMENTS steps: a steady signal must not redraw the meter.
|
||||
capture.drawFrames(20);
|
||||
expect(emissions).toBe(1);
|
||||
|
||||
subscription.unsubscribe();
|
||||
});
|
||||
});
|
||||
|
||||
/** An error with the `name` the browser would give it, not just a message. */
|
||||
function named(error: Error, name: string): Error {
|
||||
error.name = name;
|
||||
return error;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { distinctUntilChanged, Observable } from "rxjs";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
/**
|
||||
* What the microphone selector can say about the input, beyond its level.
|
||||
*
|
||||
@@ -24,6 +27,112 @@ export type MicrophoneState =
|
||||
*/
|
||||
export const METER_SEGMENTS = 24;
|
||||
|
||||
/**
|
||||
* Observes what a microphone is picking up, as the meter should show it.
|
||||
*
|
||||
* Subscribing opens the device and unsubscribing releases it, so it is held for
|
||||
* exactly as long as something is watching — the menu, and not the call.
|
||||
*
|
||||
* Opens a capture of its own rather than reading the track the call already
|
||||
* holds. That is a shortcut, recorded as S1 in the feature spec: the pre-join
|
||||
* screen freezes its audio track to the device selected when it mounted, so a
|
||||
* meter fed from that track could not follow the picker.
|
||||
*
|
||||
* The level says whether the microphone is picking anything up, which is not
|
||||
* the same as whether the user is being heard: it keeps reading while muted,
|
||||
* and the mute control is what says nothing is transmitted.
|
||||
*/
|
||||
export function observeMicrophoneState$(
|
||||
deviceId: string | undefined,
|
||||
): Observable<MicrophoneState> {
|
||||
return new Observable<MicrophoneState>((subscriber) => {
|
||||
let stream: MediaStream | undefined;
|
||||
let context: AudioContext | undefined;
|
||||
let frame: number | undefined;
|
||||
|
||||
// Safe to call more than once: unsubscribing runs it, and `start` runs it
|
||||
// again for anything the browser handed over after that.
|
||||
const release = (): void => {
|
||||
if (frame !== undefined) cancelAnimationFrame(frame);
|
||||
stream?.getTracks().forEach((track) => track.stop());
|
||||
void context?.close();
|
||||
frame = undefined;
|
||||
stream = undefined;
|
||||
context = undefined;
|
||||
};
|
||||
|
||||
const start = async (): Promise<void> => {
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio:
|
||||
deviceId === undefined ? true : { deviceId: { exact: deviceId } },
|
||||
});
|
||||
// A permission prompt outlives the subscription that asked for it, so by
|
||||
// now nobody may be watching — and `release` ran while `stream` was still
|
||||
// undefined. Nothing will call it again, so release here or the device
|
||||
// stays held, with the indicator lit and no meter on screen.
|
||||
if (subscriber.closed) return release();
|
||||
|
||||
context = new AudioContext();
|
||||
// Chrome starts the context suspended unless it was created during a
|
||||
// gesture; opening the menu is one, but resume explicitly so the meter
|
||||
// cannot silently sit at zero.
|
||||
if (context.state === "suspended") await context.resume();
|
||||
if (subscriber.closed) return release();
|
||||
|
||||
const analyser = context.createAnalyser();
|
||||
analyser.fftSize = 1024;
|
||||
context.createMediaStreamSource(stream).connect(analyser);
|
||||
const samples = new Uint8Array(analyser.fftSize);
|
||||
let displayed = 0;
|
||||
let previousFrame = performance.now();
|
||||
|
||||
const read = (): void => {
|
||||
analyser.getByteTimeDomainData(samples);
|
||||
// Root mean square of the waveform around its centre, which is the
|
||||
// loudness a listener perceives rather than the tallest spike.
|
||||
let sum = 0;
|
||||
for (const sample of samples) {
|
||||
const centred = (sample - 128) / 128;
|
||||
sum += centred * centred;
|
||||
}
|
||||
const now = performance.now();
|
||||
displayed = smoothVolume(
|
||||
displayed,
|
||||
Math.sqrt(sum / samples.length),
|
||||
now - previousFrame,
|
||||
);
|
||||
previousFrame = now;
|
||||
subscriber.next({ type: "level", level: segmentsForVolume(displayed) });
|
||||
frame = requestAnimationFrame(read);
|
||||
};
|
||||
read();
|
||||
};
|
||||
|
||||
start().catch((e: unknown) => subscriber.next(stateForFailure(e)));
|
||||
|
||||
return release;
|
||||
}).pipe(
|
||||
// Read every animation frame, but quantised to a whole number of segments,
|
||||
// so most frames say nothing new and should not reach React.
|
||||
distinctUntilChanged(
|
||||
(a, b) =>
|
||||
a.type === b.type &&
|
||||
(a.type !== "level" || b.type !== "level" || a.level === b.level),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** What a failure to open the microphone means for the person using it. */
|
||||
function stateForFailure(e: unknown): MicrophoneState {
|
||||
const name = e instanceof Error ? e.name : "";
|
||||
if (name === "NotAllowedError" || name === "SecurityError")
|
||||
return { type: "permission-denied" };
|
||||
if (name === "NotFoundError" || name === "OverconstrainedError")
|
||||
return { type: "no-device" };
|
||||
logger.error("Could not read the microphone level", e);
|
||||
return { type: "no-device" };
|
||||
}
|
||||
|
||||
/**
|
||||
* Loudness below which the microphone is treated as picking up nothing.
|
||||
*
|
||||
|
||||
@@ -8,6 +8,7 @@ import { map, type Observable, of, type SchedulerLike } from "rxjs";
|
||||
import { type RunHelpers, TestScheduler } from "rxjs/testing";
|
||||
import {
|
||||
expect,
|
||||
type Mock,
|
||||
type MockedObject,
|
||||
type MockInstance,
|
||||
onTestFinished,
|
||||
@@ -590,3 +591,92 @@ export class MockConnection extends Connection {
|
||||
public async start(): Promise<void> {}
|
||||
public async stop(): Promise<void> {}
|
||||
}
|
||||
|
||||
export interface StubbedCapture {
|
||||
getUserMedia: Mock;
|
||||
/** Hands the microphone over, as the browser does once permission is given. */
|
||||
grant: () => void;
|
||||
track: { stop: Mock };
|
||||
contexts: { close: Mock }[];
|
||||
/** Runs the animation frames the level meter reads on, in order. */
|
||||
drawFrames: (count: number) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stubs just enough of the capture and Web Audio APIs for a microphone level to
|
||||
* be read, with the grant held back so a test decides when — or whether — it
|
||||
* lands, and with animation frames driven by hand rather than by a clock.
|
||||
*
|
||||
* Call {@link restoreAudioCapture} afterwards, or every later test in the run
|
||||
* inherits the stub.
|
||||
*/
|
||||
export function stubAudioCapture(): StubbedCapture {
|
||||
const track = { stop: vi.fn() };
|
||||
const contexts: { close: Mock }[] = [];
|
||||
let grant = (): void => {};
|
||||
const granted = new Promise<MediaStream>((resolve) => {
|
||||
grant = (): void =>
|
||||
resolve({ getTracks: () => [track] } as unknown as MediaStream);
|
||||
});
|
||||
|
||||
const frames: FrameRequestCallback[] = [];
|
||||
vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => {
|
||||
frames.push(callback);
|
||||
return frames.length;
|
||||
});
|
||||
vi.stubGlobal("cancelAnimationFrame", vi.fn());
|
||||
vi.stubGlobal(
|
||||
"AudioContext",
|
||||
class {
|
||||
public readonly state = "running";
|
||||
public readonly close = vi.fn();
|
||||
public constructor() {
|
||||
contexts.push(this);
|
||||
}
|
||||
public createAnalyser(): object {
|
||||
return {
|
||||
fftSize: 1024,
|
||||
// Digital silence, which is the midpoint of the range and not zero:
|
||||
// a buffer left at zero reads as a full-scale waveform.
|
||||
getByteTimeDomainData: (samples: Uint8Array): void => {
|
||||
samples.fill(128);
|
||||
},
|
||||
};
|
||||
}
|
||||
public createMediaStreamSource(): object {
|
||||
return { connect: (): void => {} };
|
||||
}
|
||||
},
|
||||
);
|
||||
// Only this property: replacing navigator wholesale drops the getters on its
|
||||
// prototype, such as userAgent.
|
||||
Object.defineProperty(navigator, "mediaDevices", {
|
||||
configurable: true,
|
||||
value: { getUserMedia: vi.fn().mockReturnValue(granted) },
|
||||
});
|
||||
|
||||
return {
|
||||
getUserMedia: navigator.mediaDevices.getUserMedia as unknown as Mock,
|
||||
grant: () => grant(),
|
||||
track,
|
||||
contexts,
|
||||
drawFrames: (count): void => {
|
||||
for (let i = 0; i < count; i++) frames.shift()?.(i);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const realMediaDevices = Object.getOwnPropertyDescriptor(
|
||||
navigator,
|
||||
"mediaDevices",
|
||||
);
|
||||
|
||||
/** Undoes {@link stubAudioCapture}. */
|
||||
export function restoreAudioCapture(): void {
|
||||
vi.unstubAllGlobals();
|
||||
if (realMediaDevices === undefined) {
|
||||
Reflect.deleteProperty(navigator, "mediaDevices");
|
||||
} else {
|
||||
Object.defineProperty(navigator, "mediaDevices", realMediaDevices);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user