mirror of
https://github.com/vector-im/element-call.git
synced 2026-01-30 03:15:55 +00:00
Ensure call sound effects are played over the correct sink (#2863)
* Refactor to use AudioContext * Remove unused audio format. * Reduce update frequency for volume * Port to useAudioContext * Port reactionaudiorenderer to useAudioContext * Integrate raise hand sound into call event renderer. * Simplify reaction sounds * only play one sound per reaction type * Start to build out tests * fixup tests / comments * Fix reaction sound * remove console line * Remove another debug line. * fix lint * Use testing library click * lint * fix a few things * Change the way we as unknown the mock RTC session. * Lint * Fix types for MockRTCSession * value change should always be set * Update volume slider description. * Only load reaction sound effects if enabled. * cache improvements * lowercase soundMap * lint * move prefetch sounds to fix hot reload * correct docs * add a header * Wording change --------- Co-authored-by: Hugh Nimmo-Smith <hughns@element.io>
This commit is contained in:
@@ -147,7 +147,6 @@
|
||||
"screenshare_button_label": "Bildschirm teilen",
|
||||
"settings": {
|
||||
"audio_tab": {
|
||||
"effect_volume_description": "Lautstärke anpassen, mit der Reaktionen und Handmeldungen abgespielt werden",
|
||||
"effect_volume_label": "Lautstärke der Soundeffekte"
|
||||
},
|
||||
"developer_settings_label": "Entwicklereinstellungen",
|
||||
|
||||
@@ -147,7 +147,7 @@
|
||||
"screenshare_button_label": "Share screen",
|
||||
"settings": {
|
||||
"audio_tab": {
|
||||
"effect_volume_description": "Adjust the volume at which reactions and hand raised effects play",
|
||||
"effect_volume_description": "Volume for sound effects such as: joining or leaving a call, and reactions.",
|
||||
"effect_volume_label": "Sound effect volume"
|
||||
},
|
||||
"developer_settings_label": "Developer Settings",
|
||||
|
||||
@@ -16,6 +16,9 @@ interface Props {
|
||||
className?: string;
|
||||
label: string;
|
||||
value: number;
|
||||
/**
|
||||
* Event handler called when the value changes during an interaction.
|
||||
*/
|
||||
onValueChange: (value: number) => void;
|
||||
/**
|
||||
* Event handler called when the value changes at the end of an interaction.
|
||||
|
||||
@@ -115,12 +115,12 @@ function useMediaDevice(
|
||||
}, [available, preferredId, select, alwaysDefault]);
|
||||
}
|
||||
|
||||
const deviceStub: MediaDevice = {
|
||||
export const deviceStub: MediaDevice = {
|
||||
available: [],
|
||||
selectedId: undefined,
|
||||
select: () => {},
|
||||
};
|
||||
const devicesStub: MediaDevices = {
|
||||
export const devicesStub: MediaDevices = {
|
||||
audioInput: deviceStub,
|
||||
audioOutput: deviceStub,
|
||||
videoInput: deviceStub,
|
||||
|
||||
@@ -6,25 +6,30 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { render } from "@testing-library/react";
|
||||
import { beforeEach, expect, test } from "vitest";
|
||||
import {
|
||||
afterAll,
|
||||
beforeEach,
|
||||
expect,
|
||||
MockedFunction,
|
||||
test,
|
||||
vitest,
|
||||
} from "vitest";
|
||||
import { MatrixClient } from "matrix-js-sdk/src/client";
|
||||
import { ConnectionState, Room } from "livekit-client";
|
||||
import { ConnectionState } from "livekit-client";
|
||||
import { BehaviorSubject, of } from "rxjs";
|
||||
import { afterEach } from "node:test";
|
||||
import { act } from "react";
|
||||
import { act, ReactNode } from "react";
|
||||
import {
|
||||
CallMembership,
|
||||
type MatrixRTCSession,
|
||||
} from "matrix-js-sdk/src/matrixrtc";
|
||||
import { RoomMember } from "matrix-js-sdk/src/matrix";
|
||||
|
||||
import { soundEffectVolumeSetting } from "../settings/settings";
|
||||
import {
|
||||
EmittableMockLivekitRoom,
|
||||
mockLivekitRoom,
|
||||
mockLocalParticipant,
|
||||
mockMatrixRoom,
|
||||
mockMatrixRoomMember,
|
||||
mockMediaPlay,
|
||||
mockRemoteParticipant,
|
||||
mockRtcMembership,
|
||||
MockRTCSession,
|
||||
@@ -35,37 +40,69 @@ import {
|
||||
CallEventAudioRenderer,
|
||||
MAX_PARTICIPANT_COUNT_FOR_SOUND,
|
||||
} from "./CallEventAudioRenderer";
|
||||
import { useAudioContext } from "../useAudioContext";
|
||||
import { TestReactionsWrapper } from "../utils/testReactions";
|
||||
import { prefetchSounds } from "../soundUtils";
|
||||
|
||||
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
|
||||
const local = mockMatrixRoomMember(localRtcMember);
|
||||
const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
|
||||
const alice = mockMatrixRoomMember(aliceRtcMember);
|
||||
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
|
||||
const bob = mockMatrixRoomMember(bobRtcMember);
|
||||
const localParticipant = mockLocalParticipant({ identity: "" });
|
||||
const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
|
||||
const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
|
||||
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
|
||||
const bobParticipant = mockRemoteParticipant({ identity: bobId });
|
||||
|
||||
const originalPlayFn = window.HTMLMediaElement.prototype.play;
|
||||
|
||||
const enterSound = "http://localhost:3000/src/sound/join_call.ogg";
|
||||
const leaveSound = "http://localhost:3000/src/sound/left_call.ogg";
|
||||
|
||||
beforeEach(() => {
|
||||
soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
|
||||
});
|
||||
vitest.mock("../useAudioContext");
|
||||
vitest.mock("../soundUtils");
|
||||
|
||||
afterEach(() => {
|
||||
window.HTMLMediaElement.prototype.play = originalPlayFn;
|
||||
vitest.resetAllMocks();
|
||||
});
|
||||
|
||||
test("plays a sound when entering a call", () => {
|
||||
const audioIsPlaying: string[] = mockMediaPlay();
|
||||
const matrixRoomMembers = new Map(
|
||||
[local, alice, bob].map((p) => [p.userId, p]),
|
||||
afterAll(() => {
|
||||
vitest.restoreAllMocks();
|
||||
});
|
||||
|
||||
let playSound: MockedFunction<
|
||||
NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
|
||||
>;
|
||||
|
||||
beforeEach(() => {
|
||||
(prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
|
||||
sound: new ArrayBuffer(0),
|
||||
});
|
||||
playSound = vitest.fn();
|
||||
(useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
|
||||
playSound,
|
||||
});
|
||||
});
|
||||
|
||||
function TestComponent({
|
||||
rtcSession,
|
||||
vm,
|
||||
}: {
|
||||
rtcSession: MockRTCSession;
|
||||
vm: CallViewModel;
|
||||
}): ReactNode {
|
||||
return (
|
||||
<TestReactionsWrapper
|
||||
rtcSession={rtcSession as unknown as MatrixRTCSession}
|
||||
>
|
||||
<CallEventAudioRenderer vm={vm} />
|
||||
</TestReactionsWrapper>
|
||||
);
|
||||
}
|
||||
|
||||
function getMockEnv(
|
||||
members: RoomMember[],
|
||||
initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember],
|
||||
): {
|
||||
vm: CallViewModel;
|
||||
session: MockRTCSession;
|
||||
remoteRtcMemberships: BehaviorSubject<CallMembership[]>;
|
||||
} {
|
||||
const matrixRoomMembers = new Map(members.map((p) => [p.userId, p]));
|
||||
const remoteParticipants = of([aliceParticipant]);
|
||||
const liveKitRoom = mockLivekitRoom(
|
||||
{ localParticipant },
|
||||
@@ -75,221 +112,90 @@ test("plays a sound when entering a call", () => {
|
||||
client: {
|
||||
getUserId: () => localRtcMember.sender,
|
||||
getDeviceId: () => localRtcMember.deviceId,
|
||||
on: vitest.fn(),
|
||||
off: vitest.fn(),
|
||||
} as Partial<MatrixClient> as MatrixClient,
|
||||
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
|
||||
});
|
||||
|
||||
const session = new MockRTCSession(matrixRoom, localRtcMember, [
|
||||
aliceRtcMember,
|
||||
]) as unknown as MatrixRTCSession;
|
||||
|
||||
const vm = new CallViewModel(
|
||||
session,
|
||||
liveKitRoom,
|
||||
{
|
||||
kind: E2eeType.PER_PARTICIPANT,
|
||||
},
|
||||
of(ConnectionState.Connected),
|
||||
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
|
||||
initialRemoteRtcMemberships,
|
||||
);
|
||||
|
||||
render(<CallEventAudioRenderer vm={vm} />);
|
||||
expect(audioIsPlaying).toEqual([
|
||||
// Joining the call
|
||||
enterSound,
|
||||
]);
|
||||
});
|
||||
|
||||
test("plays no sound when muted", () => {
|
||||
soundEffectVolumeSetting.setValue(0);
|
||||
const audioIsPlaying: string[] = mockMediaPlay();
|
||||
const matrixRoomMembers = new Map(
|
||||
[local, alice, bob].map((p) => [p.userId, p]),
|
||||
);
|
||||
const remoteParticipants = of([aliceParticipant, bobParticipant]);
|
||||
const liveKitRoom = mockLivekitRoom(
|
||||
{ localParticipant },
|
||||
{ remoteParticipants },
|
||||
);
|
||||
|
||||
const matrixRoom = mockMatrixRoom({
|
||||
client: {
|
||||
getUserId: () => localRtcMember.sender,
|
||||
getDeviceId: () => localRtcMember.deviceId,
|
||||
} as Partial<MatrixClient> as MatrixClient,
|
||||
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
|
||||
});
|
||||
|
||||
const session = new MockRTCSession(matrixRoom, localRtcMember, [
|
||||
aliceRtcMember,
|
||||
]) as unknown as MatrixRTCSession;
|
||||
|
||||
const vm = new CallViewModel(
|
||||
session,
|
||||
liveKitRoom,
|
||||
{
|
||||
kind: E2eeType.PER_PARTICIPANT,
|
||||
},
|
||||
of(ConnectionState.Connected),
|
||||
);
|
||||
|
||||
render(<CallEventAudioRenderer vm={vm} />);
|
||||
// Play a sound when joining a call.
|
||||
expect(audioIsPlaying).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("plays a sound when a user joins", () => {
|
||||
const audioIsPlaying: string[] = mockMediaPlay();
|
||||
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
|
||||
const remoteParticipants = new Map(
|
||||
[aliceParticipant].map((p) => [p.identity, p]),
|
||||
);
|
||||
const liveKitRoom = new EmittableMockLivekitRoom({
|
||||
localParticipant,
|
||||
remoteParticipants,
|
||||
});
|
||||
|
||||
const matrixRoom = mockMatrixRoom({
|
||||
client: {
|
||||
getUserId: () => localRtcMember.sender,
|
||||
getDeviceId: () => localRtcMember.deviceId,
|
||||
} as Partial<MatrixClient> as MatrixClient,
|
||||
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
|
||||
});
|
||||
|
||||
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
|
||||
aliceRtcMember,
|
||||
]);
|
||||
// we give Bob an RTC session now, but no participant yet
|
||||
const session = new MockRTCSession(
|
||||
matrixRoom,
|
||||
localRtcMember,
|
||||
).withMemberships(
|
||||
remoteRtcMemberships.asObservable(),
|
||||
) as unknown as MatrixRTCSession;
|
||||
).withMemberships(remoteRtcMemberships);
|
||||
|
||||
const vm = new CallViewModel(
|
||||
session,
|
||||
liveKitRoom as unknown as Room,
|
||||
session as unknown as MatrixRTCSession,
|
||||
liveKitRoom,
|
||||
{
|
||||
kind: E2eeType.PER_PARTICIPANT,
|
||||
},
|
||||
of(ConnectionState.Connected),
|
||||
);
|
||||
render(<CallEventAudioRenderer vm={vm} />);
|
||||
return { vm, session, remoteRtcMemberships };
|
||||
}
|
||||
|
||||
/**
|
||||
* We don't want to play a sound when loading the call state
|
||||
* because typically this occurs in two stages. We first join
|
||||
* the call as a local participant and *then* the remote
|
||||
* participants join from our perspective. We don't want to make
|
||||
* a noise every time.
|
||||
*/
|
||||
test("plays one sound when entering a call", () => {
|
||||
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
|
||||
render(<TestComponent rtcSession={session} vm={vm} />);
|
||||
// Joining a call usually means remote participants are added later.
|
||||
act(() => {
|
||||
remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
|
||||
});
|
||||
expect(playSound).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
// TODO: Same test?
|
||||
test("plays a sound when a user joins", () => {
|
||||
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
|
||||
render(<TestComponent rtcSession={session} vm={vm} />);
|
||||
|
||||
act(() => {
|
||||
remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
|
||||
});
|
||||
// Play a sound when joining a call.
|
||||
expect(audioIsPlaying).toEqual([
|
||||
// Joining the call
|
||||
enterSound,
|
||||
// Bob joins
|
||||
enterSound,
|
||||
]);
|
||||
expect(playSound).toBeCalledWith("join");
|
||||
});
|
||||
|
||||
test("plays a sound when a user leaves", () => {
|
||||
const audioIsPlaying: string[] = mockMediaPlay();
|
||||
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
|
||||
const remoteParticipants = new Map(
|
||||
[aliceParticipant].map((p) => [p.identity, p]),
|
||||
);
|
||||
const liveKitRoom = new EmittableMockLivekitRoom({
|
||||
localParticipant,
|
||||
remoteParticipants,
|
||||
});
|
||||
|
||||
const matrixRoom = mockMatrixRoom({
|
||||
client: {
|
||||
getUserId: () => localRtcMember.sender,
|
||||
getDeviceId: () => localRtcMember.deviceId,
|
||||
} as Partial<MatrixClient> as MatrixClient,
|
||||
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
|
||||
});
|
||||
|
||||
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
|
||||
aliceRtcMember,
|
||||
]);
|
||||
|
||||
const session = new MockRTCSession(
|
||||
matrixRoom,
|
||||
localRtcMember,
|
||||
).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
|
||||
|
||||
const vm = new CallViewModel(
|
||||
session,
|
||||
liveKitRoom as unknown as Room,
|
||||
{
|
||||
kind: E2eeType.PER_PARTICIPANT,
|
||||
},
|
||||
of(ConnectionState.Connected),
|
||||
);
|
||||
render(<CallEventAudioRenderer vm={vm} />);
|
||||
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
|
||||
render(<TestComponent rtcSession={session} vm={vm} />);
|
||||
|
||||
act(() => {
|
||||
remoteRtcMemberships.next([]);
|
||||
});
|
||||
expect(audioIsPlaying).toEqual([
|
||||
// Joining the call
|
||||
enterSound,
|
||||
// Alice leaves
|
||||
leaveSound,
|
||||
]);
|
||||
expect(playSound).toBeCalledWith("left");
|
||||
});
|
||||
|
||||
test("plays no sound when the session member count is larger than the max, until decreased", () => {
|
||||
const audioIsPlaying: string[] = mockMediaPlay();
|
||||
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
|
||||
const remoteParticipants = new Map(
|
||||
[aliceParticipant].map((p) => [p.identity, p]),
|
||||
);
|
||||
|
||||
test("plays no sound when the participant list is more than the maximum size", () => {
|
||||
const mockRtcMemberships: CallMembership[] = [];
|
||||
|
||||
for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) {
|
||||
mockRtcMemberships.push(
|
||||
mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`),
|
||||
);
|
||||
}
|
||||
|
||||
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
|
||||
const { session, vm, remoteRtcMemberships } = getMockEnv(
|
||||
[local, alice],
|
||||
mockRtcMemberships,
|
||||
);
|
||||
|
||||
const liveKitRoom = new EmittableMockLivekitRoom({
|
||||
localParticipant,
|
||||
remoteParticipants,
|
||||
});
|
||||
|
||||
const matrixRoom = mockMatrixRoom({
|
||||
client: {
|
||||
getUserId: () => localRtcMember.sender,
|
||||
getDeviceId: () => localRtcMember.deviceId,
|
||||
} as Partial<MatrixClient> as MatrixClient,
|
||||
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
|
||||
});
|
||||
|
||||
const session = new MockRTCSession(
|
||||
matrixRoom,
|
||||
localRtcMember,
|
||||
).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
|
||||
|
||||
const vm = new CallViewModel(
|
||||
session,
|
||||
liveKitRoom as unknown as Room,
|
||||
{
|
||||
kind: E2eeType.PER_PARTICIPANT,
|
||||
},
|
||||
of(ConnectionState.Connected),
|
||||
);
|
||||
render(<CallEventAudioRenderer vm={vm} />);
|
||||
expect(audioIsPlaying).toEqual([]);
|
||||
// When the count drops to the max we should play the leave sound
|
||||
render(<TestComponent rtcSession={session} vm={vm} />);
|
||||
expect(playSound).not.toBeCalled();
|
||||
act(() => {
|
||||
remoteRtcMemberships.next(
|
||||
mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1),
|
||||
);
|
||||
});
|
||||
expect(audioIsPlaying).toEqual([leaveSound]);
|
||||
expect(playSound).toBeCalledWith("left");
|
||||
});
|
||||
|
||||
@@ -5,47 +5,76 @@ SPDX-License-Identifier: AGPL-3.0-only
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { ReactNode, useEffect, useRef } from "react";
|
||||
import { filter } from "rxjs";
|
||||
import { ReactNode, useDeferredValue, useEffect, useMemo } from "react";
|
||||
import { filter, interval, throttle } from "rxjs";
|
||||
|
||||
import {
|
||||
soundEffectVolumeSetting as effectSoundVolumeSetting,
|
||||
useSetting,
|
||||
} from "../settings/settings";
|
||||
import { CallViewModel } from "../state/CallViewModel";
|
||||
import enterCallSoundMp3 from "../sound/join_call.mp3";
|
||||
import enterCallSoundOgg from "../sound/join_call.ogg";
|
||||
import joinCallSoundMp3 from "../sound/join_call.mp3";
|
||||
import joinCallSoundOgg from "../sound/join_call.ogg";
|
||||
import leftCallSoundMp3 from "../sound/left_call.mp3";
|
||||
import leftCallSoundOgg from "../sound/left_call.ogg";
|
||||
import handSoundOgg from "../sound/raise_hand.ogg?url";
|
||||
import handSoundMp3 from "../sound/raise_hand.mp3?url";
|
||||
import { useAudioContext } from "../useAudioContext";
|
||||
import { prefetchSounds } from "../soundUtils";
|
||||
import { useReactions } from "../useReactions";
|
||||
import { useLatest } from "../useLatest";
|
||||
|
||||
// Do not play any sounds if the participant count has exceeded this
|
||||
// number.
|
||||
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
|
||||
export const CONCURRENT_AUDIO_CHANNELS = 2;
|
||||
export const THROTTLE_SOUND_EFFECT_MS = 500;
|
||||
|
||||
const sounds = prefetchSounds({
|
||||
join: {
|
||||
mp3: joinCallSoundMp3,
|
||||
ogg: joinCallSoundOgg,
|
||||
},
|
||||
left: {
|
||||
mp3: leftCallSoundMp3,
|
||||
ogg: leftCallSoundOgg,
|
||||
},
|
||||
raiseHand: {
|
||||
mp3: handSoundMp3,
|
||||
ogg: handSoundOgg,
|
||||
},
|
||||
});
|
||||
|
||||
export function CallEventAudioRenderer({
|
||||
vm,
|
||||
}: {
|
||||
vm: CallViewModel;
|
||||
}): ReactNode {
|
||||
const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
|
||||
const callEntered = useRef<(HTMLAudioElement | null)[]>([]);
|
||||
const callLeft = useRef<(HTMLAudioElement | null)[]>([]);
|
||||
const audioEngineCtx = useAudioContext({
|
||||
sounds,
|
||||
latencyHint: "interactive",
|
||||
});
|
||||
const audioEngineRef = useLatest(audioEngineCtx);
|
||||
|
||||
const { raisedHands } = useReactions();
|
||||
const raisedHandCount = useMemo(
|
||||
() => Object.keys(raisedHands).length,
|
||||
[raisedHands],
|
||||
);
|
||||
const previousRaisedHandCount = useDeferredValue(raisedHandCount);
|
||||
|
||||
useEffect(() => {
|
||||
if (effectSoundVolume === 0) {
|
||||
return;
|
||||
if (audioEngineRef.current && previousRaisedHandCount < raisedHandCount) {
|
||||
audioEngineRef.current.playSound("raiseHand");
|
||||
}
|
||||
}, [audioEngineRef, previousRaisedHandCount, raisedHandCount]);
|
||||
|
||||
useEffect(() => {
|
||||
const joinSub = vm.memberChanges
|
||||
.pipe(
|
||||
filter(
|
||||
({ joined, ids }) =>
|
||||
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0,
|
||||
),
|
||||
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
|
||||
)
|
||||
.subscribe(({ joined }) => {
|
||||
const availablePlayer = callEntered.current.find((v) => v?.paused);
|
||||
void availablePlayer?.play();
|
||||
.subscribe(() => {
|
||||
audioEngineRef.current?.playSound("join");
|
||||
});
|
||||
|
||||
const leftSub = vm.memberChanges
|
||||
@@ -54,64 +83,17 @@ export function CallEventAudioRenderer({
|
||||
({ ids, left }) =>
|
||||
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
|
||||
),
|
||||
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
|
||||
)
|
||||
.subscribe(() => {
|
||||
const availablePlayer = callLeft.current.find((v) => v?.paused);
|
||||
void availablePlayer?.play();
|
||||
audioEngineRef.current?.playSound("left");
|
||||
});
|
||||
|
||||
return (): void => {
|
||||
joinSub.unsubscribe();
|
||||
leftSub.unsubscribe();
|
||||
};
|
||||
}, [effectSoundVolume, callEntered, callLeft, vm]);
|
||||
}, [audioEngineRef, vm]);
|
||||
|
||||
// Set volume.
|
||||
useEffect(() => {
|
||||
callEntered.current.forEach((a) => {
|
||||
if (a) {
|
||||
a.volume = effectSoundVolume;
|
||||
}
|
||||
});
|
||||
callLeft.current.forEach((a) => {
|
||||
if (a) {
|
||||
a.volume = effectSoundVolume;
|
||||
}
|
||||
});
|
||||
}, [callEntered, callLeft, effectSoundVolume]);
|
||||
|
||||
// Do not render any audio elements if playback is disabled. Will save
|
||||
// audio file fetches.
|
||||
if (effectSoundVolume === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
// Will play as soon as it's mounted, which is what we want as this will
|
||||
// play when the call is entered.
|
||||
<>
|
||||
{Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
|
||||
<audio
|
||||
key={index}
|
||||
ref={(r) => (callEntered.current[index] = r)}
|
||||
preload="auto"
|
||||
hidden
|
||||
>
|
||||
<source src={enterCallSoundOgg} type="audio/ogg; codecs=vorbis" />
|
||||
<source src={enterCallSoundMp3} type="audio/mpeg" />
|
||||
</audio>
|
||||
))}
|
||||
{Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
|
||||
<audio
|
||||
key={index}
|
||||
ref={(r) => (callLeft.current[index] = r)}
|
||||
preload="auto"
|
||||
hidden
|
||||
>
|
||||
<source src={leftCallSoundOgg} type="audio/ogg; codecs=vorbis" />
|
||||
<source src={leftCallSoundMp3} type="audio/mpeg" />
|
||||
</audio>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
return <></>;
|
||||
}
|
||||
|
||||
@@ -331,7 +331,7 @@ export const GroupCallView: FC<Props> = ({
|
||||
<ActiveCall
|
||||
client={client}
|
||||
matrixInfo={matrixInfo}
|
||||
rtcSession={rtcSession}
|
||||
rtcSession={rtcSession as unknown as MatrixRTCSession}
|
||||
participantCount={participantCount}
|
||||
onLeave={onLeave}
|
||||
hideHeader={hideHeader}
|
||||
|
||||
@@ -19,7 +19,6 @@ import {
|
||||
TouchEvent,
|
||||
forwardRef,
|
||||
useCallback,
|
||||
useDeferredValue,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
@@ -81,11 +80,8 @@ import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout";
|
||||
import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout";
|
||||
import { GridTileViewModel, TileViewModel } from "../state/TileViewModel";
|
||||
import { ReactionsProvider, useReactions } from "../useReactions";
|
||||
import handSoundOgg from "../sound/raise_hand.ogg?url";
|
||||
import handSoundMp3 from "../sound/raise_hand.mp3?url";
|
||||
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
|
||||
import { useSwitchCamera } from "./useSwitchCamera";
|
||||
import { soundEffectVolumeSetting, useSetting } from "../settings/settings";
|
||||
import { ReactionsOverlay } from "./ReactionsOverlay";
|
||||
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
|
||||
|
||||
@@ -178,14 +174,7 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
connState,
|
||||
onShareClick,
|
||||
}) => {
|
||||
const [soundEffectVolume] = useSetting(soundEffectVolumeSetting);
|
||||
const { supportsReactions, raisedHands, sendReaction, toggleRaisedHand } =
|
||||
useReactions();
|
||||
const raisedHandCount = useMemo(
|
||||
() => Object.keys(raisedHands).length,
|
||||
[raisedHands],
|
||||
);
|
||||
const previousRaisedHandCount = useDeferredValue(raisedHandCount);
|
||||
const { supportsReactions, sendReaction, toggleRaisedHand } = useReactions();
|
||||
|
||||
useWakeLock();
|
||||
|
||||
@@ -335,25 +324,6 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
[vm],
|
||||
);
|
||||
|
||||
// Play a sound when the raised hand count increases.
|
||||
const handRaisePlayer = useRef<HTMLAudioElement>(null);
|
||||
useEffect(() => {
|
||||
if (!handRaisePlayer.current) {
|
||||
return;
|
||||
}
|
||||
if (previousRaisedHandCount < raisedHandCount) {
|
||||
handRaisePlayer.current.volume = soundEffectVolume;
|
||||
handRaisePlayer.current.play().catch((ex) => {
|
||||
logger.warn("Failed to play raise hand sound", ex);
|
||||
});
|
||||
}
|
||||
}, [
|
||||
raisedHandCount,
|
||||
handRaisePlayer,
|
||||
previousRaisedHandCount,
|
||||
soundEffectVolume,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
widget?.api.transport
|
||||
.send(
|
||||
@@ -667,10 +637,6 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
<RoomAudioRenderer />
|
||||
{renderContent()}
|
||||
<CallEventAudioRenderer vm={vm} />
|
||||
<audio ref={handRaisePlayer} preload="auto" hidden>
|
||||
<source src={handSoundOgg} type="audio/ogg; codecs=vorbis" />
|
||||
<source src={handSoundMp3} type="audio/mpeg" />
|
||||
</audio>
|
||||
<ReactionsAudioRenderer />
|
||||
<ReactionsOverlay />
|
||||
{footer}
|
||||
|
||||
@@ -6,9 +6,18 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { render } from "@testing-library/react";
|
||||
import { afterAll, expect, test } from "vitest";
|
||||
import {
|
||||
afterAll,
|
||||
beforeEach,
|
||||
expect,
|
||||
test,
|
||||
vitest,
|
||||
MockedFunction,
|
||||
Mock,
|
||||
} from "vitest";
|
||||
import { TooltipProvider } from "@vector-im/compound-web";
|
||||
import { act, ReactNode } from "react";
|
||||
import { afterEach } from "node:test";
|
||||
|
||||
import {
|
||||
MockRoom,
|
||||
@@ -16,12 +25,13 @@ import {
|
||||
TestReactionsWrapper,
|
||||
} from "../utils/testReactions";
|
||||
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
|
||||
import { GenericReaction, ReactionSet } from "../reactions";
|
||||
import {
|
||||
playReactionsSound,
|
||||
soundEffectVolumeSetting,
|
||||
} from "../settings/settings";
|
||||
import { mockMediaPlay } from "../utils/test";
|
||||
import { useAudioContext } from "../useAudioContext";
|
||||
import { GenericReaction, ReactionSet } from "../reactions";
|
||||
import { prefetchSounds } from "../soundUtils";
|
||||
|
||||
const memberUserIdAlice = "@alice:example.org";
|
||||
const memberUserIdBob = "@bob:example.org";
|
||||
@@ -50,11 +60,31 @@ function TestComponent({
|
||||
);
|
||||
}
|
||||
|
||||
const originalPlayFn = window.HTMLMediaElement.prototype.play;
|
||||
afterAll(() => {
|
||||
vitest.mock("../useAudioContext");
|
||||
vitest.mock("../soundUtils");
|
||||
|
||||
afterEach(() => {
|
||||
vitest.resetAllMocks();
|
||||
playReactionsSound.setValue(playReactionsSound.defaultValue);
|
||||
soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
|
||||
window.HTMLMediaElement.prototype.play = originalPlayFn;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
vitest.restoreAllMocks();
|
||||
});
|
||||
|
||||
let playSound: Mock<
|
||||
NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
|
||||
>;
|
||||
|
||||
beforeEach(() => {
|
||||
(prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
|
||||
sound: new ArrayBuffer(0),
|
||||
});
|
||||
playSound = vitest.fn();
|
||||
(useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
|
||||
playSound,
|
||||
});
|
||||
});
|
||||
|
||||
test("preloads all audio elements", () => {
|
||||
@@ -63,25 +93,11 @@ test("preloads all audio elements", () => {
|
||||
new MockRoom(memberUserIdAlice),
|
||||
membership,
|
||||
);
|
||||
const { container } = render(<TestComponent rtcSession={rtcSession} />);
|
||||
expect(container.getElementsByTagName("audio")).toHaveLength(
|
||||
// All reactions plus the generic sound
|
||||
ReactionSet.filter((r) => r.sound).length + 1,
|
||||
);
|
||||
});
|
||||
|
||||
test("loads no audio elements when disabled in settings", () => {
|
||||
playReactionsSound.setValue(false);
|
||||
const rtcSession = new MockRTCSession(
|
||||
new MockRoom(memberUserIdAlice),
|
||||
membership,
|
||||
);
|
||||
const { container } = render(<TestComponent rtcSession={rtcSession} />);
|
||||
expect(container.getElementsByTagName("audio")).toHaveLength(0);
|
||||
render(<TestComponent rtcSession={rtcSession} />);
|
||||
expect(prefetchSounds).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test("will play an audio sound when there is a reaction", () => {
|
||||
const audioIsPlaying: string[] = mockMediaPlay();
|
||||
playReactionsSound.setValue(true);
|
||||
const room = new MockRoom(memberUserIdAlice);
|
||||
const rtcSession = new MockRTCSession(room, membership);
|
||||
@@ -97,12 +113,10 @@ test("will play an audio sound when there is a reaction", () => {
|
||||
act(() => {
|
||||
room.testSendReaction(memberEventAlice, chosenReaction, membership);
|
||||
});
|
||||
expect(audioIsPlaying).toHaveLength(1);
|
||||
expect(audioIsPlaying[0]).toContain(chosenReaction.sound?.ogg);
|
||||
expect(playSound).toHaveBeenCalledWith(chosenReaction.name);
|
||||
});
|
||||
|
||||
test("will play the generic audio sound when there is soundless reaction", () => {
|
||||
const audioIsPlaying: string[] = mockMediaPlay();
|
||||
playReactionsSound.setValue(true);
|
||||
const room = new MockRoom(memberUserIdAlice);
|
||||
const rtcSession = new MockRTCSession(room, membership);
|
||||
@@ -118,34 +132,10 @@ test("will play the generic audio sound when there is soundless reaction", () =>
|
||||
act(() => {
|
||||
room.testSendReaction(memberEventAlice, chosenReaction, membership);
|
||||
});
|
||||
expect(audioIsPlaying).toHaveLength(1);
|
||||
expect(audioIsPlaying[0]).toContain(GenericReaction.sound?.ogg);
|
||||
});
|
||||
|
||||
test("will play an audio sound with the correct volume", () => {
|
||||
playReactionsSound.setValue(true);
|
||||
soundEffectVolumeSetting.setValue(0.5);
|
||||
const room = new MockRoom(memberUserIdAlice);
|
||||
const rtcSession = new MockRTCSession(room, membership);
|
||||
const { getByTestId } = render(<TestComponent rtcSession={rtcSession} />);
|
||||
|
||||
// Find the first reaction with a sound effect
|
||||
const chosenReaction = ReactionSet.find((r) => !!r.sound);
|
||||
if (!chosenReaction) {
|
||||
throw Error(
|
||||
"No reactions have sounds configured, this test cannot succeed",
|
||||
);
|
||||
}
|
||||
act(() => {
|
||||
room.testSendReaction(memberEventAlice, chosenReaction, membership);
|
||||
});
|
||||
expect((getByTestId(chosenReaction.name) as HTMLAudioElement).volume).toEqual(
|
||||
0.5,
|
||||
);
|
||||
expect(playSound).toHaveBeenCalledWith(GenericReaction.name);
|
||||
});
|
||||
|
||||
test("will play multiple audio sounds when there are multiple different reactions", () => {
|
||||
const audioIsPlaying: string[] = mockMediaPlay();
|
||||
playReactionsSound.setValue(true);
|
||||
|
||||
const room = new MockRoom(memberUserIdAlice);
|
||||
@@ -164,7 +154,6 @@ test("will play multiple audio sounds when there are multiple different reaction
|
||||
room.testSendReaction(memberEventBob, reaction2, membership);
|
||||
room.testSendReaction(memberEventCharlie, reaction1, membership);
|
||||
});
|
||||
expect(audioIsPlaying).toHaveLength(2);
|
||||
expect(audioIsPlaying[0]).toContain(reaction1.sound?.ogg);
|
||||
expect(audioIsPlaying[1]).toContain(reaction2.sound?.ogg);
|
||||
expect(playSound).toHaveBeenCalledWith(reaction1.name);
|
||||
expect(playSound).toHaveBeenCalledWith(reaction2.name);
|
||||
});
|
||||
|
||||
@@ -5,70 +5,67 @@ SPDX-License-Identifier: AGPL-3.0-only
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { ReactNode, useEffect, useRef } from "react";
|
||||
import { ReactNode, useDeferredValue, useEffect, useState } from "react";
|
||||
|
||||
import { useReactions } from "../useReactions";
|
||||
import {
|
||||
playReactionsSound,
|
||||
soundEffectVolumeSetting as effectSoundVolumeSetting,
|
||||
useSetting,
|
||||
} from "../settings/settings";
|
||||
import { playReactionsSound, useSetting } from "../settings/settings";
|
||||
import { GenericReaction, ReactionSet } from "../reactions";
|
||||
import { useAudioContext } from "../useAudioContext";
|
||||
import { prefetchSounds } from "../soundUtils";
|
||||
import { useLatest } from "../useLatest";
|
||||
|
||||
const soundMap = Object.fromEntries([
|
||||
...ReactionSet.filter((v) => v.sound !== undefined).map((v) => [
|
||||
v.name,
|
||||
v.sound!,
|
||||
]),
|
||||
[GenericReaction.name, GenericReaction.sound],
|
||||
]);
|
||||
|
||||
export function ReactionsAudioRenderer(): ReactNode {
|
||||
const { reactions } = useReactions();
|
||||
const [shouldPlay] = useSetting(playReactionsSound);
|
||||
const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
|
||||
const audioElements = useRef<Record<string, HTMLAudioElement | null>>({});
|
||||
const [soundCache, setSoundCache] = useState<ReturnType<
|
||||
typeof prefetchSounds
|
||||
> | null>(null);
|
||||
const audioEngineCtx = useAudioContext({
|
||||
sounds: soundCache,
|
||||
latencyHint: "interactive",
|
||||
});
|
||||
const audioEngineRef = useLatest(audioEngineCtx);
|
||||
const oldReactions = useDeferredValue(reactions);
|
||||
|
||||
useEffect(() => {
|
||||
if (!audioElements.current) {
|
||||
if (!shouldPlay || soundCache) {
|
||||
return;
|
||||
}
|
||||
// This is fine even if we load the component multiple times,
|
||||
// as the browser's cache should ensure once the media is loaded
|
||||
// once that future fetches come via the cache.
|
||||
setSoundCache(prefetchSounds(soundMap));
|
||||
}, [soundCache, shouldPlay]);
|
||||
|
||||
if (!shouldPlay) {
|
||||
useEffect(() => {
|
||||
if (!shouldPlay || !audioEngineRef.current) {
|
||||
return;
|
||||
}
|
||||
const oldReactionSet = new Set(
|
||||
Object.values(oldReactions).map((r) => r.name),
|
||||
);
|
||||
for (const reactionName of new Set(
|
||||
Object.values(reactions).map((r) => r.name),
|
||||
)) {
|
||||
const audioElement =
|
||||
audioElements.current[reactionName] ?? audioElements.current.generic;
|
||||
if (audioElement?.paused) {
|
||||
audioElement.volume = effectSoundVolume;
|
||||
void audioElement.play();
|
||||
if (oldReactionSet.has(reactionName)) {
|
||||
// Don't replay old reactions
|
||||
return;
|
||||
}
|
||||
if (soundMap[reactionName]) {
|
||||
audioEngineRef.current.playSound(reactionName);
|
||||
} else {
|
||||
// Fallback sounds.
|
||||
audioEngineRef.current.playSound("generic");
|
||||
}
|
||||
}
|
||||
}, [audioElements, shouldPlay, reactions, effectSoundVolume]);
|
||||
|
||||
// Do not render any audio elements if playback is disabled. Will save
|
||||
// audio file fetches.
|
||||
if (!shouldPlay) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// NOTE: We load all audio elements ahead of time to allow the cache
|
||||
// to be populated, rather than risk a cache miss and have the audio
|
||||
// be delayed.
|
||||
return (
|
||||
<>
|
||||
{[GenericReaction, ...ReactionSet].map(
|
||||
(r) =>
|
||||
r.sound && (
|
||||
<audio
|
||||
ref={(el) => (audioElements.current[r.name] = el)}
|
||||
data-testid={r.name}
|
||||
key={r.name}
|
||||
preload="auto"
|
||||
hidden
|
||||
>
|
||||
<source src={r.sound.ogg} type="audio/ogg; codecs=vorbis" />
|
||||
{r.sound.mp3 ? (
|
||||
<source src={r.sound.mp3} type="audio/mpeg" />
|
||||
) : null}
|
||||
</audio>
|
||||
),
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}, [audioEngineRef, shouldPlay, oldReactions, reactions]);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { ChangeEvent, FC, useCallback } from "react";
|
||||
import { ChangeEvent, FC, useCallback, useState } from "react";
|
||||
import { Trans, useTranslation } from "react-i18next";
|
||||
import { MatrixClient } from "matrix-js-sdk/src/matrix";
|
||||
import { Root as Form, Text } from "@vector-im/compound-web";
|
||||
@@ -83,8 +83,8 @@ export const SettingsModal: FC<Props> = ({
|
||||
|
||||
const devices = useMediaDevices();
|
||||
useMediaDeviceNames(devices, open);
|
||||
|
||||
const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting);
|
||||
const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
|
||||
|
||||
const audioTab: Tab<SettingsTab> = {
|
||||
key: "audio",
|
||||
@@ -107,8 +107,9 @@ export const SettingsModal: FC<Props> = ({
|
||||
<p>{t("settings.audio_tab.effect_volume_description")}</p>
|
||||
<Slider
|
||||
label={t("video_tile.volume")}
|
||||
value={soundVolume}
|
||||
onValueChange={setSoundVolume}
|
||||
value={soundVolumeRaw}
|
||||
onValueChange={setSoundVolumeRaw}
|
||||
onValueCommit={setSoundVolume}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.01}
|
||||
|
||||
63
src/soundUtils.ts
Normal file
63
src/soundUtils.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
Copyright 2024 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { logger } from "matrix-js-sdk/src/logger";
|
||||
|
||||
type SoundDefinition = { mp3?: string; ogg: string };
|
||||
|
||||
export type PrefetchedSounds<S extends string> = Promise<
|
||||
Record<S, ArrayBuffer>
|
||||
>;
|
||||
|
||||
/**
|
||||
* Determine the best format we can use to play our sounds
|
||||
* through. We prefer ogg support if possible, but will fall
|
||||
* back to MP3.
|
||||
* @returns "ogg" if the browser is likely to support it, or "mp3" otherwise.
|
||||
*/
|
||||
function getPreferredAudioFormat(): "ogg" | "mp3" {
|
||||
const a = document.createElement("audio");
|
||||
if (a.canPlayType("audio/ogg") === "maybe") {
|
||||
return "ogg";
|
||||
}
|
||||
// Otherwise just assume MP3, as that has a chance of being more widely supported.
|
||||
return "mp3";
|
||||
}
|
||||
|
||||
const preferredFormat = getPreferredAudioFormat();
|
||||
|
||||
/**
|
||||
* Prefetch sounds to be used by the AudioContext. This can
|
||||
* be called outside the scope of a component to ensure the
|
||||
* sounds load ahead of time.
|
||||
* @param sounds A set of sound files that may be played.
|
||||
* @returns A map of sound files to buffers.
|
||||
*/
|
||||
export async function prefetchSounds<S extends string>(
|
||||
sounds: Record<S, SoundDefinition>,
|
||||
): PrefetchedSounds<S> {
|
||||
const buffers: Record<string, ArrayBuffer> = {};
|
||||
await Promise.all(
|
||||
Object.entries(sounds).map(async ([name, file]) => {
|
||||
const { mp3, ogg } = file as SoundDefinition;
|
||||
// Use preferred format, fallback to ogg if no mp3 is provided.
|
||||
// Load an audio file
|
||||
const response = await fetch(
|
||||
preferredFormat === "ogg" ? ogg : (mp3 ?? ogg),
|
||||
);
|
||||
if (!response.ok) {
|
||||
// If the sound doesn't load, it's not the end of the world. We won't play
|
||||
// the sound when requested, but it's better than failing the whole application.
|
||||
logger.warn(`Could not load sound ${name}, response was not okay`);
|
||||
return;
|
||||
}
|
||||
// Decode it
|
||||
buffers[name] = await response.arrayBuffer();
|
||||
}),
|
||||
);
|
||||
return buffers as Record<S, ArrayBuffer>;
|
||||
}
|
||||
@@ -335,7 +335,7 @@ function findMatrixRoomMember(
|
||||
// must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon
|
||||
if (parts.length < 3) {
|
||||
logger.warn(
|
||||
"Livekit participants ID doesn't look like a userId:deviceId combination",
|
||||
`Livekit participants ID (${id}) doesn't look like a userId:deviceId combination`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
129
src/useAudioContext.test.tsx
Normal file
129
src/useAudioContext.test.tsx
Normal file
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
Copyright 2024 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, test, vitest } from "vitest";
|
||||
import { FC } from "react";
|
||||
import { render } from "@testing-library/react";
|
||||
import { afterEach } from "node:test";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
|
||||
import { deviceStub, MediaDevicesContext } from "./livekit/MediaDevicesContext";
|
||||
import { useAudioContext } from "./useAudioContext";
|
||||
import { soundEffectVolumeSetting } from "./settings/settings";
|
||||
|
||||
const staticSounds = Promise.resolve({
|
||||
aSound: new ArrayBuffer(0),
|
||||
});
|
||||
|
||||
const TestComponent: FC = () => {
|
||||
const audioCtx = useAudioContext({
|
||||
sounds: staticSounds,
|
||||
latencyHint: "balanced",
|
||||
});
|
||||
if (!audioCtx) {
|
||||
return null;
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<button onClick={() => audioCtx.playSound("aSound")}>Valid sound</button>
|
||||
{/* eslint-disable-next-line @typescript-eslint/no-explicit-any*/}
|
||||
<button onClick={() => audioCtx.playSound("not-valid" as any)}>
|
||||
Invalid sound
|
||||
</button>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
class MockAudioContext {
|
||||
public static testContext: MockAudioContext;
|
||||
|
||||
public constructor() {
|
||||
MockAudioContext.testContext = this;
|
||||
}
|
||||
|
||||
public gain = vitest.mocked(
|
||||
{
|
||||
connect: () => {},
|
||||
gain: {
|
||||
setValueAtTime: vitest.fn(),
|
||||
},
|
||||
},
|
||||
true,
|
||||
);
|
||||
|
||||
public setSinkId = vitest.fn().mockResolvedValue(undefined);
|
||||
public decodeAudioData = vitest.fn().mockReturnValue(1);
|
||||
public createBufferSource = vitest.fn().mockReturnValue(
|
||||
vitest.mocked({
|
||||
connect: (v: unknown) => v,
|
||||
start: () => {},
|
||||
}),
|
||||
);
|
||||
public createGain = vitest.fn().mockReturnValue(this.gain);
|
||||
public close = vitest.fn().mockResolvedValue(undefined);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vitest.unstubAllGlobals();
|
||||
});
|
||||
|
||||
test("can play a single sound", async () => {
|
||||
const user = userEvent.setup();
|
||||
vitest.stubGlobal("AudioContext", MockAudioContext);
|
||||
const { findByText } = render(<TestComponent />);
|
||||
await user.click(await findByText("Valid sound"));
|
||||
expect(
|
||||
MockAudioContext.testContext.createBufferSource,
|
||||
).toHaveBeenCalledOnce();
|
||||
});
|
||||
test("will ignore sounds that are not registered", async () => {
|
||||
const user = userEvent.setup();
|
||||
vitest.stubGlobal("AudioContext", MockAudioContext);
|
||||
const { findByText } = render(<TestComponent />);
|
||||
await user.click(await findByText("Invalid sound"));
|
||||
expect(
|
||||
MockAudioContext.testContext.createBufferSource,
|
||||
).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("will use the correct device", () => {
|
||||
vitest.stubGlobal("AudioContext", MockAudioContext);
|
||||
render(
|
||||
<MediaDevicesContext.Provider
|
||||
value={{
|
||||
audioInput: deviceStub,
|
||||
audioOutput: {
|
||||
selectedId: "chosen-device",
|
||||
available: [],
|
||||
select: () => {},
|
||||
},
|
||||
videoInput: deviceStub,
|
||||
startUsingDeviceNames: () => {},
|
||||
stopUsingDeviceNames: () => {},
|
||||
}}
|
||||
>
|
||||
<TestComponent />
|
||||
</MediaDevicesContext.Provider>,
|
||||
);
|
||||
expect(
|
||||
MockAudioContext.testContext.createBufferSource,
|
||||
).not.toHaveBeenCalled();
|
||||
expect(MockAudioContext.testContext.setSinkId).toHaveBeenCalledWith(
|
||||
"chosen-device",
|
||||
);
|
||||
});
|
||||
|
||||
test("will use the correct volume level", async () => {
|
||||
const user = userEvent.setup();
|
||||
vitest.stubGlobal("AudioContext", MockAudioContext);
|
||||
soundEffectVolumeSetting.setValue(0.33);
|
||||
const { findByText } = render(<TestComponent />);
|
||||
await user.click(await findByText("Valid sound"));
|
||||
expect(
|
||||
MockAudioContext.testContext.gain.gain.setValueAtTime,
|
||||
).toHaveBeenCalledWith(0.33, 0);
|
||||
});
|
||||
124
src/useAudioContext.tsx
Normal file
124
src/useAudioContext.tsx
Normal file
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
Copyright 2024 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { logger } from "matrix-js-sdk/src/logger";
|
||||
import { useState, useEffect } from "react";
|
||||
|
||||
import {
|
||||
soundEffectVolumeSetting as effectSoundVolumeSetting,
|
||||
useSetting,
|
||||
} from "./settings/settings";
|
||||
import { useMediaDevices } from "./livekit/MediaDevicesContext";
|
||||
import { PrefetchedSounds } from "./soundUtils";
|
||||
|
||||
/**
|
||||
* Play a sound though a given AudioContext. Will take
|
||||
* care of connecting the correct buffer and gating
|
||||
* through gain.
|
||||
* @param volume The volume to play at.
|
||||
* @param ctx The context to play through.
|
||||
* @param buffer The buffer to play.
|
||||
*/
|
||||
function playSound(
|
||||
ctx: AudioContext,
|
||||
buffer: AudioBuffer,
|
||||
volume: number,
|
||||
): void {
|
||||
const gain = ctx.createGain();
|
||||
gain.gain.setValueAtTime(volume, 0);
|
||||
const src = ctx.createBufferSource();
|
||||
src.buffer = buffer;
|
||||
src.connect(gain).connect(ctx.destination);
|
||||
src.start();
|
||||
}
|
||||
|
||||
interface Props<S extends string> {
|
||||
/**
|
||||
* The sounds to play. If no sounds should be played then
|
||||
* this can be set to null, which will prevent the audio
|
||||
* context from being created.
|
||||
*/
|
||||
sounds: PrefetchedSounds<S> | null;
|
||||
latencyHint: AudioContextLatencyCategory;
|
||||
}
|
||||
|
||||
interface UseAudioContext<S> {
|
||||
playSound(soundName: S): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an audio context which can be used to play
|
||||
* a set of preloaded sounds.
|
||||
* @param props
|
||||
* @returns Either an instance that can be used to play sounds, or null if not ready.
|
||||
*/
|
||||
export function useAudioContext<S extends string>(
|
||||
props: Props<S>,
|
||||
): UseAudioContext<S> | null {
|
||||
const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
|
||||
const devices = useMediaDevices();
|
||||
const [audioContext, setAudioContext] = useState<AudioContext>();
|
||||
const [audioBuffers, setAudioBuffers] = useState<Record<S, AudioBuffer>>();
|
||||
|
||||
useEffect(() => {
|
||||
const sounds = props.sounds;
|
||||
if (!sounds) {
|
||||
return;
|
||||
}
|
||||
const ctx = new AudioContext({
|
||||
// We want low latency for these effects.
|
||||
latencyHint: props.latencyHint,
|
||||
});
|
||||
|
||||
// We want to clone the content of our preloaded
|
||||
// sound buffers into this context. The context may
|
||||
// close during this process, so it's okay if it throws.
|
||||
(async (): Promise<void> => {
|
||||
const buffers: Record<string, AudioBuffer> = {};
|
||||
for (const [name, buffer] of Object.entries<ArrayBuffer>(await sounds)) {
|
||||
const audioBuffer = await ctx.decodeAudioData(buffer.slice(0));
|
||||
buffers[name] = audioBuffer;
|
||||
}
|
||||
setAudioBuffers(buffers as Record<S, AudioBuffer>);
|
||||
})().catch((ex) => {
|
||||
logger.debug("Failed to setup audio context", ex);
|
||||
});
|
||||
|
||||
setAudioContext(ctx);
|
||||
return (): void => {
|
||||
void ctx.close().catch((ex) => {
|
||||
logger.debug("Failed to close audio engine", ex);
|
||||
});
|
||||
setAudioContext(undefined);
|
||||
};
|
||||
}, [props.sounds, props.latencyHint]);
|
||||
|
||||
// Update the sink ID whenever we change devices.
|
||||
useEffect(() => {
|
||||
if (audioContext && "setSinkId" in audioContext) {
|
||||
// https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
|
||||
// @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
|
||||
audioContext.setSinkId(devices.audioOutput.selectedId).catch((ex) => {
|
||||
logger.warn("Unable to change sink for audio context", ex);
|
||||
});
|
||||
}
|
||||
}, [audioContext, devices]);
|
||||
|
||||
// Don't return a function until we're ready.
|
||||
if (!audioContext || !audioBuffers) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
playSound: (name): void => {
|
||||
if (!audioBuffers[name]) {
|
||||
logger.debug(`Tried to play a sound that wasn't buffered (${name})`);
|
||||
return;
|
||||
}
|
||||
return playSound(audioContext, audioBuffers[name], effectSoundVolume);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -27,9 +27,7 @@ import {
|
||||
RemoteParticipant,
|
||||
RemoteTrackPublication,
|
||||
Room as LivekitRoom,
|
||||
RoomEvent,
|
||||
} from "livekit-client";
|
||||
import { EventEmitter } from "stream";
|
||||
|
||||
import {
|
||||
LocalUserMediaViewModel,
|
||||
@@ -153,33 +151,6 @@ export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
|
||||
return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
|
||||
}
|
||||
|
||||
/**
|
||||
* A mock of a Livekit Room that can emit events.
|
||||
*/
|
||||
export class EmittableMockLivekitRoom extends EventEmitter {
|
||||
public localParticipant?: LocalParticipant;
|
||||
public remoteParticipants: Map<string, RemoteParticipant>;
|
||||
|
||||
public constructor(room: {
|
||||
localParticipant?: LocalParticipant;
|
||||
remoteParticipants: Map<string, RemoteParticipant>;
|
||||
}) {
|
||||
super();
|
||||
this.localParticipant = room.localParticipant;
|
||||
this.remoteParticipants = room.remoteParticipants ?? new Map();
|
||||
}
|
||||
|
||||
public addParticipant(remoteParticipant: RemoteParticipant): void {
|
||||
this.remoteParticipants.set(remoteParticipant.identity, remoteParticipant);
|
||||
this.emit(RoomEvent.ParticipantConnected, remoteParticipant);
|
||||
}
|
||||
|
||||
public removeParticipant(remoteParticipant: RemoteParticipant): void {
|
||||
this.remoteParticipants.delete(remoteParticipant.identity);
|
||||
this.emit(RoomEvent.ParticipantDisconnected, remoteParticipant);
|
||||
}
|
||||
}
|
||||
|
||||
export function mockLivekitRoom(
|
||||
room: Partial<LivekitRoom>,
|
||||
{
|
||||
@@ -280,15 +251,6 @@ export function mockConfig(config: Partial<ResolvedConfigOptions> = {}): void {
|
||||
});
|
||||
}
|
||||
|
||||
export function mockMediaPlay(): string[] {
|
||||
const audioIsPlaying: string[] = [];
|
||||
window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
|
||||
audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
|
||||
return Promise.resolve();
|
||||
};
|
||||
return audioIsPlaying;
|
||||
}
|
||||
|
||||
export class MockRTCSession extends TypedEventEmitter<
|
||||
MatrixRTCSessionEvent,
|
||||
MatrixRTCSessionEventHandlerMap
|
||||
|
||||
@@ -32,7 +32,7 @@ export const TestReactionsWrapper = ({
|
||||
rtcSession,
|
||||
children,
|
||||
}: PropsWithChildren<{
|
||||
rtcSession: MockRTCSession;
|
||||
rtcSession: MockRTCSession | MatrixRTCSession;
|
||||
}>): ReactNode => {
|
||||
return (
|
||||
<ReactionsProvider rtcSession={rtcSession as unknown as MatrixRTCSession}>
|
||||
@@ -203,4 +203,12 @@ export class MockRoom extends EventEmitter {
|
||||
});
|
||||
return evt.getId()!;
|
||||
}
|
||||
|
||||
public getMember(): void {
|
||||
return;
|
||||
}
|
||||
|
||||
public testGetAsMatrixRoom(): Room {
|
||||
return this as unknown as Room;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user