Ensure call sound effects are played over the correct sink (#2863)

* Refactor to use AudioContext

* Remove unused audio format.

* Reduce update frequency for volume

* Port to useAudioContext

* Port reactionaudiorenderer to useAudioContext

* Integrate raise hand sound into call event renderer.

* Simplify reaction sounds

* only play one sound per reaction type

* Start to build out tests

* fixup tests / comments

* Fix reaction sound

* remove console line

* Remove another debug line.

* fix lint

* Use testing library click

* lint

* fix a few things

* Change the way we as unknown the mock RTC session.

* Lint

* Fix types for MockRTCSession

* value change should always be set

* Update volume slider description.

* Only load reaction sound effects if enabled.

* cache improvements

* lowercase soundMap

* lint

* move prefetch sounds to fix hot reload

* correct docs

* add a header

* Wording change

---------

Co-authored-by: Hugh Nimmo-Smith <hughns@element.io>
This commit is contained in:
Will Hunt
2024-12-09 11:39:16 +00:00
committed by GitHub
parent 9d4cd211ed
commit a8a95c3f00
17 changed files with 575 additions and 446 deletions

View File

@@ -147,7 +147,6 @@
"screenshare_button_label": "Bildschirm teilen",
"settings": {
"audio_tab": {
"effect_volume_description": "Lautstärke anpassen, mit der Reaktionen und Handmeldungen abgespielt werden",
"effect_volume_label": "Lautstärke der Soundeffekte"
},
"developer_settings_label": "Entwicklereinstellungen",

View File

@@ -147,7 +147,7 @@
"screenshare_button_label": "Share screen",
"settings": {
"audio_tab": {
"effect_volume_description": "Adjust the volume at which reactions and hand raised effects play",
"effect_volume_description": "Volume for sound effects such as: joining or leaving a call, and reactions.",
"effect_volume_label": "Sound effect volume"
},
"developer_settings_label": "Developer Settings",

View File

@@ -16,6 +16,9 @@ interface Props {
className?: string;
label: string;
value: number;
/**
* Event handler called when the value changes during an interaction.
*/
onValueChange: (value: number) => void;
/**
* Event handler called when the value changes at the end of an interaction.

View File

@@ -115,12 +115,12 @@ function useMediaDevice(
}, [available, preferredId, select, alwaysDefault]);
}
const deviceStub: MediaDevice = {
export const deviceStub: MediaDevice = {
available: [],
selectedId: undefined,
select: () => {},
};
const devicesStub: MediaDevices = {
export const devicesStub: MediaDevices = {
audioInput: deviceStub,
audioOutput: deviceStub,
videoInput: deviceStub,

View File

@@ -6,25 +6,30 @@ Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { beforeEach, expect, test } from "vitest";
import {
afterAll,
beforeEach,
expect,
MockedFunction,
test,
vitest,
} from "vitest";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { ConnectionState, Room } from "livekit-client";
import { ConnectionState } from "livekit-client";
import { BehaviorSubject, of } from "rxjs";
import { afterEach } from "node:test";
import { act } from "react";
import { act, ReactNode } from "react";
import {
CallMembership,
type MatrixRTCSession,
} from "matrix-js-sdk/src/matrixrtc";
import { RoomMember } from "matrix-js-sdk/src/matrix";
import { soundEffectVolumeSetting } from "../settings/settings";
import {
EmittableMockLivekitRoom,
mockLivekitRoom,
mockLocalParticipant,
mockMatrixRoom,
mockMatrixRoomMember,
mockMediaPlay,
mockRemoteParticipant,
mockRtcMembership,
MockRTCSession,
@@ -35,37 +40,69 @@ import {
CallEventAudioRenderer,
MAX_PARTICIPANT_COUNT_FOR_SOUND,
} from "./CallEventAudioRenderer";
import { useAudioContext } from "../useAudioContext";
import { TestReactionsWrapper } from "../utils/testReactions";
import { prefetchSounds } from "../soundUtils";
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const local = mockMatrixRoomMember(localRtcMember);
const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
const alice = mockMatrixRoomMember(aliceRtcMember);
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const bob = mockMatrixRoomMember(bobRtcMember);
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
const bobParticipant = mockRemoteParticipant({ identity: bobId });
const originalPlayFn = window.HTMLMediaElement.prototype.play;
const enterSound = "http://localhost:3000/src/sound/join_call.ogg";
const leaveSound = "http://localhost:3000/src/sound/left_call.ogg";
beforeEach(() => {
soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
});
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
afterEach(() => {
window.HTMLMediaElement.prototype.play = originalPlayFn;
vitest.resetAllMocks();
});
test("plays a sound when entering a call", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map(
[local, alice, bob].map((p) => [p.userId, p]),
afterAll(() => {
vitest.restoreAllMocks();
});
let playSound: MockedFunction<
NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
>;
beforeEach(() => {
(prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
sound: new ArrayBuffer(0),
});
playSound = vitest.fn();
(useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
playSound,
});
});
function TestComponent({
rtcSession,
vm,
}: {
rtcSession: MockRTCSession;
vm: CallViewModel;
}): ReactNode {
return (
<TestReactionsWrapper
rtcSession={rtcSession as unknown as MatrixRTCSession}
>
<CallEventAudioRenderer vm={vm} />
</TestReactionsWrapper>
);
}
function getMockEnv(
members: RoomMember[],
initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember],
): {
vm: CallViewModel;
session: MockRTCSession;
remoteRtcMemberships: BehaviorSubject<CallMembership[]>;
} {
const matrixRoomMembers = new Map(members.map((p) => [p.userId, p]));
const remoteParticipants = of([aliceParticipant]);
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
@@ -75,221 +112,90 @@ test("plays a sound when entering a call", () => {
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
on: vitest.fn(),
off: vitest.fn(),
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const session = new MockRTCSession(matrixRoom, localRtcMember, [
aliceRtcMember,
]) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
initialRemoteRtcMemberships,
);
render(<CallEventAudioRenderer vm={vm} />);
expect(audioIsPlaying).toEqual([
// Joining the call
enterSound,
]);
});
test("plays no sound when muted", () => {
soundEffectVolumeSetting.setValue(0);
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map(
[local, alice, bob].map((p) => [p.userId, p]),
);
const remoteParticipants = of([aliceParticipant, bobParticipant]);
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
{ remoteParticipants },
);
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const session = new MockRTCSession(matrixRoom, localRtcMember, [
aliceRtcMember,
]) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
// Play a sound when joining a call.
expect(audioIsPlaying).toHaveLength(0);
});
test("plays a sound when a user joins", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
const remoteParticipants = new Map(
[aliceParticipant].map((p) => [p.identity, p]),
);
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
aliceRtcMember,
]);
// we give Bob an RTC session now, but no participant yet
const session = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(
remoteRtcMemberships.asObservable(),
) as unknown as MatrixRTCSession;
).withMemberships(remoteRtcMemberships);
const vm = new CallViewModel(
session,
liveKitRoom as unknown as Room,
session as unknown as MatrixRTCSession,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
return { vm, session, remoteRtcMemberships };
}
/**
* We don't want to play a sound when loading the call state
* because typically this occurs in two stages. We first join
* the call as a local participant and *then* the remote
* participants join from our perspective. We don't want to make
* a noise every time.
*/
test("plays one sound when entering a call", () => {
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
// Joining a call usually means remote participants are added later.
act(() => {
remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
});
expect(playSound).toHaveBeenCalledOnce();
});
// TODO: Same test?
test("plays a sound when a user joins", () => {
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
act(() => {
remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
});
// Play a sound when joining a call.
expect(audioIsPlaying).toEqual([
// Joining the call
enterSound,
// Bob joins
enterSound,
]);
expect(playSound).toBeCalledWith("join");
});
test("plays a sound when a user leaves", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
const remoteParticipants = new Map(
[aliceParticipant].map((p) => [p.identity, p]),
);
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
aliceRtcMember,
]);
const session = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom as unknown as Room,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
act(() => {
remoteRtcMemberships.next([]);
});
expect(audioIsPlaying).toEqual([
// Joining the call
enterSound,
// Alice leaves
leaveSound,
]);
expect(playSound).toBeCalledWith("left");
});
test("plays no sound when the session member count is larger than the max, until decreased", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
const remoteParticipants = new Map(
[aliceParticipant].map((p) => [p.identity, p]),
);
test("plays no sound when the participant list is more than the maximum size", () => {
const mockRtcMemberships: CallMembership[] = [];
for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) {
mockRtcMemberships.push(
mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`),
);
}
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
const { session, vm, remoteRtcMemberships } = getMockEnv(
[local, alice],
mockRtcMemberships,
);
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const session = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom as unknown as Room,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
expect(audioIsPlaying).toEqual([]);
// When the count drops to the max we should play the leave sound
render(<TestComponent rtcSession={session} vm={vm} />);
expect(playSound).not.toBeCalled();
act(() => {
remoteRtcMemberships.next(
mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1),
);
});
expect(audioIsPlaying).toEqual([leaveSound]);
expect(playSound).toBeCalledWith("left");
});

View File

@@ -5,47 +5,76 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ReactNode, useEffect, useRef } from "react";
import { filter } from "rxjs";
import { ReactNode, useDeferredValue, useEffect, useMemo } from "react";
import { filter, interval, throttle } from "rxjs";
import {
soundEffectVolumeSetting as effectSoundVolumeSetting,
useSetting,
} from "../settings/settings";
import { CallViewModel } from "../state/CallViewModel";
import enterCallSoundMp3 from "../sound/join_call.mp3";
import enterCallSoundOgg from "../sound/join_call.ogg";
import joinCallSoundMp3 from "../sound/join_call.mp3";
import joinCallSoundOgg from "../sound/join_call.ogg";
import leftCallSoundMp3 from "../sound/left_call.mp3";
import leftCallSoundOgg from "../sound/left_call.ogg";
import handSoundOgg from "../sound/raise_hand.ogg?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useReactions } from "../useReactions";
import { useLatest } from "../useLatest";
// Do not play any sounds if the participant count has exceeded this
// number.
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const CONCURRENT_AUDIO_CHANNELS = 2;
export const THROTTLE_SOUND_EFFECT_MS = 500;
const sounds = prefetchSounds({
join: {
mp3: joinCallSoundMp3,
ogg: joinCallSoundOgg,
},
left: {
mp3: leftCallSoundMp3,
ogg: leftCallSoundOgg,
},
raiseHand: {
mp3: handSoundMp3,
ogg: handSoundOgg,
},
});
export function CallEventAudioRenderer({
vm,
}: {
vm: CallViewModel;
}): ReactNode {
const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
const callEntered = useRef<(HTMLAudioElement | null)[]>([]);
const callLeft = useRef<(HTMLAudioElement | null)[]>([]);
const audioEngineCtx = useAudioContext({
sounds,
latencyHint: "interactive",
});
const audioEngineRef = useLatest(audioEngineCtx);
const { raisedHands } = useReactions();
const raisedHandCount = useMemo(
() => Object.keys(raisedHands).length,
[raisedHands],
);
const previousRaisedHandCount = useDeferredValue(raisedHandCount);
useEffect(() => {
if (effectSoundVolume === 0) {
return;
if (audioEngineRef.current && previousRaisedHandCount < raisedHandCount) {
audioEngineRef.current.playSound("raiseHand");
}
}, [audioEngineRef, previousRaisedHandCount, raisedHandCount]);
useEffect(() => {
const joinSub = vm.memberChanges
.pipe(
filter(
({ joined, ids }) =>
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0,
),
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(({ joined }) => {
const availablePlayer = callEntered.current.find((v) => v?.paused);
void availablePlayer?.play();
.subscribe(() => {
audioEngineRef.current?.playSound("join");
});
const leftSub = vm.memberChanges
@@ -54,64 +83,17 @@ export function CallEventAudioRenderer({
({ ids, left }) =>
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
),
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(() => {
const availablePlayer = callLeft.current.find((v) => v?.paused);
void availablePlayer?.play();
audioEngineRef.current?.playSound("left");
});
return (): void => {
joinSub.unsubscribe();
leftSub.unsubscribe();
};
}, [effectSoundVolume, callEntered, callLeft, vm]);
}, [audioEngineRef, vm]);
// Set volume.
useEffect(() => {
callEntered.current.forEach((a) => {
if (a) {
a.volume = effectSoundVolume;
}
});
callLeft.current.forEach((a) => {
if (a) {
a.volume = effectSoundVolume;
}
});
}, [callEntered, callLeft, effectSoundVolume]);
// Do not render any audio elements if playback is disabled. Will save
// audio file fetches.
if (effectSoundVolume === 0) {
return null;
}
return (
// Will play as soon as it's mounted, which is what we want as this will
// play when the call is entered.
<>
{Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
<audio
key={index}
ref={(r) => (callEntered.current[index] = r)}
preload="auto"
hidden
>
<source src={enterCallSoundOgg} type="audio/ogg; codecs=vorbis" />
<source src={enterCallSoundMp3} type="audio/mpeg" />
</audio>
))}
{Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
<audio
key={index}
ref={(r) => (callLeft.current[index] = r)}
preload="auto"
hidden
>
<source src={leftCallSoundOgg} type="audio/ogg; codecs=vorbis" />
<source src={leftCallSoundMp3} type="audio/mpeg" />
</audio>
))}
</>
);
return <></>;
}

View File

@@ -331,7 +331,7 @@ export const GroupCallView: FC<Props> = ({
<ActiveCall
client={client}
matrixInfo={matrixInfo}
rtcSession={rtcSession}
rtcSession={rtcSession as unknown as MatrixRTCSession}
participantCount={participantCount}
onLeave={onLeave}
hideHeader={hideHeader}

View File

@@ -19,7 +19,6 @@ import {
TouchEvent,
forwardRef,
useCallback,
useDeferredValue,
useEffect,
useMemo,
useRef,
@@ -81,11 +80,8 @@ import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout";
import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout";
import { GridTileViewModel, TileViewModel } from "../state/TileViewModel";
import { ReactionsProvider, useReactions } from "../useReactions";
import handSoundOgg from "../sound/raise_hand.ogg?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url";
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import { useSwitchCamera } from "./useSwitchCamera";
import { soundEffectVolumeSetting, useSetting } from "../settings/settings";
import { ReactionsOverlay } from "./ReactionsOverlay";
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
@@ -178,14 +174,7 @@ export const InCallView: FC<InCallViewProps> = ({
connState,
onShareClick,
}) => {
const [soundEffectVolume] = useSetting(soundEffectVolumeSetting);
const { supportsReactions, raisedHands, sendReaction, toggleRaisedHand } =
useReactions();
const raisedHandCount = useMemo(
() => Object.keys(raisedHands).length,
[raisedHands],
);
const previousRaisedHandCount = useDeferredValue(raisedHandCount);
const { supportsReactions, sendReaction, toggleRaisedHand } = useReactions();
useWakeLock();
@@ -335,25 +324,6 @@ export const InCallView: FC<InCallViewProps> = ({
[vm],
);
// Play a sound when the raised hand count increases.
const handRaisePlayer = useRef<HTMLAudioElement>(null);
useEffect(() => {
if (!handRaisePlayer.current) {
return;
}
if (previousRaisedHandCount < raisedHandCount) {
handRaisePlayer.current.volume = soundEffectVolume;
handRaisePlayer.current.play().catch((ex) => {
logger.warn("Failed to play raise hand sound", ex);
});
}
}, [
raisedHandCount,
handRaisePlayer,
previousRaisedHandCount,
soundEffectVolume,
]);
useEffect(() => {
widget?.api.transport
.send(
@@ -667,10 +637,6 @@ export const InCallView: FC<InCallViewProps> = ({
<RoomAudioRenderer />
{renderContent()}
<CallEventAudioRenderer vm={vm} />
<audio ref={handRaisePlayer} preload="auto" hidden>
<source src={handSoundOgg} type="audio/ogg; codecs=vorbis" />
<source src={handSoundMp3} type="audio/mpeg" />
</audio>
<ReactionsAudioRenderer />
<ReactionsOverlay />
{footer}

View File

@@ -6,9 +6,18 @@ Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { afterAll, expect, test } from "vitest";
import {
afterAll,
beforeEach,
expect,
test,
vitest,
MockedFunction,
Mock,
} from "vitest";
import { TooltipProvider } from "@vector-im/compound-web";
import { act, ReactNode } from "react";
import { afterEach } from "node:test";
import {
MockRoom,
@@ -16,12 +25,13 @@ import {
TestReactionsWrapper,
} from "../utils/testReactions";
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import { GenericReaction, ReactionSet } from "../reactions";
import {
playReactionsSound,
soundEffectVolumeSetting,
} from "../settings/settings";
import { mockMediaPlay } from "../utils/test";
import { useAudioContext } from "../useAudioContext";
import { GenericReaction, ReactionSet } from "../reactions";
import { prefetchSounds } from "../soundUtils";
const memberUserIdAlice = "@alice:example.org";
const memberUserIdBob = "@bob:example.org";
@@ -50,11 +60,31 @@ function TestComponent({
);
}
const originalPlayFn = window.HTMLMediaElement.prototype.play;
afterAll(() => {
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
afterEach(() => {
vitest.resetAllMocks();
playReactionsSound.setValue(playReactionsSound.defaultValue);
soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
window.HTMLMediaElement.prototype.play = originalPlayFn;
});
afterAll(() => {
vitest.restoreAllMocks();
});
let playSound: Mock<
NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
>;
beforeEach(() => {
(prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
sound: new ArrayBuffer(0),
});
playSound = vitest.fn();
(useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
playSound,
});
});
test("preloads all audio elements", () => {
@@ -63,25 +93,11 @@ test("preloads all audio elements", () => {
new MockRoom(memberUserIdAlice),
membership,
);
const { container } = render(<TestComponent rtcSession={rtcSession} />);
expect(container.getElementsByTagName("audio")).toHaveLength(
// All reactions plus the generic sound
ReactionSet.filter((r) => r.sound).length + 1,
);
});
test("loads no audio elements when disabled in settings", () => {
playReactionsSound.setValue(false);
const rtcSession = new MockRTCSession(
new MockRoom(memberUserIdAlice),
membership,
);
const { container } = render(<TestComponent rtcSession={rtcSession} />);
expect(container.getElementsByTagName("audio")).toHaveLength(0);
render(<TestComponent rtcSession={rtcSession} />);
expect(prefetchSounds).toHaveBeenCalledOnce();
});
test("will play an audio sound when there is a reaction", () => {
const audioIsPlaying: string[] = mockMediaPlay();
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
@@ -97,12 +113,10 @@ test("will play an audio sound when there is a reaction", () => {
act(() => {
room.testSendReaction(memberEventAlice, chosenReaction, membership);
});
expect(audioIsPlaying).toHaveLength(1);
expect(audioIsPlaying[0]).toContain(chosenReaction.sound?.ogg);
expect(playSound).toHaveBeenCalledWith(chosenReaction.name);
});
test("will play the generic audio sound when there is soundless reaction", () => {
const audioIsPlaying: string[] = mockMediaPlay();
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
@@ -118,34 +132,10 @@ test("will play the generic audio sound when there is soundless reaction", () =>
act(() => {
room.testSendReaction(memberEventAlice, chosenReaction, membership);
});
expect(audioIsPlaying).toHaveLength(1);
expect(audioIsPlaying[0]).toContain(GenericReaction.sound?.ogg);
});
test("will play an audio sound with the correct volume", () => {
playReactionsSound.setValue(true);
soundEffectVolumeSetting.setValue(0.5);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { getByTestId } = render(<TestComponent rtcSession={rtcSession} />);
// Find the first reaction with a sound effect
const chosenReaction = ReactionSet.find((r) => !!r.sound);
if (!chosenReaction) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
room.testSendReaction(memberEventAlice, chosenReaction, membership);
});
expect((getByTestId(chosenReaction.name) as HTMLAudioElement).volume).toEqual(
0.5,
);
expect(playSound).toHaveBeenCalledWith(GenericReaction.name);
});
test("will play multiple audio sounds when there are multiple different reactions", () => {
const audioIsPlaying: string[] = mockMediaPlay();
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
@@ -164,7 +154,6 @@ test("will play multiple audio sounds when there are multiple different reaction
room.testSendReaction(memberEventBob, reaction2, membership);
room.testSendReaction(memberEventCharlie, reaction1, membership);
});
expect(audioIsPlaying).toHaveLength(2);
expect(audioIsPlaying[0]).toContain(reaction1.sound?.ogg);
expect(audioIsPlaying[1]).toContain(reaction2.sound?.ogg);
expect(playSound).toHaveBeenCalledWith(reaction1.name);
expect(playSound).toHaveBeenCalledWith(reaction2.name);
});

View File

@@ -5,70 +5,67 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ReactNode, useEffect, useRef } from "react";
import { ReactNode, useDeferredValue, useEffect, useState } from "react";
import { useReactions } from "../useReactions";
import {
playReactionsSound,
soundEffectVolumeSetting as effectSoundVolumeSetting,
useSetting,
} from "../settings/settings";
import { playReactionsSound, useSetting } from "../settings/settings";
import { GenericReaction, ReactionSet } from "../reactions";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useLatest } from "../useLatest";
const soundMap = Object.fromEntries([
...ReactionSet.filter((v) => v.sound !== undefined).map((v) => [
v.name,
v.sound!,
]),
[GenericReaction.name, GenericReaction.sound],
]);
export function ReactionsAudioRenderer(): ReactNode {
const { reactions } = useReactions();
const [shouldPlay] = useSetting(playReactionsSound);
const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
const audioElements = useRef<Record<string, HTMLAudioElement | null>>({});
const [soundCache, setSoundCache] = useState<ReturnType<
typeof prefetchSounds
> | null>(null);
const audioEngineCtx = useAudioContext({
sounds: soundCache,
latencyHint: "interactive",
});
const audioEngineRef = useLatest(audioEngineCtx);
const oldReactions = useDeferredValue(reactions);
useEffect(() => {
if (!audioElements.current) {
if (!shouldPlay || soundCache) {
return;
}
// This is fine even if we load the component multiple times,
// as the browser's cache should ensure once the media is loaded
// once that future fetches come via the cache.
setSoundCache(prefetchSounds(soundMap));
}, [soundCache, shouldPlay]);
if (!shouldPlay) {
useEffect(() => {
if (!shouldPlay || !audioEngineRef.current) {
return;
}
const oldReactionSet = new Set(
Object.values(oldReactions).map((r) => r.name),
);
for (const reactionName of new Set(
Object.values(reactions).map((r) => r.name),
)) {
const audioElement =
audioElements.current[reactionName] ?? audioElements.current.generic;
if (audioElement?.paused) {
audioElement.volume = effectSoundVolume;
void audioElement.play();
if (oldReactionSet.has(reactionName)) {
// Don't replay old reactions
return;
}
if (soundMap[reactionName]) {
audioEngineRef.current.playSound(reactionName);
} else {
// Fallback sounds.
audioEngineRef.current.playSound("generic");
}
}
}, [audioElements, shouldPlay, reactions, effectSoundVolume]);
// Do not render any audio elements if playback is disabled. Will save
// audio file fetches.
if (!shouldPlay) {
return null;
}
// NOTE: We load all audio elements ahead of time to allow the cache
// to be populated, rather than risk a cache miss and have the audio
// be delayed.
return (
<>
{[GenericReaction, ...ReactionSet].map(
(r) =>
r.sound && (
<audio
ref={(el) => (audioElements.current[r.name] = el)}
data-testid={r.name}
key={r.name}
preload="auto"
hidden
>
<source src={r.sound.ogg} type="audio/ogg; codecs=vorbis" />
{r.sound.mp3 ? (
<source src={r.sound.mp3} type="audio/mpeg" />
) : null}
</audio>
),
)}
</>
);
}, [audioEngineRef, shouldPlay, oldReactions, reactions]);
return null;
}

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ChangeEvent, FC, useCallback } from "react";
import { ChangeEvent, FC, useCallback, useState } from "react";
import { Trans, useTranslation } from "react-i18next";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import { Root as Form, Text } from "@vector-im/compound-web";
@@ -83,8 +83,8 @@ export const SettingsModal: FC<Props> = ({
const devices = useMediaDevices();
useMediaDeviceNames(devices, open);
const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting);
const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
const audioTab: Tab<SettingsTab> = {
key: "audio",
@@ -107,8 +107,9 @@ export const SettingsModal: FC<Props> = ({
<p>{t("settings.audio_tab.effect_volume_description")}</p>
<Slider
label={t("video_tile.volume")}
value={soundVolume}
onValueChange={setSoundVolume}
value={soundVolumeRaw}
onValueChange={setSoundVolumeRaw}
onValueCommit={setSoundVolume}
min={0}
max={1}
step={0.01}

63
src/soundUtils.ts Normal file
View File

@@ -0,0 +1,63 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/src/logger";
type SoundDefinition = { mp3?: string; ogg: string };
export type PrefetchedSounds<S extends string> = Promise<
Record<S, ArrayBuffer>
>;
/**
* Determine the best format we can use to play our sounds
* through. We prefer ogg support if possible, but will fall
* back to MP3.
* @returns "ogg" if the browser is likely to support it, or "mp3" otherwise.
*/
function getPreferredAudioFormat(): "ogg" | "mp3" {
const a = document.createElement("audio");
if (a.canPlayType("audio/ogg") === "maybe") {
return "ogg";
}
// Otherwise just assume MP3, as that has a chance of being more widely supported.
return "mp3";
}
const preferredFormat = getPreferredAudioFormat();
/**
* Prefetch sounds to be used by the AudioContext. This can
* be called outside the scope of a component to ensure the
* sounds load ahead of time.
* @param sounds A set of sound files that may be played.
* @returns A map of sound files to buffers.
*/
export async function prefetchSounds<S extends string>(
sounds: Record<S, SoundDefinition>,
): PrefetchedSounds<S> {
const buffers: Record<string, ArrayBuffer> = {};
await Promise.all(
Object.entries(sounds).map(async ([name, file]) => {
const { mp3, ogg } = file as SoundDefinition;
// Use preferred format, fallback to ogg if no mp3 is provided.
// Load an audio file
const response = await fetch(
preferredFormat === "ogg" ? ogg : (mp3 ?? ogg),
);
if (!response.ok) {
// If the sound doesn't load, it's not the end of the world. We won't play
// the sound when requested, but it's better than failing the whole application.
logger.warn(`Could not load sound ${name}, response was not okay`);
return;
}
// Decode it
buffers[name] = await response.arrayBuffer();
}),
);
return buffers as Record<S, ArrayBuffer>;
}

View File

@@ -335,7 +335,7 @@ function findMatrixRoomMember(
// must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon
if (parts.length < 3) {
logger.warn(
"Livekit participants ID doesn't look like a userId:deviceId combination",
`Livekit participants ID (${id}) doesn't look like a userId:deviceId combination`,
);
return undefined;
}

View File

@@ -0,0 +1,129 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { expect, test, vitest } from "vitest";
import { FC } from "react";
import { render } from "@testing-library/react";
import { afterEach } from "node:test";
import userEvent from "@testing-library/user-event";
import { deviceStub, MediaDevicesContext } from "./livekit/MediaDevicesContext";
import { useAudioContext } from "./useAudioContext";
import { soundEffectVolumeSetting } from "./settings/settings";
const staticSounds = Promise.resolve({
aSound: new ArrayBuffer(0),
});
const TestComponent: FC = () => {
const audioCtx = useAudioContext({
sounds: staticSounds,
latencyHint: "balanced",
});
if (!audioCtx) {
return null;
}
return (
<>
<button onClick={() => audioCtx.playSound("aSound")}>Valid sound</button>
{/* eslint-disable-next-line @typescript-eslint/no-explicit-any*/}
<button onClick={() => audioCtx.playSound("not-valid" as any)}>
Invalid sound
</button>
</>
);
};
class MockAudioContext {
public static testContext: MockAudioContext;
public constructor() {
MockAudioContext.testContext = this;
}
public gain = vitest.mocked(
{
connect: () => {},
gain: {
setValueAtTime: vitest.fn(),
},
},
true,
);
public setSinkId = vitest.fn().mockResolvedValue(undefined);
public decodeAudioData = vitest.fn().mockReturnValue(1);
public createBufferSource = vitest.fn().mockReturnValue(
vitest.mocked({
connect: (v: unknown) => v,
start: () => {},
}),
);
public createGain = vitest.fn().mockReturnValue(this.gain);
public close = vitest.fn().mockResolvedValue(undefined);
}
afterEach(() => {
vitest.unstubAllGlobals();
});
test("can play a single sound", async () => {
const user = userEvent.setup();
vitest.stubGlobal("AudioContext", MockAudioContext);
const { findByText } = render(<TestComponent />);
await user.click(await findByText("Valid sound"));
expect(
MockAudioContext.testContext.createBufferSource,
).toHaveBeenCalledOnce();
});
test("will ignore sounds that are not registered", async () => {
const user = userEvent.setup();
vitest.stubGlobal("AudioContext", MockAudioContext);
const { findByText } = render(<TestComponent />);
await user.click(await findByText("Invalid sound"));
expect(
MockAudioContext.testContext.createBufferSource,
).not.toHaveBeenCalled();
});
test("will use the correct device", () => {
vitest.stubGlobal("AudioContext", MockAudioContext);
render(
<MediaDevicesContext.Provider
value={{
audioInput: deviceStub,
audioOutput: {
selectedId: "chosen-device",
available: [],
select: () => {},
},
videoInput: deviceStub,
startUsingDeviceNames: () => {},
stopUsingDeviceNames: () => {},
}}
>
<TestComponent />
</MediaDevicesContext.Provider>,
);
expect(
MockAudioContext.testContext.createBufferSource,
).not.toHaveBeenCalled();
expect(MockAudioContext.testContext.setSinkId).toHaveBeenCalledWith(
"chosen-device",
);
});
test("will use the correct volume level", async () => {
const user = userEvent.setup();
vitest.stubGlobal("AudioContext", MockAudioContext);
soundEffectVolumeSetting.setValue(0.33);
const { findByText } = render(<TestComponent />);
await user.click(await findByText("Valid sound"));
expect(
MockAudioContext.testContext.gain.gain.setValueAtTime,
).toHaveBeenCalledWith(0.33, 0);
});

124
src/useAudioContext.tsx Normal file
View File

@@ -0,0 +1,124 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/src/logger";
import { useState, useEffect } from "react";
import {
soundEffectVolumeSetting as effectSoundVolumeSetting,
useSetting,
} from "./settings/settings";
import { useMediaDevices } from "./livekit/MediaDevicesContext";
import { PrefetchedSounds } from "./soundUtils";
/**
* Play a sound though a given AudioContext. Will take
* care of connecting the correct buffer and gating
* through gain.
* @param volume The volume to play at.
* @param ctx The context to play through.
* @param buffer The buffer to play.
*/
function playSound(
ctx: AudioContext,
buffer: AudioBuffer,
volume: number,
): void {
const gain = ctx.createGain();
gain.gain.setValueAtTime(volume, 0);
const src = ctx.createBufferSource();
src.buffer = buffer;
src.connect(gain).connect(ctx.destination);
src.start();
}
interface Props<S extends string> {
/**
* The sounds to play. If no sounds should be played then
* this can be set to null, which will prevent the audio
* context from being created.
*/
sounds: PrefetchedSounds<S> | null;
latencyHint: AudioContextLatencyCategory;
}
interface UseAudioContext<S> {
playSound(soundName: S): void;
}
/**
* Add an audio context which can be used to play
* a set of preloaded sounds.
* @param props
* @returns Either an instance that can be used to play sounds, or null if not ready.
*/
export function useAudioContext<S extends string>(
props: Props<S>,
): UseAudioContext<S> | null {
const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
const devices = useMediaDevices();
const [audioContext, setAudioContext] = useState<AudioContext>();
const [audioBuffers, setAudioBuffers] = useState<Record<S, AudioBuffer>>();
useEffect(() => {
const sounds = props.sounds;
if (!sounds) {
return;
}
const ctx = new AudioContext({
// We want low latency for these effects.
latencyHint: props.latencyHint,
});
// We want to clone the content of our preloaded
// sound buffers into this context. The context may
// close during this process, so it's okay if it throws.
(async (): Promise<void> => {
const buffers: Record<string, AudioBuffer> = {};
for (const [name, buffer] of Object.entries<ArrayBuffer>(await sounds)) {
const audioBuffer = await ctx.decodeAudioData(buffer.slice(0));
buffers[name] = audioBuffer;
}
setAudioBuffers(buffers as Record<S, AudioBuffer>);
})().catch((ex) => {
logger.debug("Failed to setup audio context", ex);
});
setAudioContext(ctx);
return (): void => {
void ctx.close().catch((ex) => {
logger.debug("Failed to close audio engine", ex);
});
setAudioContext(undefined);
};
}, [props.sounds, props.latencyHint]);
// Update the sink ID whenever we change devices.
useEffect(() => {
if (audioContext && "setSinkId" in audioContext) {
// https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
// @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
audioContext.setSinkId(devices.audioOutput.selectedId).catch((ex) => {
logger.warn("Unable to change sink for audio context", ex);
});
}
}, [audioContext, devices]);
// Don't return a function until we're ready.
if (!audioContext || !audioBuffers) {
return null;
}
return {
playSound: (name): void => {
if (!audioBuffers[name]) {
logger.debug(`Tried to play a sound that wasn't buffered (${name})`);
return;
}
return playSound(audioContext, audioBuffers[name], effectSoundVolume);
},
};
}

View File

@@ -27,9 +27,7 @@ import {
RemoteParticipant,
RemoteTrackPublication,
Room as LivekitRoom,
RoomEvent,
} from "livekit-client";
import { EventEmitter } from "stream";
import {
LocalUserMediaViewModel,
@@ -153,33 +151,6 @@ export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
}
/**
* A mock of a Livekit Room that can emit events.
*/
export class EmittableMockLivekitRoom extends EventEmitter {
public localParticipant?: LocalParticipant;
public remoteParticipants: Map<string, RemoteParticipant>;
public constructor(room: {
localParticipant?: LocalParticipant;
remoteParticipants: Map<string, RemoteParticipant>;
}) {
super();
this.localParticipant = room.localParticipant;
this.remoteParticipants = room.remoteParticipants ?? new Map();
}
public addParticipant(remoteParticipant: RemoteParticipant): void {
this.remoteParticipants.set(remoteParticipant.identity, remoteParticipant);
this.emit(RoomEvent.ParticipantConnected, remoteParticipant);
}
public removeParticipant(remoteParticipant: RemoteParticipant): void {
this.remoteParticipants.delete(remoteParticipant.identity);
this.emit(RoomEvent.ParticipantDisconnected, remoteParticipant);
}
}
export function mockLivekitRoom(
room: Partial<LivekitRoom>,
{
@@ -280,15 +251,6 @@ export function mockConfig(config: Partial<ResolvedConfigOptions> = {}): void {
});
}
export function mockMediaPlay(): string[] {
const audioIsPlaying: string[] = [];
window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
return Promise.resolve();
};
return audioIsPlaying;
}
export class MockRTCSession extends TypedEventEmitter<
MatrixRTCSessionEvent,
MatrixRTCSessionEventHandlerMap

View File

@@ -32,7 +32,7 @@ export const TestReactionsWrapper = ({
rtcSession,
children,
}: PropsWithChildren<{
rtcSession: MockRTCSession;
rtcSession: MockRTCSession | MatrixRTCSession;
}>): ReactNode => {
return (
<ReactionsProvider rtcSession={rtcSession as unknown as MatrixRTCSession}>
@@ -203,4 +203,12 @@ export class MockRoom extends EventEmitter {
});
return evt.getId()!;
}
public getMember(): void {
return;
}
public testGetAsMatrixRoom(): Room {
return this as unknown as Room;
}
}