diff --git a/locales/de/app.json b/locales/de/app.json index 828fe3e0..67138ca5 100644 --- a/locales/de/app.json +++ b/locales/de/app.json @@ -147,7 +147,6 @@ "screenshare_button_label": "Bildschirm teilen", "settings": { "audio_tab": { - "effect_volume_description": "Lautstärke anpassen, mit der Reaktionen und Handmeldungen abgespielt werden", "effect_volume_label": "Lautstärke der Soundeffekte" }, "developer_settings_label": "Entwicklereinstellungen", diff --git a/locales/en/app.json b/locales/en/app.json index e500f66c..07dd53e3 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -147,7 +147,7 @@ "screenshare_button_label": "Share screen", "settings": { "audio_tab": { - "effect_volume_description": "Adjust the volume at which reactions and hand raised effects play", + "effect_volume_description": "Volume for sound effects such as: joining or leaving a call, and reactions.", "effect_volume_label": "Sound effect volume" }, "developer_settings_label": "Developer Settings", diff --git a/src/Slider.tsx b/src/Slider.tsx index a5eab56a..e398fba1 100644 --- a/src/Slider.tsx +++ b/src/Slider.tsx @@ -16,6 +16,9 @@ interface Props { className?: string; label: string; value: number; + /** + * Event handler called when the value changes during an interaction. + */ onValueChange: (value: number) => void; /** * Event handler called when the value changes at the end of an interaction. diff --git a/src/livekit/MediaDevicesContext.tsx b/src/livekit/MediaDevicesContext.tsx index 4977f021..d19840ef 100644 --- a/src/livekit/MediaDevicesContext.tsx +++ b/src/livekit/MediaDevicesContext.tsx @@ -115,12 +115,12 @@ function useMediaDevice( }, [available, preferredId, select, alwaysDefault]); } -const deviceStub: MediaDevice = { +export const deviceStub: MediaDevice = { available: [], selectedId: undefined, select: () => {}, }; -const devicesStub: MediaDevices = { +export const devicesStub: MediaDevices = { audioInput: deviceStub, audioOutput: deviceStub, videoInput: deviceStub, diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 78d08f60..5bb1ba19 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -6,25 +6,30 @@ Please see LICENSE in the repository root for full details. */ import { render } from "@testing-library/react"; -import { beforeEach, expect, test } from "vitest"; +import { + afterAll, + beforeEach, + expect, + MockedFunction, + test, + vitest, +} from "vitest"; import { MatrixClient } from "matrix-js-sdk/src/client"; -import { ConnectionState, Room } from "livekit-client"; +import { ConnectionState } from "livekit-client"; import { BehaviorSubject, of } from "rxjs"; import { afterEach } from "node:test"; -import { act } from "react"; +import { act, ReactNode } from "react"; import { CallMembership, type MatrixRTCSession, } from "matrix-js-sdk/src/matrixrtc"; +import { RoomMember } from "matrix-js-sdk/src/matrix"; -import { soundEffectVolumeSetting } from "../settings/settings"; import { - EmittableMockLivekitRoom, mockLivekitRoom, mockLocalParticipant, mockMatrixRoom, mockMatrixRoomMember, - mockMediaPlay, mockRemoteParticipant, mockRtcMembership, MockRTCSession, @@ -35,37 +40,69 @@ import { CallEventAudioRenderer, MAX_PARTICIPANT_COUNT_FOR_SOUND, } from "./CallEventAudioRenderer"; +import { useAudioContext } from "../useAudioContext"; +import { TestReactionsWrapper } from "../utils/testReactions"; +import { prefetchSounds } from "../soundUtils"; const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); const local = mockMatrixRoomMember(localRtcMember); const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); const alice = mockMatrixRoomMember(aliceRtcMember); const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); -const bob = mockMatrixRoomMember(bobRtcMember); const localParticipant = mockLocalParticipant({ identity: "" }); const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`; -const bobId = `${bob.userId}:${bobRtcMember.deviceId}`; const aliceParticipant = mockRemoteParticipant({ identity: aliceId }); -const bobParticipant = mockRemoteParticipant({ identity: bobId }); -const originalPlayFn = window.HTMLMediaElement.prototype.play; - -const enterSound = "http://localhost:3000/src/sound/join_call.ogg"; -const leaveSound = "http://localhost:3000/src/sound/left_call.ogg"; - -beforeEach(() => { - soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue); -}); +vitest.mock("../useAudioContext"); +vitest.mock("../soundUtils"); afterEach(() => { - window.HTMLMediaElement.prototype.play = originalPlayFn; + vitest.resetAllMocks(); }); -test("plays a sound when entering a call", () => { - const audioIsPlaying: string[] = mockMediaPlay(); - const matrixRoomMembers = new Map( - [local, alice, bob].map((p) => [p.userId, p]), +afterAll(() => { + vitest.restoreAllMocks(); +}); + +let playSound: MockedFunction< + NonNullable>["playSound"] +>; + +beforeEach(() => { + (prefetchSounds as MockedFunction).mockResolvedValue({ + sound: new ArrayBuffer(0), + }); + playSound = vitest.fn(); + (useAudioContext as MockedFunction).mockReturnValue({ + playSound, + }); +}); + +function TestComponent({ + rtcSession, + vm, +}: { + rtcSession: MockRTCSession; + vm: CallViewModel; +}): ReactNode { + return ( + + + ); +} + +function getMockEnv( + members: RoomMember[], + initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember], +): { + vm: CallViewModel; + session: MockRTCSession; + remoteRtcMemberships: BehaviorSubject; +} { + const matrixRoomMembers = new Map(members.map((p) => [p.userId, p])); const remoteParticipants = of([aliceParticipant]); const liveKitRoom = mockLivekitRoom( { localParticipant }, @@ -75,221 +112,90 @@ test("plays a sound when entering a call", () => { client: { getUserId: () => localRtcMember.sender, getDeviceId: () => localRtcMember.deviceId, + on: vitest.fn(), + off: vitest.fn(), } as Partial as MatrixClient, getMember: (userId) => matrixRoomMembers.get(userId) ?? null, }); - const session = new MockRTCSession(matrixRoom, localRtcMember, [ - aliceRtcMember, - ]) as unknown as MatrixRTCSession; - - const vm = new CallViewModel( - session, - liveKitRoom, - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), + const remoteRtcMemberships = new BehaviorSubject( + initialRemoteRtcMemberships, ); - render(); - expect(audioIsPlaying).toEqual([ - // Joining the call - enterSound, - ]); -}); - -test("plays no sound when muted", () => { - soundEffectVolumeSetting.setValue(0); - const audioIsPlaying: string[] = mockMediaPlay(); - const matrixRoomMembers = new Map( - [local, alice, bob].map((p) => [p.userId, p]), - ); - const remoteParticipants = of([aliceParticipant, bobParticipant]); - const liveKitRoom = mockLivekitRoom( - { localParticipant }, - { remoteParticipants }, - ); - - const matrixRoom = mockMatrixRoom({ - client: { - getUserId: () => localRtcMember.sender, - getDeviceId: () => localRtcMember.deviceId, - } as Partial as MatrixClient, - getMember: (userId) => matrixRoomMembers.get(userId) ?? null, - }); - - const session = new MockRTCSession(matrixRoom, localRtcMember, [ - aliceRtcMember, - ]) as unknown as MatrixRTCSession; - - const vm = new CallViewModel( - session, - liveKitRoom, - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), - ); - - render(); - // Play a sound when joining a call. - expect(audioIsPlaying).toHaveLength(0); -}); - -test("plays a sound when a user joins", () => { - const audioIsPlaying: string[] = mockMediaPlay(); - const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p])); - const remoteParticipants = new Map( - [aliceParticipant].map((p) => [p.identity, p]), - ); - const liveKitRoom = new EmittableMockLivekitRoom({ - localParticipant, - remoteParticipants, - }); - - const matrixRoom = mockMatrixRoom({ - client: { - getUserId: () => localRtcMember.sender, - getDeviceId: () => localRtcMember.deviceId, - } as Partial as MatrixClient, - getMember: (userId) => matrixRoomMembers.get(userId) ?? null, - }); - - const remoteRtcMemberships = new BehaviorSubject([ - aliceRtcMember, - ]); - // we give Bob an RTC session now, but no participant yet const session = new MockRTCSession( matrixRoom, localRtcMember, - ).withMemberships( - remoteRtcMemberships.asObservable(), - ) as unknown as MatrixRTCSession; + ).withMemberships(remoteRtcMemberships); const vm = new CallViewModel( - session, - liveKitRoom as unknown as Room, + session as unknown as MatrixRTCSession, + liveKitRoom, { kind: E2eeType.PER_PARTICIPANT, }, of(ConnectionState.Connected), ); - render(); + return { vm, session, remoteRtcMemberships }; +} + +/** + * We don't want to play a sound when loading the call state + * because typically this occurs in two stages. We first join + * the call as a local participant and *then* the remote + * participants join from our perspective. We don't want to make + * a noise every time. + */ +test("plays one sound when entering a call", () => { + const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]); + render(); + // Joining a call usually means remote participants are added later. + act(() => { + remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]); + }); + expect(playSound).toHaveBeenCalledOnce(); +}); + +// TODO: Same test? +test("plays a sound when a user joins", () => { + const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]); + render(); act(() => { remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]); }); // Play a sound when joining a call. - expect(audioIsPlaying).toEqual([ - // Joining the call - enterSound, - // Bob joins - enterSound, - ]); + expect(playSound).toBeCalledWith("join"); }); test("plays a sound when a user leaves", () => { - const audioIsPlaying: string[] = mockMediaPlay(); - const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p])); - const remoteParticipants = new Map( - [aliceParticipant].map((p) => [p.identity, p]), - ); - const liveKitRoom = new EmittableMockLivekitRoom({ - localParticipant, - remoteParticipants, - }); - - const matrixRoom = mockMatrixRoom({ - client: { - getUserId: () => localRtcMember.sender, - getDeviceId: () => localRtcMember.deviceId, - } as Partial as MatrixClient, - getMember: (userId) => matrixRoomMembers.get(userId) ?? null, - }); - - const remoteRtcMemberships = new BehaviorSubject([ - aliceRtcMember, - ]); - - const session = new MockRTCSession( - matrixRoom, - localRtcMember, - ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession; - - const vm = new CallViewModel( - session, - liveKitRoom as unknown as Room, - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), - ); - render(); + const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]); + render(); act(() => { remoteRtcMemberships.next([]); }); - expect(audioIsPlaying).toEqual([ - // Joining the call - enterSound, - // Alice leaves - leaveSound, - ]); + expect(playSound).toBeCalledWith("left"); }); -test("plays no sound when the session member count is larger than the max, until decreased", () => { - const audioIsPlaying: string[] = mockMediaPlay(); - const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p])); - const remoteParticipants = new Map( - [aliceParticipant].map((p) => [p.identity, p]), - ); - +test("plays no sound when the participant list is more than the maximum size", () => { const mockRtcMemberships: CallMembership[] = []; - for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) { mockRtcMemberships.push( mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`), ); } - const remoteRtcMemberships = new BehaviorSubject( + const { session, vm, remoteRtcMemberships } = getMockEnv( + [local, alice], mockRtcMemberships, ); - const liveKitRoom = new EmittableMockLivekitRoom({ - localParticipant, - remoteParticipants, - }); - - const matrixRoom = mockMatrixRoom({ - client: { - getUserId: () => localRtcMember.sender, - getDeviceId: () => localRtcMember.deviceId, - } as Partial as MatrixClient, - getMember: (userId) => matrixRoomMembers.get(userId) ?? null, - }); - - const session = new MockRTCSession( - matrixRoom, - localRtcMember, - ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession; - - const vm = new CallViewModel( - session, - liveKitRoom as unknown as Room, - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), - ); - render(); - expect(audioIsPlaying).toEqual([]); - // When the count drops to the max we should play the leave sound + render(); + expect(playSound).not.toBeCalled(); act(() => { remoteRtcMemberships.next( mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1), ); }); - expect(audioIsPlaying).toEqual([leaveSound]); + expect(playSound).toBeCalledWith("left"); }); diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index 4f8213cf..6f4f0359 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -5,47 +5,76 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { ReactNode, useEffect, useRef } from "react"; -import { filter } from "rxjs"; +import { ReactNode, useDeferredValue, useEffect, useMemo } from "react"; +import { filter, interval, throttle } from "rxjs"; -import { - soundEffectVolumeSetting as effectSoundVolumeSetting, - useSetting, -} from "../settings/settings"; import { CallViewModel } from "../state/CallViewModel"; -import enterCallSoundMp3 from "../sound/join_call.mp3"; -import enterCallSoundOgg from "../sound/join_call.ogg"; +import joinCallSoundMp3 from "../sound/join_call.mp3"; +import joinCallSoundOgg from "../sound/join_call.ogg"; import leftCallSoundMp3 from "../sound/left_call.mp3"; import leftCallSoundOgg from "../sound/left_call.ogg"; +import handSoundOgg from "../sound/raise_hand.ogg?url"; +import handSoundMp3 from "../sound/raise_hand.mp3?url"; +import { useAudioContext } from "../useAudioContext"; +import { prefetchSounds } from "../soundUtils"; +import { useReactions } from "../useReactions"; +import { useLatest } from "../useLatest"; // Do not play any sounds if the participant count has exceeded this // number. export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; -export const CONCURRENT_AUDIO_CHANNELS = 2; +export const THROTTLE_SOUND_EFFECT_MS = 500; + +const sounds = prefetchSounds({ + join: { + mp3: joinCallSoundMp3, + ogg: joinCallSoundOgg, + }, + left: { + mp3: leftCallSoundMp3, + ogg: leftCallSoundOgg, + }, + raiseHand: { + mp3: handSoundMp3, + ogg: handSoundOgg, + }, +}); export function CallEventAudioRenderer({ vm, }: { vm: CallViewModel; }): ReactNode { - const [effectSoundVolume] = useSetting(effectSoundVolumeSetting); - const callEntered = useRef<(HTMLAudioElement | null)[]>([]); - const callLeft = useRef<(HTMLAudioElement | null)[]>([]); + const audioEngineCtx = useAudioContext({ + sounds, + latencyHint: "interactive", + }); + const audioEngineRef = useLatest(audioEngineCtx); + + const { raisedHands } = useReactions(); + const raisedHandCount = useMemo( + () => Object.keys(raisedHands).length, + [raisedHands], + ); + const previousRaisedHandCount = useDeferredValue(raisedHandCount); useEffect(() => { - if (effectSoundVolume === 0) { - return; + if (audioEngineRef.current && previousRaisedHandCount < raisedHandCount) { + audioEngineRef.current.playSound("raiseHand"); } + }, [audioEngineRef, previousRaisedHandCount, raisedHandCount]); + + useEffect(() => { const joinSub = vm.memberChanges .pipe( filter( ({ joined, ids }) => ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0, ), + throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)), ) - .subscribe(({ joined }) => { - const availablePlayer = callEntered.current.find((v) => v?.paused); - void availablePlayer?.play(); + .subscribe(() => { + audioEngineRef.current?.playSound("join"); }); const leftSub = vm.memberChanges @@ -54,64 +83,17 @@ export function CallEventAudioRenderer({ ({ ids, left }) => ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0, ), + throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)), ) .subscribe(() => { - const availablePlayer = callLeft.current.find((v) => v?.paused); - void availablePlayer?.play(); + audioEngineRef.current?.playSound("left"); }); return (): void => { joinSub.unsubscribe(); leftSub.unsubscribe(); }; - }, [effectSoundVolume, callEntered, callLeft, vm]); + }, [audioEngineRef, vm]); - // Set volume. - useEffect(() => { - callEntered.current.forEach((a) => { - if (a) { - a.volume = effectSoundVolume; - } - }); - callLeft.current.forEach((a) => { - if (a) { - a.volume = effectSoundVolume; - } - }); - }, [callEntered, callLeft, effectSoundVolume]); - - // Do not render any audio elements if playback is disabled. Will save - // audio file fetches. - if (effectSoundVolume === 0) { - return null; - } - - return ( - // Will play as soon as it's mounted, which is what we want as this will - // play when the call is entered. - <> - {Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => ( - - ))} - {Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => ( - - ))} - - ); + return <>; } diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index b0fa8c69..9336ffdd 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -331,7 +331,7 @@ export const GroupCallView: FC = ({ = ({ connState, onShareClick, }) => { - const [soundEffectVolume] = useSetting(soundEffectVolumeSetting); - const { supportsReactions, raisedHands, sendReaction, toggleRaisedHand } = - useReactions(); - const raisedHandCount = useMemo( - () => Object.keys(raisedHands).length, - [raisedHands], - ); - const previousRaisedHandCount = useDeferredValue(raisedHandCount); + const { supportsReactions, sendReaction, toggleRaisedHand } = useReactions(); useWakeLock(); @@ -335,25 +324,6 @@ export const InCallView: FC = ({ [vm], ); - // Play a sound when the raised hand count increases. - const handRaisePlayer = useRef(null); - useEffect(() => { - if (!handRaisePlayer.current) { - return; - } - if (previousRaisedHandCount < raisedHandCount) { - handRaisePlayer.current.volume = soundEffectVolume; - handRaisePlayer.current.play().catch((ex) => { - logger.warn("Failed to play raise hand sound", ex); - }); - } - }, [ - raisedHandCount, - handRaisePlayer, - previousRaisedHandCount, - soundEffectVolume, - ]); - useEffect(() => { widget?.api.transport .send( @@ -667,10 +637,6 @@ export const InCallView: FC = ({ {renderContent()} - {footer} diff --git a/src/room/ReactionAudioRenderer.test.tsx b/src/room/ReactionAudioRenderer.test.tsx index 80ae2b38..2fec8a9a 100644 --- a/src/room/ReactionAudioRenderer.test.tsx +++ b/src/room/ReactionAudioRenderer.test.tsx @@ -6,9 +6,18 @@ Please see LICENSE in the repository root for full details. */ import { render } from "@testing-library/react"; -import { afterAll, expect, test } from "vitest"; +import { + afterAll, + beforeEach, + expect, + test, + vitest, + MockedFunction, + Mock, +} from "vitest"; import { TooltipProvider } from "@vector-im/compound-web"; import { act, ReactNode } from "react"; +import { afterEach } from "node:test"; import { MockRoom, @@ -16,12 +25,13 @@ import { TestReactionsWrapper, } from "../utils/testReactions"; import { ReactionsAudioRenderer } from "./ReactionAudioRenderer"; -import { GenericReaction, ReactionSet } from "../reactions"; import { playReactionsSound, soundEffectVolumeSetting, } from "../settings/settings"; -import { mockMediaPlay } from "../utils/test"; +import { useAudioContext } from "../useAudioContext"; +import { GenericReaction, ReactionSet } from "../reactions"; +import { prefetchSounds } from "../soundUtils"; const memberUserIdAlice = "@alice:example.org"; const memberUserIdBob = "@bob:example.org"; @@ -50,11 +60,31 @@ function TestComponent({ ); } -const originalPlayFn = window.HTMLMediaElement.prototype.play; -afterAll(() => { +vitest.mock("../useAudioContext"); +vitest.mock("../soundUtils"); + +afterEach(() => { + vitest.resetAllMocks(); playReactionsSound.setValue(playReactionsSound.defaultValue); soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue); - window.HTMLMediaElement.prototype.play = originalPlayFn; +}); + +afterAll(() => { + vitest.restoreAllMocks(); +}); + +let playSound: Mock< + NonNullable>["playSound"] +>; + +beforeEach(() => { + (prefetchSounds as MockedFunction).mockResolvedValue({ + sound: new ArrayBuffer(0), + }); + playSound = vitest.fn(); + (useAudioContext as MockedFunction).mockReturnValue({ + playSound, + }); }); test("preloads all audio elements", () => { @@ -63,25 +93,11 @@ test("preloads all audio elements", () => { new MockRoom(memberUserIdAlice), membership, ); - const { container } = render(); - expect(container.getElementsByTagName("audio")).toHaveLength( - // All reactions plus the generic sound - ReactionSet.filter((r) => r.sound).length + 1, - ); -}); - -test("loads no audio elements when disabled in settings", () => { - playReactionsSound.setValue(false); - const rtcSession = new MockRTCSession( - new MockRoom(memberUserIdAlice), - membership, - ); - const { container } = render(); - expect(container.getElementsByTagName("audio")).toHaveLength(0); + render(); + expect(prefetchSounds).toHaveBeenCalledOnce(); }); test("will play an audio sound when there is a reaction", () => { - const audioIsPlaying: string[] = mockMediaPlay(); playReactionsSound.setValue(true); const room = new MockRoom(memberUserIdAlice); const rtcSession = new MockRTCSession(room, membership); @@ -97,12 +113,10 @@ test("will play an audio sound when there is a reaction", () => { act(() => { room.testSendReaction(memberEventAlice, chosenReaction, membership); }); - expect(audioIsPlaying).toHaveLength(1); - expect(audioIsPlaying[0]).toContain(chosenReaction.sound?.ogg); + expect(playSound).toHaveBeenCalledWith(chosenReaction.name); }); test("will play the generic audio sound when there is soundless reaction", () => { - const audioIsPlaying: string[] = mockMediaPlay(); playReactionsSound.setValue(true); const room = new MockRoom(memberUserIdAlice); const rtcSession = new MockRTCSession(room, membership); @@ -118,34 +132,10 @@ test("will play the generic audio sound when there is soundless reaction", () => act(() => { room.testSendReaction(memberEventAlice, chosenReaction, membership); }); - expect(audioIsPlaying).toHaveLength(1); - expect(audioIsPlaying[0]).toContain(GenericReaction.sound?.ogg); -}); - -test("will play an audio sound with the correct volume", () => { - playReactionsSound.setValue(true); - soundEffectVolumeSetting.setValue(0.5); - const room = new MockRoom(memberUserIdAlice); - const rtcSession = new MockRTCSession(room, membership); - const { getByTestId } = render(); - - // Find the first reaction with a sound effect - const chosenReaction = ReactionSet.find((r) => !!r.sound); - if (!chosenReaction) { - throw Error( - "No reactions have sounds configured, this test cannot succeed", - ); - } - act(() => { - room.testSendReaction(memberEventAlice, chosenReaction, membership); - }); - expect((getByTestId(chosenReaction.name) as HTMLAudioElement).volume).toEqual( - 0.5, - ); + expect(playSound).toHaveBeenCalledWith(GenericReaction.name); }); test("will play multiple audio sounds when there are multiple different reactions", () => { - const audioIsPlaying: string[] = mockMediaPlay(); playReactionsSound.setValue(true); const room = new MockRoom(memberUserIdAlice); @@ -164,7 +154,6 @@ test("will play multiple audio sounds when there are multiple different reaction room.testSendReaction(memberEventBob, reaction2, membership); room.testSendReaction(memberEventCharlie, reaction1, membership); }); - expect(audioIsPlaying).toHaveLength(2); - expect(audioIsPlaying[0]).toContain(reaction1.sound?.ogg); - expect(audioIsPlaying[1]).toContain(reaction2.sound?.ogg); + expect(playSound).toHaveBeenCalledWith(reaction1.name); + expect(playSound).toHaveBeenCalledWith(reaction2.name); }); diff --git a/src/room/ReactionAudioRenderer.tsx b/src/room/ReactionAudioRenderer.tsx index cc0b4a57..15bfc90f 100644 --- a/src/room/ReactionAudioRenderer.tsx +++ b/src/room/ReactionAudioRenderer.tsx @@ -5,70 +5,67 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { ReactNode, useEffect, useRef } from "react"; +import { ReactNode, useDeferredValue, useEffect, useState } from "react"; import { useReactions } from "../useReactions"; -import { - playReactionsSound, - soundEffectVolumeSetting as effectSoundVolumeSetting, - useSetting, -} from "../settings/settings"; +import { playReactionsSound, useSetting } from "../settings/settings"; import { GenericReaction, ReactionSet } from "../reactions"; +import { useAudioContext } from "../useAudioContext"; +import { prefetchSounds } from "../soundUtils"; +import { useLatest } from "../useLatest"; + +const soundMap = Object.fromEntries([ + ...ReactionSet.filter((v) => v.sound !== undefined).map((v) => [ + v.name, + v.sound!, + ]), + [GenericReaction.name, GenericReaction.sound], +]); export function ReactionsAudioRenderer(): ReactNode { const { reactions } = useReactions(); const [shouldPlay] = useSetting(playReactionsSound); - const [effectSoundVolume] = useSetting(effectSoundVolumeSetting); - const audioElements = useRef>({}); + const [soundCache, setSoundCache] = useState | null>(null); + const audioEngineCtx = useAudioContext({ + sounds: soundCache, + latencyHint: "interactive", + }); + const audioEngineRef = useLatest(audioEngineCtx); + const oldReactions = useDeferredValue(reactions); useEffect(() => { - if (!audioElements.current) { + if (!shouldPlay || soundCache) { return; } + // This is fine even if we load the component multiple times, + // as the browser's cache should ensure once the media is loaded + // once that future fetches come via the cache. + setSoundCache(prefetchSounds(soundMap)); + }, [soundCache, shouldPlay]); - if (!shouldPlay) { + useEffect(() => { + if (!shouldPlay || !audioEngineRef.current) { return; } + const oldReactionSet = new Set( + Object.values(oldReactions).map((r) => r.name), + ); for (const reactionName of new Set( Object.values(reactions).map((r) => r.name), )) { - const audioElement = - audioElements.current[reactionName] ?? audioElements.current.generic; - if (audioElement?.paused) { - audioElement.volume = effectSoundVolume; - void audioElement.play(); + if (oldReactionSet.has(reactionName)) { + // Don't replay old reactions + return; + } + if (soundMap[reactionName]) { + audioEngineRef.current.playSound(reactionName); + } else { + // Fallback sounds. + audioEngineRef.current.playSound("generic"); } } - }, [audioElements, shouldPlay, reactions, effectSoundVolume]); - - // Do not render any audio elements if playback is disabled. Will save - // audio file fetches. - if (!shouldPlay) { - return null; - } - - // NOTE: We load all audio elements ahead of time to allow the cache - // to be populated, rather than risk a cache miss and have the audio - // be delayed. - return ( - <> - {[GenericReaction, ...ReactionSet].map( - (r) => - r.sound && ( - - ), - )} - - ); + }, [audioEngineRef, shouldPlay, oldReactions, reactions]); + return null; } diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 4ffcecf5..9d7eb5a1 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { ChangeEvent, FC, useCallback } from "react"; +import { ChangeEvent, FC, useCallback, useState } from "react"; import { Trans, useTranslation } from "react-i18next"; import { MatrixClient } from "matrix-js-sdk/src/matrix"; import { Root as Form, Text } from "@vector-im/compound-web"; @@ -83,8 +83,8 @@ export const SettingsModal: FC = ({ const devices = useMediaDevices(); useMediaDeviceNames(devices, open); - const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting); + const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume); const audioTab: Tab = { key: "audio", @@ -107,8 +107,9 @@ export const SettingsModal: FC = ({

{t("settings.audio_tab.effect_volume_description")}

= Promise< + Record +>; + +/** + * Determine the best format we can use to play our sounds + * through. We prefer ogg support if possible, but will fall + * back to MP3. + * @returns "ogg" if the browser is likely to support it, or "mp3" otherwise. + */ +function getPreferredAudioFormat(): "ogg" | "mp3" { + const a = document.createElement("audio"); + if (a.canPlayType("audio/ogg") === "maybe") { + return "ogg"; + } + // Otherwise just assume MP3, as that has a chance of being more widely supported. + return "mp3"; +} + +const preferredFormat = getPreferredAudioFormat(); + +/** + * Prefetch sounds to be used by the AudioContext. This can + * be called outside the scope of a component to ensure the + * sounds load ahead of time. + * @param sounds A set of sound files that may be played. + * @returns A map of sound files to buffers. + */ +export async function prefetchSounds( + sounds: Record, +): PrefetchedSounds { + const buffers: Record = {}; + await Promise.all( + Object.entries(sounds).map(async ([name, file]) => { + const { mp3, ogg } = file as SoundDefinition; + // Use preferred format, fallback to ogg if no mp3 is provided. + // Load an audio file + const response = await fetch( + preferredFormat === "ogg" ? ogg : (mp3 ?? ogg), + ); + if (!response.ok) { + // If the sound doesn't load, it's not the end of the world. We won't play + // the sound when requested, but it's better than failing the whole application. + logger.warn(`Could not load sound ${name}, response was not okay`); + return; + } + // Decode it + buffers[name] = await response.arrayBuffer(); + }), + ); + return buffers as Record; +} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 95762c3f..af8780b1 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -335,7 +335,7 @@ function findMatrixRoomMember( // must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon if (parts.length < 3) { logger.warn( - "Livekit participants ID doesn't look like a userId:deviceId combination", + `Livekit participants ID (${id}) doesn't look like a userId:deviceId combination`, ); return undefined; } diff --git a/src/useAudioContext.test.tsx b/src/useAudioContext.test.tsx new file mode 100644 index 00000000..5a1afe43 --- /dev/null +++ b/src/useAudioContext.test.tsx @@ -0,0 +1,129 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test, vitest } from "vitest"; +import { FC } from "react"; +import { render } from "@testing-library/react"; +import { afterEach } from "node:test"; +import userEvent from "@testing-library/user-event"; + +import { deviceStub, MediaDevicesContext } from "./livekit/MediaDevicesContext"; +import { useAudioContext } from "./useAudioContext"; +import { soundEffectVolumeSetting } from "./settings/settings"; + +const staticSounds = Promise.resolve({ + aSound: new ArrayBuffer(0), +}); + +const TestComponent: FC = () => { + const audioCtx = useAudioContext({ + sounds: staticSounds, + latencyHint: "balanced", + }); + if (!audioCtx) { + return null; + } + return ( + <> + + {/* eslint-disable-next-line @typescript-eslint/no-explicit-any*/} + + + ); +}; + +class MockAudioContext { + public static testContext: MockAudioContext; + + public constructor() { + MockAudioContext.testContext = this; + } + + public gain = vitest.mocked( + { + connect: () => {}, + gain: { + setValueAtTime: vitest.fn(), + }, + }, + true, + ); + + public setSinkId = vitest.fn().mockResolvedValue(undefined); + public decodeAudioData = vitest.fn().mockReturnValue(1); + public createBufferSource = vitest.fn().mockReturnValue( + vitest.mocked({ + connect: (v: unknown) => v, + start: () => {}, + }), + ); + public createGain = vitest.fn().mockReturnValue(this.gain); + public close = vitest.fn().mockResolvedValue(undefined); +} + +afterEach(() => { + vitest.unstubAllGlobals(); +}); + +test("can play a single sound", async () => { + const user = userEvent.setup(); + vitest.stubGlobal("AudioContext", MockAudioContext); + const { findByText } = render(); + await user.click(await findByText("Valid sound")); + expect( + MockAudioContext.testContext.createBufferSource, + ).toHaveBeenCalledOnce(); +}); +test("will ignore sounds that are not registered", async () => { + const user = userEvent.setup(); + vitest.stubGlobal("AudioContext", MockAudioContext); + const { findByText } = render(); + await user.click(await findByText("Invalid sound")); + expect( + MockAudioContext.testContext.createBufferSource, + ).not.toHaveBeenCalled(); +}); + +test("will use the correct device", () => { + vitest.stubGlobal("AudioContext", MockAudioContext); + render( + {}, + }, + videoInput: deviceStub, + startUsingDeviceNames: () => {}, + stopUsingDeviceNames: () => {}, + }} + > + + , + ); + expect( + MockAudioContext.testContext.createBufferSource, + ).not.toHaveBeenCalled(); + expect(MockAudioContext.testContext.setSinkId).toHaveBeenCalledWith( + "chosen-device", + ); +}); + +test("will use the correct volume level", async () => { + const user = userEvent.setup(); + vitest.stubGlobal("AudioContext", MockAudioContext); + soundEffectVolumeSetting.setValue(0.33); + const { findByText } = render(); + await user.click(await findByText("Valid sound")); + expect( + MockAudioContext.testContext.gain.gain.setValueAtTime, + ).toHaveBeenCalledWith(0.33, 0); +}); diff --git a/src/useAudioContext.tsx b/src/useAudioContext.tsx new file mode 100644 index 00000000..ccf4cbd5 --- /dev/null +++ b/src/useAudioContext.tsx @@ -0,0 +1,124 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { logger } from "matrix-js-sdk/src/logger"; +import { useState, useEffect } from "react"; + +import { + soundEffectVolumeSetting as effectSoundVolumeSetting, + useSetting, +} from "./settings/settings"; +import { useMediaDevices } from "./livekit/MediaDevicesContext"; +import { PrefetchedSounds } from "./soundUtils"; + +/** + * Play a sound though a given AudioContext. Will take + * care of connecting the correct buffer and gating + * through gain. + * @param volume The volume to play at. + * @param ctx The context to play through. + * @param buffer The buffer to play. + */ +function playSound( + ctx: AudioContext, + buffer: AudioBuffer, + volume: number, +): void { + const gain = ctx.createGain(); + gain.gain.setValueAtTime(volume, 0); + const src = ctx.createBufferSource(); + src.buffer = buffer; + src.connect(gain).connect(ctx.destination); + src.start(); +} + +interface Props { + /** + * The sounds to play. If no sounds should be played then + * this can be set to null, which will prevent the audio + * context from being created. + */ + sounds: PrefetchedSounds | null; + latencyHint: AudioContextLatencyCategory; +} + +interface UseAudioContext { + playSound(soundName: S): void; +} + +/** + * Add an audio context which can be used to play + * a set of preloaded sounds. + * @param props + * @returns Either an instance that can be used to play sounds, or null if not ready. + */ +export function useAudioContext( + props: Props, +): UseAudioContext | null { + const [effectSoundVolume] = useSetting(effectSoundVolumeSetting); + const devices = useMediaDevices(); + const [audioContext, setAudioContext] = useState(); + const [audioBuffers, setAudioBuffers] = useState>(); + + useEffect(() => { + const sounds = props.sounds; + if (!sounds) { + return; + } + const ctx = new AudioContext({ + // We want low latency for these effects. + latencyHint: props.latencyHint, + }); + + // We want to clone the content of our preloaded + // sound buffers into this context. The context may + // close during this process, so it's okay if it throws. + (async (): Promise => { + const buffers: Record = {}; + for (const [name, buffer] of Object.entries(await sounds)) { + const audioBuffer = await ctx.decodeAudioData(buffer.slice(0)); + buffers[name] = audioBuffer; + } + setAudioBuffers(buffers as Record); + })().catch((ex) => { + logger.debug("Failed to setup audio context", ex); + }); + + setAudioContext(ctx); + return (): void => { + void ctx.close().catch((ex) => { + logger.debug("Failed to close audio engine", ex); + }); + setAudioContext(undefined); + }; + }, [props.sounds, props.latencyHint]); + + // Update the sink ID whenever we change devices. + useEffect(() => { + if (audioContext && "setSinkId" in audioContext) { + // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId + // @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere. + audioContext.setSinkId(devices.audioOutput.selectedId).catch((ex) => { + logger.warn("Unable to change sink for audio context", ex); + }); + } + }, [audioContext, devices]); + + // Don't return a function until we're ready. + if (!audioContext || !audioBuffers) { + return null; + } + return { + playSound: (name): void => { + if (!audioBuffers[name]) { + logger.debug(`Tried to play a sound that wasn't buffered (${name})`); + return; + } + return playSound(audioContext, audioBuffers[name], effectSoundVolume); + }, + }; +} diff --git a/src/utils/test.ts b/src/utils/test.ts index dca98825..459a252e 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -27,9 +27,7 @@ import { RemoteParticipant, RemoteTrackPublication, Room as LivekitRoom, - RoomEvent, } from "livekit-client"; -import { EventEmitter } from "stream"; import { LocalUserMediaViewModel, @@ -153,33 +151,6 @@ export function mockMatrixRoom(room: Partial): MatrixRoom { return { ...mockEmitter(), ...room } as Partial as MatrixRoom; } -/** - * A mock of a Livekit Room that can emit events. - */ -export class EmittableMockLivekitRoom extends EventEmitter { - public localParticipant?: LocalParticipant; - public remoteParticipants: Map; - - public constructor(room: { - localParticipant?: LocalParticipant; - remoteParticipants: Map; - }) { - super(); - this.localParticipant = room.localParticipant; - this.remoteParticipants = room.remoteParticipants ?? new Map(); - } - - public addParticipant(remoteParticipant: RemoteParticipant): void { - this.remoteParticipants.set(remoteParticipant.identity, remoteParticipant); - this.emit(RoomEvent.ParticipantConnected, remoteParticipant); - } - - public removeParticipant(remoteParticipant: RemoteParticipant): void { - this.remoteParticipants.delete(remoteParticipant.identity); - this.emit(RoomEvent.ParticipantDisconnected, remoteParticipant); - } -} - export function mockLivekitRoom( room: Partial, { @@ -280,15 +251,6 @@ export function mockConfig(config: Partial = {}): void { }); } -export function mockMediaPlay(): string[] { - const audioIsPlaying: string[] = []; - window.HTMLMediaElement.prototype.play = async function (): Promise { - audioIsPlaying.push((this.children[0] as HTMLSourceElement).src); - return Promise.resolve(); - }; - return audioIsPlaying; -} - export class MockRTCSession extends TypedEventEmitter< MatrixRTCSessionEvent, MatrixRTCSessionEventHandlerMap diff --git a/src/utils/testReactions.tsx b/src/utils/testReactions.tsx index 84ff217b..fec3e859 100644 --- a/src/utils/testReactions.tsx +++ b/src/utils/testReactions.tsx @@ -32,7 +32,7 @@ export const TestReactionsWrapper = ({ rtcSession, children, }: PropsWithChildren<{ - rtcSession: MockRTCSession; + rtcSession: MockRTCSession | MatrixRTCSession; }>): ReactNode => { return ( @@ -203,4 +203,12 @@ export class MockRoom extends EventEmitter { }); return evt.getId()!; } + + public getMember(): void { + return; + } + + public testGetAsMatrixRoom(): Room { + return this as unknown as Room; + } }