From 77facd01e41faae55a62dd7d839fbb351a881865 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 12 Dec 2024 07:33:47 +0000 Subject: [PATCH] Add support for playing a sound when the user exits a call. (#2860) * Refactor to use AudioContext * Remove unused audio format. * Reduce update frequency for volume * Port to useAudioContext * Port reactionaudiorenderer to useAudioContext * Integrate raise hand sound into call event renderer. * Simplify reaction sounds * only play one sound per reaction type * Start to build out tests * fixup tests / comments * Fix reaction sound * remove console line * Remove another debug line. * fix lint * Use testing library click * lint * Add support for playing a sound when the user exits a call. * Port GroupCallView to useAudioContext * Remove debug bits. * asyncify * lint * lint * lint * tidy * Add test for group call view * Test widget mode too. * fix ?. * Format * Lint * Lint --------- Co-authored-by: Hugh Nimmo-Smith --- src/room/CallEventAudioRenderer.tsx | 10 +- src/room/GroupCallView.test.tsx | 153 ++++++++++++++++++++++++++++ src/room/GroupCallView.tsx | 74 +++++++++----- src/room/ReactionAudioRenderer.tsx | 4 +- src/room/RoomPage.tsx | 1 + src/rtcSessionHelpers.ts | 8 +- src/useAudioContext.test.tsx | 7 +- src/useAudioContext.tsx | 11 +- src/utils/test.ts | 12 ++- 9 files changed, 242 insertions(+), 38 deletions(-) create mode 100644 src/room/GroupCallView.test.tsx diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index 3c2e338f..a363c6f5 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -25,7 +25,7 @@ import { useLatest } from "../useLatest"; export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; export const THROTTLE_SOUND_EFFECT_MS = 500; -const sounds = prefetchSounds({ +export const callEventAudioSounds = prefetchSounds({ join: { mp3: joinCallSoundMp3, ogg: joinCallSoundOgg, @@ -46,7 +46,7 @@ export function CallEventAudioRenderer({ vm: CallViewModel; }): ReactNode { const audioEngineCtx = useAudioContext({ - sounds, + sounds: callEventAudioSounds, latencyHint: "interactive", }); const audioEngineRef = useLatest(audioEngineCtx); @@ -60,7 +60,7 @@ export function CallEventAudioRenderer({ useEffect(() => { if (audioEngineRef.current && previousRaisedHandCount < raisedHandCount) { - audioEngineRef.current.playSound("raiseHand"); + void audioEngineRef.current.playSound("raiseHand"); } }, [audioEngineRef, previousRaisedHandCount, raisedHandCount]); @@ -74,7 +74,7 @@ export function CallEventAudioRenderer({ throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)), ) .subscribe(() => { - audioEngineRef.current?.playSound("join"); + void audioEngineRef.current?.playSound("join"); }); const leftSub = vm.memberChanges @@ -86,7 +86,7 @@ export function CallEventAudioRenderer({ throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)), ) .subscribe(() => { - audioEngineRef.current?.playSound("left"); + void audioEngineRef.current?.playSound("left"); }); return (): void => { diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx new file mode 100644 index 00000000..ea2cc5cf --- /dev/null +++ b/src/room/GroupCallView.test.tsx @@ -0,0 +1,153 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { beforeEach, expect, type MockedFunction, test, vitest } from "vitest"; +import { render } from "@testing-library/react"; +import { type MatrixClient } from "matrix-js-sdk/src/client"; +import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc"; +import { of } from "rxjs"; +import { JoinRule, type RoomState } from "matrix-js-sdk/src/matrix"; +import { Router } from "react-router-dom"; +import { createBrowserHistory } from "history"; +import userEvent from "@testing-library/user-event"; + +import { type MuteStates } from "./MuteStates"; +import { prefetchSounds } from "../soundUtils"; +import { useAudioContext } from "../useAudioContext"; +import { ActiveCall } from "./InCallView"; +import { + mockMatrixRoom, + mockMatrixRoomMember, + mockRtcMembership, + MockRTCSession, +} from "../utils/test"; +import { GroupCallView } from "./GroupCallView"; +import { leaveRTCSession } from "../rtcSessionHelpers"; +import { type WidgetHelpers } from "../widget"; +import { LazyEventEmitter } from "../LazyEventEmitter"; + +vitest.mock("../soundUtils"); +vitest.mock("../useAudioContext"); +vitest.mock("./InCallView"); + +vitest.mock("../rtcSessionHelpers", async (importOriginal) => { + // TODO: perhaps there is a more elegant way to manage the type import here? + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + const orig = await importOriginal(); + vitest.spyOn(orig, "leaveRTCSession"); + return orig; +}); + +let playSound: MockedFunction< + NonNullable>["playSound"] +>; + +const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); +const carol = mockMatrixRoomMember(localRtcMember); +const roomMembers = new Map([carol].map((p) => [p.userId, p])); + +const roomId = "!foo:bar"; +const soundPromise = Promise.resolve(true); + +beforeEach(() => { + (prefetchSounds as MockedFunction).mockResolvedValue({ + sound: new ArrayBuffer(0), + }); + playSound = vitest.fn().mockReturnValue(soundPromise); + (useAudioContext as MockedFunction).mockReturnValue({ + playSound, + }); + // A trivial implementation of Active call to ensure we are testing GroupCallView exclusively here. + (ActiveCall as MockedFunction).mockImplementation( + ({ onLeave }) => { + return ( +
+ +
+ ); + }, + ); +}); + +function createGroupCallView(widget: WidgetHelpers | null): { + rtcSession: MockRTCSession; + getByText: ReturnType["getByText"]; +} { + const history = createBrowserHistory(); + const client = { + getUser: () => null, + getUserId: () => localRtcMember.sender, + getDeviceId: () => localRtcMember.deviceId, + getRoom: (rId) => (rId === roomId ? room : null), + } as Partial as MatrixClient; + const room = mockMatrixRoom({ + client, + roomId, + getMember: (userId) => roomMembers.get(userId) ?? null, + getMxcAvatarUrl: () => null, + getCanonicalAlias: () => null, + currentState: { + getJoinRule: () => JoinRule.Invite, + } as Partial as RoomState, + }); + const rtcSession = new MockRTCSession( + room, + localRtcMember, + [], + ).withMemberships(of([])); + const muteState = { + audio: { enabled: false }, + video: { enabled: false }, + } as MuteStates; + const { getByText } = render( + + + , + ); + return { + getByText, + rtcSession, + }; +} + +test("will play a leave sound asynchronously in SPA mode", async () => { + const user = userEvent.setup(); + const { getByText, rtcSession } = createGroupCallView(null); + const leaveButton = getByText("Leave"); + await user.click(leaveButton); + expect(playSound).toHaveBeenCalledWith("left"); + expect(leaveRTCSession).toHaveBeenCalledWith(rtcSession, undefined); + expect(rtcSession.leaveRoomSession).toHaveBeenCalledOnce(); +}); + +test("will play a leave sound synchronously in widget mode", async () => { + const user = userEvent.setup(); + const widget = { + api: { + setAlwaysOnScreen: async () => Promise.resolve(true), + } as Partial, + lazyActions: new LazyEventEmitter(), + }; + const { getByText, rtcSession } = createGroupCallView( + widget as WidgetHelpers, + ); + const leaveButton = getByText("Leave"); + await user.click(leaveButton); + expect(playSound).toHaveBeenCalledWith("left"); + expect(leaveRTCSession).toHaveBeenCalledWith(rtcSession, soundPromise); + expect(rtcSession.leaveRoomSession).toHaveBeenCalledOnce(); +}); diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 98bfa1a6..3ea6a9c2 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -26,7 +26,11 @@ import { Heading, Text } from "@vector-im/compound-web"; import { useTranslation } from "react-i18next"; import type { IWidgetApiRequest } from "matrix-widget-api"; -import { widget, ElementWidgetActions, type JoinCallData } from "../widget"; +import { + ElementWidgetActions, + type JoinCallData, + type WidgetHelpers, +} from "../widget"; import { FullScreenView } from "../FullScreenView"; import { LobbyView } from "./LobbyView"; import { type MatrixInfo } from "./VideoPreview"; @@ -51,6 +55,9 @@ import { InviteModal } from "./InviteModal"; import { useUrlParams } from "../UrlParams"; import { E2eeType } from "../e2ee/e2eeType"; import { Link } from "../button/Link"; +import { useAudioContext } from "../useAudioContext"; +import { callEventAudioSounds } from "./CallEventAudioRenderer"; +import { useLatest } from "../useLatest"; declare global { interface Window { @@ -67,6 +74,7 @@ interface Props { hideHeader: boolean; rtcSession: MatrixRTCSession; muteStates: MuteStates; + widget: WidgetHelpers | null; } export const GroupCallView: FC = ({ @@ -78,10 +86,16 @@ export const GroupCallView: FC = ({ hideHeader, rtcSession, muteStates, + widget, }) => { const memberships = useMatrixRTCSessionMemberships(rtcSession); const isJoined = useMatrixRTCSessionJoinState(rtcSession); - + const leaveSoundContext = useLatest( + useAudioContext({ + sounds: callEventAudioSounds, + latencyHint: "interactive", + }), + ); // This should use `useEffectEvent` (only available in experimental versions) useEffect(() => { if (memberships.length >= MUTE_PARTICIPANT_COUNT) @@ -195,14 +209,14 @@ export const GroupCallView: FC = ({ ev.detail.data as unknown as JoinCallData, ); await enterRTCSession(rtcSession, perParticipantE2EE); - widget!.api.transport.reply(ev.detail, {}); + widget.api.transport.reply(ev.detail, {}); })().catch((e) => { logger.error("Error joining RTC session", e); }); }; widget.lazyActions.on(ElementWidgetActions.JoinCall, onJoin); return (): void => { - widget!.lazyActions.off(ElementWidgetActions.JoinCall, onJoin); + widget.lazyActions.off(ElementWidgetActions.JoinCall, onJoin); }; } else { // No lobby and no preload: we enter the rtc session right away @@ -216,7 +230,7 @@ export const GroupCallView: FC = ({ void enterRTCSession(rtcSession, perParticipantE2EE); } } - }, [rtcSession, preload, skipLobby, perParticipantE2EE]); + }, [widget, rtcSession, preload, skipLobby, perParticipantE2EE]); const [left, setLeft] = useState(false); const [leaveError, setLeaveError] = useState(undefined); @@ -224,12 +238,12 @@ export const GroupCallView: FC = ({ const onLeave = useCallback( (leaveError?: Error): void => { - setLeaveError(leaveError); - setLeft(true); - + const audioPromise = leaveSoundContext.current?.playSound("left"); // In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent, // therefore we want the event to be sent instantly without getting queued/batched. const sendInstantly = !!widget; + setLeaveError(leaveError); + setLeft(true); PosthogAnalytics.instance.eventCallEnded.track( rtcSession.room.roomId, rtcSession.memberships.length, @@ -237,8 +251,12 @@ export const GroupCallView: FC = ({ rtcSession, ); - // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. - leaveRTCSession(rtcSession) + leaveRTCSession( + rtcSession, + // Wait for the sound in widget mode (it's not long) + sendInstantly && audioPromise ? audioPromise : undefined, + ) + // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. .then(() => { if ( !isPasswordlessUser && @@ -252,18 +270,25 @@ export const GroupCallView: FC = ({ logger.error("Error leaving RTC session", e); }); }, - [rtcSession, isPasswordlessUser, confineToRoom, history], + [ + widget, + rtcSession, + isPasswordlessUser, + confineToRoom, + leaveSoundContext, + history, + ], ); useEffect(() => { if (widget && isJoined) { // set widget to sticky once joined. - widget!.api.setAlwaysOnScreen(true).catch((e) => { + widget.api.setAlwaysOnScreen(true).catch((e) => { logger.error("Error calling setAlwaysOnScreen(true)", e); }); const onHangup = (ev: CustomEvent): void => { - widget!.api.transport.reply(ev.detail, {}); + widget.api.transport.reply(ev.detail, {}); // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. leaveRTCSession(rtcSession).catch((e) => { logger.error("Failed to leave RTC session", e); @@ -271,10 +296,10 @@ export const GroupCallView: FC = ({ }; widget.lazyActions.once(ElementWidgetActions.HangupCall, onHangup); return (): void => { - widget!.lazyActions.off(ElementWidgetActions.HangupCall, onHangup); + widget.lazyActions.off(ElementWidgetActions.HangupCall, onHangup); }; } - }, [isJoined, rtcSession]); + }, [widget, isJoined, rtcSession]); const onReconnect = useCallback(() => { setLeft(false); @@ -367,14 +392,17 @@ export const GroupCallView: FC = ({ leaveError ) { return ( - + <> + + ; + ); } else { // If the user is a regular user, we'll have sent them back to the homepage, diff --git a/src/room/ReactionAudioRenderer.tsx b/src/room/ReactionAudioRenderer.tsx index 6463c9d1..be24a5d6 100644 --- a/src/room/ReactionAudioRenderer.tsx +++ b/src/room/ReactionAudioRenderer.tsx @@ -60,10 +60,10 @@ export function ReactionsAudioRenderer(): ReactNode { return; } if (soundMap[reactionName]) { - audioEngineRef.current.playSound(reactionName); + void audioEngineRef.current.playSound(reactionName); } else { // Fallback sounds. - audioEngineRef.current.playSound("generic"); + void audioEngineRef.current.playSound("generic"); } } }, [audioEngineRef, shouldPlay, oldReactions, reactions]); diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index 8c88b985..d8973c20 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -98,6 +98,7 @@ export const RoomPage: FC = () => { case "loaded": return ( , ): Promise => { // we need to wait until the callEnded event is tracked on posthog. // Otherwise the iFrame gets killed before the callEnded event got tracked. @@ -132,6 +133,8 @@ const widgetPostHangupProcedure = async ( logger.error("Failed to set call widget `alwaysOnScreen` to false", e); } + // Wait for any last bits before hanging up. + await promiseBeforeHangup; // We send the hangup event after the memberships have been updated // calling leaveRTCSession. // We need to wait because this makes the client hosting this widget killing the IFrame. @@ -140,9 +143,12 @@ const widgetPostHangupProcedure = async ( export async function leaveRTCSession( rtcSession: MatrixRTCSession, + promiseBeforeHangup?: Promise, ): Promise { await rtcSession.leaveRoomSession(); if (widget) { - await widgetPostHangupProcedure(widget); + await widgetPostHangupProcedure(widget, promiseBeforeHangup); + } else { + await promiseBeforeHangup; } } diff --git a/src/useAudioContext.test.tsx b/src/useAudioContext.test.tsx index 2b9b0982..565208b1 100644 --- a/src/useAudioContext.test.tsx +++ b/src/useAudioContext.test.tsx @@ -29,9 +29,11 @@ const TestComponent: FC = () => { } return ( <> - + {/* eslint-disable-next-line @typescript-eslint/no-explicit-any*/} - @@ -61,6 +63,7 @@ class MockAudioContext { vitest.mocked({ connect: (v: unknown) => v, start: () => {}, + addEventListener: (_name: string, cb: () => void) => cb(), }), ); public createGain = vitest.fn().mockReturnValue(this.gain); diff --git a/src/useAudioContext.tsx b/src/useAudioContext.tsx index 1580f32a..656b7460 100644 --- a/src/useAudioContext.tsx +++ b/src/useAudioContext.tsx @@ -22,18 +22,21 @@ import { type PrefetchedSounds } from "./soundUtils"; * @param volume The volume to play at. * @param ctx The context to play through. * @param buffer The buffer to play. + * @returns A promise that resolves when the sound has finished playing. */ -function playSound( +async function playSound( ctx: AudioContext, buffer: AudioBuffer, volume: number, -): void { +): Promise { const gain = ctx.createGain(); gain.gain.setValueAtTime(volume, 0); const src = ctx.createBufferSource(); src.buffer = buffer; src.connect(gain).connect(ctx.destination); + const p = new Promise((r) => src.addEventListener("ended", () => r())); src.start(); + return p; } interface Props { @@ -47,7 +50,7 @@ interface Props { } interface UseAudioContext { - playSound(soundName: S): void; + playSound(soundName: S): Promise; } /** @@ -113,7 +116,7 @@ export function useAudioContext( return null; } return { - playSound: (name): void => { + playSound: async (name): Promise => { if (!audioBuffers[name]) { logger.debug(`Tried to play a sound that wasn't buffered (${name})`); return; diff --git a/src/utils/test.ts b/src/utils/test.ts index ca6a5fce..1cd21f01 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; -import { expect, vi } from "vitest"; +import { expect, vi, vitest } from "vitest"; import { type RoomMember, type Room as MatrixRoom, @@ -258,6 +258,12 @@ export class MockRTCSession extends TypedEventEmitter< MatrixRTCSessionEvent, MatrixRTCSessionEventHandlerMap > { + public readonly statistics = { + counters: {}, + }; + + public leaveRoomSession = vitest.fn().mockResolvedValue(undefined); + public constructor( public readonly room: Room, private localMembership: CallMembership, @@ -266,6 +272,10 @@ export class MockRTCSession extends TypedEventEmitter< super(); } + public isJoined(): true { + return true; + } + public withMemberships( rtcMembers: Observable[]>, ): MockRTCSession {