Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-01-30 03:15:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for playing a sound when the user exits a call. (#2860)

Browse Source 
* Refactor to use AudioContext

* Remove unused audio format.

* Reduce update frequency for volume

* Port to useAudioContext

* Port reactionaudiorenderer to useAudioContext

* Integrate raise hand sound into call event renderer.

* Simplify reaction sounds

* only play one sound per reaction type

* Start to build out tests

* fixup tests / comments

* Fix reaction sound

* remove console line

* Remove another debug line.

* fix lint

* Use testing library click

* lint

* Add support for playing a sound when the user exits a call.

* Port GroupCallView to useAudioContext

* Remove debug bits.

* asyncify

* lint

* lint

* lint

* tidy

* Add test for group call view

* Test widget mode too.

* fix ?.

* Format

* Lint

* Lint

---------

Co-authored-by: Hugh Nimmo-Smith <hughns@element.io>
This commit is contained in:
Will Hunt
2024-12-12 07:33:47 +00:00
committed by GitHub
parent 6c81f69590
commit 77facd01e4
9 changed files with 242 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/room/CallEventAudioRenderer.tsx
View File

@@ -25,7 +25,7 @@ import { useLatest } from "../useLatest";
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const THROTTLE_SOUND_EFFECT_MS = 500;
const sounds = prefetchSounds({
export const callEventAudioSounds = prefetchSounds({
join: {
mp3: joinCallSoundMp3,
ogg: joinCallSoundOgg,
@@ -46,7 +46,7 @@ export function CallEventAudioRenderer({
vm: CallViewModel;
}): ReactNode {
const audioEngineCtx = useAudioContext({
sounds,
sounds: callEventAudioSounds,
latencyHint: "interactive",
});
const audioEngineRef = useLatest(audioEngineCtx);
@@ -60,7 +60,7 @@ export function CallEventAudioRenderer({
useEffect(() => {
if (audioEngineRef.current && previousRaisedHandCount < raisedHandCount) {
audioEngineRef.current.playSound("raiseHand");
void audioEngineRef.current.playSound("raiseHand");
}
}, [audioEngineRef, previousRaisedHandCount, raisedHandCount]);
@@ -74,7 +74,7 @@ export function CallEventAudioRenderer({
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(() => {
audioEngineRef.current?.playSound("join");
void audioEngineRef.current?.playSound("join");
});
const leftSub = vm.memberChanges
@@ -86,7 +86,7 @@ export function CallEventAudioRenderer({
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(() => {
audioEngineRef.current?.playSound("left");
void audioEngineRef.current?.playSound("left");
});
return (): void => {

153
src/room/GroupCallView.test.tsx Normal file
View File

@@ -0,0 +1,153 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { beforeEach, expect, type MockedFunction, test, vitest } from "vitest";
import { render } from "@testing-library/react";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";
import { of } from "rxjs";
import { JoinRule, type RoomState } from "matrix-js-sdk/src/matrix";
import { Router } from "react-router-dom";
import { createBrowserHistory } from "history";
import userEvent from "@testing-library/user-event";
import { type MuteStates } from "./MuteStates";
import { prefetchSounds } from "../soundUtils";
import { useAudioContext } from "../useAudioContext";
import { ActiveCall } from "./InCallView";
import {
mockMatrixRoom,
mockMatrixRoomMember,
mockRtcMembership,
MockRTCSession,
} from "../utils/test";
import { GroupCallView } from "./GroupCallView";
import { leaveRTCSession } from "../rtcSessionHelpers";
import { type WidgetHelpers } from "../widget";
import { LazyEventEmitter } from "../LazyEventEmitter";
vitest.mock("../soundUtils");
vitest.mock("../useAudioContext");
vitest.mock("./InCallView");
vitest.mock("../rtcSessionHelpers", async (importOriginal) => {
// TODO: perhaps there is a more elegant way to manage the type import here?
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
const orig = await importOriginal<typeof import("../rtcSessionHelpers")>();
vitest.spyOn(orig, "leaveRTCSession");
return orig;
});
let playSound: MockedFunction<
NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
>;
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const carol = mockMatrixRoomMember(localRtcMember);
const roomMembers = new Map([carol].map((p) => [p.userId, p]));
const roomId = "!foo:bar";
const soundPromise = Promise.resolve(true);
beforeEach(() => {
(prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
sound: new ArrayBuffer(0),
});
playSound = vitest.fn().mockReturnValue(soundPromise);
(useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
playSound,
});
// A trivial implementation of Active call to ensure we are testing GroupCallView exclusively here.
(ActiveCall as MockedFunction<typeof ActiveCall>).mockImplementation(
({ onLeave }) => {
return (
<div>
<button onClick={() => onLeave()}>Leave</button>
</div>
);
},
);
});
function createGroupCallView(widget: WidgetHelpers | null): {
rtcSession: MockRTCSession;
getByText: ReturnType<typeof render>["getByText"];
} {
const history = createBrowserHistory();
const client = {
getUser: () => null,
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
getRoom: (rId) => (rId === roomId ? room : null),
} as Partial<MatrixClient> as MatrixClient;
const room = mockMatrixRoom({
client,
roomId,
getMember: (userId) => roomMembers.get(userId) ?? null,
getMxcAvatarUrl: () => null,
getCanonicalAlias: () => null,
currentState: {
getJoinRule: () => JoinRule.Invite,
} as Partial<RoomState> as RoomState,
});
const rtcSession = new MockRTCSession(
room,
localRtcMember,
[],
).withMemberships(of([]));
const muteState = {
audio: { enabled: false },
video: { enabled: false },
} as MuteStates;
const { getByText } = render(
<Router history={history}>
<GroupCallView
client={client}
isPasswordlessUser={false}
confineToRoom={false}
preload={false}
skipLobby={false}
hideHeader={true}
rtcSession={rtcSession as unknown as MatrixRTCSession}
muteStates={muteState}
widget={widget}
/>
</Router>,
);
return {
getByText,
rtcSession,
};
}
test("will play a leave sound asynchronously in SPA mode", async () => {
const user = userEvent.setup();
const { getByText, rtcSession } = createGroupCallView(null);
const leaveButton = getByText("Leave");
await user.click(leaveButton);
expect(playSound).toHaveBeenCalledWith("left");
expect(leaveRTCSession).toHaveBeenCalledWith(rtcSession, undefined);
expect(rtcSession.leaveRoomSession).toHaveBeenCalledOnce();
});
test("will play a leave sound synchronously in widget mode", async () => {
const user = userEvent.setup();
const widget = {
api: {
setAlwaysOnScreen: async () => Promise.resolve(true),
} as Partial<WidgetHelpers["api"]>,
lazyActions: new LazyEventEmitter(),
};
const { getByText, rtcSession } = createGroupCallView(
widget as WidgetHelpers,
);
const leaveButton = getByText("Leave");
await user.click(leaveButton);
expect(playSound).toHaveBeenCalledWith("left");
expect(leaveRTCSession).toHaveBeenCalledWith(rtcSession, soundPromise);
expect(rtcSession.leaveRoomSession).toHaveBeenCalledOnce();
});

74
src/room/GroupCallView.tsx
View File

@@ -26,7 +26,11 @@ import { Heading, Text } from "@vector-im/compound-web";
import { useTranslation } from "react-i18next";
import type { IWidgetApiRequest } from "matrix-widget-api";
import { widget, ElementWidgetActions, type JoinCallData } from "../widget";
import {
ElementWidgetActions,
type JoinCallData,
type WidgetHelpers,
} from "../widget";
import { FullScreenView } from "../FullScreenView";
import { LobbyView } from "./LobbyView";
import { type MatrixInfo } from "./VideoPreview";
@@ -51,6 +55,9 @@ import { InviteModal } from "./InviteModal";
import { useUrlParams } from "../UrlParams";
import { E2eeType } from "../e2ee/e2eeType";
import { Link } from "../button/Link";
import { useAudioContext } from "../useAudioContext";
import { callEventAudioSounds } from "./CallEventAudioRenderer";
import { useLatest } from "../useLatest";
declare global {
interface Window {
@@ -67,6 +74,7 @@ interface Props {
hideHeader: boolean;
rtcSession: MatrixRTCSession;
muteStates: MuteStates;
widget: WidgetHelpers | null;
}
export const GroupCallView: FC<Props> = ({
@@ -78,10 +86,16 @@ export const GroupCallView: FC<Props> = ({
hideHeader,
rtcSession,
muteStates,
widget,
}) => {
const memberships = useMatrixRTCSessionMemberships(rtcSession);
const isJoined = useMatrixRTCSessionJoinState(rtcSession);
const leaveSoundContext = useLatest(
useAudioContext({
sounds: callEventAudioSounds,
latencyHint: "interactive",
}),
);
// This should use `useEffectEvent` (only available in experimental versions)
useEffect(() => {
if (memberships.length >= MUTE_PARTICIPANT_COUNT)
@@ -195,14 +209,14 @@ export const GroupCallView: FC<Props> = ({
ev.detail.data as unknown as JoinCallData,
);
await enterRTCSession(rtcSession, perParticipantE2EE);
widget!.api.transport.reply(ev.detail, {});
widget.api.transport.reply(ev.detail, {});
})().catch((e) => {
logger.error("Error joining RTC session", e);
});
};
widget.lazyActions.on(ElementWidgetActions.JoinCall, onJoin);
return (): void => {
widget!.lazyActions.off(ElementWidgetActions.JoinCall, onJoin);
widget.lazyActions.off(ElementWidgetActions.JoinCall, onJoin);
};
} else {
// No lobby and no preload: we enter the rtc session right away
@@ -216,7 +230,7 @@ export const GroupCallView: FC<Props> = ({
void enterRTCSession(rtcSession, perParticipantE2EE);
}
}
}, [rtcSession, preload, skipLobby, perParticipantE2EE]);
}, [widget, rtcSession, preload, skipLobby, perParticipantE2EE]);
const [left, setLeft] = useState(false);
const [leaveError, setLeaveError] = useState<Error | undefined>(undefined);
@@ -224,12 +238,12 @@ export const GroupCallView: FC<Props> = ({
const onLeave = useCallback(
(leaveError?: Error): void => {
setLeaveError(leaveError);
setLeft(true);
const audioPromise = leaveSoundContext.current?.playSound("left");
// In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent,
// therefore we want the event to be sent instantly without getting queued/batched.
const sendInstantly = !!widget;
setLeaveError(leaveError);
setLeft(true);
PosthogAnalytics.instance.eventCallEnded.track(
rtcSession.room.roomId,
rtcSession.memberships.length,
@@ -237,8 +251,12 @@ export const GroupCallView: FC<Props> = ({
rtcSession,
);
// Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts.
leaveRTCSession(rtcSession)
leaveRTCSession(
rtcSession,
// Wait for the sound in widget mode (it's not long)
sendInstantly && audioPromise ? audioPromise : undefined,
)
// Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts.
.then(() => {
if (
!isPasswordlessUser &&
@@ -252,18 +270,25 @@ export const GroupCallView: FC<Props> = ({
logger.error("Error leaving RTC session", e);
});
},
[rtcSession, isPasswordlessUser, confineToRoom, history],
[
widget,
rtcSession,
isPasswordlessUser,
confineToRoom,
leaveSoundContext,
history,
],
);
useEffect(() => {
if (widget && isJoined) {
// set widget to sticky once joined.
widget!.api.setAlwaysOnScreen(true).catch((e) => {
widget.api.setAlwaysOnScreen(true).catch((e) => {
logger.error("Error calling setAlwaysOnScreen(true)", e);
});
const onHangup = (ev: CustomEvent<IWidgetApiRequest>): void => {
widget!.api.transport.reply(ev.detail, {});
widget.api.transport.reply(ev.detail, {});
// Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts.
leaveRTCSession(rtcSession).catch((e) => {
logger.error("Failed to leave RTC session", e);
@@ -271,10 +296,10 @@ export const GroupCallView: FC<Props> = ({
};
widget.lazyActions.once(ElementWidgetActions.HangupCall, onHangup);
return (): void => {
widget!.lazyActions.off(ElementWidgetActions.HangupCall, onHangup);
widget.lazyActions.off(ElementWidgetActions.HangupCall, onHangup);
};
}
}, [isJoined, rtcSession]);
}, [widget, isJoined, rtcSession]);
const onReconnect = useCallback(() => {
setLeft(false);
@@ -367,14 +392,17 @@ export const GroupCallView: FC<Props> = ({
leaveError
) {
return (
<CallEndedView
endedCallId={rtcSession.room.roomId}
client={client}
isPasswordlessUser={isPasswordlessUser}
confineToRoom={confineToRoom}
leaveError={leaveError}
reconnect={onReconnect}
/>
<>
<CallEndedView
endedCallId={rtcSession.room.roomId}
client={client}
isPasswordlessUser={isPasswordlessUser}
confineToRoom={confineToRoom}
leaveError={leaveError}
reconnect={onReconnect}
/>
;
</>
);
} else {
// If the user is a regular user, we'll have sent them back to the homepage,

4
src/room/ReactionAudioRenderer.tsx
View File

@@ -60,10 +60,10 @@ export function ReactionsAudioRenderer(): ReactNode {
return;
}
if (soundMap[reactionName]) {
audioEngineRef.current.playSound(reactionName);
void audioEngineRef.current.playSound(reactionName);
} else {
// Fallback sounds.
audioEngineRef.current.playSound("generic");
void audioEngineRef.current.playSound("generic");
}
}
}, [audioEngineRef, shouldPlay, oldReactions, reactions]);

1
src/room/RoomPage.tsx
View File

@@ -98,6 +98,7 @@ export const RoomPage: FC = () => {
case "loaded":
return (
<GroupCallView
widget={widget}
client={client!}
rtcSession={groupCallState.rtcSession}
isPasswordlessUser={passwordlessUser}

8
src/rtcSessionHelpers.ts
View File

@@ -120,6 +120,7 @@ export async function enterRTCSession(
const widgetPostHangupProcedure = async (
widget: WidgetHelpers,
promiseBeforeHangup?: Promise<unknown>,
): Promise<void> => {
// we need to wait until the callEnded event is tracked on posthog.
// Otherwise the iFrame gets killed before the callEnded event got tracked.
@@ -132,6 +133,8 @@ const widgetPostHangupProcedure = async (
logger.error("Failed to set call widget `alwaysOnScreen` to false", e);
}
// Wait for any last bits before hanging up.
await promiseBeforeHangup;
// We send the hangup event after the memberships have been updated
// calling leaveRTCSession.
// We need to wait because this makes the client hosting this widget killing the IFrame.
@@ -140,9 +143,12 @@ const widgetPostHangupProcedure = async (
export async function leaveRTCSession(
rtcSession: MatrixRTCSession,
promiseBeforeHangup?: Promise<unknown>,
): Promise<void> {
await rtcSession.leaveRoomSession();
if (widget) {
await widgetPostHangupProcedure(widget);
await widgetPostHangupProcedure(widget, promiseBeforeHangup);
} else {
await promiseBeforeHangup;
}
}

7
src/useAudioContext.test.tsx
View File

@@ -29,9 +29,11 @@ const TestComponent: FC = () => {
}
return (
<>
<button onClick={() => audioCtx.playSound("aSound")}>Valid sound</button>
<button onClick={() => void audioCtx.playSound("aSound")}>
Valid sound
</button>
{/* eslint-disable-next-line @typescript-eslint/no-explicit-any*/}
<button onClick={() => audioCtx.playSound("not-valid" as any)}>
<button onClick={() => void audioCtx.playSound("not-valid" as any)}>
Invalid sound
</button>
</>
@@ -61,6 +63,7 @@ class MockAudioContext {
vitest.mocked({
connect: (v: unknown) => v,
start: () => {},
addEventListener: (_name: string, cb: () => void) => cb(),
}),
);
public createGain = vitest.fn().mockReturnValue(this.gain);

11
src/useAudioContext.tsx
View File

@@ -22,18 +22,21 @@ import { type PrefetchedSounds } from "./soundUtils";
* @param volume The volume to play at.
* @param ctx The context to play through.
* @param buffer The buffer to play.
* @returns A promise that resolves when the sound has finished playing.
*/
function playSound(
async function playSound(
ctx: AudioContext,
buffer: AudioBuffer,
volume: number,
): void {
): Promise<void> {
const gain = ctx.createGain();
gain.gain.setValueAtTime(volume, 0);
const src = ctx.createBufferSource();
src.buffer = buffer;
src.connect(gain).connect(ctx.destination);
const p = new Promise<void>((r) => src.addEventListener("ended", () => r()));
src.start();
return p;
}
interface Props<S extends string> {
@@ -47,7 +50,7 @@ interface Props<S extends string> {
}
interface UseAudioContext<S> {
playSound(soundName: S): void;
playSound(soundName: S): Promise<void>;
}
/**
@@ -113,7 +116,7 @@ export function useAudioContext<S extends string>(
return null;
}
return {
playSound: (name): void => {
playSound: async (name): Promise<void> => {
if (!audioBuffers[name]) {
logger.debug(`Tried to play a sound that wasn't buffered (${name})`);
return;

12
src/utils/test.ts
View File

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { map, type Observable, of, type SchedulerLike } from "rxjs";
import { type RunHelpers, TestScheduler } from "rxjs/testing";
import { expect, vi } from "vitest";
import { expect, vi, vitest } from "vitest";
import {
type RoomMember,
type Room as MatrixRoom,
@@ -258,6 +258,12 @@ export class MockRTCSession extends TypedEventEmitter<
MatrixRTCSessionEvent,
MatrixRTCSessionEventHandlerMap
> {
public readonly statistics = {
counters: {},
};
public leaveRoomSession = vitest.fn().mockResolvedValue(undefined);
public constructor(
public readonly room: Room,
private localMembership: CallMembership,
@@ -266,6 +272,10 @@ export class MockRTCSession extends TypedEventEmitter<
super();
}
public isJoined(): true {
return true;
}
public withMemberships(
rtcMembers: Observable<Partial<CallMembership>[]>,
): MockRTCSession {