= ({
{t("settings.audio_tab.effect_volume_description")}
= Promise<
+ Record
+>;
+
+/**
+ * Determine the best format we can use to play our sounds
+ * through. We prefer ogg support if possible, but will fall
+ * back to MP3.
+ * @returns "ogg" if the browser is likely to support it, or "mp3" otherwise.
+ */
+function getPreferredAudioFormat(): "ogg" | "mp3" {
+ const a = document.createElement("audio");
+ if (a.canPlayType("audio/ogg") === "maybe") {
+ return "ogg";
+ }
+ // Otherwise just assume MP3, as that has a chance of being more widely supported.
+ return "mp3";
+}
+
+const preferredFormat = getPreferredAudioFormat();
+
+/**
+ * Prefetch sounds to be used by the AudioContext. This can
+ * be called outside the scope of a component to ensure the
+ * sounds load ahead of time.
+ * @param sounds A set of sound files that may be played.
+ * @returns A map of sound files to buffers.
+ */
+export async function prefetchSounds(
+ sounds: Record,
+): PrefetchedSounds {
+ const buffers: Record = {};
+ await Promise.all(
+ Object.entries(sounds).map(async ([name, file]) => {
+ const { mp3, ogg } = file as SoundDefinition;
+ // Use preferred format, fallback to ogg if no mp3 is provided.
+ // Load an audio file
+ const response = await fetch(
+ preferredFormat === "ogg" ? ogg : (mp3 ?? ogg),
+ );
+ if (!response.ok) {
+ // If the sound doesn't load, it's not the end of the world. We won't play
+ // the sound when requested, but it's better than failing the whole application.
+ logger.warn(`Could not load sound ${name}, response was not okay`);
+ return;
+ }
+ // Decode it
+ buffers[name] = await response.arrayBuffer();
+ }),
+ );
+ return buffers as Record;
+}
diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts
index 95762c3f..af8780b1 100644
--- a/src/state/CallViewModel.ts
+++ b/src/state/CallViewModel.ts
@@ -335,7 +335,7 @@ function findMatrixRoomMember(
// must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon
if (parts.length < 3) {
logger.warn(
- "Livekit participants ID doesn't look like a userId:deviceId combination",
+ `Livekit participants ID (${id}) doesn't look like a userId:deviceId combination`,
);
return undefined;
}
diff --git a/src/useAudioContext.test.tsx b/src/useAudioContext.test.tsx
new file mode 100644
index 00000000..5a1afe43
--- /dev/null
+++ b/src/useAudioContext.test.tsx
@@ -0,0 +1,129 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { expect, test, vitest } from "vitest";
+import { FC } from "react";
+import { render } from "@testing-library/react";
+import { afterEach } from "node:test";
+import userEvent from "@testing-library/user-event";
+
+import { deviceStub, MediaDevicesContext } from "./livekit/MediaDevicesContext";
+import { useAudioContext } from "./useAudioContext";
+import { soundEffectVolumeSetting } from "./settings/settings";
+
+const staticSounds = Promise.resolve({
+ aSound: new ArrayBuffer(0),
+});
+
+const TestComponent: FC = () => {
+ const audioCtx = useAudioContext({
+ sounds: staticSounds,
+ latencyHint: "balanced",
+ });
+ if (!audioCtx) {
+ return null;
+ }
+ return (
+ <>
+
+ {/* eslint-disable-next-line @typescript-eslint/no-explicit-any*/}
+
+
+ );
+};
+
+class MockAudioContext {
+ public static testContext: MockAudioContext;
+
+ public constructor() {
+ MockAudioContext.testContext = this;
+ }
+
+ public gain = vitest.mocked(
+ {
+ connect: () => {},
+ gain: {
+ setValueAtTime: vitest.fn(),
+ },
+ },
+ true,
+ );
+
+ public setSinkId = vitest.fn().mockResolvedValue(undefined);
+ public decodeAudioData = vitest.fn().mockReturnValue(1);
+ public createBufferSource = vitest.fn().mockReturnValue(
+ vitest.mocked({
+ connect: (v: unknown) => v,
+ start: () => {},
+ }),
+ );
+ public createGain = vitest.fn().mockReturnValue(this.gain);
+ public close = vitest.fn().mockResolvedValue(undefined);
+}
+
+afterEach(() => {
+ vitest.unstubAllGlobals();
+});
+
+test("can play a single sound", async () => {
+ const user = userEvent.setup();
+ vitest.stubGlobal("AudioContext", MockAudioContext);
+ const { findByText } = render();
+ await user.click(await findByText("Valid sound"));
+ expect(
+ MockAudioContext.testContext.createBufferSource,
+ ).toHaveBeenCalledOnce();
+});
+test("will ignore sounds that are not registered", async () => {
+ const user = userEvent.setup();
+ vitest.stubGlobal("AudioContext", MockAudioContext);
+ const { findByText } = render();
+ await user.click(await findByText("Invalid sound"));
+ expect(
+ MockAudioContext.testContext.createBufferSource,
+ ).not.toHaveBeenCalled();
+});
+
+test("will use the correct device", () => {
+ vitest.stubGlobal("AudioContext", MockAudioContext);
+ render(
+ {},
+ },
+ videoInput: deviceStub,
+ startUsingDeviceNames: () => {},
+ stopUsingDeviceNames: () => {},
+ }}
+ >
+
+ ,
+ );
+ expect(
+ MockAudioContext.testContext.createBufferSource,
+ ).not.toHaveBeenCalled();
+ expect(MockAudioContext.testContext.setSinkId).toHaveBeenCalledWith(
+ "chosen-device",
+ );
+});
+
+test("will use the correct volume level", async () => {
+ const user = userEvent.setup();
+ vitest.stubGlobal("AudioContext", MockAudioContext);
+ soundEffectVolumeSetting.setValue(0.33);
+ const { findByText } = render();
+ await user.click(await findByText("Valid sound"));
+ expect(
+ MockAudioContext.testContext.gain.gain.setValueAtTime,
+ ).toHaveBeenCalledWith(0.33, 0);
+});
diff --git a/src/useAudioContext.tsx b/src/useAudioContext.tsx
new file mode 100644
index 00000000..ccf4cbd5
--- /dev/null
+++ b/src/useAudioContext.tsx
@@ -0,0 +1,124 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { logger } from "matrix-js-sdk/src/logger";
+import { useState, useEffect } from "react";
+
+import {
+ soundEffectVolumeSetting as effectSoundVolumeSetting,
+ useSetting,
+} from "./settings/settings";
+import { useMediaDevices } from "./livekit/MediaDevicesContext";
+import { PrefetchedSounds } from "./soundUtils";
+
+/**
+ * Play a sound though a given AudioContext. Will take
+ * care of connecting the correct buffer and gating
+ * through gain.
+ * @param volume The volume to play at.
+ * @param ctx The context to play through.
+ * @param buffer The buffer to play.
+ */
+function playSound(
+ ctx: AudioContext,
+ buffer: AudioBuffer,
+ volume: number,
+): void {
+ const gain = ctx.createGain();
+ gain.gain.setValueAtTime(volume, 0);
+ const src = ctx.createBufferSource();
+ src.buffer = buffer;
+ src.connect(gain).connect(ctx.destination);
+ src.start();
+}
+
+interface Props {
+ /**
+ * The sounds to play. If no sounds should be played then
+ * this can be set to null, which will prevent the audio
+ * context from being created.
+ */
+ sounds: PrefetchedSounds | null;
+ latencyHint: AudioContextLatencyCategory;
+}
+
+interface UseAudioContext {
+ playSound(soundName: S): void;
+}
+
+/**
+ * Add an audio context which can be used to play
+ * a set of preloaded sounds.
+ * @param props
+ * @returns Either an instance that can be used to play sounds, or null if not ready.
+ */
+export function useAudioContext(
+ props: Props,
+): UseAudioContext | null {
+ const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
+ const devices = useMediaDevices();
+ const [audioContext, setAudioContext] = useState();
+ const [audioBuffers, setAudioBuffers] = useState>();
+
+ useEffect(() => {
+ const sounds = props.sounds;
+ if (!sounds) {
+ return;
+ }
+ const ctx = new AudioContext({
+ // We want low latency for these effects.
+ latencyHint: props.latencyHint,
+ });
+
+ // We want to clone the content of our preloaded
+ // sound buffers into this context. The context may
+ // close during this process, so it's okay if it throws.
+ (async (): Promise => {
+ const buffers: Record = {};
+ for (const [name, buffer] of Object.entries(await sounds)) {
+ const audioBuffer = await ctx.decodeAudioData(buffer.slice(0));
+ buffers[name] = audioBuffer;
+ }
+ setAudioBuffers(buffers as Record);
+ })().catch((ex) => {
+ logger.debug("Failed to setup audio context", ex);
+ });
+
+ setAudioContext(ctx);
+ return (): void => {
+ void ctx.close().catch((ex) => {
+ logger.debug("Failed to close audio engine", ex);
+ });
+ setAudioContext(undefined);
+ };
+ }, [props.sounds, props.latencyHint]);
+
+ // Update the sink ID whenever we change devices.
+ useEffect(() => {
+ if (audioContext && "setSinkId" in audioContext) {
+ // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
+ // @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
+ audioContext.setSinkId(devices.audioOutput.selectedId).catch((ex) => {
+ logger.warn("Unable to change sink for audio context", ex);
+ });
+ }
+ }, [audioContext, devices]);
+
+ // Don't return a function until we're ready.
+ if (!audioContext || !audioBuffers) {
+ return null;
+ }
+ return {
+ playSound: (name): void => {
+ if (!audioBuffers[name]) {
+ logger.debug(`Tried to play a sound that wasn't buffered (${name})`);
+ return;
+ }
+ return playSound(audioContext, audioBuffers[name], effectSoundVolume);
+ },
+ };
+}
diff --git a/src/utils/matrix.ts b/src/utils/matrix.ts
index d3821a3f..63b6ef67 100644
--- a/src/utils/matrix.ts
+++ b/src/utils/matrix.ts
@@ -333,15 +333,3 @@ export function getRelativeRoomUrl(
: "";
return `/room/#${roomPart}?${generateUrlSearchParams(roomId, encryptionSystem, viaServers).toString()}`;
}
-
-export function getAvatarUrl(
- client: MatrixClient,
- mxcUrl: string,
- avatarSize = 96,
-): string {
- const width = Math.floor(avatarSize * window.devicePixelRatio);
- const height = Math.floor(avatarSize * window.devicePixelRatio);
- // scale is more suitable for larger sizes
- const resizeMethod = avatarSize <= 96 ? "crop" : "scale";
- return mxcUrl && client.mxcUrlToHttp(mxcUrl, width, height, resizeMethod)!;
-}
diff --git a/src/utils/test.ts b/src/utils/test.ts
index dca98825..459a252e 100644
--- a/src/utils/test.ts
+++ b/src/utils/test.ts
@@ -27,9 +27,7 @@ import {
RemoteParticipant,
RemoteTrackPublication,
Room as LivekitRoom,
- RoomEvent,
} from "livekit-client";
-import { EventEmitter } from "stream";
import {
LocalUserMediaViewModel,
@@ -153,33 +151,6 @@ export function mockMatrixRoom(room: Partial): MatrixRoom {
return { ...mockEmitter(), ...room } as Partial as MatrixRoom;
}
-/**
- * A mock of a Livekit Room that can emit events.
- */
-export class EmittableMockLivekitRoom extends EventEmitter {
- public localParticipant?: LocalParticipant;
- public remoteParticipants: Map;
-
- public constructor(room: {
- localParticipant?: LocalParticipant;
- remoteParticipants: Map;
- }) {
- super();
- this.localParticipant = room.localParticipant;
- this.remoteParticipants = room.remoteParticipants ?? new Map();
- }
-
- public addParticipant(remoteParticipant: RemoteParticipant): void {
- this.remoteParticipants.set(remoteParticipant.identity, remoteParticipant);
- this.emit(RoomEvent.ParticipantConnected, remoteParticipant);
- }
-
- public removeParticipant(remoteParticipant: RemoteParticipant): void {
- this.remoteParticipants.delete(remoteParticipant.identity);
- this.emit(RoomEvent.ParticipantDisconnected, remoteParticipant);
- }
-}
-
export function mockLivekitRoom(
room: Partial,
{
@@ -280,15 +251,6 @@ export function mockConfig(config: Partial = {}): void {
});
}
-export function mockMediaPlay(): string[] {
- const audioIsPlaying: string[] = [];
- window.HTMLMediaElement.prototype.play = async function (): Promise {
- audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
- return Promise.resolve();
- };
- return audioIsPlaying;
-}
-
export class MockRTCSession extends TypedEventEmitter<
MatrixRTCSessionEvent,
MatrixRTCSessionEventHandlerMap
diff --git a/src/utils/testReactions.tsx b/src/utils/testReactions.tsx
index 84ff217b..fec3e859 100644
--- a/src/utils/testReactions.tsx
+++ b/src/utils/testReactions.tsx
@@ -32,7 +32,7 @@ export const TestReactionsWrapper = ({
rtcSession,
children,
}: PropsWithChildren<{
- rtcSession: MockRTCSession;
+ rtcSession: MockRTCSession | MatrixRTCSession;
}>): ReactNode => {
return (
@@ -203,4 +203,12 @@ export class MockRoom extends EventEmitter {
});
return evt.getId()!;
}
+
+ public getMember(): void {
+ return;
+ }
+
+ public testGetAsMatrixRoom(): Room {
+ return this as unknown as Room;
+ }
}