Port to useAudioContext

src/room/CallEventAudioRenderer.tsx

@@ -5,147 +5,59 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { ReactNode, useEffect, useMemo, useRef, useState } from "react";
+import { ReactNode, useEffect } from "react";
-import { filter, interval, skip, throttle } from "rxjs";
+import { debounce, filter, interval, skip, throttle } from "rxjs";
-import { logger } from "matrix-js-sdk/src/logger";
-
-import {
-  soundEffectVolumeSetting as effectSoundVolumeSetting,
-  useSetting,
-} from "../settings/settings";
 import { CallViewModel } from "../state/CallViewModel";
 import joinCallSoundMp3 from "../sound/join_call.mp3";
 import joinCallSoundOgg from "../sound/join_call.ogg";
 import leftCallSoundMp3 from "../sound/left_call.mp3";
 import leftCallSoundOgg from "../sound/left_call.ogg";
-import { useMediaDevices } from "../livekit/MediaDevicesContext";
+import { prefetchSounds, useAudioContext } from "../useAudioContext";
-import { useLatest } from "../useLatest";
 
 // Do not play any sounds if the participant count has exceeded this
 // number.
 export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
-export const DEBOUNCE_SOUND_EFFECT_MS = 500;
+export const THROTTLE_SOUND_EFFECT_MS = 500;
+export const DEBOUNCE_SOUND_EFFECT_MS = 150;
 
-async function loadAudioBuffer(filename: string) {
+const Sounds = prefetchSounds({
-  // Load an audio file
+  join: {
-  const response = await fetch(filename);
+    mp3: joinCallSoundMp3,
-  if (!response.ok) {
+    ogg: joinCallSoundOgg,
-    throw Error("Could not load sound, resposne was not okay");
+  },
-  }
+  left: {
-  // Decode it
+    mp3: leftCallSoundMp3,
-  return await await response.arrayBuffer();
+    ogg: leftCallSoundOgg,
-}
+  },
-
+});
-function playSound(
-  volume: number,
-  ctx?: AudioContext,
-  buffer?: AudioBuffer,
-): void {
-  if (!ctx || !buffer) {
-    return;
-  }
-  const gain = ctx.createGain();
-  gain.gain.setValueAtTime(volume, 0);
-  const src = ctx.createBufferSource();
-  src.buffer = buffer;
-  src.connect(gain).connect(ctx.destination);
-  src.start();
-}
-
-function getPreferredAudioFormat() {
-  const a = document.createElement("audio");
-  if (a.canPlayType("audio/ogg") === "maybe") {
-    return "ogg";
-  }
-  // Otherwise just assume MP3, as that has a chance of being more widely supported.
-  return "mp3";
-}
-
-// We prefer to load these sounds ahead of time, so there
-// is no delay on call join.
-const preferredFormat = getPreferredAudioFormat();
-const JoinSoundBufferPromise = loadAudioBuffer(
-  preferredFormat === "ogg" ? joinCallSoundOgg : joinCallSoundMp3,
-);
-const LeftSoundBufferPromise = loadAudioBuffer(
-  preferredFormat === "ogg" ? leftCallSoundOgg : leftCallSoundMp3,
-);
 
 export function CallEventAudioRenderer({
   vm,
 }: {
   vm: CallViewModel;
 }): ReactNode {
-  const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
+  const audioEngineCtx = useAudioContext({
-  const devices = useMediaDevices();
+    sounds: Sounds,
-  const audioSourceElement = useRef<HTMLAudioElement>(null);
+    latencyHint: "interactive",
-  const [audioContext, setAudioContext] = useState<AudioContext>();
+  });
-  const [joinCallBuffer, setJoinSoundNode] = useState<AudioBuffer>();
-  const [leaveCallBuffer, setLeaveSoundNode] = useState<AudioBuffer>();
 
   useEffect(() => {
-    const ctx = new AudioContext({
+    if (!audioEngineCtx) {
-      // We want low latency for these effects.
+      return;
-      latencyHint: "interactive",
-      // XXX: Types don't include this yet.
-      ...{ sinkId: devices.audioOutput.selectedId },
-    });
-    const controller = new AbortController();
-    (async () => {
-      controller.signal.throwIfAborted();
-      const enterCall = await ctx.decodeAudioData(
-        (await JoinSoundBufferPromise).slice(0),
-      );
-      controller.signal.throwIfAborted();
-      const leaveCall = await ctx.decodeAudioData(
-        (await LeftSoundBufferPromise).slice(0),
-      );
-      controller.signal.throwIfAborted();
-      setJoinSoundNode(enterCall);
-      setLeaveSoundNode(leaveCall);
-    })().catch((ex) => {
-      logger.debug("Failed to setup audio context", ex);
-    });
-
-    setAudioContext(ctx);
-    return () => {
-      controller.abort("Closing");
-      void ctx.close().catch((ex) => {
-        logger.debug("Failed to close audio engine", ex);
-      });
-      setAudioContext(undefined);
-    };
-  }, []);
-
-  // Update the sink ID whenever we change devices.
-  useEffect(() => {
-    if (audioContext && "setSinkId" in audioContext) {
-      // setSinkId doesn't exist in types but does exist for some browsers.
-      // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
-      // @ts-ignore
-      audioContext.setSinkId(devices.audioOutput.selectedId).catch((ex) => {
-        logger.warn("Unable to change sink for audio context", ex);
-      });
     }
-  }, [audioContext, devices]);
 
-  // Prevent a rerender when t he
-  const soundVolume = useLatest(effectSoundVolume);
-
-  useEffect(() => {
     const joinSub = vm.memberChanges
       .pipe(
         filter(
           ({ joined, ids }) =>
-            // Only play when more than one person is in the room.
+            ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0,
-            ids.length > 1 &&
-            ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND &&
-            joined.length > 0,
         ),
-        throttle((_) => interval(DEBOUNCE_SOUND_EFFECT_MS)),
+        throttle((_) => interval(THROTTLE_SOUND_EFFECT_MS)),
+        debounce((_) => interval(DEBOUNCE_SOUND_EFFECT_MS)),
       )
-      .subscribe(() => {
+      .subscribe((prev) => {
-        playSound(soundVolume.current, audioContext, joinCallBuffer);
+        console.log("Playing join sound for", ...prev.joined, "|", prev);
+        audioEngineCtx.playSound("join");
       });
 
     const leftSub = vm.memberChanges
@@ -154,17 +66,18 @@ export function CallEventAudioRenderer({
           ({ ids, left }) =>
             ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
         ),
-        throttle((_) => interval(DEBOUNCE_SOUND_EFFECT_MS)),
+        throttle((_) => interval(THROTTLE_SOUND_EFFECT_MS)),
+        debounce((_) => interval(DEBOUNCE_SOUND_EFFECT_MS)),
       )
       .subscribe(() => {
-        playSound(soundVolume.current, audioContext, leaveCallBuffer);
+        audioEngineCtx.playSound("left");
       });
 
     return (): void => {
       joinSub.unsubscribe();
       leftSub.unsubscribe();
     };
-  }, [joinCallBuffer, leaveCallBuffer, soundVolume, vm]);
+  }, [audioEngineCtx, vm]);
 
-  return <audio ref={audioSourceElement} hidden />;
+  return <></>;
 }

src/useAudioContext.tsx

@@ -0,0 +1,143 @@
+import { logger } from "matrix-js-sdk/src/logger";
+import { useState, useEffect } from "react";
+import {
+  soundEffectVolumeSetting as effectSoundVolumeSetting,
+  useSetting,
+} from "./settings/settings";
+import { useMediaDevices } from "./livekit/MediaDevicesContext";
+
+type SoundDefinition = { mp3: string; ogg: string };
+
+async function fetchBuffer(filename: string) {
+  // Load an audio file
+  const response = await fetch(filename);
+  if (!response.ok) {
+    throw Error("Could not load sound, resposne was not okay");
+  }
+  // Decode it
+  return await await response.arrayBuffer();
+}
+
+function playSound(
+  volume: number,
+  ctx?: AudioContext,
+  buffer?: AudioBuffer,
+): void {
+  if (!ctx || !buffer) {
+    return;
+  }
+  const gain = ctx.createGain();
+  gain.gain.setValueAtTime(volume, 0);
+  const src = ctx.createBufferSource();
+  src.buffer = buffer;
+  src.connect(gain).connect(ctx.destination);
+  src.start();
+}
+
+function getPreferredAudioFormat() {
+  const a = document.createElement("audio");
+  if (a.canPlayType("audio/ogg") === "maybe") {
+    return "ogg";
+  }
+  // Otherwise just assume MP3, as that has a chance of being more widely supported.
+  return "mp3";
+}
+
+type PrefetchedSounds<S extends string> = Promise<Record<string, ArrayBuffer>>;
+
+// We prefer to load these sounds ahead of time, so there
+// is no delay on call join.
+const PreferredFormat = getPreferredAudioFormat();
+
+/**
+ * Prefetch sounds to be used by the AudioContext. This should
+ * be called outside the scope of a component to ensure the
+ * sounds load ahead of time.
+ * @param sounds A set of sound files that may be played.
+ * @returns A map of sound files to buffers.
+ */
+export async function prefetchSounds<S extends string>(
+  sounds: Record<S, SoundDefinition>,
+): PrefetchedSounds<S> {
+  logger.debug(`Loading sounds`);
+  const buffers: Record<string, ArrayBuffer> = {};
+  await Promise.all(
+    Object.entries(sounds).map(async ([name, file]) => {
+      const { mp3, ogg } = file as SoundDefinition;
+      buffers[name] = await fetchBuffer(PreferredFormat === "ogg" ? ogg : mp3);
+    }),
+  );
+  return buffers as Record<S, ArrayBuffer>;
+}
+
+interface Props<S extends string> {
+  sounds: PrefetchedSounds<S>;
+  latencyHint: AudioContextLatencyCategory;
+}
+
+interface UseAudioContext<S> {
+  playSound(soundName: S): void;
+}
+
+export function useAudioContext<S extends string>(
+  props: Props<S>,
+): UseAudioContext<S> | null {
+  const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
+  const devices = useMediaDevices();
+  const [audioContext, setAudioContext] = useState<AudioContext>();
+  const [audioBuffers, setAudioBuffers] = useState<Record<S, AudioBuffer>>();
+
+  useEffect(() => {
+    const ctx = new AudioContext({
+      // We want low latency for these effects.
+      latencyHint: props.latencyHint,
+      // XXX: Types don't include this yet.
+      ...{ sinkId: devices.audioOutput.selectedId },
+    });
+    const controller = new AbortController();
+    (async () => {
+      const buffers: Record<string, AudioBuffer> = {};
+      controller.signal.throwIfAborted();
+      for (const [name, buffer] of Object.entries(await props.sounds)) {
+        controller.signal.throwIfAborted();
+        const audioBuffer = await ctx.decodeAudioData(buffer.slice(0));
+        buffers[name] = audioBuffer;
+        // Store as we go.
+        setAudioBuffers(buffers as Record<S, AudioBuffer>);
+      }
+    })().catch((ex) => {
+      logger.debug("Failed to setup audio context", ex);
+    });
+
+    setAudioContext(ctx);
+    return () => {
+      controller.abort("Closing");
+      void ctx.close().catch((ex) => {
+        logger.debug("Failed to close audio engine", ex);
+      });
+      setAudioContext(undefined);
+    };
+  }, []);
+
+  // Update the sink ID whenever we change devices.
+  useEffect(() => {
+    if (audioContext && "setSinkId" in audioContext) {
+      // setSinkId doesn't exist in types but does exist for some browsers.
+      // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
+      // @ts-ignore
+      audioContext.setSinkId(devices.audioOutput.selectedId).catch((ex) => {
+        logger.warn("Unable to change sink for audio context", ex);
+      });
+    }
+  }, [audioContext, devices]);
+
+  if (!audioContext || !audioBuffers) {
+    logger.debug("Audio not ready yet");
+    return null;
+  }
+  return {
+    playSound: (name) => {
+      playSound(effectSoundVolume, audioContext, audioBuffers[name]);
+    },
+  };
+}