Merge branch 'livekit' into toger5/tiles_based_on_rtc_member

locales/de/app.json

@@ -68,8 +68,12 @@
     "username": "Benutzername",
     "video": "Video"
   },
-  "crypto_version": "Krypto-Version:{{version}}",
+  "developer_mode": {
-  "device_id": "Geräte-ID: {{id}}",
+    "crypto_version": "Krypto-Version:{{version}}",
+    "device_id": "Geräte-ID: {{id}}",
+    "duplicate_tiles_label": "Anzahl zusätzlicher Kachelkopien pro Teilnehmer",
+    "matrix_id": "Matrix-ID: {{id}}"
+  },
   "disconnected_banner": "Die Verbindung zum Server wurde getrennt.",
   "full_screen_view_description": "<0>Übermittelte Problemberichte helfen uns, Fehler zu beheben.</0>",
   "full_screen_view_h1": "<0>Hoppla, etwas ist schiefgelaufen.</0>",
@@ -111,7 +115,6 @@
   "login_auth_links_prompt": "Noch nicht registriert?",
   "login_subheading": "Weiter zu Element",
   "login_title": "Anmelden",
-  "matrix_id": "Matrix-ID: {{id}}",
   "microphone_off": "Mikrofon aus",
   "microphone_on": "Mikrofon an",
   "mute_microphone_button_label": "Mikrofon stumm schalten",
@@ -149,7 +152,6 @@
     "developer_settings_label": "Entwicklereinstellungen",
     "developer_settings_label_description": "Zeige die Entwicklereinstellungen im Einstellungsfenster.",
     "developer_tab_title": "Entwickler",
-    "duplicate_tiles_label": "Anzahl zusätzlicher Kachelkopien pro Teilnehmer",
     "feedback_tab_body": "Falls du auf Probleme stößt oder einfach nur eine Rückmeldung geben möchtest, sende uns bitte eine kurze Beschreibung.",
     "feedback_tab_description_label": "Deine Rückmeldung",
     "feedback_tab_h4": "Rückmeldung geben",

locales/en-GB/app.json

@@ -68,8 +68,13 @@
     "username": "Username",
     "video": "Video"
   },
-  "crypto_version": "Crypto version: {{version}}",
+  "developer_mode": {
-  "device_id": "Device ID: {{id}}",
+    "crypto_version": "Crypto version: {{version}}",
+    "device_id": "Device ID: {{id}}",
+    "duplicate_tiles_label": "Number of additional tile copies per participant",
+    "hostname": "Hostname: {{hostname}}",
+    "matrix_id": "Matrix ID: {{id}}"
+  },
   "disconnected_banner": "Connectivity to the server has been lost.",
   "full_screen_view_description": "<0>Submitting debug logs will help us track down the problem.</0>",
   "full_screen_view_h1": "<0>Oops, something's gone wrong.</0>",
@@ -111,7 +116,6 @@
   "login_auth_links_prompt": "Not registered yet?",
   "login_subheading": "To continue to Element",
   "login_title": "Login",
-  "matrix_id": "Matrix ID: {{id}}",
   "microphone_off": "Microphone off",
   "microphone_on": "Microphone on",
   "mute_microphone_button_label": "Mute microphone",
@@ -149,7 +153,6 @@
     "developer_settings_label": "Developer Settings",
     "developer_settings_label_description": "Expose developer settings in the settings window.",
     "developer_tab_title": "Developer",
-    "duplicate_tiles_label": "Number of additional tile copies per participant",
     "feedback_tab_body": "If you are experiencing issues or simply would like to provide some feedback, please send us a short description below.",
     "feedback_tab_description_label": "Your feedback",
     "feedback_tab_h4": "Submit feedback",

src/room/CallEventAudioRenderer.test.tsx

@@ -0,0 +1,225 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { render } from "@testing-library/react";
+import { beforeEach, expect, test } from "vitest";
+import { MatrixClient } from "matrix-js-sdk/src/client";
+import { ConnectionState, RemoteParticipant, Room } from "livekit-client";
+import { of } from "rxjs";
+import { afterEach } from "node:test";
+import { act } from "react";
+
+import { soundEffectVolumeSetting } from "../settings/settings";
+import {
+  EmittableMockLivekitRoom,
+  mockLivekitRoom,
+  mockLocalParticipant,
+  mockMatrixRoom,
+  mockMatrixRoomMember,
+  mockMediaPlay,
+  mockRemoteParticipant,
+} from "../utils/test";
+import { E2eeType } from "../e2ee/e2eeType";
+import { CallViewModel } from "../state/CallViewModel";
+import {
+  CallEventAudioRenderer,
+  MAX_PARTICIPANT_COUNT_FOR_SOUND,
+} from "./CallEventAudioRenderer";
+
+const alice = mockMatrixRoomMember({ userId: "@alice:example.org" });
+const bob = mockMatrixRoomMember({ userId: "@bob:example.org" });
+const aliceId = `${alice.userId}:AAAA`;
+const bobId = `${bob.userId}:BBBB`;
+const localParticipant = mockLocalParticipant({ identity: "" });
+const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
+const bobParticipant = mockRemoteParticipant({ identity: bobId });
+
+const originalPlayFn = window.HTMLMediaElement.prototype.play;
+
+const enterSound = "http://localhost:3000/src/sound/join_call.ogg";
+const leaveSound = "http://localhost:3000/src/sound/left_call.ogg";
+
+beforeEach(() => {
+  soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
+});
+
+afterEach(() => {
+  window.HTMLMediaElement.prototype.play = originalPlayFn;
+});
+
+test("plays a sound when entering a call", () => {
+  const audioIsPlaying: string[] = mockMediaPlay();
+  const members = new Map([alice, bob].map((p) => [p.userId, p]));
+  const remoteParticipants = of([aliceParticipant]);
+  const liveKitRoom = mockLivekitRoom(
+    { localParticipant },
+    { remoteParticipants },
+  );
+
+  const vm = new CallViewModel(
+    mockMatrixRoom({
+      client: {
+        getUserId: () => "@carol:example.org",
+      } as Partial<MatrixClient> as MatrixClient,
+      getMember: (userId) => members.get(userId) ?? null,
+    }),
+    liveKitRoom,
+    {
+      kind: E2eeType.PER_PARTICIPANT,
+    },
+    of(ConnectionState.Connected),
+  );
+
+  render(<CallEventAudioRenderer vm={vm} />);
+  expect(audioIsPlaying).toEqual([
+    // Joining the call
+    enterSound,
+  ]);
+});
+
+test("plays no sound when muted", () => {
+  soundEffectVolumeSetting.setValue(0);
+  const audioIsPlaying: string[] = mockMediaPlay();
+  const members = new Map([alice, bob].map((p) => [p.userId, p]));
+  const remoteParticipants = of([aliceParticipant, bobParticipant]);
+  const liveKitRoom = mockLivekitRoom(
+    { localParticipant },
+    { remoteParticipants },
+  );
+
+  const vm = new CallViewModel(
+    mockMatrixRoom({
+      client: {
+        getUserId: () => "@carol:example.org",
+      } as Partial<MatrixClient> as MatrixClient,
+      getMember: (userId) => members.get(userId) ?? null,
+    }),
+    liveKitRoom,
+    {
+      kind: E2eeType.PER_PARTICIPANT,
+    },
+    of(ConnectionState.Connected),
+  );
+
+  render(<CallEventAudioRenderer vm={vm} />);
+  // Play a sound when joining a call.
+  expect(audioIsPlaying).toHaveLength(0);
+});
+
+test("plays a sound when a user joins", () => {
+  const audioIsPlaying: string[] = mockMediaPlay();
+  const members = new Map([alice].map((p) => [p.userId, p]));
+  const remoteParticipants = new Map(
+    [aliceParticipant].map((p) => [p.identity, p]),
+  );
+  const liveKitRoom = new EmittableMockLivekitRoom({
+    localParticipant,
+    remoteParticipants,
+  });
+
+  const vm = new CallViewModel(
+    mockMatrixRoom({
+      client: {
+        getUserId: () => "@carol:example.org",
+      } as Partial<MatrixClient> as MatrixClient,
+      getMember: (userId) => members.get(userId) ?? null,
+    }),
+    liveKitRoom as unknown as Room,
+    {
+      kind: E2eeType.PER_PARTICIPANT,
+    },
+    of(ConnectionState.Connected),
+  );
+  render(<CallEventAudioRenderer vm={vm} />);
+
+  act(() => {
+    liveKitRoom.addParticipant(bobParticipant);
+  });
+  // Play a sound when joining a call.
+  expect(audioIsPlaying).toEqual([
+    // Joining the call
+    enterSound,
+    // Bob leaves
+    enterSound,
+  ]);
+});
+
+test("plays a sound when a user leaves", () => {
+  const audioIsPlaying: string[] = mockMediaPlay();
+  const members = new Map([alice].map((p) => [p.userId, p]));
+  const remoteParticipants = new Map(
+    [aliceParticipant].map((p) => [p.identity, p]),
+  );
+  const liveKitRoom = new EmittableMockLivekitRoom({
+    localParticipant,
+    remoteParticipants,
+  });
+
+  const vm = new CallViewModel(
+    mockMatrixRoom({
+      client: {
+        getUserId: () => "@carol:example.org",
+      } as Partial<MatrixClient> as MatrixClient,
+      getMember: (userId) => members.get(userId) ?? null,
+    }),
+    liveKitRoom as unknown as Room,
+    {
+      kind: E2eeType.PER_PARTICIPANT,
+    },
+    of(ConnectionState.Connected),
+  );
+  render(<CallEventAudioRenderer vm={vm} />);
+
+  act(() => {
+    liveKitRoom.removeParticipant(aliceParticipant);
+  });
+  expect(audioIsPlaying).toEqual([
+    // Joining the call
+    enterSound,
+    // Alice leaves
+    leaveSound,
+  ]);
+});
+
+test("plays no sound when the participant list", () => {
+  const audioIsPlaying: string[] = mockMediaPlay();
+  const members = new Map([alice].map((p) => [p.userId, p]));
+  const remoteParticipants = new Map<string, RemoteParticipant>([
+    [aliceParticipant.identity, aliceParticipant],
+    ...Array.from({ length: MAX_PARTICIPANT_COUNT_FOR_SOUND - 1 }).map<
+      [string, RemoteParticipant]
+    >((_, index) => {
+      const p = mockRemoteParticipant({ identity: `user${index}` });
+      return [p.identity, p];
+    }),
+  ]);
+  const liveKitRoom = new EmittableMockLivekitRoom({
+    localParticipant,
+    remoteParticipants,
+  });
+
+  const vm = new CallViewModel(
+    mockMatrixRoom({
+      client: {
+        getUserId: () => "@carol:example.org",
+      } as Partial<MatrixClient> as MatrixClient,
+      getMember: (userId) => members.get(userId) ?? null,
+    }),
+    liveKitRoom as unknown as Room,
+    {
+      kind: E2eeType.PER_PARTICIPANT,
+    },
+    of(ConnectionState.Connected),
+  );
+  render(<CallEventAudioRenderer vm={vm} />);
+  expect(audioIsPlaying).toEqual([]);
+  // When the count drops
+  act(() => {
+    liveKitRoom.removeParticipant(aliceParticipant);
+  });
+  expect(audioIsPlaying).toEqual([leaveSound]);
+});

src/room/CallEventAudioRenderer.tsx

@@ -0,0 +1,117 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { ReactNode, useEffect, useRef } from "react";
+import { filter } from "rxjs";
+
+import {
+  soundEffectVolumeSetting as effectSoundVolumeSetting,
+  useSetting,
+} from "../settings/settings";
+import { CallViewModel } from "../state/CallViewModel";
+import enterCallSoundMp3 from "../sound/join_call.mp3";
+import enterCallSoundOgg from "../sound/join_call.ogg";
+import leftCallSoundMp3 from "../sound/left_call.mp3";
+import leftCallSoundOgg from "../sound/left_call.ogg";
+
+// Do not play any sounds if the participant count has exceeded this
+// number.
+export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
+export const CONCURRENT_AUDIO_CHANNELS = 2;
+
+export function CallEventAudioRenderer({
+  vm,
+}: {
+  vm: CallViewModel;
+}): ReactNode {
+  const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
+  const callEntered = useRef<(HTMLAudioElement | null)[]>([]);
+  const callLeft = useRef<(HTMLAudioElement | null)[]>([]);
+
+  useEffect(() => {
+    if (effectSoundVolume === 0) {
+      return;
+    }
+    const joinSub = vm.memberChanges
+      .pipe(
+        filter(
+          ({ joined, ids }) =>
+            ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0,
+        ),
+      )
+      .subscribe(({ joined }) => {
+        const availablePlayer = callEntered.current.find((v) => v?.paused);
+        void availablePlayer?.play();
+      });
+
+    const leftSub = vm.memberChanges
+      .pipe(
+        filter(
+          ({ ids, left }) =>
+            ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
+        ),
+      )
+      .subscribe(() => {
+        const availablePlayer = callLeft.current.find((v) => v?.paused);
+        void availablePlayer?.play();
+      });
+
+    return (): void => {
+      joinSub.unsubscribe();
+      leftSub.unsubscribe();
+    };
+  }, [effectSoundVolume, callEntered, callLeft, vm]);
+
+  // Set volume.
+  useEffect(() => {
+    callEntered.current.forEach((a) => {
+      if (a) {
+        a.volume = effectSoundVolume;
+      }
+    });
+    callLeft.current.forEach((a) => {
+      if (a) {
+        a.volume = effectSoundVolume;
+      }
+    });
+  }, [callEntered, callLeft, effectSoundVolume]);
+
+  // Do not render any audio elements if playback is disabled. Will save
+  // audio file fetches.
+  if (effectSoundVolume === 0) {
+    return null;
+  }
+
+  return (
+    // Will play as soon as it's mounted, which is what we want as this will
+    // play when the call is entered.
+    <>
+      {Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
+        <audio
+          key={index}
+          ref={(r) => (callEntered.current[index] = r)}
+          preload="auto"
+          hidden
+        >
+          <source src={enterCallSoundOgg} type="audio/ogg; codecs=vorbis" />
+          <source src={enterCallSoundMp3} type="audio/mpeg" />
+        </audio>
+      ))}
+      {Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
+        <audio
+          key={index}
+          ref={(r) => (callLeft.current[index] = r)}
+          preload="auto"
+          hidden
+        >
+          <source src={leftCallSoundOgg} type="audio/ogg; codecs=vorbis" />
+          <source src={leftCallSoundMp3} type="audio/mpeg" />
+        </audio>
+      ))}
+    </>
+  );
+}

src/room/InCallView.tsx

@@ -87,6 +87,7 @@ import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
 import { useSwitchCamera } from "./useSwitchCamera";
 import { soundEffectVolumeSetting, useSetting } from "../settings/settings";
 import { ReactionsOverlay } from "./ReactionsOverlay";
+import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 
@@ -665,6 +666,7 @@ export const InCallView: FC<InCallViewProps> = ({
         ))}
       <RoomAudioRenderer />
       {renderContent()}
+      <CallEventAudioRenderer vm={vm} />
       <audio ref={handRaisePlayer} preload="auto" hidden>
         <source src={handSoundOgg} type="audio/ogg; codecs=vorbis" />
         <source src={handSoundMp3} type="audio/mpeg" />

src/room/ReactionAudioRenderer.test.tsx

@@ -21,6 +21,7 @@ import {
   playReactionsSound,
   soundEffectVolumeSetting,
 } from "../settings/settings";
+import { mockMediaPlay } from "../utils/test";
 
 const memberUserIdAlice = "@alice:example.org";
 const memberUserIdBob = "@bob:example.org";
@@ -80,11 +81,7 @@ test("loads no audio elements when disabled in settings", () => {
 });
 
 test("will play an audio sound when there is a reaction", () => {
-  const audioIsPlaying: string[] = [];
+  const audioIsPlaying: string[] = mockMediaPlay();
-  window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
-    audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
-    return Promise.resolve();
-  };
   playReactionsSound.setValue(true);
   const room = new MockRoom(memberUserIdAlice);
   const rtcSession = new MockRTCSession(room, membership);
@@ -105,11 +102,7 @@ test("will play an audio sound when there is a reaction", () => {
 });
 
 test("will play the generic audio sound when there is soundless reaction", () => {
-  const audioIsPlaying: string[] = [];
+  const audioIsPlaying: string[] = mockMediaPlay();
-  window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
-    audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
-    return Promise.resolve();
-  };
   playReactionsSound.setValue(true);
   const room = new MockRoom(memberUserIdAlice);
   const rtcSession = new MockRTCSession(room, membership);
@@ -152,11 +145,7 @@ test("will play an audio sound with the correct volume", () => {
 });
 
 test("will play multiple audio sounds when there are multiple different reactions", () => {
-  const audioIsPlaying: string[] = [];
+  const audioIsPlaying: string[] = mockMediaPlay();
-  window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
-    audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
-    return Promise.resolve();
-  };
   playReactionsSound.setValue(true);
 
   const room = new MockRoom(memberUserIdAlice);

src/settings/SettingsModal.tsx

@@ -195,6 +195,11 @@ export const SettingsModal: FC<Props> = ({
     name: t("settings.developer_tab_title"),
     content: (
       <>
+        <p>
+          {t("developer_mode.hostname", {
+            hostname: window.location.hostname || "unknown",
+          })}
+        </p>
         <p>
           {t("version", {
             productName: import.meta.env.VITE_PRODUCT_NAME || "Element Call",
@@ -202,17 +207,17 @@ export const SettingsModal: FC<Props> = ({
           })}
         </p>
         <p>
-          {t("crypto_version", {
+          {t("developer_mode.crypto_version", {
             version: client.getCrypto()?.getVersion() || "unknown",
           })}
         </p>
         <p>
-          {t("matrix_id", {
+          {t("developer_mode.matrix_id", {
             id: client.getUserId() || "unknown",
           })}
         </p>
         <p>
-          {t("device_id", {
+          {t("developer_mode.device_id", {
             id: client.getDeviceId() || "unknown",
           })}
         </p>
@@ -220,7 +225,7 @@ export const SettingsModal: FC<Props> = ({
           <InputField
             id="duplicateTiles"
             type="number"
-            label={t("settings.duplicate_tiles_label")}
+            label={t("developer_mode.duplicate_tiles_label")}
             value={duplicateTiles.toString()}
             onChange={useCallback(
               (event: ChangeEvent<HTMLInputElement>): void => {

src/sound/LICENCE.md

@@ -19,4 +19,6 @@ The following sound effects have been originally created by Element.
 - `end_talk`
 - `start_talk_local`
 - `start_talk_remote`
+- `join_call`
+- `end_call`
 - `reactions/rock`

src/state/CallViewModel.ts

@@ -545,6 +545,19 @@ export class CallViewModel extends ViewModel {
     ),
   );
 
+  public readonly memberChanges = this.userMedia
+    .pipe(map((mediaItems) => mediaItems.map((m) => m.id)))
+    .pipe(
+      scan<string[], { ids: string[]; joined: string[]; left: string[] }>(
+        (prev, ids) => {
+          const left = prev.ids.filter((id) => !ids.includes(id));
+          const joined = ids.filter((id) => !prev.ids.includes(id));
+          return { ids, joined, left };
+        },
+        { ids: [], joined: [], left: [] },
+      ),
+    );
+
   /**
    * List of MediaItems that we want to display, that are of type ScreenShare
    */

src/utils/test.ts

@@ -27,7 +27,9 @@ import {
   RemoteParticipant,
   RemoteTrackPublication,
   Room as LivekitRoom,
+  RoomEvent,
 } from "livekit-client";
+import { EventEmitter } from "stream";
 
 import {
   LocalUserMediaViewModel,
@@ -151,6 +153,33 @@ export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
   return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
 }
 
+/**
+ * A mock of a Livekit Room that can emit events.
+ */
+export class EmittableMockLivekitRoom extends EventEmitter {
+  public localParticipant?: LocalParticipant;
+  public remoteParticipants: Map<string, RemoteParticipant>;
+
+  public constructor(room: {
+    localParticipant?: LocalParticipant;
+    remoteParticipants: Map<string, RemoteParticipant>;
+  }) {
+    super();
+    this.localParticipant = room.localParticipant;
+    this.remoteParticipants = room.remoteParticipants ?? new Map();
+  }
+
+  public addParticipant(remoteParticipant: RemoteParticipant): void {
+    this.remoteParticipants.set(remoteParticipant.identity, remoteParticipant);
+    this.emit(RoomEvent.ParticipantConnected, remoteParticipant);
+  }
+
+  public removeParticipant(remoteParticipant: RemoteParticipant): void {
+    this.remoteParticipants.delete(remoteParticipant.identity);
+    this.emit(RoomEvent.ParticipantDisconnected, remoteParticipant);
+  }
+}
+
 export function mockLivekitRoom(
   room: Partial<LivekitRoom>,
   {
@@ -251,6 +280,15 @@ export function mockConfig(config: Partial<ResolvedConfigOptions> = {}): void {
   });
 }
 
+export function mockMediaPlay(): string[] {
+  const audioIsPlaying: string[] = [];
+  window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
+    audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
+    return Promise.resolve();
+  };
+  return audioIsPlaying;
+}
+
 export class MockRTCSession extends TypedEventEmitter<
   MatrixRTCSessionEvent,
   MatrixRTCSessionEventHandlerMap
@@ -276,4 +314,4 @@ export class MockRTCSession extends TypedEventEmitter<
 
     return this;
   }
-}
+}