Make video tiles be based on MatrixRTC member not LiveKit participants (#2701)

* make tiles based on rtc member * display missing lk participant + fix tile multiplier * add show_non_member_participants config option * per member tiles * merge fixes * linter * linter and tests * tests * adapt tests (wip) * Remove unused keys * Fix optionality of nonMemberItemCount * video is optional * Mock RTC members * Lint * Merge fixes * Fix user id * Add explicit types for public fields * isRTCParticipantAvailable => isLiveKitParticipantAvailable * isLiveKitParticipantAvailable * Readonly * More keys removal * Make local field based on view model class not observable * Wording * Fix RTC members in tes * Tests again * Lint * Disable showing non-member tiles by default * Duplicate screen sharing tiles like we used to * Lint * Revert function reordering * Remove throttleTime from bad merge * Cleanup * Tidy config of show non-member settings * tidy up handling of local rtc member in tests * tidy up test init * Fix mocks * Cleanup * Apply local override where participant not yet known * Handle no visible media id * Assertions for one-on-one view * Remove isLiveKitParticipantAvailable and show via encryption status * Handle no local media (yet) * Remove unused effect for setting * Tidy settings * Avoid case of one-to-one layout with missing local or remote * Iterate * Remove option to show non-member tiles to simplify code review * Remove unused code * Remove more remnants of show-non-member-tiles * iterate * back * Fix unit test * Refactor * Expose TestScheduler as global * Fix incorrect type assertion * Simplify speaking observer * Fix * Whitespace * Make it clear that we are mocking MatrixRTC memberships * Test case for only showing tiles for MatrixRTC session members * Simplify diff * Simplify diff These changes are in https://github.com/element-hq/element-call/pull/2809 * . * Whitespaces * Use asObservable when exposing subject * Show "waiting for media..." when no participant * Additional test case * Don't show "waiting for media..." in case of local participant * Make the loading state more subtle - instead of a label we show a animated gradient * Use correct key for matrix rtc foci in code comment. (#2838) * Update src/tile/SpotlightTile.tsx Co-authored-by: Timo <16718859+toger5@users.noreply.github.com> * Update src/state/CallViewModel.ts Co-authored-by: Timo <16718859+toger5@users.noreply.github.com> * Make the purpose of BaseMediaViewModel.local explicit * Use named object instead of unnamed array for spotlightAndPip * Refactor spotlightAndPip into spotlight and pip * Use if statement instead of ternary for readability in spotlight and pip logic * Review feedback * Fix tests for CallEventAudioRenderer * Lint * Revert "Make the loading state more subtle" This reverts commit 765f7b4f31. * Update src/state/CallViewModel.ts Co-authored-by: Timo <16718859+toger5@users.noreply.github.com> * Fix spelling * Remove a non-null assertion that failed at runtime --------- Co-authored-by: Hugh Nimmo-Smith <hughns@element.io> Co-authored-by: Hugh Nimmo-Smith <hughns@users.noreply.github.com>
2026-07-24 19:19:21 +00:00 · 2024-12-06 12:28:37 +01:00
parent 21b62dbd89
commit 43c81a2758
16 changed files with 729 additions and 307 deletions
--- a/src/room/CallEventAudioRenderer.test.tsx
+++ b/src/room/CallEventAudioRenderer.test.tsx
@@ -8,10 +8,14 @@ Please see LICENSE in the repository root for full details.
 import { render } from "@testing-library/react";
 import { beforeEach, expect, test } from "vitest";
 import { MatrixClient } from "matrix-js-sdk/src/client";
-import { ConnectionState, RemoteParticipant, Room } from "livekit-client";
-import { of } from "rxjs";
+import { ConnectionState, Room } from "livekit-client";
+import { BehaviorSubject, of } from "rxjs";
 import { afterEach } from "node:test";
 import { act } from "react";
+import {
+  CallMembership,
+  type MatrixRTCSession,
+} from "matrix-js-sdk/src/matrixrtc";

 import { soundEffectVolumeSetting } from "../settings/settings";
 import {
@@ -22,6 +26,8 @@ import {
  mockMatrixRoomMember,
  mockMediaPlay,
  mockRemoteParticipant,
+  mockRtcMembership,
+  MockRTCSession,
 } from "../utils/test";
 import { E2eeType } from "../e2ee/e2eeType";
 import { CallViewModel } from "../state/CallViewModel";
@@ -30,11 +36,15 @@ import {
  MAX_PARTICIPANT_COUNT_FOR_SOUND,
 } from "./CallEventAudioRenderer";

-const alice = mockMatrixRoomMember({ userId: "@alice:example.org" });
-const bob = mockMatrixRoomMember({ userId: "@bob:example.org" });
-const aliceId = `${alice.userId}:AAAA`;
-const bobId = `${bob.userId}:BBBB`;
+const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
+const local = mockMatrixRoomMember(localRtcMember);
+const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
+const alice = mockMatrixRoomMember(aliceRtcMember);
+const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
+const bob = mockMatrixRoomMember(bobRtcMember);
 const localParticipant = mockLocalParticipant({ identity: "" });
+const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
+const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
 const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
 const bobParticipant = mockRemoteParticipant({ identity: bobId });

@@ -53,20 +63,28 @@ afterEach(() => {

 test("plays a sound when entering a call", () => {
  const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice, bob].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map(
+    [local, alice, bob].map((p) => [p.userId, p]),
+  );
  const remoteParticipants = of([aliceParticipant]);
  const liveKitRoom = mockLivekitRoom(
    { localParticipant },
    { remoteParticipants },
  );
+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const session = new MockRTCSession(matrixRoom, localRtcMember, [
+    aliceRtcMember,
+  ]) as unknown as MatrixRTCSession;

  const vm = new CallViewModel(
-    mockMatrixRoom({
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
+    session,
    liveKitRoom,
    {
      kind: E2eeType.PER_PARTICIPANT,
@@ -84,20 +102,29 @@ test("plays a sound when entering a call", () => {
 test("plays no sound when muted", () => {
  soundEffectVolumeSetting.setValue(0);
  const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice, bob].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map(
+    [local, alice, bob].map((p) => [p.userId, p]),
+  );
  const remoteParticipants = of([aliceParticipant, bobParticipant]);
  const liveKitRoom = mockLivekitRoom(
    { localParticipant },
    { remoteParticipants },
  );

+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const session = new MockRTCSession(matrixRoom, localRtcMember, [
+    aliceRtcMember,
+  ]) as unknown as MatrixRTCSession;
+
  const vm = new CallViewModel(
-    mockMatrixRoom({
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
+    session,
    liveKitRoom,
    {
      kind: E2eeType.PER_PARTICIPANT,
@@ -112,7 +139,7 @@ test("plays no sound when muted", () => {

 test("plays a sound when a user joins", () => {
  const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
  const remoteParticipants = new Map(
    [aliceParticipant].map((p) => [p.identity, p]),
  );
@@ -121,13 +148,27 @@ test("plays a sound when a user joins", () => {
    remoteParticipants,
  });

+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
+    aliceRtcMember,
+  ]);
+  // we give Bob an RTC session now, but no participant yet
+  const session = new MockRTCSession(
+    matrixRoom,
+    localRtcMember,
+  ).withMemberships(
+    remoteRtcMemberships.asObservable(),
+  ) as unknown as MatrixRTCSession;
+
  const vm = new CallViewModel(
-    mockMatrixRoom({
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
+    session,
    liveKitRoom as unknown as Room,
    {
      kind: E2eeType.PER_PARTICIPANT,
@@ -137,20 +178,20 @@ test("plays a sound when a user joins", () => {
  render(<CallEventAudioRenderer vm={vm} />);

  act(() => {
-    liveKitRoom.addParticipant(bobParticipant);
+    remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
  });
  // Play a sound when joining a call.
  expect(audioIsPlaying).toEqual([
    // Joining the call
    enterSound,
-    // Bob leaves
+    // Bob joins
    enterSound,
  ]);
 });

 test("plays a sound when a user leaves", () => {
  const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
  const remoteParticipants = new Map(
    [aliceParticipant].map((p) => [p.identity, p]),
  );
@@ -159,13 +200,25 @@ test("plays a sound when a user leaves", () => {
    remoteParticipants,
  });

+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
+    aliceRtcMember,
+  ]);
+
+  const session = new MockRTCSession(
+    matrixRoom,
+    localRtcMember,
+  ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
+
  const vm = new CallViewModel(
-    mockMatrixRoom({
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
+    session,
    liveKitRoom as unknown as Room,
    {
      kind: E2eeType.PER_PARTICIPANT,
@@ -175,7 +228,7 @@ test("plays a sound when a user leaves", () => {
  render(<CallEventAudioRenderer vm={vm} />);

  act(() => {
-    liveKitRoom.removeParticipant(aliceParticipant);
+    remoteRtcMemberships.next([]);
  });
  expect(audioIsPlaying).toEqual([
    // Joining the call
@@ -185,30 +238,45 @@ test("plays a sound when a user leaves", () => {
  ]);
 });

-test("plays no sound when the participant list", () => {
+test("plays no sound when the session member count is larger than the max, until decreased", () => {
  const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice].map((p) => [p.userId, p]));
-  const remoteParticipants = new Map<string, RemoteParticipant>([
-    [aliceParticipant.identity, aliceParticipant],
-    ...Array.from({ length: MAX_PARTICIPANT_COUNT_FOR_SOUND - 1 }).map<
-      [string, RemoteParticipant]
-    >((_, index) => {
-      const p = mockRemoteParticipant({ identity: `user${index}` });
-      return [p.identity, p];
-    }),
-  ]);
+  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
+  const remoteParticipants = new Map(
+    [aliceParticipant].map((p) => [p.identity, p]),
+  );
+
+  const mockRtcMemberships: CallMembership[] = [];
+
+  for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) {
+    mockRtcMemberships.push(
+      mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`),
+    );
+  }
+
+  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
+    mockRtcMemberships,
+  );
+
  const liveKitRoom = new EmittableMockLivekitRoom({
    localParticipant,
    remoteParticipants,
  });

+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const session = new MockRTCSession(
+    matrixRoom,
+    localRtcMember,
+  ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
+
  const vm = new CallViewModel(
-    mockMatrixRoom({
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
+    session,
    liveKitRoom as unknown as Room,
    {
      kind: E2eeType.PER_PARTICIPANT,
@@ -217,9 +285,11 @@ test("plays no sound when the participant list", () => {
  );
  render(<CallEventAudioRenderer vm={vm} />);
  expect(audioIsPlaying).toEqual([]);
-  // When the count drops
+  // When the count drops to the max we should play the leave sound
  act(() => {
-    liveKitRoom.removeParticipant(aliceParticipant);
+    remoteRtcMemberships.next(
+      mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1),
+    );
  });
  expect(audioIsPlaying).toEqual([leaveSound]);
 });
--- a/src/room/InCallView.tsx
+++ b/src/room/InCallView.tsx
@@ -124,7 +124,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
  useEffect(() => {
    if (livekitRoom !== undefined) {
      const vm = new CallViewModel(
-        props.rtcSession.room,
+        props.rtcSession,
        livekitRoom,
        props.e2eeSystem,
        connStateObservable,
@@ -132,12 +132,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
      setVm(vm);
      return (): void => vm.destroy();
    }
-  }, [
-    props.rtcSession.room,
-    livekitRoom,
-    props.e2eeSystem,
-    connStateObservable,
-  ]);
+  }, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable]);

  if (livekitRoom === undefined || vm === null) return null;