Make video tiles be based on MatrixRTC member not LiveKit participants (#2701)

* make tiles based on rtc member * display missing lk participant + fix tile multiplier * add show_non_member_participants config option * per member tiles * merge fixes * linter * linter and tests * tests * adapt tests (wip) * Remove unused keys * Fix optionality of nonMemberItemCount * video is optional * Mock RTC members * Lint * Merge fixes * Fix user id * Add explicit types for public fields * isRTCParticipantAvailable => isLiveKitParticipantAvailable * isLiveKitParticipantAvailable * Readonly * More keys removal * Make local field based on view model class not observable * Wording * Fix RTC members in tes * Tests again * Lint * Disable showing non-member tiles by default * Duplicate screen sharing tiles like we used to * Lint * Revert function reordering * Remove throttleTime from bad merge * Cleanup * Tidy config of show non-member settings * tidy up handling of local rtc member in tests * tidy up test init * Fix mocks * Cleanup * Apply local override where participant not yet known * Handle no visible media id * Assertions for one-on-one view * Remove isLiveKitParticipantAvailable and show via encryption status * Handle no local media (yet) * Remove unused effect for setting * Tidy settings * Avoid case of one-to-one layout with missing local or remote * Iterate * Remove option to show non-member tiles to simplify code review * Remove unused code * Remove more remnants of show-non-member-tiles * iterate * back * Fix unit test * Refactor * Expose TestScheduler as global * Fix incorrect type assertion * Simplify speaking observer * Fix * Whitespace * Make it clear that we are mocking MatrixRTC memberships * Test case for only showing tiles for MatrixRTC session members * Simplify diff * Simplify diff These changes are in https://github.com/element-hq/element-call/pull/2809 * . * Whitespaces * Use asObservable when exposing subject * Show "waiting for media..." when no participant * Additional test case * Don't show "waiting for media..." in case of local participant * Make the loading state more subtle - instead of a label we show a animated gradient * Use correct key for matrix rtc foci in code comment. (#2838) * Update src/tile/SpotlightTile.tsx Co-authored-by: Timo <16718859+toger5@users.noreply.github.com> * Update src/state/CallViewModel.ts Co-authored-by: Timo <16718859+toger5@users.noreply.github.com> * Make the purpose of BaseMediaViewModel.local explicit * Use named object instead of unnamed array for spotlightAndPip * Refactor spotlightAndPip into spotlight and pip * Use if statement instead of ternary for readability in spotlight and pip logic * Review feedback * Fix tests for CallEventAudioRenderer * Lint * Revert "Make the loading state more subtle" This reverts commit 765f7b4f31. * Update src/state/CallViewModel.ts Co-authored-by: Timo <16718859+toger5@users.noreply.github.com> * Fix spelling * Remove a non-null assertion that failed at runtime --------- Co-authored-by: Hugh Nimmo-Smith <hughns@element.io> Co-authored-by: Hugh Nimmo-Smith <hughns@users.noreply.github.com>
2026-03-07 05:47:03 +00:00 · 2024-12-06 12:28:37 +01:00
parent 21b62dbd89
commit 43c81a2758
16 changed files with 729 additions and 307 deletions
--- a/locales/en/app.json
+++ b/locales/en/app.json
@@ -194,6 +194,7 @@
    "expand": "Expand",
    "mute_for_me": "Mute for me",
    "muted_for_me": "Muted for me",
-    "volume": "Volume"
+    "volume": "Volume",
+    "waiting_for_media": "Waiting for media..."
  }
 }
--- a/src/config/ConfigOptions.ts
+++ b/src/config/ConfigOptions.ts
@@ -51,7 +51,7 @@ export interface ConfigOptions {
    // a livekit service url in the client well-known.
    // The well known needs to be formatted like so:
    // {"type":"livekit", "livekit_service_url":"https://livekit.example.com"}
-    // and stored under the key: "livekit_focus"
+    // and stored under the key: "org.matrix.msc4143.rtc_foci"
    livekit_service_url: string;
  };

--- a/src/room/CallEventAudioRenderer.test.tsx
+++ b/src/room/CallEventAudioRenderer.test.tsx
@@ -8,10 +8,14 @@ Please see LICENSE in the repository root for full details.
 import { render } from "@testing-library/react";
 import { beforeEach, expect, test } from "vitest";
 import { MatrixClient } from "matrix-js-sdk/src/client";
-import { ConnectionState, RemoteParticipant, Room } from "livekit-client";
-import { of } from "rxjs";
+import { ConnectionState, Room } from "livekit-client";
+import { BehaviorSubject, of } from "rxjs";
 import { afterEach } from "node:test";
 import { act } from "react";
+import {
+  CallMembership,
+  type MatrixRTCSession,
+} from "matrix-js-sdk/src/matrixrtc";

 import { soundEffectVolumeSetting } from "../settings/settings";
 import {
@@ -22,6 +26,8 @@ import {
  mockMatrixRoomMember,
  mockMediaPlay,
  mockRemoteParticipant,
+  mockRtcMembership,
+  MockRTCSession,
 } from "../utils/test";
 import { E2eeType } from "../e2ee/e2eeType";
 import { CallViewModel } from "../state/CallViewModel";
@@ -30,11 +36,15 @@ import {
  MAX_PARTICIPANT_COUNT_FOR_SOUND,
 } from "./CallEventAudioRenderer";

-const alice = mockMatrixRoomMember({ userId: "@alice:example.org" });
-const bob = mockMatrixRoomMember({ userId: "@bob:example.org" });
-const aliceId = `${alice.userId}:AAAA`;
-const bobId = `${bob.userId}:BBBB`;
+const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
+const local = mockMatrixRoomMember(localRtcMember);
+const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
+const alice = mockMatrixRoomMember(aliceRtcMember);
+const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
+const bob = mockMatrixRoomMember(bobRtcMember);
 const localParticipant = mockLocalParticipant({ identity: "" });
+const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
+const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
 const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
 const bobParticipant = mockRemoteParticipant({ identity: bobId });

@@ -53,20 +63,28 @@ afterEach(() => {

 test("plays a sound when entering a call", () => {
  const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice, bob].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map(
+    [local, alice, bob].map((p) => [p.userId, p]),
+  );
  const remoteParticipants = of([aliceParticipant]);
  const liveKitRoom = mockLivekitRoom(
    { localParticipant },
    { remoteParticipants },
  );
+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const session = new MockRTCSession(matrixRoom, localRtcMember, [
+    aliceRtcMember,
+  ]) as unknown as MatrixRTCSession;

  const vm = new CallViewModel(
-    mockMatrixRoom({
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
+    session,
    liveKitRoom,
    {
      kind: E2eeType.PER_PARTICIPANT,
@@ -84,20 +102,29 @@ test("plays a sound when entering a call", () => {
 test("plays no sound when muted", () => {
  soundEffectVolumeSetting.setValue(0);
  const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice, bob].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map(
+    [local, alice, bob].map((p) => [p.userId, p]),
+  );
  const remoteParticipants = of([aliceParticipant, bobParticipant]);
  const liveKitRoom = mockLivekitRoom(
    { localParticipant },
    { remoteParticipants },
  );

+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const session = new MockRTCSession(matrixRoom, localRtcMember, [
+    aliceRtcMember,
+  ]) as unknown as MatrixRTCSession;
+
  const vm = new CallViewModel(
-    mockMatrixRoom({
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
+    session,
    liveKitRoom,
    {
      kind: E2eeType.PER_PARTICIPANT,
@@ -112,7 +139,7 @@ test("plays no sound when muted", () => {

 test("plays a sound when a user joins", () => {
  const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
  const remoteParticipants = new Map(
    [aliceParticipant].map((p) => [p.identity, p]),
  );
@@ -121,13 +148,27 @@ test("plays a sound when a user joins", () => {
    remoteParticipants,
  });

+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
+    aliceRtcMember,
+  ]);
+  // we give Bob an RTC session now, but no participant yet
+  const session = new MockRTCSession(
+    matrixRoom,
+    localRtcMember,
+  ).withMemberships(
+    remoteRtcMemberships.asObservable(),
+  ) as unknown as MatrixRTCSession;
+
  const vm = new CallViewModel(
-    mockMatrixRoom({
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
+    session,
    liveKitRoom as unknown as Room,
    {
      kind: E2eeType.PER_PARTICIPANT,
@@ -137,20 +178,20 @@ test("plays a sound when a user joins", () => {
  render(<CallEventAudioRenderer vm={vm} />);

  act(() => {
-    liveKitRoom.addParticipant(bobParticipant);
+    remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
  });
  // Play a sound when joining a call.
  expect(audioIsPlaying).toEqual([
    // Joining the call
    enterSound,
-    // Bob leaves
+    // Bob joins
    enterSound,
  ]);
 });

 test("plays a sound when a user leaves", () => {
  const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
  const remoteParticipants = new Map(
    [aliceParticipant].map((p) => [p.identity, p]),
  );
@@ -159,13 +200,25 @@ test("plays a sound when a user leaves", () => {
    remoteParticipants,
  });

+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
+    aliceRtcMember,
+  ]);
+
+  const session = new MockRTCSession(
+    matrixRoom,
+    localRtcMember,
+  ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
+
  const vm = new CallViewModel(
-    mockMatrixRoom({
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
+    session,
    liveKitRoom as unknown as Room,
    {
      kind: E2eeType.PER_PARTICIPANT,
@@ -175,7 +228,7 @@ test("plays a sound when a user leaves", () => {
  render(<CallEventAudioRenderer vm={vm} />);

  act(() => {
-    liveKitRoom.removeParticipant(aliceParticipant);
+    remoteRtcMemberships.next([]);
  });
  expect(audioIsPlaying).toEqual([
    // Joining the call
@@ -185,30 +238,45 @@ test("plays a sound when a user leaves", () => {
  ]);
 });

-test("plays no sound when the participant list", () => {
+test("plays no sound when the session member count is larger than the max, until decreased", () => {
  const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice].map((p) => [p.userId, p]));
-  const remoteParticipants = new Map<string, RemoteParticipant>([
-    [aliceParticipant.identity, aliceParticipant],
-    ...Array.from({ length: MAX_PARTICIPANT_COUNT_FOR_SOUND - 1 }).map<
-      [string, RemoteParticipant]
-    >((_, index) => {
-      const p = mockRemoteParticipant({ identity: `user${index}` });
-      return [p.identity, p];
-    }),
-  ]);
+  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
+  const remoteParticipants = new Map(
+    [aliceParticipant].map((p) => [p.identity, p]),
+  );
+
+  const mockRtcMemberships: CallMembership[] = [];
+
+  for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) {
+    mockRtcMemberships.push(
+      mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`),
+    );
+  }
+
+  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
+    mockRtcMemberships,
+  );
+
  const liveKitRoom = new EmittableMockLivekitRoom({
    localParticipant,
    remoteParticipants,
  });

+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const session = new MockRTCSession(
+    matrixRoom,
+    localRtcMember,
+  ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
+
  const vm = new CallViewModel(
-    mockMatrixRoom({
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
+    session,
    liveKitRoom as unknown as Room,
    {
      kind: E2eeType.PER_PARTICIPANT,
@@ -217,9 +285,11 @@ test("plays no sound when the participant list", () => {
  );
  render(<CallEventAudioRenderer vm={vm} />);
  expect(audioIsPlaying).toEqual([]);
-  // When the count drops
+  // When the count drops to the max we should play the leave sound
  act(() => {
-    liveKitRoom.removeParticipant(aliceParticipant);
+    remoteRtcMemberships.next(
+      mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1),
+    );
  });
  expect(audioIsPlaying).toEqual([leaveSound]);
 });
--- a/src/room/InCallView.tsx
+++ b/src/room/InCallView.tsx
@@ -124,7 +124,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
  useEffect(() => {
    if (livekitRoom !== undefined) {
      const vm = new CallViewModel(
-        props.rtcSession.room,
+        props.rtcSession,
        livekitRoom,
        props.e2eeSystem,
        connStateObservable,
@@ -132,12 +132,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
      setVm(vm);
      return (): void => vm.destroy();
    }
-  }, [
-    props.rtcSession.room,
-    livekitRoom,
-    props.e2eeSystem,
-    connStateObservable,
-  ]);
+  }, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable]);

  if (livekitRoom === undefined || vm === null) return null;

--- a/src/state/CallViewModel.test.ts
+++ b/src/state/CallViewModel.test.ts
@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */

-import { test, vi, onTestFinished } from "vitest";
+import { test, vi, onTestFinished, it } from "vitest";
 import {
  combineLatest,
  debounceTime,
@@ -25,6 +25,7 @@ import {
 } from "livekit-client";
 import * as ComponentsCore from "@livekit/components-core";
 import { isEqual } from "lodash-es";
+import { CallMembership, MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";

 import { CallViewModel, Layout } from "./CallViewModel";
 import {
@@ -34,6 +35,8 @@ import {
  mockMatrixRoomMember,
  mockRemoteParticipant,
  withTestScheduler,
+  mockRtcMembership,
+  MockRTCSession,
 } from "../utils/test";
 import {
  ECAddonConnectionState,
@@ -43,14 +46,19 @@ import { E2eeType } from "../e2ee/e2eeType";

 vi.mock("@livekit/components-core");

-const alice = mockMatrixRoomMember({ userId: "@alice:example.org" });
-const bob = mockMatrixRoomMember({ userId: "@bob:example.org" });
-const carol = mockMatrixRoomMember({ userId: "@carol:example.org" });
-const dave = mockMatrixRoomMember({ userId: "@dave:example.org" });
+const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
+const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
+const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
+const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD");

-const aliceId = `${alice.userId}:AAAA`;
-const bobId = `${bob.userId}:BBBB`;
-const daveId = `${dave.userId}:DDDD`;
+const alice = mockMatrixRoomMember(aliceRtcMember);
+const bob = mockMatrixRoomMember(bobRtcMember);
+const carol = mockMatrixRoomMember(localRtcMember);
+const dave = mockMatrixRoomMember(daveRtcMember);
+
+const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
+const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
+const daveId = `${dave.userId}:${daveRtcMember.deviceId}`;

 const localParticipant = mockLocalParticipant({ identity: "" });
 const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
@@ -65,7 +73,9 @@ const bobSharingScreen = mockRemoteParticipant({
 });
 const daveParticipant = mockRemoteParticipant({ identity: daveId });

-const members = new Map([alice, bob, carol, dave].map((p) => [p.userId, p]));
+const roomMembers = new Map(
+  [alice, bob, carol, dave].map((p) => [p.userId, p]),
+);

 export interface GridLayoutSummary {
  type: "grid";
@@ -173,10 +183,23 @@ function summarizeLayout(l: Observable<Layout>): Observable<LayoutSummary> {

 function withCallViewModel(
  remoteParticipants: Observable<RemoteParticipant[]>,
+  rtcMembers: Observable<Partial<CallMembership>[]>,
  connectionState: Observable<ECConnectionState>,
  speaking: Map<Participant, Observable<boolean>>,
  continuation: (vm: CallViewModel) => void,
 ): void {
+  const room = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => roomMembers.get(userId) ?? null,
+  });
+  const rtcSession = new MockRTCSession(
+    room,
+    localRtcMember,
+    [],
+  ).withMemberships(rtcMembers);
  const participantsSpy = vi
    .spyOn(ComponentsCore, "connectedParticipantsObserver")
    .mockReturnValue(remoteParticipants);
@@ -209,12 +232,7 @@ function withCallViewModel(
  );

  const vm = new CallViewModel(
-    mockMatrixRoom({
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
+    rtcSession as unknown as MatrixRTCSession,
    liveKitRoom,
    {
      kind: E2eeType.PER_PARTICIPANT,
@@ -247,6 +265,7 @@ test("participants are retained during a focus switch", () => {
        a: [aliceParticipant, bobParticipant],
        b: [],
      }),
+      of([aliceRtcMember, bobRtcMember]),
      hot(connectionInputMarbles, {
        c: ConnectionState.Connected,
        s: ECAddonConnectionState.ECSwitchingFocus,
@@ -288,6 +307,7 @@ test("screen sharing activates spotlight layout", () => {
        c: [aliceSharingScreen, bobSharingScreen],
        d: [aliceParticipant, bobSharingScreen],
      }),
+      of([aliceRtcMember, bobRtcMember]),
      of(ConnectionState.Connected),
      new Map(),
      (vm) => {
@@ -356,7 +376,7 @@ test("participants stay in the same order unless to appear/disappear", () => {
    const modeInputMarbles = "     a";
    // First Bob speaks, then Dave, then Alice
    const aSpeakingInputMarbles = "n- 1998ms - 1999ms y";
-    const bSpeakingInputMarbles = "ny 1998ms n 1999ms ";
+    const bSpeakingInputMarbles = "ny 1998ms n 1999ms -";
    const dSpeakingInputMarbles = "n- 1998ms y 1999ms n";
    // Nothing should change when Bob speaks, because Bob is already on screen.
    // When Dave speaks he should switch with Alice because she's the one who
@@ -366,6 +386,7 @@ test("participants stay in the same order unless to appear/disappear", () => {

    withCallViewModel(
      of([aliceParticipant, bobParticipant, daveParticipant]),
+      of([aliceRtcMember, bobRtcMember, daveRtcMember]),
      of(ConnectionState.Connected),
      new Map([
        [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })],
@@ -427,6 +448,7 @@ test("spotlight speakers swap places", () => {

    withCallViewModel(
      of([aliceParticipant, bobParticipant, daveParticipant]),
+      of([aliceRtcMember, bobRtcMember, daveRtcMember]),
      of(ConnectionState.Connected),
      new Map([
        [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })],
@@ -475,6 +497,7 @@ test("layout enters picture-in-picture mode when requested", () => {

    withCallViewModel(
      of([aliceParticipant, bobParticipant]),
+      of([aliceRtcMember, bobRtcMember]),
      of(ConnectionState.Connected),
      new Map(),
      (vm) => {
@@ -515,6 +538,7 @@ test("spotlight remembers whether it's expanded", () => {

    withCallViewModel(
      of([aliceParticipant, bobParticipant]),
+      of([aliceRtcMember, bobRtcMember]),
      of(ConnectionState.Connected),
      new Map(),
      (vm) => {
@@ -559,3 +583,104 @@ test("spotlight remembers whether it's expanded", () => {
    );
  });
 });
+
+test("participants must have a MatrixRTCSession to be visible", () => {
+  withTestScheduler(({ hot, expectObservable }) => {
+    // iterate through a number of combinations of participants and MatrixRTC memberships
+    // Bob never has an MatrixRTC membership
+    const scenarioInputMarbles = " abcdec";
+    // Bob should never be visible
+    const expectedLayoutMarbles = "a-bc-b";
+
+    withCallViewModel(
+      hot(scenarioInputMarbles, {
+        a: [],
+        b: [bobParticipant],
+        c: [aliceParticipant, bobParticipant],
+        d: [aliceParticipant, daveParticipant, bobParticipant],
+        e: [aliceParticipant, daveParticipant, bobSharingScreen],
+      }),
+      hot(scenarioInputMarbles, {
+        a: [],
+        b: [],
+        c: [aliceRtcMember],
+        d: [aliceRtcMember, daveRtcMember],
+        e: [aliceRtcMember, daveRtcMember],
+      }),
+      of(ConnectionState.Connected),
+      new Map(),
+      (vm) => {
+        vm.setGridMode("grid");
+        expectObservable(summarizeLayout(vm.layout)).toBe(
+          expectedLayoutMarbles,
+          {
+            a: {
+              type: "grid",
+              spotlight: undefined,
+              grid: ["local:0"],
+            },
+            b: {
+              type: "one-on-one",
+              local: "local:0",
+              remote: `${aliceId}:0`,
+            },
+            c: {
+              type: "grid",
+              spotlight: undefined,
+              grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
+            },
+          },
+        );
+      },
+    );
+  });
+});
+
+it("should show at least one tile per MatrixRTCSession", () => {
+  withTestScheduler(({ hot, expectObservable }) => {
+    // iterate through some combinations of MatrixRTC memberships
+    const scenarioInputMarbles = " abcd";
+    // There should always be one tile for each MatrixRTCSession
+    const expectedLayoutMarbles = "abcd";
+
+    withCallViewModel(
+      of([]),
+      hot(scenarioInputMarbles, {
+        a: [],
+        b: [aliceRtcMember],
+        c: [aliceRtcMember, daveRtcMember],
+        d: [daveRtcMember],
+      }),
+      of(ConnectionState.Connected),
+      new Map(),
+      (vm) => {
+        vm.setGridMode("grid");
+        expectObservable(summarizeLayout(vm.layout)).toBe(
+          expectedLayoutMarbles,
+          {
+            a: {
+              type: "grid",
+              spotlight: undefined,
+              grid: ["local:0"],
+            },
+            b: {
+              type: "one-on-one",
+              local: "local:0",
+              remote: `${aliceId}:0`,
+            },
+            c: {
+              type: "grid",
+              spotlight: undefined,
+              grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
+            },
+            d: {
+              type: "one-on-one",
+              local: "local:0",
+              remote: `${daveId}:0`,
+            },
+          },
+        );
+      },
+    );
+  });
+});
--- a/src/state/CallViewModel.ts
+++ b/src/state/CallViewModel.ts
@@ -18,12 +18,9 @@ import {
  RemoteParticipant,
  Track,
 } from "livekit-client";
+import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix";
 import {
-  Room as MatrixRoom,
-  RoomMember,
-  RoomStateEvent,
-} from "matrix-js-sdk/src/matrix";
-import {
+  BehaviorSubject,
  EMPTY,
  Observable,
  Subject,
@@ -49,6 +46,10 @@ import {
  withLatestFrom,
 } from "rxjs";
 import { logger } from "matrix-js-sdk/src/logger";
+import {
+  MatrixRTCSession,
+  MatrixRTCSessionEvent,
+} from "matrix-js-sdk/src/matrixrtc";

 import { ViewModel } from "./ViewModel";
 import {
@@ -222,41 +223,67 @@ interface LayoutScanState {
 class UserMedia {
  private readonly scope = new ObservableScope();
  public readonly vm: UserMediaViewModel;
+  private readonly participant: BehaviorSubject<
+    LocalParticipant | RemoteParticipant | undefined
+  >;
+
  public readonly speaker: Observable<boolean>;
  public readonly presenter: Observable<boolean>;
-
  public constructor(
    public readonly id: string,
    member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: LocalParticipant | RemoteParticipant | undefined,
    encryptionSystem: EncryptionSystem,
    livekitRoom: LivekitRoom,
  ) {
-    this.vm = participant.isLocal
-      ? new LocalUserMediaViewModel(
-          id,
-          member,
-          participant as LocalParticipant,
-          encryptionSystem,
-          livekitRoom,
-        )
-      : new RemoteUserMediaViewModel(
-          id,
-          member,
-          participant as RemoteParticipant,
-          encryptionSystem,
-          livekitRoom,
-        );
+    this.participant = new BehaviorSubject(participant);
+
+    if (participant?.isLocal) {
+      this.vm = new LocalUserMediaViewModel(
+        this.id,
+        member,
+        this.participant.asObservable() as Observable<LocalParticipant>,
+        encryptionSystem,
+        livekitRoom,
+      );
+    } else {
+      this.vm = new RemoteUserMediaViewModel(
+        id,
+        member,
+        this.participant.asObservable() as Observable<
+          RemoteParticipant | undefined
+        >,
+        encryptionSystem,
+        livekitRoom,
+      );
+    }

    this.speaker = observeSpeaker(this.vm.speaking).pipe(this.scope.state());

-    this.presenter = observeParticipantEvents(
-      participant,
-      ParticipantEvent.TrackPublished,
-      ParticipantEvent.TrackUnpublished,
-      ParticipantEvent.LocalTrackPublished,
-      ParticipantEvent.LocalTrackUnpublished,
-    ).pipe(map((p) => p.isScreenShareEnabled));
+    this.presenter = this.participant.pipe(
+      switchMap(
+        (p) =>
+          (p &&
+            observeParticipantEvents(
+              p,
+              ParticipantEvent.TrackPublished,
+              ParticipantEvent.TrackUnpublished,
+              ParticipantEvent.LocalTrackPublished,
+              ParticipantEvent.LocalTrackUnpublished,
+            ).pipe(map((p) => p.isScreenShareEnabled))) ??
+          of(false),
+      ),
+      this.scope.state(),
+    );
+  }
+
+  public updateParticipant(
+    newParticipant: LocalParticipant | RemoteParticipant | undefined,
+  ): void {
+    if (this.participant.value !== newParticipant) {
+      // Update the BehaviourSubject in the UserMedia.
+      this.participant.next(newParticipant);
+    }
  }

  public destroy(): void {
@@ -267,6 +294,9 @@ class UserMedia {

 class ScreenShare {
  public readonly vm: ScreenShareViewModel;
+  private readonly participant: BehaviorSubject<
+    LocalParticipant | RemoteParticipant
+  >;

  public constructor(
    id: string,
@@ -275,12 +305,15 @@ class ScreenShare {
    encryptionSystem: EncryptionSystem,
    liveKitRoom: LivekitRoom,
  ) {
+    this.participant = new BehaviorSubject(participant);
+
    this.vm = new ScreenShareViewModel(
      id,
      member,
-      participant,
+      this.participant.asObservable(),
      encryptionSystem,
      liveKitRoom,
+      participant.isLocal,
    );
  }

@@ -317,11 +350,11 @@ function findMatrixRoomMember(
 export class CallViewModel extends ViewModel {
  public readonly localVideo: Observable<LocalVideoTrack | null> =
    observeTrackReference(
-      this.livekitRoom.localParticipant,
+      of(this.livekitRoom.localParticipant),
      Track.Source.Camera,
    ).pipe(
      map((trackRef) => {
-        const track = trackRef.publication?.track;
+        const track = trackRef?.publication?.track;
        return track instanceof LocalVideoTrack ? track : null;
      }),
    );
@@ -401,49 +434,87 @@ export class CallViewModel extends ViewModel {
    this.remoteParticipants,
    observeParticipantMedia(this.livekitRoom.localParticipant),
    duplicateTiles.value,
-    // Also react to changes in the list of members
-    fromEvent(this.matrixRoom, RoomStateEvent.Update).pipe(startWith(null)),
+    // Also react to changes in the MatrixRTC session list.
+    // The session list will also be update if a room membership changes.
+    // No additional RoomState event listener needs to be set up.
+    fromEvent(
+      this.matrixRTCSession,
+      MatrixRTCSessionEvent.MembershipsChanged,
+    ).pipe(startWith(null)),
  ]).pipe(
    scan(
      (
        prevItems,
-        [remoteParticipants, { participant: localParticipant }, duplicateTiles],
+        [
+          remoteParticipants,
+          { participant: localParticipant },
+          duplicateTiles,
+          _membershipsChanged,
+        ],
      ) => {
        const newItems = new Map(
          function* (this: CallViewModel): Iterable<[string, MediaItem]> {
-            for (const p of [localParticipant, ...remoteParticipants]) {
-              const id = p === localParticipant ? "local" : p.identity;
-              const member = findMatrixRoomMember(this.matrixRoom, id);
-              if (member === undefined)
-                logger.warn(
-                  `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`,
-                );
+            // m.rtc.members are the basis for calculating what is visible in the call
+            for (const rtcMember of this.matrixRTCSession.memberships) {
+              const room = this.matrixRTCSession.room;
+              // WARN! This is not exactly the sender but the user defined in the state key.
+              // This will be available once we change to the new "member as object" format in the MatrixRTC object.
+              let livekitParticipantId =
+                rtcMember.sender + ":" + rtcMember.deviceId;

-              // Create as many tiles for this participant as called for by
-              // the duplicateTiles option
+              let participant:
+                | LocalParticipant
+                | RemoteParticipant
+                | undefined = undefined;
+              if (
+                rtcMember.sender === room.client.getUserId()! &&
+                rtcMember.deviceId === room.client.getDeviceId()
+              ) {
+                livekitParticipantId = "local";
+                participant = localParticipant;
+              } else {
+                participant = remoteParticipants.find(
+                  (p) => p.identity === livekitParticipantId,
+                );
+              }
+
+              const member = findMatrixRoomMember(room, livekitParticipantId);
+              if (!member) {
+                logger.error(
+                  "Could not find member for media id: ",
+                  livekitParticipantId,
+                );
+              }
              for (let i = 0; i < 1 + duplicateTiles; i++) {
-                const userMediaId = `${id}:${i}`;
+                const indexedMediaId = `${livekitParticipantId}:${i}`;
+                const prevMedia = prevItems.get(indexedMediaId);
+                if (prevMedia && prevMedia instanceof UserMedia) {
+                  prevMedia.updateParticipant(participant);
+                }
                yield [
-                  userMediaId,
-                  prevItems.get(userMediaId) ??
+                  indexedMediaId,
+                  // We create UserMedia with or without a participant.
+                  // This will be the initial value of a BehaviourSubject.
+                  // Once a participant appears we will update the BehaviourSubject. (see above)
+                  prevMedia ??
                    new UserMedia(
-                      userMediaId,
+                      indexedMediaId,
                      member,
-                      p,
+                      participant,
                      this.encryptionSystem,
                      this.livekitRoom,
                    ),
                ];

-                if (p.isScreenShareEnabled) {
-                  const screenShareId = `${userMediaId}:screen-share`;
+                if (participant?.isScreenShareEnabled) {
+                  const screenShareId = `${indexedMediaId}:screen-share`;
                  yield [
                    screenShareId,
                    prevItems.get(screenShareId) ??
                      new ScreenShare(
                        screenShareId,
                        member,
-                        p,
+                        participant,
                        this.encryptionSystem,
                        this.livekitRoom,
                      ),
@@ -454,7 +525,6 @@ export class CallViewModel extends ViewModel {
          }.bind(this)(),
        );

-        for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy();
        return newItems;
      },
      new Map<string, MediaItem>(),
@@ -488,11 +558,6 @@ export class CallViewModel extends ViewModel {
      ),
    );

-  private readonly localUserMedia: Observable<LocalUserMediaViewModel> =
-    this.mediaItems.pipe(
-      map((ms) => ms.find((m) => m.vm.local)!.vm as LocalUserMediaViewModel),
-    );
-
  /**
   * List of MediaItems that we want to display, that are of type ScreenShare
   */
@@ -504,7 +569,7 @@ export class CallViewModel extends ViewModel {
      this.scope.state(),
    );

-  private readonly spotlightSpeaker: Observable<UserMediaViewModel> =
+  private readonly spotlightSpeaker: Observable<UserMediaViewModel | null> =
    this.userMedia.pipe(
      switchMap((mediaItems) =>
        mediaItems.length === 0
@@ -515,7 +580,7 @@ export class CallViewModel extends ViewModel {
              ),
            ),
      ),
-      scan<(readonly [UserMedia, boolean])[], UserMedia, null>(
+      scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>(
        (prev, mediaItems) => {
          // Only remote users that are still in the call should be sticky
          const [stickyMedia, stickySpeaking] =
@@ -532,11 +597,11 @@ export class CallViewModel extends ViewModel {
                // Otherwise, spotlight an arbitrary remote user
                mediaItems.find(([m]) => !m.vm.local)?.[0] ??
                // Otherwise, spotlight the local user
-                mediaItems.find(([m]) => m.vm.local)![0]);
+                mediaItems.find(([m]) => m.vm.local)?.[0]);
        },
        null,
      ),
-      map((speaker) => speaker.vm),
+      map((speaker) => speaker?.vm ?? null),
      this.scope.state(),
    );

@@ -576,37 +641,57 @@ export class CallViewModel extends ViewModel {
    }),
  );

-  private readonly spotlightAndPip: Observable<
-    [Observable<MediaViewModel[]>, Observable<UserMediaViewModel | null>]
-  > = this.screenShares.pipe(
-    map((screenShares) =>
-      screenShares.length > 0
-        ? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const)
-        : ([
-            this.spotlightSpeaker.pipe(map((speaker) => [speaker!])),
-            this.spotlightSpeaker.pipe(
-              switchMap((speaker) =>
-                speaker.local
-                  ? of(null)
-                  : this.localUserMedia.pipe(
-                      switchMap((vm) =>
-                        vm.alwaysShow.pipe(
-                          map((alwaysShow) => (alwaysShow ? vm : null)),
-                        ),
-                      ),
-                    ),
-              ),
-            ),
-          ] as const),
-    ),
-  );
-
  private readonly spotlight: Observable<MediaViewModel[]> =
-    this.spotlightAndPip.pipe(
-      switchMap(([spotlight]) => spotlight),
+    this.screenShares.pipe(
+      switchMap((screenShares) => {
+        if (screenShares.length > 0) {
+          return of(screenShares.map((m) => m.vm));
+        }
+
+        return this.spotlightSpeaker.pipe(
+          map((speaker) => (speaker ? [speaker] : [])),
+        );
+      }),
      this.scope.state(),
    );

+  private readonly pip: Observable<UserMediaViewModel | null> = combineLatest([
+    this.screenShares,
+    this.spotlightSpeaker,
+    this.mediaItems,
+  ]).pipe(
+    switchMap(([screenShares, spotlight, mediaItems]) => {
+      if (screenShares.length > 0) {
+        return this.spotlightSpeaker;
+      }
+      if (!spotlight || spotlight.local) {
+        return of(null);
+      }
+
+      const localUserMedia = mediaItems.find(
+        (m) => m.vm instanceof LocalUserMediaViewModel,
+      ) as UserMedia | undefined;
+
+      const localUserMediaViewModel = localUserMedia?.vm as
+        | LocalUserMediaViewModel
+        | undefined;
+
+      if (!localUserMediaViewModel) {
+        return of(null);
+      }
+      return localUserMediaViewModel.alwaysShow.pipe(
+        map((alwaysShow) => {
+          if (alwaysShow) {
+            return localUserMediaViewModel;
+          }
+
+          return null;
+        }),
+      );
+    }),
+    this.scope.state(),
+  );
+
  private readonly hasRemoteScreenShares: Observable<boolean> =
    this.spotlight.pipe(
      map((spotlight) =>
@@ -615,9 +700,6 @@ export class CallViewModel extends ViewModel {
      distinctUntilChanged(),
    );

-  private readonly pip: Observable<UserMediaViewModel | null> =
-    this.spotlightAndPip.pipe(switchMap(([, pip]) => pip));
-
  private readonly pipEnabled: Observable<boolean> = setPipEnabled.pipe(
    startWith(false),
  );
@@ -721,15 +803,16 @@ export class CallViewModel extends ViewModel {
    this.mediaItems.pipe(
      map((mediaItems) => {
        if (mediaItems.length !== 2) return null;
-        const local = mediaItems.find((vm) => vm.vm.local)!
-          .vm as LocalUserMediaViewModel;
+        const local = mediaItems.find((vm) => vm.vm.local)?.vm as
+          | LocalUserMediaViewModel
+          | undefined;
        const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as
          | RemoteUserMediaViewModel
          | undefined;
        // There might not be a remote tile if there are screen shares, or if
        // only the local user is in the call and they're using the duplicate
        // tiles option
-        if (remote === undefined) return null;
+        if (!remote || !local) return null;

        return { type: "one-on-one", local, remote };
      }),
@@ -1010,7 +1093,7 @@ export class CallViewModel extends ViewModel {

  public constructor(
    // A call is permanently tied to a single Matrix room and LiveKit room
-    private readonly matrixRoom: MatrixRoom,
+    private readonly matrixRTCSession: MatrixRTCSession,
    private readonly livekitRoom: LivekitRoom,
    private readonly encryptionSystem: EncryptionSystem,
    private readonly connectionState: Observable<ECConnectionState>,
--- a/src/state/MediaViewModel.test.ts
+++ b/src/state/MediaViewModel.test.ts
@@ -8,14 +8,17 @@ Please see LICENSE in the repository root for full details.
 import { expect, test, vi } from "vitest";

 import {
+  mockRtcMembership,
  withLocalMedia,
  withRemoteMedia,
  withTestScheduler,
 } from "../utils/test";

+const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
+
 test("control a participant's volume", async () => {
  const setVolumeSpy = vi.fn();
-  await withRemoteMedia({}, { setVolume: setVolumeSpy }, (vm) =>
+  await withRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy }, (vm) =>
    withTestScheduler(({ expectObservable, schedule }) => {
      schedule("-ab---c---d|", {
        a() {
@@ -60,7 +63,7 @@ test("control a participant's volume", async () => {
 });

 test("toggle fit/contain for a participant's video", async () => {
-  await withRemoteMedia({}, {}, (vm) =>
+  await withRemoteMedia(rtcMembership, {}, {}, (vm) =>
    withTestScheduler(({ expectObservable, schedule }) => {
      schedule("-ab|", {
        a: () => vm.toggleFitContain(),
@@ -76,17 +79,21 @@ test("toggle fit/contain for a participant's video", async () => {
 });

 test("local media remembers whether it should always be shown", async () => {
-  await withLocalMedia({}, (vm) =>
+  await withLocalMedia(rtcMembership, {}, (vm) =>
    withTestScheduler(({ expectObservable, schedule }) => {
      schedule("-a|", { a: () => vm.setAlwaysShow(false) });
      expectObservable(vm.alwaysShow).toBe("ab", { a: true, b: false });
    }),
  );
  // Next local media should start out *not* always shown
-  await withLocalMedia({}, (vm) =>
-    withTestScheduler(({ expectObservable, schedule }) => {
-      schedule("-a|", { a: () => vm.setAlwaysShow(true) });
-      expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true });
-    }),
+  await withLocalMedia(
+    rtcMembership,
+
+    {},
+    (vm) =>
+      withTestScheduler(({ expectObservable, schedule }) => {
+        schedule("-a|", { a: () => vm.setAlwaysShow(true) });
+        expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true });
+      }),
  );
 });
--- a/src/state/MediaViewModel.ts
+++ b/src/state/MediaViewModel.ts
@@ -32,7 +32,6 @@ import {
  Observable,
  Subject,
  combineLatest,
-  distinctUntilChanged,
  distinctUntilKeyChanged,
  filter,
  fromEvent,
@@ -40,7 +39,6 @@ import {
  map,
  merge,
  of,
-  shareReplay,
  startWith,
  switchMap,
  throttleTime,
@@ -77,16 +75,24 @@ export function useDisplayName(vm: MediaViewModel): string {
 }

 export function observeTrackReference(
-  participant: Participant,
+  participant: Observable<Participant | undefined>,
  source: Track.Source,
-): Observable<TrackReferenceOrPlaceholder> {
-  return observeParticipantMedia(participant).pipe(
-    map(() => ({
-      participant,
-      publication: participant.getTrackPublication(source),
-      source,
-    })),
-    distinctUntilKeyChanged("publication"),
+): Observable<TrackReferenceOrPlaceholder | undefined> {
+  return participant.pipe(
+    switchMap((p) => {
+      if (p) {
+        return observeParticipantMedia(p).pipe(
+          map(() => ({
+            participant: p,
+            publication: p.getTrackPublication(source),
+            source,
+          })),
+          distinctUntilKeyChanged("publication"),
+        );
+      } else {
+        return of(undefined);
+      }
+    }),
  );
 }

@@ -105,11 +111,11 @@ function observeRemoteTrackReceivingOkay(
  };

  return combineLatest([
-    observeTrackReference(participant, source),
+    observeTrackReference(of(participant), source),
    interval(1000).pipe(startWith(0)),
  ]).pipe(
    switchMap(async ([trackReference]) => {
-      const track = trackReference.publication?.track;
+      const track = trackReference?.publication?.track;
      if (!track || !(track instanceof RemoteTrack)) {
        return undefined;
      }
@@ -200,14 +206,10 @@ export enum EncryptionStatus {
 }

 abstract class BaseMediaViewModel extends ViewModel {
-  /**
-   * Whether the media belongs to the local user.
-   */
-  public readonly local = this.participant.isLocal;
  /**
   * The LiveKit video track for this media.
   */
-  public readonly video: Observable<TrackReferenceOrPlaceholder>;
+  public readonly video: Observable<TrackReferenceOrPlaceholder | undefined>;
  /**
   * Whether there should be a warning that this media is unencrypted.
   */
@@ -215,6 +217,11 @@ abstract class BaseMediaViewModel extends ViewModel {

  public readonly encryptionStatus: Observable<EncryptionStatus>;

+  /**
+   * Whether this media corresponds to the local participant.
+   */
+  public abstract readonly local: boolean;
+
  public constructor(
    /**
     * An opaque identifier for this media.
@@ -226,7 +233,12 @@ abstract class BaseMediaViewModel extends ViewModel {
    // TODO: Fully separate the data layer from the UI layer by keeping the
    // member object internal
    public readonly member: RoomMember | undefined,
-    protected readonly participant: LocalParticipant | RemoteParticipant,
+    // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through
+    // livekit.
+    protected readonly participant: Observable<
+      LocalParticipant | RemoteParticipant | undefined
+    >,
+
    encryptionSystem: EncryptionSystem,
    audioSource: AudioSource,
    videoSource: VideoSource,
@@ -243,69 +255,72 @@ abstract class BaseMediaViewModel extends ViewModel {
      [audio, this.video],
      (a, v) =>
        encryptionSystem.kind !== E2eeType.NONE &&
-        (a.publication?.isEncrypted === false ||
-          v.publication?.isEncrypted === false),
-    ).pipe(
-      distinctUntilChanged(),
-      shareReplay({ bufferSize: 1, refCount: false }),
-    );
+        (a?.publication?.isEncrypted === false ||
+          v?.publication?.isEncrypted === false),
+    ).pipe(this.scope.state());

-    if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) {
-      this.encryptionStatus = of(EncryptionStatus.Okay).pipe(
-        this.scope.state(),
-      );
-    } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
-      this.encryptionStatus = combineLatest([
-        encryptionErrorObservable(
-          livekitRoom,
-          participant,
-          encryptionSystem,
-          "MissingKey",
-        ),
-        encryptionErrorObservable(
-          livekitRoom,
-          participant,
-          encryptionSystem,
-          "InvalidKey",
-        ),
-        observeRemoteTrackReceivingOkay(participant, audioSource),
-        observeRemoteTrackReceivingOkay(participant, videoSource),
-      ]).pipe(
-        map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
-          if (keyMissing) return EncryptionStatus.KeyMissing;
-          if (keyInvalid) return EncryptionStatus.KeyInvalid;
-          if (audioOkay || videoOkay) return EncryptionStatus.Okay;
-          return undefined; // no change
-        }),
-        filter((x) => !!x),
-        startWith(EncryptionStatus.Connecting),
-        this.scope.state(),
-      );
-    } else {
-      this.encryptionStatus = combineLatest([
-        encryptionErrorObservable(
-          livekitRoom,
-          participant,
-          encryptionSystem,
-          "InvalidKey",
-        ),
-        observeRemoteTrackReceivingOkay(participant, audioSource),
-        observeRemoteTrackReceivingOkay(participant, videoSource),
-      ]).pipe(
-        map(
-          ([keyInvalid, audioOkay, videoOkay]):
-            | EncryptionStatus
-            | undefined => {
-            if (keyInvalid) return EncryptionStatus.PasswordInvalid;
-            if (audioOkay || videoOkay) return EncryptionStatus.Okay;
-            return undefined; // no change
-          },
-        ),
-        filter((x) => !!x),
-        startWith(EncryptionStatus.Connecting),
-        this.scope.state(),
-      );
-    }
+    this.encryptionStatus = this.participant.pipe(
+      switchMap((participant): Observable<EncryptionStatus> => {
+        if (!participant) {
+          return of(EncryptionStatus.Connecting);
+        } else if (
+          participant.isLocal ||
+          encryptionSystem.kind === E2eeType.NONE
+        ) {
+          return of(EncryptionStatus.Okay);
+        } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
+          return combineLatest([
+            encryptionErrorObservable(
+              livekitRoom,
+              participant,
+              encryptionSystem,
+              "MissingKey",
+            ),
+            encryptionErrorObservable(
+              livekitRoom,
+              participant,
+              encryptionSystem,
+              "InvalidKey",
+            ),
+            observeRemoteTrackReceivingOkay(participant, audioSource),
+            observeRemoteTrackReceivingOkay(participant, videoSource),
+          ]).pipe(
+            map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
+              if (keyMissing) return EncryptionStatus.KeyMissing;
+              if (keyInvalid) return EncryptionStatus.KeyInvalid;
+              if (audioOkay || videoOkay) return EncryptionStatus.Okay;
+              return undefined; // no change
+            }),
+            filter((x) => !!x),
+            startWith(EncryptionStatus.Connecting),
+          );
+        } else {
+          return combineLatest([
+            encryptionErrorObservable(
+              livekitRoom,
+              participant,
+              encryptionSystem,
+              "InvalidKey",
+            ),
+            observeRemoteTrackReceivingOkay(participant, audioSource),
+            observeRemoteTrackReceivingOkay(participant, videoSource),
+          ]).pipe(
+            map(
+              ([keyInvalid, audioOkay, videoOkay]):
+                | EncryptionStatus
+                | undefined => {
+                if (keyInvalid) return EncryptionStatus.PasswordInvalid;
+                if (audioOkay || videoOkay) return EncryptionStatus.Okay;
+                return undefined; // no change
+              },
+            ),
+            filter((x) => !!x),
+            startWith(EncryptionStatus.Connecting),
+          );
+        }
+      }),
+      this.scope.state(),
+    );
  }
 }

@@ -324,11 +339,14 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
  /**
   * Whether the participant is speaking.
   */
-  public readonly speaking = observeParticipantEvents(
-    this.participant,
-    ParticipantEvent.IsSpeakingChanged,
-  ).pipe(
-    map((p) => p.isSpeaking),
+  public readonly speaking = this.participant.pipe(
+    switchMap((p) =>
+      p
+        ? observeParticipantEvents(p, ParticipantEvent.IsSpeakingChanged).pipe(
+            map((p) => p.isSpeaking),
+          )
+        : of(false),
+    ),
    this.scope.state(),
  );

@@ -350,7 +368,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
  public constructor(
    id: string,
    member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: Observable<LocalParticipant | RemoteParticipant | undefined>,
    encryptionSystem: EncryptionSystem,
    livekitRoom: LivekitRoom,
  ) {
@@ -364,18 +382,25 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
      livekitRoom,
    );

-    const media = observeParticipantMedia(participant).pipe(this.scope.state());
+    const media = participant.pipe(
+      switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)),
+      this.scope.state(),
+    );
    this.audioEnabled = media.pipe(
-      map((m) => m.microphoneTrack?.isMuted === false),
+      map((m) => m?.microphoneTrack?.isMuted === false),
    );
    this.videoEnabled = media.pipe(
-      map((m) => m.cameraTrack?.isMuted === false),
+      map((m) => m?.cameraTrack?.isMuted === false),
    );
  }

  public toggleFitContain(): void {
    this._cropVideo.next(!this._cropVideo.value);
  }
+
+  public get local(): boolean {
+    return this instanceof LocalUserMediaViewModel;
+  }
 }

 /**
@@ -387,7 +412,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
   */
  public readonly mirror = this.video.pipe(
    switchMap((v) => {
-      const track = v.publication?.track;
+      const track = v?.publication?.track;
      if (!(track instanceof LocalTrack)) return of(false);
      // Watch for track restarts, because they indicate a camera switch
      return fromEvent(track, TrackEvent.Restarted).pipe(
@@ -409,7 +434,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
  public constructor(
    id: string,
    member: RoomMember | undefined,
-    participant: LocalParticipant,
+    participant: Observable<LocalParticipant | undefined>,
    encryptionSystem: EncryptionSystem,
    livekitRoom: LivekitRoom,
  ) {
@@ -470,18 +495,17 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
  public constructor(
    id: string,
    member: RoomMember | undefined,
-    participant: RemoteParticipant,
+    participant: Observable<RemoteParticipant | undefined>,
    encryptionSystem: EncryptionSystem,
    livekitRoom: LivekitRoom,
  ) {
    super(id, member, participant, encryptionSystem, livekitRoom);

    // Sync the local volume with LiveKit
-    this.localVolume
-      .pipe(this.scope.bind())
-      .subscribe((volume) =>
-        (this.participant as RemoteParticipant).setVolume(volume),
-      );
+    combineLatest([
+      participant,
+      this.localVolume.pipe(this.scope.bind()),
+    ]).subscribe(([p, volume]) => p && p.setVolume(volume));
  }

  public toggleLocallyMuted(): void {
@@ -504,9 +528,10 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
  public constructor(
    id: string,
    member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: Observable<LocalParticipant | RemoteParticipant>,
    encryptionSystem: EncryptionSystem,
    livekitRoom: LivekitRoom,
+    public readonly local: boolean,
  ) {
    super(
      id,
--- a/src/tile/GridTile.test.tsx
+++ b/src/tile/GridTile.test.tsx
@@ -13,7 +13,7 @@ import { of } from "rxjs";
 import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";

 import { GridTile } from "./GridTile";
-import { withRemoteMedia } from "../utils/test";
+import { mockRtcMembership, withRemoteMedia } from "../utils/test";
 import { GridTileViewModel } from "../state/TileViewModel";
 import { ReactionsProvider } from "../useReactions";

@@ -25,6 +25,7 @@ global.IntersectionObserver = class MockIntersectionObserver {

 test("GridTile is accessible", async () => {
  await withRemoteMedia(
+    mockRtcMembership("@alice:example.org", "AAAA"),
    {
      rawDisplayName: "Alice",
      getMxcAvatarUrl: () => "mxc://adfsg",
--- a/src/tile/GridTile.tsx
+++ b/src/tile/GridTile.tsx
@@ -175,6 +175,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
        raisedHandTime={handRaised}
        currentReaction={currentReaction}
        raisedHandOnClick={raisedHandOnClick}
+        localParticipant={vm.local}
        {...props}
      />
    );
--- a/src/tile/MediaView.module.css
+++ b/src/tile/MediaView.module.css
@@ -74,9 +74,9 @@ unconditionally select the container so we can use cqmin units */
    calc(var(--media-view-border-radius) - var(--cpd-space-3x))
  );
  display: grid;
-  grid-template-columns: 1fr auto;
+  grid-template-columns: 30px 1fr 30px;
  grid-template-rows: 1fr auto;
-  grid-template-areas: "status status" "nameTag button";
+  grid-template-areas: "reactions status ." "nameTag nameTag button";
  gap: var(--cpd-space-1x);
  place-items: start;
 }
@@ -101,8 +101,8 @@ unconditionally select the container so we can use cqmin units */
  grid-area: status;
  justify-self: center;
  align-self: start;
-  padding: var(--cpd-space-1x);
-  padding-block: var(--cpd-space-1x);
+  padding: var(--cpd-space-2x);
+  padding-block: var(--cpd-space-2x);
  color: var(--cpd-color-text-primary);
  background-color: var(--cpd-color-bg-canvas-default);
  display: flex;
@@ -116,6 +116,12 @@ unconditionally select the container so we can use cqmin units */
  text-align: center;
 }

+.reactions {
+  grid-area: reactions;
+  display: flex;
+  gap: var(--cpd-space-1x);
+}
+
 .nameTag > svg,
 .nameTag > span {
  flex-shrink: 0;
--- a/src/tile/MediaView.test.tsx
+++ b/src/tile/MediaView.test.tsx
@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */

-import { describe, expect, test } from "vitest";
+import { describe, expect, it, test } from "vitest";
 import { render, screen } from "@testing-library/react";
 import { axe } from "vitest-axe";
 import { TooltipProvider } from "@vector-im/compound-web";
@@ -42,6 +42,7 @@ describe("MediaView", () => {
    unencryptedWarning: false,
    video: trackReference,
    member: undefined,
+    localParticipant: false,
  };

  test("is accessible", async () => {
@@ -59,6 +60,25 @@ describe("MediaView", () => {
    });
  });

+  describe("with no participant", () => {
+    it("shows avatar for local user", () => {
+      render(
+        <MediaView {...baseProps} video={undefined} localParticipant={true} />,
+      );
+      expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
+      expect(screen.queryAllByText("video_tile.waiting_for_media").length).toBe(
+        0,
+      );
+    });
+    it("shows avatar and label for remote user", () => {
+      render(
+        <MediaView {...baseProps} video={undefined} localParticipant={false} />,
+      );
+      expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
+      expect(screen.getByText("video_tile.waiting_for_media")).toBeVisible();
+    });
+  });
+
  describe("name tag", () => {
    test("is shown with name", () => {
      render(<MediaView {...baseProps} displayName="Bob" />);
--- a/src/tile/MediaView.tsx
+++ b/src/tile/MediaView.tsx
@@ -28,7 +28,7 @@ interface Props extends ComponentProps<typeof animated.div> {
  style?: ComponentProps<typeof animated.div>["style"];
  targetWidth: number;
  targetHeight: number;
-  video: TrackReferenceOrPlaceholder;
+  video: TrackReferenceOrPlaceholder | undefined;
  videoFit: "cover" | "contain";
  mirror: boolean;
  member: RoomMember | undefined;
@@ -41,6 +41,7 @@ interface Props extends ComponentProps<typeof animated.div> {
  raisedHandTime?: Date;
  currentReaction?: ReactionOption;
  raisedHandOnClick?: () => void;
+  localParticipant: boolean;
 }

 export const MediaView = forwardRef<HTMLDivElement, Props>(
@@ -63,6 +64,7 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
      raisedHandTime,
      currentReaction,
      raisedHandOnClick,
+      localParticipant,
      ...props
    },
    ref,
@@ -90,21 +92,21 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
            size={avatarSize}
            src={member?.getMxcAvatarUrl()}
            className={styles.avatar}
-            style={{ display: videoEnabled ? "none" : "initial" }}
+            style={{ display: video && videoEnabled ? "none" : "initial" }}
          />
-          {video.publication !== undefined && (
+          {video?.publication !== undefined && (
            <VideoTrack
              trackRef={video}
              // There's no reason for this to be focusable
              tabIndex={-1}
              disablePictureInPicture
-              style={{ display: videoEnabled ? "block" : "none" }}
+              style={{ display: video && videoEnabled ? "block" : "none" }}
              data-testid="video"
            />
          )}
        </div>
        <div className={styles.fg}>
-          <div style={{ display: "flex", gap: "var(--cpd-space-1x)" }}>
+          <div className={styles.reactions}>
            <RaisedHandIndicator
              raisedHandTime={raisedHandTime}
              miniature={avatarSize < 96}
@@ -118,6 +120,11 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
              />
            )}
          </div>
+          {!video && !localParticipant && (
+            <div className={styles.status}>
+              {t("video_tile.waiting_for_media")}
+            </div>
+          )}
          {/* TODO: Bring this back once encryption status is less broken */}
          {/*encryptionStatus !== EncryptionStatus.Okay && (
            <div className={styles.status}>
--- a/src/tile/SpotlightTile.test.tsx
+++ b/src/tile/SpotlightTile.test.tsx
@@ -12,7 +12,11 @@ import userEvent from "@testing-library/user-event";
 import { of } from "rxjs";

 import { SpotlightTile } from "./SpotlightTile";
-import { withLocalMedia, withRemoteMedia } from "../utils/test";
+import {
+  mockRtcMembership,
+  withLocalMedia,
+  withRemoteMedia,
+} from "../utils/test";
 import { SpotlightTileViewModel } from "../state/TileViewModel";

 global.IntersectionObserver = class MockIntersectionObserver {
@@ -22,6 +26,7 @@ global.IntersectionObserver = class MockIntersectionObserver {

 test("SpotlightTile is accessible", async () => {
  await withRemoteMedia(
+    mockRtcMembership("@alice:example.org", "AAAA"),
    {
      rawDisplayName: "Alice",
      getMxcAvatarUrl: () => "mxc://adfsg",
@@ -29,6 +34,7 @@ test("SpotlightTile is accessible", async () => {
    {},
    async (vm1) => {
      await withLocalMedia(
+        mockRtcMembership("@bob:example.org", "BBBB"),
        {
          rawDisplayName: "Bob",
          getMxcAvatarUrl: () => "mxc://dlskf",
--- a/src/tile/SpotlightTile.tsx
+++ b/src/tile/SpotlightTile.tsx
@@ -49,12 +49,13 @@ interface SpotlightItemBaseProps {
  "data-id": string;
  targetWidth: number;
  targetHeight: number;
-  video: TrackReferenceOrPlaceholder;
+  video: TrackReferenceOrPlaceholder | undefined;
  member: RoomMember | undefined;
  unencryptedWarning: boolean;
  encryptionStatus: EncryptionStatus;
  displayName: string;
  "aria-hidden"?: boolean;
+  localParticipant: boolean;
 }

 interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps {
@@ -163,6 +164,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
      displayName,
      encryptionStatus,
      "aria-hidden": ariaHidden,
+      localParticipant: vm.local,
    };

    return vm instanceof ScreenShareViewModel ? (
@@ -210,7 +212,9 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
    const ref = useMergedRefs(ourRef, theirRef);
    const maximised = useObservableEagerState(vm.maximised);
    const media = useObservableEagerState(vm.media);
-    const [visibleId, setVisibleId] = useState(media[0].id);
+    const [visibleId, setVisibleId] = useState<string | undefined>(
+      media[0]?.id,
+    );
    const latestMedia = useLatest(media);
    const latestVisibleId = useLatest(visibleId);
    const visibleIndex = media.findIndex((vm) => vm.id === visibleId);
--- a/src/utils/test.ts
+++ b/src/utils/test.ts
@@ -7,7 +7,20 @@ Please see LICENSE in the repository root for full details.
 import { map, Observable, of, SchedulerLike } from "rxjs";
 import { RunHelpers, TestScheduler } from "rxjs/testing";
 import { expect, vi } from "vitest";
-import { RoomMember, Room as MatrixRoom } from "matrix-js-sdk/src/matrix";
+import {
+  RoomMember,
+  Room as MatrixRoom,
+  MatrixEvent,
+  Room,
+  TypedEventEmitter,
+} from "matrix-js-sdk/src/matrix";
+import {
+  CallMembership,
+  Focus,
+  MatrixRTCSessionEvent,
+  MatrixRTCSessionEventHandlerMap,
+  SessionMembershipData,
+} from "matrix-js-sdk/src/matrixrtc";
 import {
  LocalParticipant,
  LocalTrackPublication,
@@ -100,11 +113,40 @@ function mockEmitter<T>(): EmitterMock<T> {
  };
 }

+export function mockRtcMembership(
+  user: string | RoomMember,
+  deviceId: string,
+  callId = "",
+  fociPreferred: Focus[] = [],
+  focusActive: Focus = { type: "oldest_membership" },
+  membership: Partial<SessionMembershipData> = {},
+): CallMembership {
+  const data: SessionMembershipData = {
+    application: "m.call",
+    call_id: callId,
+    device_id: deviceId,
+    foci_preferred: fociPreferred,
+    focus_active: focusActive,
+    ...membership,
+  };
+  const event = new MatrixEvent({
+    sender: typeof user === "string" ? user : user.userId,
+  });
+  return new CallMembership(event, data);
+}
+
 // Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are
 // rather simple, but if one util to mock a member is good enough for us, maybe
 // it's useful for matrix-js-sdk consumers in general.
-export function mockMatrixRoomMember(member: Partial<RoomMember>): RoomMember {
-  return { ...mockEmitter(), ...member } as RoomMember;
+export function mockMatrixRoomMember(
+  rtcMembership: CallMembership,
+  member: Partial<RoomMember> = {},
+): RoomMember {
+  return {
+    ...mockEmitter(),
+    userId: rtcMembership.sender,
+    ...member,
+  } as RoomMember;
 }

 export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
@@ -174,14 +216,15 @@ export function mockLocalParticipant(
 }

 export async function withLocalMedia(
-  member: Partial<RoomMember>,
+  localRtcMember: CallMembership,
+  roomMember: Partial<RoomMember>,
  continuation: (vm: LocalUserMediaViewModel) => void | Promise<void>,
 ): Promise<void> {
  const localParticipant = mockLocalParticipant({});
  const vm = new LocalUserMediaViewModel(
    "local",
-    mockMatrixRoomMember(member),
-    localParticipant,
+    mockMatrixRoomMember(localRtcMember, roomMember),
+    of(localParticipant),
    {
      kind: E2eeType.PER_PARTICIPANT,
    },
@@ -208,15 +251,16 @@ export function mockRemoteParticipant(
 }

 export async function withRemoteMedia(
-  member: Partial<RoomMember>,
+  localRtcMember: CallMembership,
+  roomMember: Partial<RoomMember>,
  participant: Partial<RemoteParticipant>,
  continuation: (vm: RemoteUserMediaViewModel) => void | Promise<void>,
 ): Promise<void> {
  const remoteParticipant = mockRemoteParticipant(participant);
  const vm = new RemoteUserMediaViewModel(
    "remote",
-    mockMatrixRoomMember(member),
-    remoteParticipant,
+    mockMatrixRoomMember(localRtcMember, roomMember),
+    of(remoteParticipant),
    {
      kind: E2eeType.PER_PARTICIPANT,
    },
@@ -244,3 +288,30 @@ export function mockMediaPlay(): string[] {
  };
  return audioIsPlaying;
 }
+
+export class MockRTCSession extends TypedEventEmitter<
+  MatrixRTCSessionEvent,
+  MatrixRTCSessionEventHandlerMap
+> {
+  public constructor(
+    public readonly room: Room,
+    private localMembership: CallMembership,
+    public memberships: CallMembership[] = [],
+  ) {
+    super();
+  }
+
+  public withMemberships(
+    rtcMembers: Observable<Partial<CallMembership>[]>,
+  ): MockRTCSession {
+    rtcMembers.subscribe((m) => {
+      const old = this.memberships;
+      // always prepend the local participant
+      const updated = [this.localMembership, ...(m as CallMembership[])];
+      this.memberships = updated;
+      this.emit(MatrixRTCSessionEvent.MembershipsChanged, old, updated);
+    });
+
+    return this;
+  }
+}