prettier

public/locales/en-GB/app.json

@@ -146,11 +146,13 @@
     "feedback_tab_thank_you": "Thanks, we received your feedback!",
     "feedback_tab_title": "Feedback",
     "more_tab_title": "More",
+    "non_member_tiles": "Show non member tiles",
     "opt_in_description": "<0></0><1></1>You may withdraw consent by unchecking this box. If you are currently in a call, this setting will take effect at the end of the call.",
     "preferences_tab_body": "Here you can configure extra options for an improved experience",
     "preferences_tab_h4": "Preferences",
     "preferences_tab_show_hand_raised_timer_description": "Show a timer when a participant raises their hand",
     "preferences_tab_show_hand_raised_timer_label": "Show hand raise duration",
+    "show_media_keys": "Show media encryption keys",
     "speaker_device_selection_label": "Speaker"
   },
   "star_rating_input_label_one": "{{count}} stars",

src/App.tsx

@@ -28,6 +28,8 @@ import { Initializer } from "./initializer";
 import { MediaDevicesProvider } from "./livekit/MediaDevicesContext";
 import { widget } from "./widget";
 import { useTheme } from "./useTheme";
+import { nonMemberTiles } from "./settings/settings";
+import { Config } from "./config/Config";
 
 const SentryRoute = Sentry.withSentryRouting(Route);
 
@@ -71,6 +73,13 @@ export const App: FC<AppProps> = ({ history }) => {
       .catch(logger.error);
   });
 
+  // Update settings to use the non member tile information from the config if set
+  useEffect(() => {
+    if (loaded && Config.get().show_non_member_tiles) {
+      nonMemberTiles.setValue(true);
+    }
+  });
+
   const errorPage = <CrashView />;
 
   return (

src/Header.tsx

@@ -117,6 +117,7 @@ interface RoomHeaderInfoProps {
   avatarUrl: string | null;
   encrypted: boolean;
   participantCount: number | null;
+  nonMemberItemCount: number | null;
 }
 
 export const RoomHeaderInfo: FC<RoomHeaderInfoProps> = ({
@@ -125,6 +126,7 @@ export const RoomHeaderInfo: FC<RoomHeaderInfoProps> = ({
   avatarUrl,
   encrypted,
   participantCount,
+  nonMemberItemCount,
 }) => {
   const { t } = useTranslation();
   const size = useMediaQuery("(max-width: 550px)") ? "sm" : "lg";
@@ -157,7 +159,8 @@ export const RoomHeaderInfo: FC<RoomHeaderInfoProps> = ({
             aria-label={t("header_participants_label")}
           />
           <Text as="span" size="sm" weight="medium">
-            {t("participant_count", { count: participantCount ?? 0 })}
+            {t("participant_count", { count: participantCount ?? 0 })}{" "}
+            {(nonMemberItemCount ?? 0) > 0 && <>(+ {nonMemberItemCount})</>}
           </Text>
         </div>
       )}

src/config/ConfigOptions.ts

@@ -112,6 +112,7 @@ export interface ResolvedConfigOptions extends ConfigOptions {
     enable_video: boolean;
   };
   app_prompt: boolean;
+  show_non_member_tiles: boolean;
 }
 
 export const DEFAULT_CONFIG: ResolvedConfigOptions = {
@@ -127,4 +128,5 @@ export const DEFAULT_CONFIG: ResolvedConfigOptions = {
     enable_video: true,
   },
   app_prompt: true,
+  show_non_member_tiles: false,
 };

src/room/InCallView.tsx

@@ -118,7 +118,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
   useEffect(() => {
     if (livekitRoom !== undefined) {
       const vm = new CallViewModel(
-        props.rtcSession.room,
+        props.rtcSession,
         livekitRoom,
         props.e2eeSystem,
         connStateObservable,
@@ -126,12 +126,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
       setVm(vm);
       return (): void => vm.destroy();
     }
-  }, [
-    props.rtcSession.room,
-    livekitRoom,
-    props.e2eeSystem,
-    connStateObservable,
-  ]);
+  }, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable]);
 
   if (livekitRoom === undefined || vm === null) return null;
 
@@ -194,6 +189,7 @@ export const InCallView: FC<InCallViewProps> = ({
     }
   }, [connState, onLeave]);
 
+  const nonMemberItemCount = useObservableEagerState(vm.nonMemberItemCount);
   const containerRef1 = useRef<HTMLDivElement | null>(null);
   const [containerRef2, bounds] = useMeasure();
   const boundsValid = bounds.height > 0;
@@ -633,6 +629,7 @@ export const InCallView: FC<InCallViewProps> = ({
                 avatarUrl={matrixInfo.roomAvatar}
                 encrypted={matrixInfo.e2eeSystem.kind !== E2eeType.NONE}
                 participantCount={participantCount}
+                nonMemberItemCount={nonMemberItemCount}
               />
             </LeftNav>
             <RightNav>

src/settings/SettingsModal.tsx

@@ -27,6 +27,8 @@ import {
   useSetting,
   developerSettingsTab as developerSettingsTabSetting,
   duplicateTiles as duplicateTilesSetting,
+  nonMemberTiles as nonMemberTilesSetting,
+  showMediaKeys as showMediaKeysSetting,
   useOptInAnalytics,
 } from "./settings";
 import { isFirefox } from "../Platform";
@@ -68,6 +70,10 @@ export const SettingsModal: FC<Props> = ({
   );
   const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting);
 
+  const [nonMemberTiles, setNonMemberTiles] = useSetting(nonMemberTilesSetting);
+
+  const [showMediaKeys, setShowMediaKeys] = useSetting(showMediaKeysSetting);
+
   // Generate a `SelectInput` with a list of devices for a given device kind.
   const generateDeviceSelection = (
     devices: MediaDevice,
@@ -236,6 +242,34 @@ export const SettingsModal: FC<Props> = ({
             )}
           />
         </FieldRow>
+        <FieldRow>
+          <InputField
+            id="nonMemberTiles"
+            type="checkbox"
+            label={t("settings.non_member_tiles")}
+            checked={nonMemberTiles}
+            onChange={useCallback(
+              (event: ChangeEvent<HTMLInputElement>): void => {
+                setNonMemberTiles(event.target.checked);
+              },
+              [setNonMemberTiles],
+            )}
+          />
+        </FieldRow>
+        <FieldRow>
+          <InputField
+            id="showMediaKeys"
+            type="checkbox"
+            label={t("settings.show_media_keys")}
+            checked={showMediaKeys}
+            onChange={useCallback(
+              (event: ChangeEvent<HTMLInputElement>): void => {
+                setShowMediaKeys(event.target.checked);
+              },
+              [setShowMediaKeys],
+            )}
+          />
+        </FieldRow>
       </>
     ),
   };

src/settings/settings.ts

@@ -72,6 +72,10 @@ export const developerSettingsTab = new Setting(
 
 export const duplicateTiles = new Setting("duplicate-tiles", 0);
 
+export const nonMemberTiles = new Setting("non-member-tiles", false);
+
+export const showMediaKeys = new Setting("non-member-tiles", false);
+
 export const audioInput = new Setting<string | undefined>(
   "audio-input",
   undefined,

src/state/CallViewModel.ts

@@ -18,12 +18,9 @@ import {
   RemoteParticipant,
   Track,
 } from "livekit-client";
+import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix";
 import {
-  Room as MatrixRoom,
-  RoomMember,
-  RoomStateEvent,
-} from "matrix-js-sdk/src/matrix";
-import {
+  BehaviorSubject,
   EMPTY,
   Observable,
   Subject,
@@ -51,6 +48,10 @@ import {
   withLatestFrom,
 } from "rxjs";
 import { logger } from "matrix-js-sdk/src/logger";
+import {
+  MatrixRTCSession,
+  MatrixRTCSessionEvent,
+} from "matrix-js-sdk/src/matrixrtc";
 
 import { ViewModel } from "./ViewModel";
 import {
@@ -67,7 +68,7 @@ import {
 } from "./MediaViewModel";
 import { accumulate, finalizeValue } from "../utils/observable";
 import { ObservableScope } from "./ObservableScope";
-import { duplicateTiles } from "../settings/settings";
+import { duplicateTiles, nonMemberTiles } from "../settings/settings";
 import { isFirefox } from "../Platform";
 import { setPipEnabled } from "../controls";
 import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
@@ -164,28 +165,40 @@ enum SortingBin {
 class UserMedia {
   private readonly scope = new ObservableScope();
   public readonly vm: UserMediaViewModel;
+  public participant: BehaviorSubject<
+    LocalParticipant | RemoteParticipant | undefined
+  >;
+
   public readonly speaker: Observable<boolean>;
   public readonly presenter: Observable<boolean>;
-
   public constructor(
     public readonly id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: LocalParticipant | RemoteParticipant | undefined,
     encryptionSystem: EncryptionSystem,
+    rtcSession: MatrixRTCSession,
   ) {
-    this.vm = participant.isLocal
-      ? new LocalUserMediaViewModel(
-          id,
-          member,
-          participant as LocalParticipant,
-          encryptionSystem,
-        )
-      : new RemoteUserMediaViewModel(
-          id,
-          member,
-          participant as RemoteParticipant,
-          encryptionSystem,
-        );
+    this.participant = new BehaviorSubject(participant);
+
+    if (participant && participant.isLocal) {
+      this.vm = new LocalUserMediaViewModel(
+        this.id,
+        member,
+        this.participant.asObservable() as Observable<LocalParticipant>,
+        encryptionSystem,
+        rtcSession,
+      );
+    } else {
+      this.vm = new RemoteUserMediaViewModel(
+        id,
+        member,
+        this.participant.asObservable() as Observable<
+          RemoteParticipant | undefined
+        >,
+        encryptionSystem,
+        rtcSession,
+      );
+    }
 
     this.speaker = this.vm.speaking.pipe(
       // Require 1 s of continuous speaking to become a speaker, and 60 s of
@@ -195,7 +208,7 @@ class UserMedia {
           timer(s ? 1000 : 60000),
           // If the speaking flag resets to its original value during this time,
           // end the silencing window to stick with that original value
-          this.vm.speaking.pipe(filter((s1) => s1 !== s)),
+          this.vm!.speaking.pipe(filter((s1) => s1 !== s)),
         ),
       ),
       startWith(false),
@@ -205,13 +218,21 @@ class UserMedia {
       this.scope.state(),
     );
 
-    this.presenter = observeParticipantEvents(
-      participant,
-      ParticipantEvent.TrackPublished,
-      ParticipantEvent.TrackUnpublished,
-      ParticipantEvent.LocalTrackPublished,
-      ParticipantEvent.LocalTrackUnpublished,
-    ).pipe(map((p) => p.isScreenShareEnabled));
+    this.presenter = this.participant.pipe(
+      switchMap(
+        (p) =>
+          (p &&
+            observeParticipantEvents(
+              p,
+              ParticipantEvent.TrackPublished,
+              ParticipantEvent.TrackUnpublished,
+              ParticipantEvent.LocalTrackPublished,
+              ParticipantEvent.LocalTrackUnpublished,
+            ).pipe(map((p) => p.isScreenShareEnabled))) ??
+          of(false),
+      ),
+      this.scope.state(),
+    );
   }
 
   public destroy(): void {
@@ -222,6 +243,7 @@ class UserMedia {
 
 class ScreenShare {
   public readonly vm: ScreenShareViewModel;
+  private participant: BehaviorSubject<LocalParticipant | RemoteParticipant>;
 
   public constructor(
     id: string,
@@ -229,10 +251,12 @@ class ScreenShare {
     participant: LocalParticipant | RemoteParticipant,
     encryptionSystem: EncryptionSystem,
   ) {
+    this.participant = new BehaviorSubject(participant);
+
     this.vm = new ScreenShareViewModel(
       id,
       member,
-      participant,
+      this.participant.asObservable(),
       encryptionSystem,
     );
   }
@@ -244,7 +268,7 @@ class ScreenShare {
 
 type MediaItem = UserMedia | ScreenShare;
 
-function findMatrixMember(
+function findMatrixRoomMember(
   room: MatrixRoom,
   id: string,
 ): RoomMember | undefined {
@@ -270,11 +294,11 @@ function findMatrixMember(
 export class CallViewModel extends ViewModel {
   public readonly localVideo: Observable<LocalVideoTrack | null> =
     observeTrackReference(
-      this.livekitRoom.localParticipant,
+      of(this.livekitRoom.localParticipant),
       Track.Source.Camera,
     ).pipe(
       map((trackRef) => {
-        const track = trackRef.publication?.track;
+        const track = trackRef?.publication?.track;
         return track instanceof LocalVideoTrack ? track : null;
       }),
     );
@@ -340,63 +364,160 @@ export class CallViewModel extends ViewModel {
       },
     );
 
+  public readonly nonMemberItemCount = new BehaviorSubject<number>(0);
+
   private readonly mediaItems: Observable<MediaItem[]> = combineLatest([
     this.remoteParticipants,
     observeParticipantMedia(this.livekitRoom.localParticipant),
     duplicateTiles.value,
-    // Also react to changes in the list of members
-    fromEvent(this.matrixRoom, RoomStateEvent.Update).pipe(startWith(null)),
+    // Also react to changes in the MatrixRTC session list:
+    fromEvent(
+      this.matrixRTCSession,
+      MatrixRTCSessionEvent.MembershipsChanged,
+    ).pipe(startWith(null)),
+    nonMemberTiles.value,
   ]).pipe(
     scan(
       (
         prevItems,
-        [remoteParticipants, { participant: localParticipant }, duplicateTiles],
+        [
+          remoteParticipants,
+          { participant: localParticipant },
+          duplicateTiles,
+          _participantChange,
+          nonMemberTiles,
+        ],
       ) => {
         const newItems = new Map(
           function* (this: CallViewModel): Iterable<[string, MediaItem]> {
-            for (const p of [localParticipant, ...remoteParticipants]) {
-              const id = p === localParticipant ? "local" : p.identity;
-              const member = findMatrixMember(this.matrixRoom, id);
-              if (member === undefined)
-                logger.warn(
-                  `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`,
+            for (const rtcMember of this.matrixRTCSession.memberships) {
+              const room = this.matrixRTCSession.room;
+              // WARN! This is not exactly the sender but the user defined in the state key.
+              // This will be available once we change to the new "member as object" format in the MatrixRTC object.
+              let mediaId = rtcMember.sender + ":" + rtcMember.deviceId;
+              let participant = undefined;
+              if (
+                rtcMember.sender === room.client.getUserId()! &&
+                rtcMember.deviceId === room.client.getDeviceId()
+              ) {
+                mediaId = "local";
+                participant = localParticipant;
+              } else {
+                participant = remoteParticipants.find(
+                  (p) => p.identity === mediaId,
                 );
+              }
 
-              // Create as many tiles for this participant as called for by
-              // the duplicateTiles option
+              const member = findMatrixRoomMember(room, mediaId);
+              if (!member) {
+                logger.error("Could not find member for media id: ", mediaId);
+              }
               for (let i = 0; i < 1 + duplicateTiles; i++) {
-                const userMediaId = `${id}:${i}`;
+                const indexedMediaId = `${mediaId}:${i}`;
+                let prevMedia = prevItems.get(indexedMediaId);
+                if (prevMedia && prevMedia instanceof UserMedia) {
+                  if (prevMedia.participant.value !== participant) {
+                    // Update the BahviourSubject in the UserMedia.
+                    prevMedia.participant.next(participant);
+                  }
+                  if (prevMedia.vm.member === undefined) {
+                    // We have a previous media created because of the `debugShowNonMember` flag.
+                    // In this case we actually replace the media item.
+                    // This "hack" never occurs if we do not use the `debugShowNonMember` debugging
+                    // option and if we always find a room member for each rtc member (which also
+                    // only fails if we have a fundamental problem)
+                    prevMedia = undefined;
+                  }
+                }
                 yield [
-                  userMediaId,
-                  prevItems.get(userMediaId) ??
+                  indexedMediaId,
+                  // We create UserMedia with or without a participant.
+                  // This will be the initial value of a BehaviourSubject.
+                  // Once a participant appears we will update the BehaviourSubject. (see above)
+                  prevMedia ??
                     new UserMedia(
-                      userMediaId,
+                      indexedMediaId,
                       member,
-                      p,
+                      participant,
+                      this.encryptionSystem,
+                      this.matrixRTCSession,
+                    ),
+                ];
+              }
+              if (participant && participant.isScreenShareEnabled) {
+                const screenShareId = `${mediaId}:screen-share`;
+                yield [
+                  screenShareId,
+                  prevItems.get(screenShareId) ??
+                    new ScreenShare(
+                      screenShareId,
+                      member,
+                      participant,
                       this.encryptionSystem,
                     ),
                 ];
-
-                if (p.isScreenShareEnabled) {
-                  const screenShareId = `${userMediaId}:screen-share`;
-                  yield [
-                    screenShareId,
-                    prevItems.get(screenShareId) ??
-                      new ScreenShare(
-                        screenShareId,
-                        member,
-                        p,
-                        this.encryptionSystem,
-                      ),
-                  ];
-                }
               }
             }
           }.bind(this)(),
         );
 
-        for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy();
-        return newItems;
+        // Generate non member items (items without a corresponding MatrixRTC member)
+        // Those items should not be rendered, they are participants in livekit that do not have a corresponding
+        // matrix rtc members. This cannot be any good:
+        //  - A malicious user impersonates someone
+        //  - Someone injects abusive content
+        //  - The user cannot have encryption keys so it makes no sense to participate
+        // We can only trust users that have a matrixRTC member event.
+        //
+        // This is still available as a debug option. This can be useful
+        //  - If one wants to test scalability using the livekit cli.
+        //  - If an experimental project does not yet do the matrixRTC bits.
+        //  - If someone wants to debug if the LK connection works but matrixRTC room state failed to arrive.
+        const debugShowNonMember = nonMemberTiles; //Config.get().show_non_member_tiles;
+        const newNonMemberItems = debugShowNonMember
+          ? new Map(
+              function* (this: CallViewModel): Iterable<[string, MediaItem]> {
+                for (const participant of remoteParticipants) {
+                  for (let i = 0; i < 1 + duplicateTiles; i++) {
+                    const maybeNonMemberParticipantId =
+                      participant.identity + ":" + i;
+                    if (!newItems.has(maybeNonMemberParticipantId)) {
+                      const nonMemberId = maybeNonMemberParticipantId;
+                      yield [
+                        nonMemberId,
+                        // We create UserMedia with or without a participant.
+                        // This will be the initial value of a BehaviourSubject.
+                        // Once a participant appears we will update the BehaviourSubject. (see above)
+                        prevItems.get(nonMemberId) ??
+                          new UserMedia(
+                            nonMemberId,
+                            undefined,
+                            participant,
+                            this.encryptionSystem,
+                            this.matrixRTCSession,
+                          ),
+                      ];
+                    }
+                  }
+                }
+              }.bind(this)(),
+            )
+          : new Map();
+        if (newNonMemberItems.size > 0) {
+          logger.debug("Added NonMember items: ", newNonMemberItems);
+        }
+        const newNonMemberItemCount =
+          newNonMemberItems.size / (1 + duplicateTiles);
+        if (this.nonMemberItemCount.value !== newNonMemberItemCount)
+          this.nonMemberItemCount.next(newNonMemberItemCount);
+
+        const combinedNew = new Map([
+          ...newNonMemberItems.entries(),
+          ...newItems.entries(),
+        ]);
+
+        for (const [id, t] of prevItems) if (!combinedNew.has(id)) t.destroy();
+        return combinedNew;
       },
       new Map<string, MediaItem>(),
     ),
@@ -432,42 +553,43 @@ export class CallViewModel extends ViewModel {
       distinctUntilChanged(),
     );
 
-  private readonly spotlightSpeaker: Observable<UserMediaViewModel> =
-    this.userMedia.pipe(
-      switchMap((mediaItems) =>
-        mediaItems.length === 0
-          ? of([])
-          : combineLatest(
-              mediaItems.map((m) =>
-                m.vm.speaking.pipe(map((s) => [m, s] as const)),
-              ),
+  private readonly spotlightSpeaker: Observable<
+    UserMediaViewModel | undefined
+  > = this.userMedia.pipe(
+    switchMap((mediaItems) =>
+      mediaItems.length === 0
+        ? of([])
+        : combineLatest(
+            mediaItems.map((m) =>
+              m.vm.speaking.pipe(map((s) => [m, s] as const)),
             ),
-      ),
-      scan<(readonly [UserMedia, boolean])[], UserMedia, null>(
-        (prev, mediaItems) => {
-          // Only remote users that are still in the call should be sticky
-          const [stickyMedia, stickySpeaking] =
-            (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || [];
-          // Decide who to spotlight:
-          // If the previous speaker is still speaking, stick with them rather
-          // than switching eagerly to someone else
-          return stickySpeaking
-            ? stickyMedia!
-            : // Otherwise, select any remote user who is speaking
-              (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ??
-                // Otherwise, stick with the person who was last speaking
-                stickyMedia ??
-                // Otherwise, spotlight an arbitrary remote user
-                mediaItems.find(([m]) => !m.vm.local)?.[0] ??
-                // Otherwise, spotlight the local user
-                mediaItems.find(([m]) => m.vm.local)![0]);
-        },
-        null,
-      ),
-      map((speaker) => speaker.vm),
-      this.scope.state(),
-      throttleTime(1600, undefined, { leading: true, trailing: true }),
-    );
+          ),
+    ),
+    scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>(
+      (prev, mediaItems) => {
+        // Only remote users that are still in the call should be sticky
+        const [stickyMedia, stickySpeaking] =
+          (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || [];
+        // Decide who to spotlight:
+        // If the previous speaker is still speaking, stick with them rather
+        // than switching eagerly to someone else
+        return stickySpeaking
+          ? stickyMedia!
+          : // Otherwise, select any remote user who is speaking
+            (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ??
+              // Otherwise, stick with the person who was last speaking
+              stickyMedia ??
+              // Otherwise, spotlight an arbitrary remote user
+              mediaItems.find(([m]) => !m.vm.local)?.[0] ??
+              // Otherwise, spotlight the local user
+              mediaItems.find(([m]) => m.vm.local)?.[0]);
+      },
+      null,
+    ),
+    map((speaker) => speaker?.vm),
+    this.scope.state(),
+    throttleTime(1600, undefined, { leading: true, trailing: true }),
+  );
 
   private readonly grid: Observable<UserMediaViewModel[]> = this.userMedia.pipe(
     switchMap((mediaItems) => {
@@ -510,20 +632,29 @@ export class CallViewModel extends ViewModel {
   > = this.screenShares.pipe(
     map((screenShares) =>
       screenShares.length > 0
-        ? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const)
+        ? ([
+            of(screenShares.map((m) => m.vm)),
+            this.spotlightSpeaker.pipe(
+              map((speaker) => (speaker && speaker) ?? null),
+            ),
+          ] as const)
         : ([
-            this.spotlightSpeaker.pipe(map((speaker) => [speaker!])),
+            this.spotlightSpeaker.pipe(
+              map((speaker) => (speaker && [speaker]) ?? []),
+            ),
             this.spotlightSpeaker.pipe(
               switchMap((speaker) =>
-                speaker.local
-                  ? of(null)
-                  : this.localUserMedia.pipe(
-                      switchMap((vm) =>
-                        vm.alwaysShow.pipe(
-                          map((alwaysShow) => (alwaysShow ? vm : null)),
+                speaker
+                  ? speaker.local
+                    ? of(null)
+                    : this.localUserMedia.pipe(
+                        switchMap((vm) =>
+                          vm.alwaysShow.pipe(
+                            map((alwaysShow) => (alwaysShow ? vm : null)),
+                          ),
                         ),
-                      ),
-                    ),
+                      )
+                  : of(null),
               ),
             ),
           ] as const),
@@ -843,7 +974,7 @@ export class CallViewModel extends ViewModel {
 
   public constructor(
     // A call is permanently tied to a single Matrix room and LiveKit room
-    private readonly matrixRoom: MatrixRoom,
+    private readonly matrixRTCSession: MatrixRTCSession,
     private readonly livekitRoom: LivekitRoom,
     private readonly encryptionSystem: EncryptionSystem,
     private readonly connectionState: Observable<ECConnectionState>,

src/state/MediaViewModel.ts

@@ -37,6 +37,10 @@ import {
   switchMap,
 } from "rxjs";
 import { useEffect } from "react";
+import {
+  MatrixRTCSession,
+  MatrixRTCSessionEvent,
+} from "matrix-js-sdk/src/matrixrtc";
 
 import { ViewModel } from "./ViewModel";
 import { useReactiveState } from "../useReactiveState";
@@ -68,33 +72,50 @@ export function useDisplayName(vm: MediaViewModel): string {
 }
 
 export function observeTrackReference(
-  participant: Participant,
+  participant: Observable<Participant | undefined>,
   source: Track.Source,
-): Observable<TrackReferenceOrPlaceholder> {
-  return observeParticipantMedia(participant).pipe(
-    map(() => ({
-      participant,
-      publication: participant.getTrackPublication(source),
-      source,
-    })),
-    distinctUntilKeyChanged("publication"),
+): Observable<TrackReferenceOrPlaceholder | undefined> {
+  const obs = participant.pipe(
+    switchMap((p) => {
+      if (p) {
+        return observeParticipantMedia(p).pipe(
+          map(() => ({
+            participant: p,
+            publication: p.getTrackPublication(source),
+            source,
+          })),
+          distinctUntilKeyChanged("publication"),
+        );
+      } else {
+        return of(undefined);
+      }
+    }),
   );
+  return obs;
 }
 
 abstract class BaseMediaViewModel extends ViewModel {
   /**
    * Whether the media belongs to the local user.
    */
-  public readonly local = this.participant.isLocal;
+  public readonly local = this.participant.pipe(
+    // We can assume, that the user is not local if the participant is undefined
+    // We assume the local LK participant will always be available.
+    map((p) => p?.isLocal ?? false),
+  );
   /**
    * The LiveKit video track for this media.
    */
-  public readonly video: Observable<TrackReferenceOrPlaceholder>;
+  public readonly video: Observable<TrackReferenceOrPlaceholder | undefined>;
   /**
    * Whether there should be a warning that this media is unencrypted.
    */
   public readonly unencryptedWarning: Observable<boolean>;
 
+  public readonly isRTCParticipantAvailable = this.participant.pipe(
+    map((p) => !!p),
+  );
+
   public constructor(
     /**
      * An opaque identifier for this media.
@@ -106,7 +127,12 @@ abstract class BaseMediaViewModel extends ViewModel {
     // TODO: Fully separate the data layer from the UI layer by keeping the
     // member object internal
     public readonly member: RoomMember | undefined,
-    protected readonly participant: LocalParticipant | RemoteParticipant,
+    // We dont necassarly have a participant if a user connects via MatrixRTC but not (not yet) through
+    // livekit.
+    protected readonly participant: Observable<
+      LocalParticipant | RemoteParticipant | undefined
+    >,
+
     encryptionSystem: EncryptionSystem,
     audioSource: AudioSource,
     videoSource: VideoSource,
@@ -122,8 +148,8 @@ abstract class BaseMediaViewModel extends ViewModel {
       [audio, this.video],
       (a, v) =>
         encryptionSystem.kind !== E2eeType.NONE &&
-        (a.publication?.isEncrypted === false ||
-          v.publication?.isEncrypted === false),
+        (a?.publication?.isEncrypted === false ||
+          v?.publication?.isEncrypted === false),
     ).pipe(this.scope.state());
   }
 }
@@ -143,12 +169,20 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
   /**
    * Whether the participant is speaking.
    */
-  public readonly speaking = observeParticipantEvents(
-    this.participant,
-    ParticipantEvent.IsSpeakingChanged,
-  ).pipe(
-    map((p) => p.isSpeaking),
-    this.scope.state(),
+  public readonly speaking = this.participant.pipe(
+    switchMap((p) => {
+      if (p) {
+        return observeParticipantEvents(
+          p,
+          ParticipantEvent.IsSpeakingChanged,
+        ).pipe(
+          map((p) => p.isSpeaking),
+          this.scope.state(),
+        );
+      } else {
+        return of(false);
+      }
+    }),
   );
 
   /**
@@ -166,11 +200,16 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
    */
   public readonly cropVideo: Observable<boolean> = this._cropVideo;
 
+  public readonly keys = new BehaviorSubject(
+    [] as { index: number; key: Uint8Array }[],
+  );
+
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: Observable<LocalParticipant | RemoteParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
+    rtcSession: MatrixRTCSession,
   ) {
     super(
       id,
@@ -181,12 +220,32 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
       Track.Source.Camera,
     );
 
-    const media = observeParticipantMedia(participant).pipe(this.scope.state());
+    combineLatest([
+      participant,
+      fromEvent(rtcSession, MatrixRTCSessionEvent.EncryptionKeyChanged).pipe(
+        startWith(null),
+      ),
+    ]).subscribe(([par, ev]) => {
+      for (const participantKeys of rtcSession.getEncryptionKeys()) {
+        if (participantKeys[0] === par?.identity) {
+          this.keys.next(
+            Array.from(participantKeys[1].entries()).map(([i, k]) => {
+              return { index: i, key: k };
+            }),
+          );
+        }
+      }
+    });
+
+    const media = participant.pipe(
+      switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)),
+      this.scope.state(),
+    );
     this.audioEnabled = media.pipe(
-      map((m) => m.microphoneTrack?.isMuted === false),
+      map((m) => m?.microphoneTrack?.isMuted === false),
     );
     this.videoEnabled = media.pipe(
-      map((m) => m.cameraTrack?.isMuted === false),
+      map((m) => m?.cameraTrack?.isMuted === false),
     );
   }
 
@@ -204,7 +263,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
    */
   public readonly mirror = this.video.pipe(
     switchMap((v) => {
-      const track = v.publication?.track;
+      const track = v?.publication?.track;
       if (!(track instanceof LocalTrack)) return of(false);
       // Watch for track restarts, because they indicate a camera switch
       return fromEvent(track, TrackEvent.Restarted).pipe(
@@ -226,10 +285,11 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant,
+    participant: Observable<LocalParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
+    rtcSession: MatrixRTCSession,
   ) {
-    super(id, member, participant, encryptionSystem);
+    super(id, member, participant, encryptionSystem, rtcSession);
   }
 }
 
@@ -286,17 +346,17 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: RemoteParticipant,
+    participant: Observable<RemoteParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
+    rtcSession: MatrixRTCSession,
   ) {
-    super(id, member, participant, encryptionSystem);
+    super(id, member, participant, encryptionSystem, rtcSession);
 
     // Sync the local volume with LiveKit
-    this.localVolume
-      .pipe(this.scope.bind())
-      .subscribe((volume) =>
-        (this.participant as RemoteParticipant).setVolume(volume),
-      );
+    combineLatest([
+      participant,
+      this.localVolume.pipe(this.scope.bind()),
+    ]).subscribe(([p, volume]) => p && p.setVolume(volume));
   }
 
   public toggleLocallyMuted(): void {
@@ -319,7 +379,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: Observable<LocalParticipant | RemoteParticipant>,
     encryptionSystem: EncryptionSystem,
   ) {
     super(

src/tile/GridTile.tsx

@@ -84,6 +84,10 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
     const videoEnabled = useObservableEagerState(vm.videoEnabled);
     const speaking = useObservableEagerState(vm.speaking);
     const cropVideo = useObservableEagerState(vm.cropVideo);
+    const keys = useObservableEagerState(vm.keys);
+    const isRTCParticipantAvailable = useObservableEagerState(
+      vm.isRTCParticipantAvailable,
+    );
     const onSelectFitContain = useCallback(
       (e: Event) => {
         e.preventDefault();
@@ -118,6 +122,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
         ref={ref}
         video={video}
         member={vm.member}
+        keys={keys}
         unencryptedWarning={unencryptedWarning}
         videoEnabled={videoEnabled && showVideo}
         videoFit={cropVideo ? "cover" : "contain"}
@@ -134,7 +139,10 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
             className={styles.muteIcon}
           />
         }
-        displayName={displayName}
+        displayName={
+          displayName +
+          (isRTCParticipantAvailable ? "" : " missing Livekit Participant...")
+        }
         primaryButton={
           <Menu
             open={menuOpen}
@@ -226,6 +234,7 @@ const RemoteUserMediaTile = forwardRef<
   const { t } = useTranslation();
   const locallyMuted = useObservableEagerState(vm.locallyMuted);
   const localVolume = useObservableEagerState(vm.localVolume);
+
   const onSelectMute = useCallback(
     (e: Event) => {
       e.preventDefault();
@@ -248,6 +257,9 @@ const RemoteUserMediaTile = forwardRef<
       mirror={false}
       menuStart={
         <>
+          {/* {isRTCParticipantAvailable
+            ? "is available"
+            : "Loading RTC participant"} */}
           <ToggleMenuItem
             Icon={MicOffIcon}
             label={t("video_tile.mute_for_me")}

src/tile/MediaView.tsx

@@ -7,8 +7,8 @@ Please see LICENSE in the repository root for full details.
 
 import { TrackReferenceOrPlaceholder } from "@livekit/components-core";
 import { animated } from "@react-spring/web";
-import { RoomMember } from "matrix-js-sdk/src/matrix";
-import { ComponentProps, ReactNode, forwardRef } from "react";
+import { encodeUnpaddedBase64, RoomMember } from "matrix-js-sdk/src/matrix";
+import { ComponentProps, FC, ReactNode, forwardRef } from "react";
 import { useTranslation } from "react-i18next";
 import classNames from "classnames";
 import { VideoTrack } from "@livekit/components-react";
@@ -18,17 +18,22 @@ import { ErrorIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
 import styles from "./MediaView.module.css";
 import { Avatar } from "../Avatar";
 import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
-import { showHandRaisedTimer, useSetting } from "../settings/settings";
+import {
+  showHandRaisedTimer,
+  showMediaKeys as showMediaKeysSettings,
+  useSetting,
+} from "../settings/settings";
 
 interface Props extends ComponentProps<typeof animated.div> {
   className?: string;
   style?: ComponentProps<typeof animated.div>["style"];
   targetWidth: number;
   targetHeight: number;
-  video: TrackReferenceOrPlaceholder;
+  video: TrackReferenceOrPlaceholder | undefined;
   videoFit: "cover" | "contain";
   mirror: boolean;
   member: RoomMember | undefined;
+  keys: { index: number; key: Uint8Array }[];
   videoEnabled: boolean;
   unencryptedWarning: boolean;
   nameTagLeadingIcon?: ReactNode;
@@ -48,6 +53,7 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
       videoFit,
       mirror,
       member,
+      keys,
       videoEnabled,
       unencryptedWarning,
       nameTagLeadingIcon,
@@ -60,6 +66,7 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
   ) => {
     const { t } = useTranslation();
     const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer);
+    const [showMediaKeys] = useSetting(showMediaKeysSettings);
 
     const avatarSize = Math.round(Math.min(targetWidth, targetHeight) / 2);
 
@@ -83,7 +90,7 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
             src={member?.getMxcAvatarUrl()}
             className={styles.avatar}
           />
-          {video.publication !== undefined && (
+          {video?.publication !== undefined && (
             <VideoTrack
               trackRef={video}
               // There's no reason for this to be focusable
@@ -98,11 +105,13 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
             minature={avatarSize < 96}
             showTimer={handRaiseTimerVisible}
           />
+          {keys && showMediaKeys && <MediaKeyList keys={keys} />}
           <div className={styles.nameTag}>
             {nameTagLeadingIcon}
             <Text as="span" size="sm" weight="medium" className={styles.name}>
               {displayName}
             </Text>
+
             {unencryptedWarning && (
               <Tooltip
                 label={t("common.unencrypted")}
@@ -123,5 +132,45 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
     );
   },
 );
+interface MediaKeyListProps {
+  keys: {
+    index: number;
+    key: Uint8Array;
+  }[];
+}
 
+export const MediaKeyList: FC<MediaKeyListProps> = ({ keys }) => {
+  return (
+    <div
+      style={{
+        display: "flex",
+        flexDirection: "column",
+        height: "70%",
+        overflow: "scroll",
+        color: "white",
+      }}
+    >
+      {keys.map(({ index, key }) => (
+        <div
+          style={{
+            display: "flex",
+            flexDirection: "column",
+            backgroundColor: "#000000c0",
+            margin: "3px",
+            padding: "5px",
+            borderRadius: "5px",
+          }}
+          key={index}
+        >
+          <Text as="span" size="sm">
+            index:{index}
+          </Text>
+          <Text as="span" size="xs">
+            key:{key ? encodeUnpaddedBase64(key) : "unavailable"}
+          </Text>
+        </div>
+      ))}
+    </div>
+  );
+};
 MediaView.displayName = "MediaView";