make tiles based on rtc member

src/room/InCallView.tsx

@@ -118,7 +118,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
   useEffect(() => {
     if (livekitRoom !== undefined) {
       const vm = new CallViewModel(
-        props.rtcSession.room,
+        props.rtcSession,
         livekitRoom,
         props.e2eeSystem,
         connStateObservable,
@@ -127,7 +127,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
       return (): void => vm.destroy();
     }
   }, [
-    props.rtcSession.room,
+    props.rtcSession,
     livekitRoom,
     props.e2eeSystem,
     connStateObservable,

src/state/CallViewModel.ts

@@ -18,12 +18,9 @@ import {
   RemoteParticipant,
   Track,
 } from "livekit-client";
+import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix";
 import {
-  Room as MatrixRoom,
+  BehaviorSubject,
-  RoomMember,
-  RoomStateEvent,
-} from "matrix-js-sdk/src/matrix";
-import {
   EMPTY,
   Observable,
   Subject,
@@ -51,6 +48,10 @@ import {
   withLatestFrom,
 } from "rxjs";
 import { logger } from "matrix-js-sdk/src/logger";
+import {
+  MatrixRTCSession,
+  MatrixRTCSessionEvent,
+} from "matrix-js-sdk/src/matrixrtc";
 
 import { ViewModel } from "./ViewModel";
 import {
@@ -164,28 +165,37 @@ enum SortingBin {
 class UserMedia {
   private readonly scope = new ObservableScope();
   public readonly vm: UserMediaViewModel;
+  public participant: BehaviorSubject<
+    LocalParticipant | RemoteParticipant | undefined
+  >;
+
   public readonly speaker: Observable<boolean>;
   public readonly presenter: Observable<boolean>;
-
   public constructor(
     public readonly id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: LocalParticipant | RemoteParticipant | undefined,
     encryptionSystem: EncryptionSystem,
   ) {
-    this.vm = participant.isLocal
+    this.participant = new BehaviorSubject(participant);
-      ? new LocalUserMediaViewModel(
+
-          id,
+    if (participant && participant.isLocal) {
-          member,
+      this.vm = new LocalUserMediaViewModel(
-          participant as LocalParticipant,
+        this.id,
-          encryptionSystem,
+        member,
-        )
+        this.participant.asObservable() as Observable<LocalParticipant>,
-      : new RemoteUserMediaViewModel(
+        encryptionSystem,
-          id,
+      );
-          member,
+    } else {
-          participant as RemoteParticipant,
+      this.vm = new RemoteUserMediaViewModel(
-          encryptionSystem,
+        id,
-        );
+        member,
+        this.participant.asObservable() as Observable<
+          RemoteParticipant | undefined
+        >,
+        encryptionSystem,
+      );
+    }
 
     this.speaker = this.vm.speaking.pipe(
       // Require 1 s of continuous speaking to become a speaker, and 60 s of
@@ -195,7 +205,7 @@ class UserMedia {
           timer(s ? 1000 : 60000),
           // If the speaking flag resets to its original value during this time,
           // end the silencing window to stick with that original value
-          this.vm.speaking.pipe(filter((s1) => s1 !== s)),
+          this.vm!.speaking.pipe(filter((s1) => s1 !== s)),
         ),
       ),
       startWith(false),
@@ -205,13 +215,21 @@ class UserMedia {
       this.scope.state(),
     );
 
-    this.presenter = observeParticipantEvents(
+    this.presenter = this.participant.pipe(
-      participant,
+      switchMap(
-      ParticipantEvent.TrackPublished,
+        (p) =>
-      ParticipantEvent.TrackUnpublished,
+          (p &&
-      ParticipantEvent.LocalTrackPublished,
+            observeParticipantEvents(
-      ParticipantEvent.LocalTrackUnpublished,
+              p,
-    ).pipe(map((p) => p.isScreenShareEnabled));
+              ParticipantEvent.TrackPublished,
+              ParticipantEvent.TrackUnpublished,
+              ParticipantEvent.LocalTrackPublished,
+              ParticipantEvent.LocalTrackUnpublished,
+            ).pipe(map((p) => p.isScreenShareEnabled))) ??
+          of(false),
+      ),
+      this.scope.state(),
+    );
   }
 
   public destroy(): void {
@@ -222,6 +240,7 @@ class UserMedia {
 
 class ScreenShare {
   public readonly vm: ScreenShareViewModel;
+  private participant: BehaviorSubject<LocalParticipant | RemoteParticipant>;
 
   public constructor(
     id: string,
@@ -229,10 +248,12 @@ class ScreenShare {
     participant: LocalParticipant | RemoteParticipant,
     encryptionSystem: EncryptionSystem,
   ) {
+    this.participant = new BehaviorSubject(participant);
+
     this.vm = new ScreenShareViewModel(
       id,
       member,
-      participant,
+      this.participant.asObservable(),
       encryptionSystem,
     );
   }
@@ -244,7 +265,7 @@ class ScreenShare {
 
 type MediaItem = UserMedia | ScreenShare;
 
-function findMatrixMember(
+function findMatrixRoomMember(
   room: MatrixRoom,
   id: string,
 ): RoomMember | undefined {
@@ -344,8 +365,15 @@ export class CallViewModel extends ViewModel {
     this.remoteParticipants,
     observeParticipantMedia(this.livekitRoom.localParticipant),
     duplicateTiles.value,
-    // Also react to changes in the list of members
+    // Also react to changes in the MatrixRTC session list:
-    fromEvent(this.matrixRoom, RoomStateEvent.Update).pipe(startWith(null)),
+    fromEvent(
+      this.matrixRTCSession,
+      MatrixRTCSessionEvent.MembershipsChanged,
+    ).pipe(startWith(null)),
+    // fromEvent(
+    //   this.matrixRTCSession,
+    //   MatrixRTCSessionEvent.EncryptionKeyChanged,
+    // ).pipe(startWith(null)),
   ]).pipe(
     scan(
       (
@@ -354,42 +382,64 @@ export class CallViewModel extends ViewModel {
       ) => {
         const newItems = new Map(
           function* (this: CallViewModel): Iterable<[string, MediaItem]> {
-            for (const p of [localParticipant, ...remoteParticipants]) {
+            for (const rtcMember of this.matrixRTCSession.memberships) {
-              const id = p === localParticipant ? "local" : p.identity;
+              const room = this.matrixRTCSession.room;
-              const member = findMatrixMember(this.matrixRoom, id);
+              // WARN! This is not exactly the sender but the user defined in the state key.
-              if (member === undefined)
+              // This will be available once we change to the new "member as object" format in the MatrixRTC object.
-                logger.warn(
+              let mediaId = rtcMember.sender + ":" + rtcMember.deviceId;
-                  `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`,
+              let participant = undefined;
+              if (
+                rtcMember.sender === room.client.getUserId()! &&
+                rtcMember.deviceId === room.client.getDeviceId()
+              ) {
+                mediaId = "local";
+                participant = localParticipant;
+              } else {
+                participant = remoteParticipants.find(
+                  (p) => p.identity === mediaId,
                 );
+              }
+
+              const member = findMatrixRoomMember(room, mediaId);
 
-              // Create as many tiles for this participant as called for by
-              // the duplicateTiles option
               for (let i = 0; i < 1 + duplicateTiles; i++) {
-                const userMediaId = `${id}:${i}`;
+                const indexedMediaId = `${mediaId}:${i}`;
+                const prevMedia = prevItems.get(indexedMediaId);
+                if (prevMedia && prevMedia instanceof UserMedia) {
+                  if (
+                    prevMedia.participant.value === undefined &&
+                    participant !== undefined
+                  ) {
+                    // Update the BahviourSubject in the UserMedia.
+                    prevMedia.participant.next(participant);
+                  }
+                }
                 yield [
-                  userMediaId,
+                  indexedMediaId,
-                  prevItems.get(userMediaId) ??
+                  // We create UserMedia with or without a participant.
+                  // This will be the initial value of a BehaviourSubject.
+                  // Once a participant appears we will update the BehaviourSubject. (see above)
+                  prevMedia ??
                     new UserMedia(
-                      userMediaId,
+                      mediaId,
                       member,
-                      p,
+                      participant,
+                      this.encryptionSystem,
+                    ),
+                ];
+              }
+              if (participant && participant.isScreenShareEnabled) {
+                const screenShareId = `${mediaId}:screen-share`;
+                yield [
+                  screenShareId,
+                  prevItems.get(screenShareId) ??
+                    new ScreenShare(
+                      screenShareId,
+                      member,
+                      participant,
                       this.encryptionSystem,
                     ),
                 ];
-
-                if (p.isScreenShareEnabled) {
-                  const screenShareId = `${userMediaId}:screen-share`;
-                  yield [
-                    screenShareId,
-                    prevItems.get(screenShareId) ??
-                      new ScreenShare(
-                        screenShareId,
-                        member,
-                        p,
-                        this.encryptionSystem,
-                      ),
-                  ];
-                }
               }
             }
           }.bind(this)(),
@@ -432,42 +482,43 @@ export class CallViewModel extends ViewModel {
       distinctUntilChanged(),
     );
 
-  private readonly spotlightSpeaker: Observable<UserMediaViewModel> =
+  private readonly spotlightSpeaker: Observable<
-    this.userMedia.pipe(
+    UserMediaViewModel | undefined
-      switchMap((mediaItems) =>
+  > = this.userMedia.pipe(
-        mediaItems.length === 0
+    switchMap((mediaItems) =>
-          ? of([])
+      mediaItems.length === 0
-          : combineLatest(
+        ? of([])
-              mediaItems.map((m) =>
+        : combineLatest(
-                m.vm.speaking.pipe(map((s) => [m, s] as const)),
+            mediaItems.map((m) =>
-              ),
+              m.vm.speaking.pipe(map((s) => [m, s] as const)),
             ),
-      ),
+          ),
-      scan<(readonly [UserMedia, boolean])[], UserMedia, null>(
+    ),
-        (prev, mediaItems) => {
+    scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>(
-          // Only remote users that are still in the call should be sticky
+      (prev, mediaItems) => {
-          const [stickyMedia, stickySpeaking] =
+        // Only remote users that are still in the call should be sticky
-            (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || [];
+        const [stickyMedia, stickySpeaking] =
-          // Decide who to spotlight:
+          (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || [];
-          // If the previous speaker is still speaking, stick with them rather
+        // Decide who to spotlight:
-          // than switching eagerly to someone else
+        // If the previous speaker is still speaking, stick with them rather
-          return stickySpeaking
+        // than switching eagerly to someone else
-            ? stickyMedia!
+        return stickySpeaking
-            : // Otherwise, select any remote user who is speaking
+          ? stickyMedia!
-              (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ??
+          : // Otherwise, select any remote user who is speaking
-                // Otherwise, stick with the person who was last speaking
+            (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ??
-                stickyMedia ??
+              // Otherwise, stick with the person who was last speaking
-                // Otherwise, spotlight an arbitrary remote user
+              stickyMedia ??
-                mediaItems.find(([m]) => !m.vm.local)?.[0] ??
+              // Otherwise, spotlight an arbitrary remote user
-                // Otherwise, spotlight the local user
+              mediaItems.find(([m]) => !m.vm.local)?.[0] ??
-                mediaItems.find(([m]) => m.vm.local)![0]);
+              // Otherwise, spotlight the local user
-        },
+              mediaItems.find(([m]) => m.vm.local)?.[0]);
-        null,
+      },
-      ),
+      null,
-      map((speaker) => speaker.vm),
+    ),
-      this.scope.state(),
+    map((speaker) => speaker?.vm),
-      throttleTime(1600, undefined, { leading: true, trailing: true }),
+    this.scope.state(),
-    );
+    throttleTime(1600, undefined, { leading: true, trailing: true }),
+  );
 
   private readonly grid: Observable<UserMediaViewModel[]> = this.userMedia.pipe(
     switchMap((mediaItems) => {
@@ -510,20 +561,29 @@ export class CallViewModel extends ViewModel {
   > = this.screenShares.pipe(
     map((screenShares) =>
       screenShares.length > 0
-        ? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const)
+        ? ([
+            of(screenShares.map((m) => m.vm)),
+            this.spotlightSpeaker.pipe(
+              map((speaker) => (speaker && speaker) ?? null),
+            ),
+          ] as const)
         : ([
-            this.spotlightSpeaker.pipe(map((speaker) => [speaker!])),
+            this.spotlightSpeaker.pipe(
+              map((speaker) => (speaker && [speaker]) ?? []),
+            ),
             this.spotlightSpeaker.pipe(
               switchMap((speaker) =>
-                speaker.local
+                speaker
-                  ? of(null)
+                  ? speaker.local
-                  : this.localUserMedia.pipe(
+                    ? of(null)
-                      switchMap((vm) =>
+                    : this.localUserMedia.pipe(
-                        vm.alwaysShow.pipe(
+                        switchMap((vm) =>
-                          map((alwaysShow) => (alwaysShow ? vm : null)),
+                          vm.alwaysShow.pipe(
+                            map((alwaysShow) => (alwaysShow ? vm : null)),
+                          ),
                         ),
-                      ),
+                      )
-                    ),
+                  : of(null),
               ),
             ),
           ] as const),
@@ -843,7 +903,7 @@ export class CallViewModel extends ViewModel {
 
   public constructor(
     // A call is permanently tied to a single Matrix room and LiveKit room
-    private readonly matrixRoom: MatrixRoom,
+    private readonly matrixRTCSession: MatrixRTCSession,
     private readonly livekitRoom: LivekitRoom,
     private readonly encryptionSystem: EncryptionSystem,
     private readonly connectionState: Observable<ECConnectionState>,

src/state/MediaViewModel.ts

@@ -68,33 +68,50 @@ export function useDisplayName(vm: MediaViewModel): string {
 }
 
 export function observeTrackReference(
-  participant: Participant,
+  participant: Observable<Participant | undefined>,
   source: Track.Source,
-): Observable<TrackReferenceOrPlaceholder> {
+): Observable<TrackReferenceOrPlaceholder | undefined> {
-  return observeParticipantMedia(participant).pipe(
+  const obs = participant.pipe(
-    map(() => ({
+    switchMap((p) => {
-      participant,
+      if (p) {
-      publication: participant.getTrackPublication(source),
+        return observeParticipantMedia(p).pipe(
-      source,
+          map(() => ({
-    })),
+            participant: p,
-    distinctUntilKeyChanged("publication"),
+            publication: p.getTrackPublication(source),
+            source,
+          })),
+          distinctUntilKeyChanged("publication"),
+        );
+      } else {
+        return of(undefined);
+      }
+    }),
   );
+  return obs;
 }
 
 abstract class BaseMediaViewModel extends ViewModel {
   /**
    * Whether the media belongs to the local user.
    */
-  public readonly local = this.participant.isLocal;
+  public readonly local = this.participant.pipe(
+    // We can assume, that the user is not local if the participant is undefined
+    // We assume the local LK participant will always be available.
+    map((p) => p?.isLocal ?? false),
+  );
   /**
    * The LiveKit video track for this media.
    */
-  public readonly video: Observable<TrackReferenceOrPlaceholder>;
+  public readonly video: Observable<TrackReferenceOrPlaceholder | undefined>;
   /**
    * Whether there should be a warning that this media is unencrypted.
    */
   public readonly unencryptedWarning: Observable<boolean>;
 
+  public readonly isRTCParticipantAvailable = this.participant.pipe(
+    map((p) => !!p),
+  );
+
   public constructor(
     /**
      * An opaque identifier for this media.
@@ -106,7 +123,12 @@ abstract class BaseMediaViewModel extends ViewModel {
     // TODO: Fully separate the data layer from the UI layer by keeping the
     // member object internal
     public readonly member: RoomMember | undefined,
-    protected readonly participant: LocalParticipant | RemoteParticipant,
+    // We dont necassarly have a participant if a user connects via MatrixRTC but not (not yet) through
+    // livekit.
+    protected readonly participant: Observable<
+      LocalParticipant | RemoteParticipant | undefined
+    >,
+
     encryptionSystem: EncryptionSystem,
     audioSource: AudioSource,
     videoSource: VideoSource,
@@ -122,8 +144,8 @@ abstract class BaseMediaViewModel extends ViewModel {
       [audio, this.video],
       (a, v) =>
         encryptionSystem.kind !== E2eeType.NONE &&
-        (a.publication?.isEncrypted === false ||
+        (a?.publication?.isEncrypted === false ||
-          v.publication?.isEncrypted === false),
+          v?.publication?.isEncrypted === false),
     ).pipe(this.scope.state());
   }
 }
@@ -143,12 +165,20 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
   /**
    * Whether the participant is speaking.
    */
-  public readonly speaking = observeParticipantEvents(
+  public readonly speaking = this.participant.pipe(
-    this.participant,
+    switchMap((p) => {
-    ParticipantEvent.IsSpeakingChanged,
+      if (p) {
-  ).pipe(
+        return observeParticipantEvents(
-    map((p) => p.isSpeaking),
+          p,
-    this.scope.state(),
+          ParticipantEvent.IsSpeakingChanged,
+        ).pipe(
+          map((p) => p.isSpeaking),
+          this.scope.state(),
+        );
+      } else {
+        return of(false);
+      }
+    }),
   );
 
   /**
@@ -169,7 +199,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: Observable<LocalParticipant | RemoteParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
   ) {
     super(
@@ -181,12 +211,17 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
       Track.Source.Camera,
     );
 
-    const media = observeParticipantMedia(participant).pipe(this.scope.state());
+    // const media = observeParticipantMedia(participant).pipe(this.scope.state());
+
+    const media = participant.pipe(
+      switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)),
+      this.scope.state(),
+    );
     this.audioEnabled = media.pipe(
-      map((m) => m.microphoneTrack?.isMuted === false),
+      map((m) => m?.microphoneTrack?.isMuted === false),
     );
     this.videoEnabled = media.pipe(
-      map((m) => m.cameraTrack?.isMuted === false),
+      map((m) => m?.cameraTrack?.isMuted === false),
     );
   }
 
@@ -204,7 +239,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
    */
   public readonly mirror = this.video.pipe(
     switchMap((v) => {
-      const track = v.publication?.track;
+      const track = v?.publication?.track;
       if (!(track instanceof LocalTrack)) return of(false);
       // Watch for track restarts, because they indicate a camera switch
       return fromEvent(track, TrackEvent.Restarted).pipe(
@@ -226,7 +261,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant,
+    participant: Observable<LocalParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
   ) {
     super(id, member, participant, encryptionSystem);
@@ -286,17 +321,16 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: RemoteParticipant,
+    participant: Observable<RemoteParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
   ) {
     super(id, member, participant, encryptionSystem);
 
     // Sync the local volume with LiveKit
-    this.localVolume
+    combineLatest([
-      .pipe(this.scope.bind())
+      participant,
-      .subscribe((volume) =>
+      this.localVolume.pipe(this.scope.bind()),
-        (this.participant as RemoteParticipant).setVolume(volume),
+    ]).subscribe(([p, volume]) => p && p.setVolume(volume));
-      );
   }
 
   public toggleLocallyMuted(): void {
@@ -319,7 +353,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: Observable<LocalParticipant | RemoteParticipant>,
     encryptionSystem: EncryptionSystem,
   ) {
     super(