Prototyping of showing video tile for every call membership

...even if they don't have a LK participant

package.json

@@ -23,6 +23,7 @@
     "@livekit/components-core": "^0.10.0",
     "@livekit/components-react": "^2.0.0",
     "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz",
+    "@nkzw/use-relative-time": "^1.1.0",
     "@opentelemetry/api": "^1.4.0",
     "@opentelemetry/context-zone": "^1.9.1",
     "@opentelemetry/exporter-jaeger": "^1.9.1",

src/room/InCallView.tsx

@@ -238,7 +238,7 @@ export const InCallView: FC<InCallViewProps> = subscribe(
     const noControls = reducedControls && bounds.height <= 400;
 
     const vm = useCallViewModel(
-      rtcSession.room,
+      rtcSession,
       livekitRoom,
       matrixInfo.e2eeSystem.kind !== E2eeType.NONE,
       connState,

src/state/CallViewModel.ts

@@ -52,6 +52,8 @@ import {
 } from "rxjs";
 import { StateObservable, state } from "@react-rxjs/core";
 import { logger } from "matrix-js-sdk/src/logger";
+import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
+import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership";
 
 import { ViewModel } from "./ViewModel";
 import { useObservable } from "./useObservable";
@@ -64,6 +66,7 @@ import {
   MediaViewModel,
   UserMediaViewModel,
   ScreenShareViewModel,
+  MembershipOnlyViewModel,
 } from "./MediaViewModel";
 import { finalizeValue } from "../observable-utils";
 import { ObservableScope } from "./ObservableScope";
@@ -204,25 +207,45 @@ class ScreenShare {
   }
 }
 
-type MediaItem = UserMedia | ScreenShare;
+class MembershipOnly {
+  public readonly vm: MembershipOnlyViewModel;
 
-function findMatrixMember(
+  public constructor(member: RoomMember) {
-  room: MatrixRoom,
+    this.vm = new MembershipOnlyViewModel(member);
-  id: string,
+  }
-): RoomMember | undefined {
+
+  public destroy(): void {
+    this.vm.destroy();
+  }
+}
+
+type MediaItem = UserMedia | ScreenShare | MembershipOnly;
+
+function matrixUserIdFromParticipantId(id: string): string | undefined {
   if (!id) return undefined;
-
   const parts = id.split(":");
   // must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon
   if (parts.length < 3) {
     logger.warn(
-      "Livekit participants ID doesn't look like a userId:deviceId combination",
+      `Livekit participants ID doesn't look like a userId:deviceId combination: ${id}`,
     );
     return undefined;
   }
 
   parts.pop();
   const userId = parts.join(":");
+  return userId;
+}
+
+function findMatrixMember(
+  room: MatrixRoom,
+  id: string,
+): RoomMember | undefined {
+  const userId = matrixUserIdFromParticipantId(id);
+
+  if (!userId) {
+    return undefined;
+  }
 
   return room.getMember(userId) ?? undefined;
 }
@@ -304,6 +327,24 @@ export class CallViewModel extends ViewModel {
       },
     );
 
+  private readonly membershipsWithoutParticipant = combineLatest([
+    of(this.rtcSession.memberships),
+    this.remoteParticipants,
+    of(this.livekitRoom.localParticipant),
+  ]).pipe(
+    scan((prev, [memberships, remoteParticipants, localParticipant]) => {
+      const participantIds = new Set(
+        remoteParticipants.map((p) =>
+          matrixUserIdFromParticipantId(p.identity),
+        ),
+      );
+      participantIds.add(
+        matrixUserIdFromParticipantId(localParticipant.identity),
+      );
+      return memberships.filter((m) => !participantIds.has(m.sender ?? ""));
+    }, [] as CallMembership[]),
+  );
+
   private readonly mediaItems: StateObservable<MediaItem[]> = state(
     combineLatest([
       this.remoteParticipants,
@@ -495,90 +536,136 @@ export class CallViewModel extends ViewModel {
       combineLatest([
         this.remoteParticipants,
         observeParticipantMedia(this.livekitRoom.localParticipant),
+        this.membershipsWithoutParticipant,
       ]).pipe(
-        scan((ts, [remoteParticipants, { participant: localParticipant }]) => {
+        scan(
-          const ps = [localParticipant, ...remoteParticipants];
+          (
-          const tilesById = new Map(ts.map((t) => [t.id, t]));
+            ts,
-          const now = Date.now();
+            [
-          let allGhosts = true;
+              remoteParticipants,
+              { participant: localParticipant },
+              membershipsWithoutParticipant,
+            ],
+          ) => {
+            const ps = [
+              localParticipant,
+              ...remoteParticipants,
+              ...membershipsWithoutParticipant,
+            ];
+            const tilesById = new Map(ts.map((t) => [t.id, t]));
+            const now = Date.now();
+            let allGhosts = true;
 
-          const newTiles = ps.flatMap((p) => {
+            const newTiles = ps.flatMap((p) => {
-            const userMediaId = p.identity;
+              if (p instanceof CallMembership) {
-            const member = findMatrixMember(this.matrixRoom, userMediaId);
+                const userId = p.sender ?? "";
-            allGhosts &&= member === undefined;
+                const member = this.matrixRoom.getMember(userId);
-            const spokeRecently =
+                if (!member) {
-              p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000;
+                  logger.warn(
+                    `Ruh, roh! No matrix member found for call membership '${userId}': ignoring`,
+                  );
+                  return [];
+                }
+                const membershipOnlyVm =
+                  tilesById.get(userId)?.data ??
+                  new MembershipOnlyViewModel(member);
+                tilesById.delete(userId);
 
-            // We always start with a local participant with the empty string as
+                const membershipOnlyTile: TileDescriptor<MediaViewModel> = {
-            // their ID before we're connected, this is fine and we'll be in
+                  id: userId,
-            // "all ghosts" mode.
+                  focused: false,
-            if (userMediaId !== "" && member === undefined) {
+                  isPresenter: false,
-              logger.warn(
+                  isSpeaker: false,
-                `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`,
+                  hasVideo: false,
-              );
+                  local: false,
-            }
+                  largeBaseSize: false,
+                  data: membershipOnlyVm,
+                };
+                return [membershipOnlyTile];
+              }
 
-            const userMediaVm =
+              const userMediaId = p.identity;
-              tilesById.get(userMediaId)?.data ??
+              const member = findMatrixMember(this.matrixRoom, userMediaId);
-              new UserMediaViewModel(userMediaId, member, p, this.encrypted);
+              allGhosts &&= member === undefined;
-            tilesById.delete(userMediaId);
+              const spokeRecently =
+                p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000;
 
-            const userMediaTile: TileDescriptor<MediaViewModel> = {
+              // We always start with a local participant with the empty string as
-              id: userMediaId,
+              // their ID before we're connected, this is fine and we'll be in
-              focused: false,
+              // "all ghosts" mode.
-              isPresenter: p.isScreenShareEnabled,
+              if (userMediaId !== "" && member === undefined) {
-              isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal,
+                logger.warn(
-              hasVideo: p.isCameraEnabled,
+                  `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`,
-              local: p.isLocal,
-              largeBaseSize: false,
-              data: userMediaVm,
-            };
-
-            if (p.isScreenShareEnabled) {
-              const screenShareId = `${userMediaId}:screen-share`;
-              const screenShareVm =
-                tilesById.get(screenShareId)?.data ??
-                new ScreenShareViewModel(
-                  screenShareId,
-                  member,
-                  p,
-                  this.encrypted,
                 );
-              tilesById.delete(screenShareId);
+              }
 
-              const screenShareTile: TileDescriptor<MediaViewModel> = {
+              const userMediaVm =
-                id: screenShareId,
+                tilesById.get(userMediaId)?.data ??
-                focused: true,
+                new UserMediaViewModel(userMediaId, member, p, this.encrypted);
-                isPresenter: false,
+              tilesById.delete(userMediaId);
-                isSpeaker: false,
+
-                hasVideo: true,
+              const userMediaTile: TileDescriptor<MediaViewModel> = {
+                id: userMediaId,
+                focused: false,
+                isPresenter: p.isScreenShareEnabled,
+                isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal,
+                hasVideo: p.isCameraEnabled,
                 local: p.isLocal,
-                largeBaseSize: true,
+                largeBaseSize: false,
-                placeNear: userMediaId,
+                data: userMediaVm,
-                data: screenShareVm,
               };
-              return [userMediaTile, screenShareTile];
-            } else {
-              return [userMediaTile];
-            }
-          });
 
-          // Any tiles left in the map are unused and should be destroyed
+              if (p.isScreenShareEnabled) {
-          for (const t of tilesById.values()) t.data.destroy();
+                const screenShareId = `${userMediaId}:screen-share`;
+                const screenShareVm =
+                  tilesById.get(screenShareId)?.data ??
+                  new ScreenShareViewModel(
+                    screenShareId,
+                    member,
+                    p,
+                    this.encrypted,
+                  );
+                tilesById.delete(screenShareId);
 
-          // If every item is a ghost, that probably means we're still connecting
+                const screenShareTile: TileDescriptor<MediaViewModel> = {
-          // and shouldn't bother showing anything yet
+                  id: screenShareId,
-          return allGhosts ? [] : newTiles;
+                  focused: true,
-        }, [] as TileDescriptor<MediaViewModel>[]),
+                  isPresenter: false,
+                  isSpeaker: false,
+                  hasVideo: true,
+                  local: p.isLocal,
+                  largeBaseSize: true,
+                  placeNear: userMediaId,
+                  data: screenShareVm,
+                };
+                return [userMediaTile, screenShareTile];
+              } else {
+                return [userMediaTile];
+              }
+            });
+
+            // Any tiles left in the map are unused and should be destroyed
+            for (const t of tilesById.values()) t.data.destroy();
+
+            // If every item is a ghost, that probably means we're still connecting
+            // and shouldn't bother showing anything yet
+            return allGhosts ? [] : newTiles;
+          },
+          [] as TileDescriptor<MediaViewModel>[],
+        ),
         finalizeValue((ts) => {
           for (const t of ts) t.data.destroy();
         }),
       ),
     );
 
+  private get matrixRoom(): MatrixRoom {
+    return this.rtcSession.room;
+  }
+
   public constructor(
     // A call is permanently tied to a single Matrix room and LiveKit room
-    private readonly matrixRoom: MatrixRoom,
+    private readonly rtcSession: MatrixRTCSession,
     private readonly livekitRoom: LivekitRoom,
     private readonly encrypted: boolean,
     private readonly connectionState: Observable<ECConnectionState>,
@@ -588,25 +675,25 @@ export class CallViewModel extends ViewModel {
 }
 
 export function useCallViewModel(
-  matrixRoom: MatrixRoom,
+  rtcSession: MatrixRTCSession,
   livekitRoom: LivekitRoom,
   encrypted: boolean,
   connectionState: ECConnectionState,
 ): CallViewModel {
-  const prevMatrixRoom = usePrevious(matrixRoom);
+  const prevRTCSession = usePrevious(rtcSession);
   const prevLivekitRoom = usePrevious(livekitRoom);
   const prevEncrypted = usePrevious(encrypted);
   const connectionStateObservable = useObservable(connectionState);
 
   const vm = useRef<CallViewModel>();
   if (
-    matrixRoom !== prevMatrixRoom ||
+    rtcSession !== prevRTCSession ||
     livekitRoom !== prevLivekitRoom ||
     encrypted !== prevEncrypted
   ) {
     vm.current?.destroy();
     vm.current = new CallViewModel(
-      matrixRoom,
+      rtcSession,
       livekitRoom,
       encrypted,
       connectionStateObservable,

src/state/MediaViewModel.ts

@@ -110,7 +110,10 @@ abstract class BaseMediaViewModel extends ViewModel {
 /**
  * Some participant's media.
  */
-export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel;
+export type MediaViewModel =
+  | UserMediaViewModel
+  | ScreenShareViewModel
+  | MembershipOnlyViewModel;
 
 /**
  * Some participant's user media.
@@ -239,3 +242,15 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
     );
   }
 }
+
+/**
+ * Placeholder for a call membership that does not have a LiveKit participant associated with it.
+ */
+export class MembershipOnlyViewModel extends ViewModel {
+  public id: string;
+  public local = false;
+  public constructor(public readonly member: RoomMember) {
+    super();
+    this.id = member.userId;
+  }
+}

src/video-grid/VideoTile.tsx

@@ -53,6 +53,7 @@ import {
   Menu,
 } from "@vector-im/compound-web";
 import { useStateObservable } from "@react-rxjs/core";
+import useRelativeTime from "@nkzw/use-relative-time";
 
 import { Avatar } from "../Avatar";
 import styles from "./VideoTile.module.css";
@@ -61,6 +62,7 @@ import {
   ScreenShareViewModel,
   MediaViewModel,
   UserMediaViewModel,
+  MembershipOnlyViewModel,
 } from "../state/MediaViewModel";
 import { subscribe } from "../state/subscribe";
 import { useMergedRefs } from "../useMergedRefs";
@@ -72,7 +74,7 @@ interface TileProps {
   style?: ComponentProps<typeof animated.div>["style"];
   targetWidth: number;
   targetHeight: number;
-  video: TrackReferenceOrPlaceholder;
+  video?: TrackReferenceOrPlaceholder;
   member: RoomMember | undefined;
   videoEnabled: boolean;
   maximised: boolean;
@@ -110,6 +112,10 @@ const Tile = forwardRef<HTMLDivElement, TileProps>(
     const { t } = useTranslation();
     const mergedRef = useMergedRefs(tileRef, ref);
 
+    const joinedCallTime = member?.events.member?.getTs() ?? 0;
+
+    const joinedCallAgo = useRelativeTime(joinedCallTime ?? 0);
+
     return (
       <animated.div
         className={classNames(styles.videoTile, className, {
@@ -129,7 +135,7 @@ const Tile = forwardRef<HTMLDivElement, TileProps>(
             src={member?.getMxcAvatarUrl()}
             className={styles.avatar}
           />
-          {video.publication !== undefined && (
+          {video?.publication !== undefined && (
             <VideoTrack
               trackRef={video}
               // There's no reason for this to be focusable
@@ -137,12 +143,21 @@ const Tile = forwardRef<HTMLDivElement, TileProps>(
               disablePictureInPicture
             />
           )}
+          {!video && member && joinedCallTime > 0 && (
+            <div style={{ textAlign: "center" }}>
+              <span title={member.userId}>{nameTag}</span> joined the call{" "}
+              <span title={new Date(joinedCallTime).toLocaleString()}>
+                {joinedCallAgo}
+              </span>{" "}
+              but is currently unreachable. Are they having connection problems?
+            </div>
+          )}
         </div>
         <div className={styles.fg}>
           <div className={styles.nameTag}>
             {nameTagLeadingIcon}
             <Text as="span" size="sm" weight="medium" className={styles.name}>
-              {nameTag}
+              <span title={member?.userId}>{nameTag}</span>{" "}
             </Text>
             {unencryptedWarning && (
               <Tooltip
@@ -402,6 +417,75 @@ const ScreenShareTile = subscribe<ScreenShareTileProps, HTMLDivElement>(
 
 ScreenShareTile.displayName = "ScreenShareTile";
 
+interface MembershipOnlyTileProps {
+  vm: MembershipOnlyViewModel;
+  className?: string;
+  style?: ComponentProps<typeof animated.div>["style"];
+  targetWidth: number;
+  targetHeight: number;
+  nameTag: string;
+  displayName: string;
+  maximised: boolean;
+  fullscreen: boolean;
+  onToggleFullscreen: (itemId: string) => void;
+}
+
+const MembershipOnlyTile = subscribe<MembershipOnlyTileProps, HTMLDivElement>(
+  (
+    {
+      vm,
+      className,
+      style,
+      targetWidth,
+      targetHeight,
+      nameTag,
+      displayName,
+      maximised,
+      fullscreen,
+      onToggleFullscreen,
+    },
+    ref,
+  ) => {
+    const { t } = useTranslation();
+    const onClickFullScreen = useCallback(
+      () => onToggleFullscreen(vm.id),
+      [onToggleFullscreen, vm],
+    );
+
+    const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon;
+
+    return (
+      <Tile
+        ref={ref}
+        className={classNames(className, styles.membershipOnly)}
+        style={style}
+        targetWidth={targetWidth}
+        targetHeight={targetHeight}
+        member={vm.member}
+        videoEnabled={true}
+        maximised={maximised}
+        unencryptedWarning={false}
+        nameTag={nameTag}
+        displayName={displayName}
+        primaryButton={
+          <button
+            aria-label={
+              fullscreen
+                ? t("video_tile.full_screen")
+                : t("video_tile.exit_full_screen")
+            }
+            onClick={onClickFullScreen}
+          >
+            <FullScreenIcon aria-hidden width={20} height={20} />
+          </button>
+        }
+      />
+    );
+  },
+);
+
+MembershipOnlyTile.displayName = "MembershipOnlyTile";
+
 interface Props {
   vm: MediaViewModel;
   maximised: boolean;
@@ -471,7 +555,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
           showSpeakingIndicator={showSpeakingIndicator}
         />
       );
-    } else {
+    } else if (vm instanceof ScreenShareViewModel) {
       return (
         <ScreenShareTile
           ref={ref}
@@ -487,6 +571,22 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
           onToggleFullscreen={onToggleFullscreen}
         />
       );
+    } else if (vm instanceof MembershipOnlyViewModel) {
+      return (
+        <MembershipOnlyTile
+          ref={ref}
+          className={className}
+          style={style}
+          vm={vm}
+          targetWidth={targetWidth}
+          targetHeight={targetHeight}
+          nameTag={nameTag}
+          displayName={displayName}
+          maximised={maximised}
+          fullscreen={fullscreen}
+          onToggleFullscreen={onToggleFullscreen}
+        />
+      );
     }
   },
 );

yarn.lock

@@ -1912,6 +1912,11 @@
   version "3.2.14"
   resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz#acd96c00a881d0f462e1f97a56c73742c8dbc984"
 
+"@nkzw/use-relative-time@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@nkzw/use-relative-time/-/use-relative-time-1.1.0.tgz#7eb52dc4c7ceaf49a1f05ecfbf725313cf5c1724"
+  integrity sha512-ogCL62FvScpRpsZUuaN6Jt0xPGRv62atQUNGyMcX+nZs4H5Fs5K1iA3MbSmkJ1y0n/N0RIRLc3VAp8o46lq2CA==
+
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"