Prototyping of showing video tile for every call membership

...even if they don't have a LK participant
2026-04-06 07:20:25 +00:00 · 2024-07-17 11:08:46 +01:00
parent 486430d1f0
commit 76a494464e
6 changed files with 290 additions and 82 deletions
--- a/package.json
+++ b/package.json
@@ -23,6 +23,7 @@
    "@livekit/components-core": "^0.10.0",
    "@livekit/components-react": "^2.0.0",
    "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz",
+    "@nkzw/use-relative-time": "^1.1.0",
    "@opentelemetry/api": "^1.4.0",
    "@opentelemetry/context-zone": "^1.9.1",
    "@opentelemetry/exporter-jaeger": "^1.9.1",
--- a/src/room/InCallView.tsx
+++ b/src/room/InCallView.tsx
@@ -238,7 +238,7 @@ export const InCallView: FC<InCallViewProps> = subscribe(
    const noControls = reducedControls && bounds.height <= 400;

    const vm = useCallViewModel(
-      rtcSession.room,
+      rtcSession,
      livekitRoom,
      matrixInfo.e2eeSystem.kind !== E2eeType.NONE,
      connState,
--- a/src/state/CallViewModel.ts
+++ b/src/state/CallViewModel.ts
@@ -52,6 +52,8 @@ import {
 } from "rxjs";
 import { StateObservable, state } from "@react-rxjs/core";
 import { logger } from "matrix-js-sdk/src/logger";
+import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
+import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership";

 import { ViewModel } from "./ViewModel";
 import { useObservable } from "./useObservable";
@@ -64,6 +66,7 @@ import {
  MediaViewModel,
  UserMediaViewModel,
  ScreenShareViewModel,
+  MembershipOnlyViewModel,
 } from "./MediaViewModel";
 import { finalizeValue } from "../observable-utils";
 import { ObservableScope } from "./ObservableScope";
@@ -204,25 +207,45 @@ class ScreenShare {
  }
 }

-type MediaItem = UserMedia | ScreenShare;
+class MembershipOnly {
+  public readonly vm: MembershipOnlyViewModel;

-function findMatrixMember(
-  room: MatrixRoom,
-  id: string,
-): RoomMember | undefined {
+  public constructor(member: RoomMember) {
+    this.vm = new MembershipOnlyViewModel(member);
+  }
+
+  public destroy(): void {
+    this.vm.destroy();
+  }
+}
+
+type MediaItem = UserMedia | ScreenShare | MembershipOnly;
+
+function matrixUserIdFromParticipantId(id: string): string | undefined {
  if (!id) return undefined;
-
  const parts = id.split(":");
  // must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon
  if (parts.length < 3) {
    logger.warn(
-      "Livekit participants ID doesn't look like a userId:deviceId combination",
+      `Livekit participants ID doesn't look like a userId:deviceId combination: ${id}`,
    );
    return undefined;
  }

  parts.pop();
  const userId = parts.join(":");
+  return userId;
+}
+
+function findMatrixMember(
+  room: MatrixRoom,
+  id: string,
+): RoomMember | undefined {
+  const userId = matrixUserIdFromParticipantId(id);
+
+  if (!userId) {
+    return undefined;
+  }

  return room.getMember(userId) ?? undefined;
 }
@@ -304,6 +327,24 @@ export class CallViewModel extends ViewModel {
      },
    );

+  private readonly membershipsWithoutParticipant = combineLatest([
+    of(this.rtcSession.memberships),
+    this.remoteParticipants,
+    of(this.livekitRoom.localParticipant),
+  ]).pipe(
+    scan((prev, [memberships, remoteParticipants, localParticipant]) => {
+      const participantIds = new Set(
+        remoteParticipants.map((p) =>
+          matrixUserIdFromParticipantId(p.identity),
+        ),
+      );
+      participantIds.add(
+        matrixUserIdFromParticipantId(localParticipant.identity),
+      );
+      return memberships.filter((m) => !participantIds.has(m.sender ?? ""));
+    }, [] as CallMembership[]),
+  );
+
  private readonly mediaItems: StateObservable<MediaItem[]> = state(
    combineLatest([
      this.remoteParticipants,
@@ -495,90 +536,136 @@ export class CallViewModel extends ViewModel {
      combineLatest([
        this.remoteParticipants,
        observeParticipantMedia(this.livekitRoom.localParticipant),
+        this.membershipsWithoutParticipant,
      ]).pipe(
-        scan((ts, [remoteParticipants, { participant: localParticipant }]) => {
-          const ps = [localParticipant, ...remoteParticipants];
-          const tilesById = new Map(ts.map((t) => [t.id, t]));
-          const now = Date.now();
-          let allGhosts = true;
+        scan(
+          (
+            ts,
+            [
+              remoteParticipants,
+              { participant: localParticipant },
+              membershipsWithoutParticipant,
+            ],
+          ) => {
+            const ps = [
+              localParticipant,
+              ...remoteParticipants,
+              ...membershipsWithoutParticipant,
+            ];
+            const tilesById = new Map(ts.map((t) => [t.id, t]));
+            const now = Date.now();
+            let allGhosts = true;

-          const newTiles = ps.flatMap((p) => {
-            const userMediaId = p.identity;
-            const member = findMatrixMember(this.matrixRoom, userMediaId);
-            allGhosts &&= member === undefined;
-            const spokeRecently =
-              p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000;
+            const newTiles = ps.flatMap((p) => {
+              if (p instanceof CallMembership) {
+                const userId = p.sender ?? "";
+                const member = this.matrixRoom.getMember(userId);
+                if (!member) {
+                  logger.warn(
+                    `Ruh, roh! No matrix member found for call membership '${userId}': ignoring`,
+                  );
+                  return [];
+                }
+                const membershipOnlyVm =
+                  tilesById.get(userId)?.data ??
+                  new MembershipOnlyViewModel(member);
+                tilesById.delete(userId);

-            // We always start with a local participant with the empty string as
-            // their ID before we're connected, this is fine and we'll be in
-            // "all ghosts" mode.
-            if (userMediaId !== "" && member === undefined) {
-              logger.warn(
-                `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`,
-              );
-            }
+                const membershipOnlyTile: TileDescriptor<MediaViewModel> = {
+                  id: userId,
+                  focused: false,
+                  isPresenter: false,
+                  isSpeaker: false,
+                  hasVideo: false,
+                  local: false,
+                  largeBaseSize: false,
+                  data: membershipOnlyVm,
+                };
+                return [membershipOnlyTile];
+              }

-            const userMediaVm =
-              tilesById.get(userMediaId)?.data ??
-              new UserMediaViewModel(userMediaId, member, p, this.encrypted);
-            tilesById.delete(userMediaId);
+              const userMediaId = p.identity;
+              const member = findMatrixMember(this.matrixRoom, userMediaId);
+              allGhosts &&= member === undefined;
+              const spokeRecently =
+                p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000;

-            const userMediaTile: TileDescriptor<MediaViewModel> = {
-              id: userMediaId,
-              focused: false,
-              isPresenter: p.isScreenShareEnabled,
-              isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal,
-              hasVideo: p.isCameraEnabled,
-              local: p.isLocal,
-              largeBaseSize: false,
-              data: userMediaVm,
-            };
-
-            if (p.isScreenShareEnabled) {
-              const screenShareId = `${userMediaId}:screen-share`;
-              const screenShareVm =
-                tilesById.get(screenShareId)?.data ??
-                new ScreenShareViewModel(
-                  screenShareId,
-                  member,
-                  p,
-                  this.encrypted,
+              // We always start with a local participant with the empty string as
+              // their ID before we're connected, this is fine and we'll be in
+              // "all ghosts" mode.
+              if (userMediaId !== "" && member === undefined) {
+                logger.warn(
+                  `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`,
                );
-              tilesById.delete(screenShareId);
+              }

-              const screenShareTile: TileDescriptor<MediaViewModel> = {
-                id: screenShareId,
-                focused: true,
-                isPresenter: false,
-                isSpeaker: false,
-                hasVideo: true,
+              const userMediaVm =
+                tilesById.get(userMediaId)?.data ??
+                new UserMediaViewModel(userMediaId, member, p, this.encrypted);
+              tilesById.delete(userMediaId);
+
+              const userMediaTile: TileDescriptor<MediaViewModel> = {
+                id: userMediaId,
+                focused: false,
+                isPresenter: p.isScreenShareEnabled,
+                isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal,
+                hasVideo: p.isCameraEnabled,
                local: p.isLocal,
-                largeBaseSize: true,
-                placeNear: userMediaId,
-                data: screenShareVm,
+                largeBaseSize: false,
+                data: userMediaVm,
              };
-              return [userMediaTile, screenShareTile];
-            } else {
-              return [userMediaTile];
-            }
-          });

-          // Any tiles left in the map are unused and should be destroyed
-          for (const t of tilesById.values()) t.data.destroy();
+              if (p.isScreenShareEnabled) {
+                const screenShareId = `${userMediaId}:screen-share`;
+                const screenShareVm =
+                  tilesById.get(screenShareId)?.data ??
+                  new ScreenShareViewModel(
+                    screenShareId,
+                    member,
+                    p,
+                    this.encrypted,
+                  );
+                tilesById.delete(screenShareId);

-          // If every item is a ghost, that probably means we're still connecting
-          // and shouldn't bother showing anything yet
-          return allGhosts ? [] : newTiles;
-        }, [] as TileDescriptor<MediaViewModel>[]),
+                const screenShareTile: TileDescriptor<MediaViewModel> = {
+                  id: screenShareId,
+                  focused: true,
+                  isPresenter: false,
+                  isSpeaker: false,
+                  hasVideo: true,
+                  local: p.isLocal,
+                  largeBaseSize: true,
+                  placeNear: userMediaId,
+                  data: screenShareVm,
+                };
+                return [userMediaTile, screenShareTile];
+              } else {
+                return [userMediaTile];
+              }
+            });
+
+            // Any tiles left in the map are unused and should be destroyed
+            for (const t of tilesById.values()) t.data.destroy();
+
+            // If every item is a ghost, that probably means we're still connecting
+            // and shouldn't bother showing anything yet
+            return allGhosts ? [] : newTiles;
+          },
+          [] as TileDescriptor<MediaViewModel>[],
+        ),
        finalizeValue((ts) => {
          for (const t of ts) t.data.destroy();
        }),
      ),
    );

+  private get matrixRoom(): MatrixRoom {
+    return this.rtcSession.room;
+  }
+
  public constructor(
    // A call is permanently tied to a single Matrix room and LiveKit room
-    private readonly matrixRoom: MatrixRoom,
+    private readonly rtcSession: MatrixRTCSession,
    private readonly livekitRoom: LivekitRoom,
    private readonly encrypted: boolean,
    private readonly connectionState: Observable<ECConnectionState>,
@@ -588,25 +675,25 @@ export class CallViewModel extends ViewModel {
 }

 export function useCallViewModel(
-  matrixRoom: MatrixRoom,
+  rtcSession: MatrixRTCSession,
  livekitRoom: LivekitRoom,
  encrypted: boolean,
  connectionState: ECConnectionState,
 ): CallViewModel {
-  const prevMatrixRoom = usePrevious(matrixRoom);
+  const prevRTCSession = usePrevious(rtcSession);
  const prevLivekitRoom = usePrevious(livekitRoom);
  const prevEncrypted = usePrevious(encrypted);
  const connectionStateObservable = useObservable(connectionState);

  const vm = useRef<CallViewModel>();
  if (
-    matrixRoom !== prevMatrixRoom ||
+    rtcSession !== prevRTCSession ||
    livekitRoom !== prevLivekitRoom ||
    encrypted !== prevEncrypted
  ) {
    vm.current?.destroy();
    vm.current = new CallViewModel(
-      matrixRoom,
+      rtcSession,
      livekitRoom,
      encrypted,
      connectionStateObservable,
--- a/src/state/MediaViewModel.ts
+++ b/src/state/MediaViewModel.ts
@@ -110,7 +110,10 @@ abstract class BaseMediaViewModel extends ViewModel {
 /**
 * Some participant's media.
 */
-export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel;
+export type MediaViewModel =
+  | UserMediaViewModel
+  | ScreenShareViewModel
+  | MembershipOnlyViewModel;

 /**
 * Some participant's user media.
@@ -239,3 +242,15 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
    );
  }
 }
+
+/**
+ * Placeholder for a call membership that does not have a LiveKit participant associated with it.
+ */
+export class MembershipOnlyViewModel extends ViewModel {
+  public id: string;
+  public local = false;
+  public constructor(public readonly member: RoomMember) {
+    super();
+    this.id = member.userId;
+  }
+}
--- a/src/video-grid/VideoTile.tsx
+++ b/src/video-grid/VideoTile.tsx
@@ -53,6 +53,7 @@ import {
  Menu,
 } from "@vector-im/compound-web";
 import { useStateObservable } from "@react-rxjs/core";
+import useRelativeTime from "@nkzw/use-relative-time";

 import { Avatar } from "../Avatar";
 import styles from "./VideoTile.module.css";
@@ -61,6 +62,7 @@ import {
  ScreenShareViewModel,
  MediaViewModel,
  UserMediaViewModel,
+  MembershipOnlyViewModel,
 } from "../state/MediaViewModel";
 import { subscribe } from "../state/subscribe";
 import { useMergedRefs } from "../useMergedRefs";
@@ -72,7 +74,7 @@ interface TileProps {
  style?: ComponentProps<typeof animated.div>["style"];
  targetWidth: number;
  targetHeight: number;
-  video: TrackReferenceOrPlaceholder;
+  video?: TrackReferenceOrPlaceholder;
  member: RoomMember | undefined;
  videoEnabled: boolean;
  maximised: boolean;
@@ -110,6 +112,10 @@ const Tile = forwardRef<HTMLDivElement, TileProps>(
    const { t } = useTranslation();
    const mergedRef = useMergedRefs(tileRef, ref);

+    const joinedCallTime = member?.events.member?.getTs() ?? 0;
+
+    const joinedCallAgo = useRelativeTime(joinedCallTime ?? 0);
+
    return (
      <animated.div
        className={classNames(styles.videoTile, className, {
@@ -129,7 +135,7 @@ const Tile = forwardRef<HTMLDivElement, TileProps>(
            src={member?.getMxcAvatarUrl()}
            className={styles.avatar}
          />
-          {video.publication !== undefined && (
+          {video?.publication !== undefined && (
            <VideoTrack
              trackRef={video}
              // There's no reason for this to be focusable
@@ -137,12 +143,21 @@ const Tile = forwardRef<HTMLDivElement, TileProps>(
              disablePictureInPicture
            />
          )}
+          {!video && member && joinedCallTime > 0 && (
+            <div style={{ textAlign: "center" }}>
+              <span title={member.userId}>{nameTag}</span> joined the call{" "}
+              <span title={new Date(joinedCallTime).toLocaleString()}>
+                {joinedCallAgo}
+              </span>{" "}
+              but is currently unreachable. Are they having connection problems?
+            </div>
+          )}
        </div>
        <div className={styles.fg}>
          <div className={styles.nameTag}>
            {nameTagLeadingIcon}
            <Text as="span" size="sm" weight="medium" className={styles.name}>
-              {nameTag}
+              <span title={member?.userId}>{nameTag}</span>{" "}
            </Text>
            {unencryptedWarning && (
              <Tooltip
@@ -402,6 +417,75 @@ const ScreenShareTile = subscribe<ScreenShareTileProps, HTMLDivElement>(

 ScreenShareTile.displayName = "ScreenShareTile";

+interface MembershipOnlyTileProps {
+  vm: MembershipOnlyViewModel;
+  className?: string;
+  style?: ComponentProps<typeof animated.div>["style"];
+  targetWidth: number;
+  targetHeight: number;
+  nameTag: string;
+  displayName: string;
+  maximised: boolean;
+  fullscreen: boolean;
+  onToggleFullscreen: (itemId: string) => void;
+}
+
+const MembershipOnlyTile = subscribe<MembershipOnlyTileProps, HTMLDivElement>(
+  (
+    {
+      vm,
+      className,
+      style,
+      targetWidth,
+      targetHeight,
+      nameTag,
+      displayName,
+      maximised,
+      fullscreen,
+      onToggleFullscreen,
+    },
+    ref,
+  ) => {
+    const { t } = useTranslation();
+    const onClickFullScreen = useCallback(
+      () => onToggleFullscreen(vm.id),
+      [onToggleFullscreen, vm],
+    );
+
+    const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon;
+
+    return (
+      <Tile
+        ref={ref}
+        className={classNames(className, styles.membershipOnly)}
+        style={style}
+        targetWidth={targetWidth}
+        targetHeight={targetHeight}
+        member={vm.member}
+        videoEnabled={true}
+        maximised={maximised}
+        unencryptedWarning={false}
+        nameTag={nameTag}
+        displayName={displayName}
+        primaryButton={
+          <button
+            aria-label={
+              fullscreen
+                ? t("video_tile.full_screen")
+                : t("video_tile.exit_full_screen")
+            }
+            onClick={onClickFullScreen}
+          >
+            <FullScreenIcon aria-hidden width={20} height={20} />
+          </button>
+        }
+      />
+    );
+  },
+);
+
+MembershipOnlyTile.displayName = "MembershipOnlyTile";
+
 interface Props {
  vm: MediaViewModel;
  maximised: boolean;
@@ -471,7 +555,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
          showSpeakingIndicator={showSpeakingIndicator}
        />
      );
-    } else {
+    } else if (vm instanceof ScreenShareViewModel) {
      return (
        <ScreenShareTile
          ref={ref}
@@ -487,6 +571,22 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
          onToggleFullscreen={onToggleFullscreen}
        />
      );
+    } else if (vm instanceof MembershipOnlyViewModel) {
+      return (
+        <MembershipOnlyTile
+          ref={ref}
+          className={className}
+          style={style}
+          vm={vm}
+          targetWidth={targetWidth}
+          targetHeight={targetHeight}
+          nameTag={nameTag}
+          displayName={displayName}
+          maximised={maximised}
+          fullscreen={fullscreen}
+          onToggleFullscreen={onToggleFullscreen}
+        />
+      );
    }
  },
 );
--- a/yarn.lock
+++ b/yarn.lock
@@ -1912,6 +1912,11 @@
  version "3.2.14"
  resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz#acd96c00a881d0f462e1f97a56c73742c8dbc984"

+"@nkzw/use-relative-time@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@nkzw/use-relative-time/-/use-relative-time-1.1.0.tgz#7eb52dc4c7ceaf49a1f05ecfbf725313cf5c1724"
+  integrity sha512-ogCL62FvScpRpsZUuaN6Jt0xPGRv62atQUNGyMcX+nZs4H5Fs5K1iA3MbSmkJ1y0n/N0RIRLc3VAp8o46lq2CA==
+
 "@nodelib/fs.scandir@2.1.5":
  version "2.1.5"
  resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"