diff --git a/package.json b/package.json index ae7a259c..1780075f 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "@livekit/components-core": "^0.10.0", "@livekit/components-react": "^2.0.0", "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz", + "@nkzw/use-relative-time": "^1.1.0", "@opentelemetry/api": "^1.4.0", "@opentelemetry/context-zone": "^1.9.1", "@opentelemetry/exporter-jaeger": "^1.9.1", diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 94edace7..77151be0 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -238,7 +238,7 @@ export const InCallView: FC = subscribe( const noControls = reducedControls && bounds.height <= 400; const vm = useCallViewModel( - rtcSession.room, + rtcSession, livekitRoom, matrixInfo.e2eeSystem.kind !== E2eeType.NONE, connState, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 4ad2f024..1bd7613a 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -52,6 +52,8 @@ import { } from "rxjs"; import { StateObservable, state } from "@react-rxjs/core"; import { logger } from "matrix-js-sdk/src/logger"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership"; import { ViewModel } from "./ViewModel"; import { useObservable } from "./useObservable"; @@ -64,6 +66,7 @@ import { MediaViewModel, UserMediaViewModel, ScreenShareViewModel, + MembershipOnlyViewModel, } from "./MediaViewModel"; import { finalizeValue } from "../observable-utils"; import { ObservableScope } from "./ObservableScope"; @@ -204,25 +207,45 @@ class ScreenShare { } } -type MediaItem = UserMedia | ScreenShare; +class MembershipOnly { + public readonly vm: MembershipOnlyViewModel; -function findMatrixMember( - room: MatrixRoom, - id: string, -): RoomMember | undefined { + public constructor(member: RoomMember) { + this.vm = new MembershipOnlyViewModel(member); + } + + public destroy(): void { + this.vm.destroy(); + } +} + +type MediaItem = UserMedia | ScreenShare | MembershipOnly; + +function matrixUserIdFromParticipantId(id: string): string | undefined { if (!id) return undefined; - const parts = id.split(":"); // must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon if (parts.length < 3) { logger.warn( - "Livekit participants ID doesn't look like a userId:deviceId combination", + `Livekit participants ID doesn't look like a userId:deviceId combination: ${id}`, ); return undefined; } parts.pop(); const userId = parts.join(":"); + return userId; +} + +function findMatrixMember( + room: MatrixRoom, + id: string, +): RoomMember | undefined { + const userId = matrixUserIdFromParticipantId(id); + + if (!userId) { + return undefined; + } return room.getMember(userId) ?? undefined; } @@ -304,6 +327,24 @@ export class CallViewModel extends ViewModel { }, ); + private readonly membershipsWithoutParticipant = combineLatest([ + of(this.rtcSession.memberships), + this.remoteParticipants, + of(this.livekitRoom.localParticipant), + ]).pipe( + scan((prev, [memberships, remoteParticipants, localParticipant]) => { + const participantIds = new Set( + remoteParticipants.map((p) => + matrixUserIdFromParticipantId(p.identity), + ), + ); + participantIds.add( + matrixUserIdFromParticipantId(localParticipant.identity), + ); + return memberships.filter((m) => !participantIds.has(m.sender ?? "")); + }, [] as CallMembership[]), + ); + private readonly mediaItems: StateObservable = state( combineLatest([ this.remoteParticipants, @@ -495,90 +536,136 @@ export class CallViewModel extends ViewModel { combineLatest([ this.remoteParticipants, observeParticipantMedia(this.livekitRoom.localParticipant), + this.membershipsWithoutParticipant, ]).pipe( - scan((ts, [remoteParticipants, { participant: localParticipant }]) => { - const ps = [localParticipant, ...remoteParticipants]; - const tilesById = new Map(ts.map((t) => [t.id, t])); - const now = Date.now(); - let allGhosts = true; + scan( + ( + ts, + [ + remoteParticipants, + { participant: localParticipant }, + membershipsWithoutParticipant, + ], + ) => { + const ps = [ + localParticipant, + ...remoteParticipants, + ...membershipsWithoutParticipant, + ]; + const tilesById = new Map(ts.map((t) => [t.id, t])); + const now = Date.now(); + let allGhosts = true; - const newTiles = ps.flatMap((p) => { - const userMediaId = p.identity; - const member = findMatrixMember(this.matrixRoom, userMediaId); - allGhosts &&= member === undefined; - const spokeRecently = - p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000; + const newTiles = ps.flatMap((p) => { + if (p instanceof CallMembership) { + const userId = p.sender ?? ""; + const member = this.matrixRoom.getMember(userId); + if (!member) { + logger.warn( + `Ruh, roh! No matrix member found for call membership '${userId}': ignoring`, + ); + return []; + } + const membershipOnlyVm = + tilesById.get(userId)?.data ?? + new MembershipOnlyViewModel(member); + tilesById.delete(userId); - // We always start with a local participant with the empty string as - // their ID before we're connected, this is fine and we'll be in - // "all ghosts" mode. - if (userMediaId !== "" && member === undefined) { - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`, - ); - } + const membershipOnlyTile: TileDescriptor = { + id: userId, + focused: false, + isPresenter: false, + isSpeaker: false, + hasVideo: false, + local: false, + largeBaseSize: false, + data: membershipOnlyVm, + }; + return [membershipOnlyTile]; + } - const userMediaVm = - tilesById.get(userMediaId)?.data ?? - new UserMediaViewModel(userMediaId, member, p, this.encrypted); - tilesById.delete(userMediaId); + const userMediaId = p.identity; + const member = findMatrixMember(this.matrixRoom, userMediaId); + allGhosts &&= member === undefined; + const spokeRecently = + p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000; - const userMediaTile: TileDescriptor = { - id: userMediaId, - focused: false, - isPresenter: p.isScreenShareEnabled, - isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal, - hasVideo: p.isCameraEnabled, - local: p.isLocal, - largeBaseSize: false, - data: userMediaVm, - }; - - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; - const screenShareVm = - tilesById.get(screenShareId)?.data ?? - new ScreenShareViewModel( - screenShareId, - member, - p, - this.encrypted, + // We always start with a local participant with the empty string as + // their ID before we're connected, this is fine and we'll be in + // "all ghosts" mode. + if (userMediaId !== "" && member === undefined) { + logger.warn( + `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`, ); - tilesById.delete(screenShareId); + } - const screenShareTile: TileDescriptor = { - id: screenShareId, - focused: true, - isPresenter: false, - isSpeaker: false, - hasVideo: true, + const userMediaVm = + tilesById.get(userMediaId)?.data ?? + new UserMediaViewModel(userMediaId, member, p, this.encrypted); + tilesById.delete(userMediaId); + + const userMediaTile: TileDescriptor = { + id: userMediaId, + focused: false, + isPresenter: p.isScreenShareEnabled, + isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal, + hasVideo: p.isCameraEnabled, local: p.isLocal, - largeBaseSize: true, - placeNear: userMediaId, - data: screenShareVm, + largeBaseSize: false, + data: userMediaVm, }; - return [userMediaTile, screenShareTile]; - } else { - return [userMediaTile]; - } - }); - // Any tiles left in the map are unused and should be destroyed - for (const t of tilesById.values()) t.data.destroy(); + if (p.isScreenShareEnabled) { + const screenShareId = `${userMediaId}:screen-share`; + const screenShareVm = + tilesById.get(screenShareId)?.data ?? + new ScreenShareViewModel( + screenShareId, + member, + p, + this.encrypted, + ); + tilesById.delete(screenShareId); - // If every item is a ghost, that probably means we're still connecting - // and shouldn't bother showing anything yet - return allGhosts ? [] : newTiles; - }, [] as TileDescriptor[]), + const screenShareTile: TileDescriptor = { + id: screenShareId, + focused: true, + isPresenter: false, + isSpeaker: false, + hasVideo: true, + local: p.isLocal, + largeBaseSize: true, + placeNear: userMediaId, + data: screenShareVm, + }; + return [userMediaTile, screenShareTile]; + } else { + return [userMediaTile]; + } + }); + + // Any tiles left in the map are unused and should be destroyed + for (const t of tilesById.values()) t.data.destroy(); + + // If every item is a ghost, that probably means we're still connecting + // and shouldn't bother showing anything yet + return allGhosts ? [] : newTiles; + }, + [] as TileDescriptor[], + ), finalizeValue((ts) => { for (const t of ts) t.data.destroy(); }), ), ); + private get matrixRoom(): MatrixRoom { + return this.rtcSession.room; + } + public constructor( // A call is permanently tied to a single Matrix room and LiveKit room - private readonly matrixRoom: MatrixRoom, + private readonly rtcSession: MatrixRTCSession, private readonly livekitRoom: LivekitRoom, private readonly encrypted: boolean, private readonly connectionState: Observable, @@ -588,25 +675,25 @@ export class CallViewModel extends ViewModel { } export function useCallViewModel( - matrixRoom: MatrixRoom, + rtcSession: MatrixRTCSession, livekitRoom: LivekitRoom, encrypted: boolean, connectionState: ECConnectionState, ): CallViewModel { - const prevMatrixRoom = usePrevious(matrixRoom); + const prevRTCSession = usePrevious(rtcSession); const prevLivekitRoom = usePrevious(livekitRoom); const prevEncrypted = usePrevious(encrypted); const connectionStateObservable = useObservable(connectionState); const vm = useRef(); if ( - matrixRoom !== prevMatrixRoom || + rtcSession !== prevRTCSession || livekitRoom !== prevLivekitRoom || encrypted !== prevEncrypted ) { vm.current?.destroy(); vm.current = new CallViewModel( - matrixRoom, + rtcSession, livekitRoom, encrypted, connectionStateObservable, diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index db11017e..7417f3f7 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -110,7 +110,10 @@ abstract class BaseMediaViewModel extends ViewModel { /** * Some participant's media. */ -export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; +export type MediaViewModel = + | UserMediaViewModel + | ScreenShareViewModel + | MembershipOnlyViewModel; /** * Some participant's user media. @@ -239,3 +242,15 @@ export class ScreenShareViewModel extends BaseMediaViewModel { ); } } + +/** + * Placeholder for a call membership that does not have a LiveKit participant associated with it. + */ +export class MembershipOnlyViewModel extends ViewModel { + public id: string; + public local = false; + public constructor(public readonly member: RoomMember) { + super(); + this.id = member.userId; + } +} diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index d4a7442e..13be0b96 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -53,6 +53,7 @@ import { Menu, } from "@vector-im/compound-web"; import { useStateObservable } from "@react-rxjs/core"; +import useRelativeTime from "@nkzw/use-relative-time"; import { Avatar } from "../Avatar"; import styles from "./VideoTile.module.css"; @@ -61,6 +62,7 @@ import { ScreenShareViewModel, MediaViewModel, UserMediaViewModel, + MembershipOnlyViewModel, } from "../state/MediaViewModel"; import { subscribe } from "../state/subscribe"; import { useMergedRefs } from "../useMergedRefs"; @@ -72,7 +74,7 @@ interface TileProps { style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; - video: TrackReferenceOrPlaceholder; + video?: TrackReferenceOrPlaceholder; member: RoomMember | undefined; videoEnabled: boolean; maximised: boolean; @@ -110,6 +112,10 @@ const Tile = forwardRef( const { t } = useTranslation(); const mergedRef = useMergedRefs(tileRef, ref); + const joinedCallTime = member?.events.member?.getTs() ?? 0; + + const joinedCallAgo = useRelativeTime(joinedCallTime ?? 0); + return ( ( src={member?.getMxcAvatarUrl()} className={styles.avatar} /> - {video.publication !== undefined && ( + {video?.publication !== undefined && ( ( disablePictureInPicture /> )} + {!video && member && joinedCallTime > 0 && ( +
+ {nameTag} joined the call{" "} + + {joinedCallAgo} + {" "} + but is currently unreachable. Are they having connection problems? +
+ )}
{nameTagLeadingIcon} - {nameTag} + {nameTag}{" "} {unencryptedWarning && ( ( ScreenShareTile.displayName = "ScreenShareTile"; +interface MembershipOnlyTileProps { + vm: MembershipOnlyViewModel; + className?: string; + style?: ComponentProps["style"]; + targetWidth: number; + targetHeight: number; + nameTag: string; + displayName: string; + maximised: boolean; + fullscreen: boolean; + onToggleFullscreen: (itemId: string) => void; +} + +const MembershipOnlyTile = subscribe( + ( + { + vm, + className, + style, + targetWidth, + targetHeight, + nameTag, + displayName, + maximised, + fullscreen, + onToggleFullscreen, + }, + ref, + ) => { + const { t } = useTranslation(); + const onClickFullScreen = useCallback( + () => onToggleFullscreen(vm.id), + [onToggleFullscreen, vm], + ); + + const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon; + + return ( + + + + } + /> + ); + }, +); + +MembershipOnlyTile.displayName = "MembershipOnlyTile"; + interface Props { vm: MediaViewModel; maximised: boolean; @@ -471,7 +555,7 @@ export const VideoTile = forwardRef( showSpeakingIndicator={showSpeakingIndicator} /> ); - } else { + } else if (vm instanceof ScreenShareViewModel) { return ( ( onToggleFullscreen={onToggleFullscreen} /> ); + } else if (vm instanceof MembershipOnlyViewModel) { + return ( + + ); } }, ); diff --git a/yarn.lock b/yarn.lock index 9121b235..7827f0a1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1912,6 +1912,11 @@ version "3.2.14" resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz#acd96c00a881d0f462e1f97a56c73742c8dbc984" +"@nkzw/use-relative-time@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@nkzw/use-relative-time/-/use-relative-time-1.1.0.tgz#7eb52dc4c7ceaf49a1f05ecfbf725313cf5c1724" + integrity sha512-ogCL62FvScpRpsZUuaN6Jt0xPGRv62atQUNGyMcX+nZs4H5Fs5K1iA3MbSmkJ1y0n/N0RIRLc3VAp8o46lq2CA== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"