diff --git a/locales/en/app.json b/locales/en/app.json index 9b1a5675..f5749cf7 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -249,6 +249,8 @@ "version": "{{productName}} version: {{version}}", "video_tile": { "always_show": "Always show", + "call_ended": "Call ended", + "calling": "Calling…", "camera_starting": "Video loading...", "collapse": "Collapse", "expand": "Expand", diff --git a/playwright/widget/voice-call-dm.spec.ts b/playwright/widget/voice-call-dm.spec.ts index 6a8473cf..cc0b4e53 100644 --- a/playwright/widget/voice-call-dm.spec.ts +++ b/playwright/widget/voice-call-dm.spec.ts @@ -34,9 +34,12 @@ widgetTest( .locator('iframe[title="Element Call"]') .contentFrame(); - // We should show a ringing overlay, let's check for that + // We should show a ringing tile, let's check for that await expect( - brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`), + brooksFrame + .getByTestId("videoTile") + .filter({ has: brooksFrame.getByText(whistler.displayName) }) + .filter({ has: brooksFrame.getByText("Calling…") }), ).toBeVisible(); await expect(whistler.page.getByText("Incoming voice call")).toBeVisible(); @@ -125,9 +128,12 @@ widgetTest( .locator('iframe[title="Element Call"]') .contentFrame(); - // We should show a ringing overlay, let's check for that + // We should show a ringing tile, let's check for that await expect( - brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`), + brooksFrame + .getByTestId("videoTile") + .filter({ has: brooksFrame.getByText(whistler.displayName) }) + .filter({ has: brooksFrame.getByText("Calling…") }), ).toBeVisible(); await expect(whistler.page.getByText("Incoming video call")).toBeVisible(); @@ -216,9 +222,12 @@ widgetTest( .locator('iframe[title="Element Call"]') .contentFrame(); - // We should show a ringing overlay, let's check for that + // We should show a ringing tile, let's check for that await expect( - brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`), + brooksFrame + .getByTestId("videoTile") + .filter({ has: brooksFrame.getByText(whistler.displayName) }) + .filter({ has: brooksFrame.getByText("Calling…") }), ).toBeVisible(); await expect(whistler.page.getByText("Incoming video call")).toBeVisible(); diff --git a/src/grid/OneOnOneLayout.tsx b/src/grid/OneOnOneLayout.tsx index 6c5ae69f..fd9c0a65 100644 --- a/src/grid/OneOnOneLayout.tsx +++ b/src/grid/OneOnOneLayout.tsx @@ -51,15 +51,15 @@ export const makeOneOnOneLayout: CallLayout = ({ return (
= ({ const { showControls } = useUrlParams(); const muteAllAudio = useBehavior(muteAllAudio$); - // Call pickup state and display names are needed for waiting overlay/sounds - const callPickupState = useBehavior(vm.callPickupState$); // Preload a waiting and decline sounds const pickupPhaseSoundCache = useInitial(async () => { @@ -239,6 +236,7 @@ export const InCallView: FC = ({ latencyHint: "interactive", muted: muteAllAudio, }); + const latestPickupPhaseAudio = useLatest(pickupPhaseAudio); const audioEnabled = useBehavior(muteStates.audio.enabled$); const videoEnabled = useBehavior(muteStates.video.enabled$); @@ -257,6 +255,7 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); + const ringing = useBehavior(vm.ringing$); const audioParticipants = useBehavior(vm.livekitRoomItems$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); @@ -271,7 +270,6 @@ export const InCallView: FC = ({ const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); const sharingScreen = useBehavior(vm.sharingScreen$); - const ringOverlay = useBehavior(vm.ringOverlay$); const fatalCallError = useBehavior(vm.fatalError$); // Stop the rendering and throw for the error boundary if (fatalCallError) { @@ -279,58 +277,21 @@ export const InCallView: FC = ({ throw fatalCallError; } - // We need to set the proper timings on the animation based upon the sound length. - const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1; - useEffect((): (() => void) => { - // The CSS animation includes the delay, so we must double the length of the sound. - window.document.body.style.setProperty( - "--call-ring-duration-s", - `${ringDuration * 2}s`, - ); - window.document.body.style.setProperty( - "--call-ring-delay-s", - `${ringDuration}s`, - ); - // Remove properties when we unload. - return () => { - window.document.body.style.removeProperty("--call-ring-duration-s"); - window.document.body.style.removeProperty("--call-ring-delay-s"); - }; - }, [pickupPhaseAudio?.soundDuration, ringDuration]); - - // When waiting for pickup, loop a waiting sound + // While ringing, loop the ringtone useEffect((): void | (() => void) => { - if (callPickupState !== "ringing" || !pickupPhaseAudio) return; - const endSound = pickupPhaseAudio.playSoundLooping("waiting", ringDuration); - return () => { - void endSound().catch((e) => { - logger.error("Failed to stop ringing sound", e); - }); - }; - }, [callPickupState, pickupPhaseAudio, ringDuration]); - - // Waiting UI overlay - const waitingOverlay: JSX.Element | null = useMemo(() => { - return ringOverlay ? ( -
-
-
- -
- - {ringOverlay.text} - -
-
- ) : null; - }, [ringOverlay]); + const audio = latestPickupPhaseAudio.current; + if (ringing && audio) { + const endSound = audio.playSoundLooping( + "waiting", + audio.soundDuration["waiting"] ?? 1, + ); + return () => { + void endSound().catch((e) => { + logger.error("Failed to stop ringing sound", e); + }); + }; + } + }, [ringing, latestPickupPhaseAudio]); const onViewClick = useCallback( (e: ReactMouseEvent) => { @@ -764,7 +725,6 @@ export const InCallView: FC = ({ {reconnectingToast} {earpieceOverlay} - {waitingOverlay} {footer} {layout.type !== "pip" && ( <> diff --git a/src/room/WaitingForJoin.module.css b/src/room/WaitingForJoin.module.css deleted file mode 100644 index a598e482..00000000 --- a/src/room/WaitingForJoin.module.css +++ /dev/null @@ -1,61 +0,0 @@ -.overlay { - position: absolute; - inset: 0; - display: flex; - align-items: center; - justify-content: center; - pointer-events: none; -} - -.content { - display: flex; - flex-direction: column; - align-items: center; - gap: 16px; -} - -.pulse { - position: relative; - height: 90px; -} - -.pulse::before { - content: ""; - position: absolute; - inset: -12px; - border-radius: 9999px; - border: 12px solid rgba(255, 255, 255, 0.6); - animation: pulse var(--call-ring-duration-s) ease-out infinite; - animation-delay: 1s; - opacity: 0; -} - -.text { - color: var(--cpd-color-text-on-solid-primary); -} - -@keyframes pulse { - 0% { - transform: scale(0.95); - opacity: 0.7; - transform: scale(0); - opacity: 1; - } - 35% { - transform: scale(1.15); - opacity: 0.15; - } - 50% { - transform: scale(1.2); - opacity: 0; - } - 50.01% { - transform: scale(0); - } - 85% { - transform: scale(0); - } - 100% { - transform: scale(0); - } -} diff --git a/src/state/CallViewModel/CallNotificationLifecycle.ts b/src/state/CallViewModel/CallNotificationLifecycle.ts index 44ce2e43..3e06108f 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.ts @@ -89,7 +89,6 @@ export interface Props { * `callPickupState$` The current call pickup state of the call. * - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership. * Then we can conclude if we were the first one to join or not. - * This may also be set if we are disconnected. * - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening). * - "timeout": No-one picked up in the defined time this call should be ringing on others devices. * The call failed. If desired this can be used as a trigger to exit the call. @@ -131,15 +130,9 @@ export function createCallNotificationLifecycle$({ ) as Behavior>; /** - * Whenever the RTC session tells us that it intends to ring the remote - * participant's devices, this emits an Observable tracking the current state of - * that ringing process. + * The state of the current ringing attempt, if the RTC session is indeed + * ringing the remote participant's devices. Otherwise `null`. */ - // This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$` - // has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`. - // A behavior will emit the latest observable with the running timer to new subscribers. - // see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if - // `ring$` would not be a behavior. const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> = scope.behavior( sentCallNotification$.pipe( diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 5ee679b0..aca3ee7b 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -46,9 +46,11 @@ import { } from "../../utils/test.ts"; import { E2eeType } from "../../e2ee/e2eeType.ts"; import { + alice, aliceId, aliceParticipant, aliceRtcMember, + aliceUserId, bobId, bobRtcMember, local, @@ -140,8 +142,8 @@ export interface SpotlightExpandedLayoutSummary { export interface OneOnOneLayoutSummary { type: "one-on-one"; - local: string; - remote: string; + spotlight: string; + pip: string; } export interface PipLayoutSummary { @@ -194,11 +196,11 @@ function summarizeLayout$(l$: Observable): Observable { ); case "one-on-one": return combineLatest( - [l.local.media$, l.remote.media$], - (local, remote) => ({ + [l.spotlight.media$, l.pip.media$], + (spotlight, pip) => ({ type: l.type, - local: local.id, - remote: remote.id, + spotlight: spotlight.id, + pip: pip.id, }), ); case "pip": @@ -537,8 +539,8 @@ describe.each([ b: { // In a larger window, expect the normal one-on-one layout type: "one-on-one", - local: `${localId}:0`, - remote: `${aliceId}:0`, + pip: `${localId}:0`, + spotlight: `${aliceId}:0`, }, c: { // In a PiP-sized window, we of course expect a PiP layout @@ -840,8 +842,8 @@ describe.each([ }, b: { type: "one-on-one", - local: `${localId}:0`, - remote: `${aliceId}:0`, + pip: `${localId}:0`, + spotlight: `${aliceId}:0`, }, c: { type: "grid", @@ -883,8 +885,8 @@ describe.each([ }, b: { type: "one-on-one", - local: `${localId}:0`, - remote: `${aliceId}:0`, + pip: `${localId}:0`, + spotlight: `${aliceId}:0`, }, c: { type: "grid", @@ -893,8 +895,8 @@ describe.each([ }, d: { type: "one-on-one", - local: `${localId}:0`, - remote: `${daveId}:0`, + pip: `${localId}:0`, + spotlight: `${daveId}:0`, }, }, ); @@ -1087,83 +1089,81 @@ describe.each([ }); }); - describe("waitForCallPickup$", () => { - it.skip("regression test: does stop ringing in case livekitConnectionState$ emits after didSendCallNotification$ has already emitted", () => { - withTestScheduler(({ schedule, expectObservable, behavior }) => { - withCallViewModel( - { - livekitConnectionState$: behavior("d 9ms c", { - d: ConnectionState.Disconnected, - c: ConnectionState.Connected, - }), - }, - (vm, rtcSession) => { - // Fire a call notification IMMEDIATELY (its important for this test, that this happens before the livekitConnectionState$ emits) - schedule("n", { - n: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif1", 30), - ); - }, - }); + test("recipient has placeholder tile while ringing or timed out", () => { + withTestScheduler(({ schedule, expectObservable }) => { + withCallViewModel( + { + roomMembers: [alice, local], // Simulate a DM + }, + (vm, rtcSession) => { + // Fire a ringing notification + schedule("n", { + n: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif1", 30), + ); + }, + }); - expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { - a: "unknown", - b: "ringing", - c: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); + // Should ring for 30ms and then time out + expectObservable(vm.ringing$).toBe("(ny) 26ms n", yesNo); + // Layout should show placeholder media for the participant we're + // ringing the entire time (even once timed out) + expectObservable(summarizeLayout$(vm.layout$)).toBe("a", { + a: { + type: "one-on-one", + spotlight: `${localId}:0`, + pip: `ringing:${aliceUserId}`, + }, + }); + }, + { waitForCallPickup: true }, + ); }); + }); - it.skip("ringing -> unknown if we get disconnected", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - const connectionState$ = new BehaviorSubject(ConnectionState.Connected); - // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) - withCallViewModel( - { - remoteParticipants$: behavior("a 19ms b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a 19ms b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - livekitConnectionState$: connectionState$, - }, - (vm, rtcSession) => { - // Notify at 5ms so we enter ringing, then get disconnected 5ms later - schedule(" 5ms r 5ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif2", 100), - ); - }, - d: () => { - connectionState$.next(ConnectionState.Disconnected); - }, - }); + test("recipient's placeholder tile is replaced by their real tile once they answer", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + withCallViewModel( + { + // Alice answers after 20ms + rtcMembers$: behavior("a 20ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + roomMembers: [alice, local], // Simulate a DM + }, + (vm, rtcSession) => { + // Fire a ringing notification + schedule("n", { + n: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif1", 30), + ); + }, + }); - expectObservable(vm.callPickupState$).toBe("a 4ms b 5ms c", { - a: "unknown", - b: "ringing", - c: "unknown", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); + // Should ring until Alice joins + expectObservable(vm.ringing$).toBe("(ny) 17ms n", yesNo); + // Layout should show placeholder media for the participant we're + // ringing the entire time + expectObservable(summarizeLayout$(vm.layout$)).toBe("a 20ms b", { + a: { + type: "one-on-one", + spotlight: `${localId}:0`, + pip: `ringing:${aliceUserId}`, + }, + b: { + type: "one-on-one", + spotlight: `${aliceId}:0`, + pip: `${localId}:0`, + }, + }); + }, + { waitForCallPickup: true }, + ); }); }); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 18e49d0a..8b4d19fb 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -128,7 +128,6 @@ import { createSentCallNotification$, } from "./CallNotificationLifecycle.ts"; import { - createDMMember$, createMatrixMemberMetadata$, createRoomMembers$, } from "./remoteMembers/MatrixMemberMetadata.ts"; @@ -137,12 +136,17 @@ import { type Connection } from "./remoteMembers/Connection.ts"; import { createLayoutModeSwitch } from "./LayoutSwitch.ts"; import { createWrappedUserMedia, - type MediaItem, type WrappedUserMediaViewModel, -} from "../media/MediaItem.ts"; +} from "../media/WrappedUserMediaViewModel.ts"; import { type ScreenShareViewModel } from "../media/ScreenShareViewModel.ts"; import { type UserMediaViewModel } from "../media/UserMediaViewModel.ts"; import { type MediaViewModel } from "../media/MediaViewModel.ts"; +import { type LocalUserMediaViewModel } from "../media/LocalUserMediaViewModel.ts"; +import { type RemoteUserMediaViewModel } from "../media/RemoteUserMediaViewModel.ts"; +import { + createRingingMedia, + type RingingMediaViewModel, +} from "../media/RingingMediaViewModel.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO @@ -210,11 +214,10 @@ export type LivekitRoomItem = { export interface CallViewModel { // lifecycle autoLeave$: Observable; - // TODO if we are in "unknown" state we need a loading rendering (or empty screen) - // Otherwise it looks like we already connected and only than the ringing starts which is weird. - callPickupState$: Behavior< - "unknown" | "ringing" | "timeout" | "decline" | "success" | null - >; + /** + * Whether we are ringing a call recipient. + */ + ringing$: Behavior; /** Observable that emits when the user should leave the call (hangup pressed, widget action, error). * THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is * - by ending the scope @@ -289,13 +292,6 @@ export interface CallViewModel { /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ reactions$: Behavior>; - ringOverlay$: Behavior; // sounds and events joinSoundEffect$: Observable; leaveSoundEffect$: Observable; @@ -611,40 +607,6 @@ export function createCallViewModel$( matrixRoomMembers$, ); - const dmMember$ = createDMMember$(scope, matrixRoomMembers$, matrixRoom); - const noUserToCallInRoom$ = scope.behavior( - matrixRoomMembers$.pipe( - map( - (roomMembersMap) => - roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined, - ), - ), - ); - - const ringOverlay$ = scope.behavior( - combineLatest([noUserToCallInRoom$, dmMember$, callPickupState$]).pipe( - map(([noUserToCallInRoom, dmMember, callPickupState]) => { - // No overlay if not in ringing state - if (callPickupState !== "ringing" || noUserToCallInRoom) return null; - - const name = dmMember ? dmMember.rawDisplayName : matrixRoom.name; - const id = dmMember ? dmMember.userId : matrixRoom.roomId; - const text = dmMember - ? `Waiting for ${name} to join…` - : "Waiting for other participants…"; - const avatarMxc = dmMember - ? (dmMember.getMxcAvatarUrl?.() ?? undefined) - : (matrixRoom.getMxcAvatarUrl() ?? undefined); - return { - name: name ?? id, - idForAvatar: id, - text, - avatarMxc, - }; - }), - ), - ); - const allConnections$ = scope.behavior( connectionManager.connectionManagerData$.pipe(map((d) => d.value)), ); @@ -720,7 +682,7 @@ export function createCallViewModel$( matrixLivekitMembers$, duplicateTiles.value$, ]).pipe( - // Generate a collection of MediaItems from the list of expected (whether + // Generate a collection of user media from the list of expected (whether // present or missing) LiveKit participants. generateItems( "CallViewModel userMedia$", @@ -793,32 +755,67 @@ export function createCallViewModel$( ), ); + const ringingMedia$ = scope.behavior( + combineLatest([userMedia$, matrixRoomMembers$, callPickupState$]).pipe( + generateItems( + "CallViewModel ringingMedia$", + function* ([userMedia, roomMembers, callPickupState]) { + if ( + callPickupState === "ringing" || + callPickupState === "timeout" || + callPickupState === "decline" + ) { + for (const member of roomMembers.values()) { + if (!userMedia.some((vm) => vm.userId === member.userId)) + yield { + keys: [member.userId], + data: callPickupState, + }; + } + } + }, + (scope, pickupState$, userId) => + createRingingMedia({ + id: `ringing:${userId}`, + userId, + displayName$: scope.behavior( + matrixRoomMembers$.pipe( + map((members) => members.get(userId)?.rawDisplayName || userId), + ), + ), + mxcAvatarUrl$: + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + pickupState$, + muteStates, + }), + ), + distinctUntilChanged(shallowEquals), + tap((ringingMedia) => { + if (ringingMedia.length > 1) + // Warn that UI may do something unexpected in this case + logger.warn( + `Ringing more than one participant is not supported (ringing ${ringingMedia.map((vm) => vm.userId).join(", ")})`, + ); + }), + ), + ); + /** - * List of all media items (user media and screen share media) that we want - * tiles for. + * All screen share media that we want to display. */ - const mediaItems$ = scope.behavior( + const screenShares$ = scope.behavior( userMedia$.pipe( switchMap((userMedia) => userMedia.length === 0 ? of([]) : combineLatest( userMedia.map((m) => m.screenShares$), - (...screenShares) => [...userMedia, ...screenShares.flat(1)], + (...screenShares) => screenShares.flat(1), ), ), ), ); - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - const screenShares$ = scope.behavior( - mediaItems$.pipe( - map((mediaItems) => mediaItems.filter((m) => m.type === "screen share")), - ), - ); - const joinSoundEffect$ = userMedia$.pipe( pairwise(), filter( @@ -931,40 +928,20 @@ export function createCallViewModel$( ), ); - const spotlight$ = scope.behavior( - screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) return of(screenShares); - - return spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), + /** + * Local user media suitable for displaying in a PiP (undefined if not found + * or if user prefers to not see themselves). + */ + const localUserMediaForPip$ = scope.behavior< + LocalUserMediaViewModel | undefined + >( + userMedia$.pipe( + switchMap((userMedia) => { + const localUserMedia = userMedia.find( + (m): m is WrappedUserMediaViewModel & LocalUserMediaViewModel => + m.type === "user" && m.local, ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - const pip$ = scope.behavior( - combineLatest([ - // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits - screenShares$, - spotlightSpeaker$, - mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(undefined); - } - - const localUserMedia = mediaItems.find( - (m) => m.type === "user" && m.local, - ); - if (!localUserMedia) { - return of(undefined); - } + if (!localUserMedia) return of(undefined); return localUserMedia.alwaysShow$.pipe( map((alwaysShow) => (alwaysShow ? localUserMedia : undefined)), ); @@ -972,6 +949,39 @@ export function createCallViewModel$( ), ); + const spotlightAndPip$ = scope.behavior<{ + spotlight: MediaViewModel[]; + pip$: Behavior; + }>( + ringingMedia$.pipe( + switchMap((ringingMedia) => { + if (ringingMedia.length > 0) + return of({ spotlight: ringingMedia, pip$: localUserMediaForPip$ }); + + return screenShares$.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) + return of({ spotlight: screenShares, pip$: spotlightSpeaker$ }); + + return spotlightSpeaker$.pipe( + map((speaker) => ({ + spotlight: speaker ? [speaker] : [], + pip$: localUserMediaForPip$, + })), + ); + }), + ); + }), + ), + ); + + const spotlight$ = scope.behavior( + spotlightAndPip$.pipe( + map(({ spotlight }) => spotlight), + distinctUntilChanged(shallowEquals), + ), + ); + const hasRemoteScreenShares$ = scope.behavior( spotlight$.pipe( map((spotlight) => @@ -1054,24 +1064,61 @@ export function createCallViewModel$( })); const spotlightExpandedLayoutMedia$: Observable = - combineLatest([spotlight$, pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); + spotlightAndPip$.pipe( + switchMap(({ spotlight, pip$ }) => + pip$.pipe( + map((pip) => ({ + type: "spotlight-expanded" as const, + spotlight, + pip: pip ?? undefined, + })), + ), + ), + ); const oneOnOneLayoutMedia$: Observable = - mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.type === "user" && vm.local); - const remote = mediaItems.find((vm) => vm.type === "user" && !vm.local); - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; + userMedia$.pipe( + switchMap((userMedia) => { + if (userMedia.length <= 2) { + const local = userMedia.find( + (vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel => + vm.type === "user" && vm.local, + ); - return { type: "one-on-one", local, remote }; + if (local !== undefined) { + const remote = userMedia.find( + ( + vm, + ): vm is WrappedUserMediaViewModel & RemoteUserMediaViewModel => + vm.type === "user" && !vm.local, + ); + + if (remote !== undefined) + return of({ + type: "one-on-one" as const, + spotlight: remote, + pip: local, + }); + + // If there's no other user media in the call (could still happen in + // this branch due to the duplicate tiles option), we could possibly + // show ringing media instead + if (userMedia.length === 1) + return ringingMedia$.pipe( + map((ringingMedia) => { + return ringingMedia.length === 1 + ? { + type: "one-on-one" as const, + spotlight: local, + pip: ringingMedia[0], + } + : null; + }), + ); + } + } + + return of(null); }), ); @@ -1482,8 +1529,9 @@ export function createCallViewModel$( return { autoLeave$: autoLeave$, - callPickupState$: callPickupState$, - ringOverlay$: ringOverlay$, + ringing$: scope.behavior( + callPickupState$.pipe(map((state) => state === "ringing")), + ), leave$: leave$, hangup: (): void => userHangup$.next(), join: localMembership.requestJoinAndPublish, diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index b6f53275..b6bf8a9a 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -17,7 +17,7 @@ import { import { SyncState } from "matrix-js-sdk/lib/sync"; import { BehaviorSubject, type Observable, map, of } from "rxjs"; import { onTestFinished, vi } from "vitest"; -import { ClientEvent, type MatrixClient } from "matrix-js-sdk"; +import { ClientEvent, type RoomMember, type MatrixClient } from "matrix-js-sdk"; import EventEmitter from "events"; import * as ComponentsCore from "@livekit/components-core"; @@ -63,15 +63,10 @@ const carol = local; const dave = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "Dave" }); -const roomMembers = new Map( - [alice, aliceDoppelganger, bob, bobZeroWidthSpace, carol, dave, daveRTL].map( - (p) => [p.userId, p], - ), -); - export interface CallViewModelInputs { remoteParticipants$: Behavior; rtcMembers$: Behavior[]>; + roomMembers: RoomMember[]; livekitConnectionState$: Behavior; speaking: Map>; mediaDevices: MediaDevices; @@ -86,6 +81,15 @@ export function withCallViewModel(mode: MatrixRTCMode) { { remoteParticipants$ = constant([]), rtcMembers$ = constant([localRtcMember]), + roomMembers = [ + alice, + aliceDoppelganger, + bob, + bobZeroWidthSpace, + carol, + dave, + daveRTL, + ], livekitConnectionState$: connectionState$ = constant( ConnectionState.Connected, ), @@ -128,8 +132,8 @@ export function withCallViewModel(mode: MatrixRTCMode) { return syncState; } })() as Partial as MatrixClient, - getMembers: () => Array.from(roomMembers.values()), - getMembersWithMembership: () => Array.from(roomMembers.values()), + getMembers: () => roomMembers, + getMembersWithMembership: () => roomMembers, }); const rtcSession = new MockRTCSession(room, []).withMemberships( rtcMembers$, diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts index c1a7a499..d9be2d35 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts @@ -54,31 +54,6 @@ export function createRoomMembers$( ); } -/** - * creates the member that this DM is with in case it is a DM (two members) otherwise null - */ -export function createDMMember$( - scope: ObservableScope, - roomMembers$: Behavior, - matrixRoom: MatrixRoom, -): Behavior | null> { - // We cannot use the normal direct check from matrix since we do not have access to the account data. - // use primitive member count === 2 check instead. - return scope.behavior( - roomMembers$.pipe( - map((membersMap) => { - // primitive appraoch do to no access to account data. - const isDM = membersMap.size === 2; - if (!isDM) return null; - return matrixRoom.getMember(matrixRoom.guessDMUserId()); - }), - ), - ); -} - /** * Displayname for each member of the call. This will disambiguate * any displayname that clashes with another member. Only members diff --git a/src/state/OneOnOneLayout.ts b/src/state/OneOnOneLayout.ts index 10268945..27fa4439 100644 --- a/src/state/OneOnOneLayout.ts +++ b/src/state/OneOnOneLayout.ts @@ -16,14 +16,14 @@ export function oneOnOneLayout( prevTiles: TileStore, ): [OneOnOneLayout, TileStore] { const update = prevTiles.from(2); - update.registerGridTile(media.local); - update.registerGridTile(media.remote); + update.registerGridTile(media.pip); + update.registerGridTile(media.spotlight); const tiles = update.build(); return [ { type: media.type, - local: tiles.gridTilesByMedia.get(media.local)!, - remote: tiles.gridTilesByMedia.get(media.remote)!, + spotlight: tiles.gridTilesByMedia.get(media.spotlight)!, + pip: tiles.gridTilesByMedia.get(media.pip)!, }, tiles, ]; diff --git a/src/state/TileStore.ts b/src/state/TileStore.ts index a954eb4e..300e6bd2 100644 --- a/src/state/TileStore.ts +++ b/src/state/TileStore.ts @@ -13,6 +13,7 @@ import { fillGaps } from "../utils/iter"; import { debugTileLayout } from "../settings/settings"; import { type MediaViewModel } from "./media/MediaViewModel"; import { type UserMediaViewModel } from "./media/UserMediaViewModel"; +import { type RingingMediaViewModel } from "./media/RingingMediaViewModel"; function debugEntries(entries: GridTileData[]): string[] { return entries.map((e) => e.media.displayName$.value); @@ -48,8 +49,10 @@ class SpotlightTileData { } class GridTileData { - private readonly media$: BehaviorSubject; - public get media(): UserMediaViewModel { + private readonly media$: BehaviorSubject< + UserMediaViewModel | RingingMediaViewModel + >; + public get media(): UserMediaViewModel | RingingMediaViewModel { return this.media$.value; } public set media(value: UserMediaViewModel) { @@ -58,7 +61,7 @@ class GridTileData { public readonly vm: GridTileViewModel; - public constructor(media: UserMediaViewModel) { + public constructor(media: UserMediaViewModel | RingingMediaViewModel) { this.media$ = new BehaviorSubject(media); this.vm = new GridTileViewModel(this.media$); } @@ -178,7 +181,9 @@ export class TileStoreBuilder { * Sets up a grid tile for the given media. If this is never called for some * media, then that media will have no grid tile. */ - public registerGridTile(media: UserMediaViewModel): void { + public registerGridTile( + media: UserMediaViewModel | RingingMediaViewModel, + ): void { if (DEBUG_ENABLED) logger.debug( `[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`, @@ -187,7 +192,11 @@ export class TileStoreBuilder { if (this.spotlight !== null) { // We actually *don't* want spotlight speakers to appear in both the // spotlight and the grid, so they're filtered out here - if (!media.local && this.spotlight.media.includes(media)) return; + if ( + !(media.type === "user" && media.local) && + this.spotlight.media.includes(media) + ) + return; // When the spotlight speaker changes, we would see one grid tile appear // and another grid tile disappear. This would be an undesirable layout // shift, so instead what we do is take the speaker's grid tile and swap diff --git a/src/state/TileViewModel.ts b/src/state/TileViewModel.ts index 8b13c685..eeec0c88 100644 --- a/src/state/TileViewModel.ts +++ b/src/state/TileViewModel.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { type Behavior } from "./Behavior"; import { type MediaViewModel } from "./media/MediaViewModel"; +import { type RingingMediaViewModel } from "./media/RingingMediaViewModel"; import { type UserMediaViewModel } from "./media/UserMediaViewModel"; let nextId = 0; @@ -17,7 +18,11 @@ function createId(): string { export class GridTileViewModel { public readonly id = createId(); - public constructor(public readonly media$: Behavior) {} + public constructor( + public readonly media$: Behavior< + UserMediaViewModel | RingingMediaViewModel + >, + ) {} } export class SpotlightTileViewModel { diff --git a/src/state/layout-types.ts b/src/state/layout-types.ts index 33796f66..2e779057 100644 --- a/src/state/layout-types.ts +++ b/src/state/layout-types.ts @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts"; import { type MediaViewModel } from "./media/MediaViewModel.ts"; -import { type RemoteUserMediaViewModel } from "./media/RemoteUserMediaViewModel.ts"; +import { type RingingMediaViewModel } from "./media/RingingMediaViewModel.ts"; import { type UserMediaViewModel } from "./media/UserMediaViewModel.ts"; import { type GridTileViewModel, @@ -40,8 +40,8 @@ export interface SpotlightExpandedLayoutMedia { export interface OneOnOneLayoutMedia { type: "one-on-one"; - local: LocalUserMediaViewModel; - remote: RemoteUserMediaViewModel; + spotlight: UserMediaViewModel; + pip: LocalUserMediaViewModel | RingingMediaViewModel; } export interface PipLayoutMedia { @@ -86,8 +86,8 @@ export interface SpotlightExpandedLayout { export interface OneOnOneLayout { type: "one-on-one"; - local: GridTileViewModel; - remote: GridTileViewModel; + spotlight: GridTileViewModel; + pip: GridTileViewModel; } export interface PipLayout { diff --git a/src/state/media/MediaViewModel.ts b/src/state/media/MediaViewModel.ts index bdc4875b..9a253d81 100644 --- a/src/state/media/MediaViewModel.ts +++ b/src/state/media/MediaViewModel.ts @@ -7,13 +7,17 @@ Please see LICENSE in the repository root for full details. */ import { type Behavior } from "../Behavior"; +import { type RingingMediaViewModel } from "./RingingMediaViewModel"; import { type ScreenShareViewModel } from "./ScreenShareViewModel"; import { type UserMediaViewModel } from "./UserMediaViewModel"; /** * A participant's media. */ -export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; +export type MediaViewModel = + | UserMediaViewModel + | ScreenShareViewModel + | RingingMediaViewModel; /** * Properties which are common to all MediaViewModels. diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts index e7f57b59..969da899 100644 --- a/src/state/media/MemberMediaViewModel.ts +++ b/src/state/media/MemberMediaViewModel.ts @@ -38,6 +38,8 @@ import { type ObservableScope } from "../ObservableScope"; import { observeTrackReference$ } from "../observeTrackReference"; import { E2eeType } from "../../e2ee/e2eeType"; import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats"; +import { type UserMediaViewModel } from "./UserMediaViewModel"; +import { type ScreenShareViewModel } from "./ScreenShareViewModel"; // TODO: Encryption status is kinda broken and thus unused right now. Remove? export enum EncryptionStatus { @@ -49,9 +51,9 @@ export enum EncryptionStatus { } /** - * Media belonging to an active member of the RTC session. + * Properties common to all MemberMediaViewModels. */ -export interface MemberMediaViewModel extends BaseMediaViewModel { +export interface BaseMemberMediaViewModel extends BaseMediaViewModel { /** * The LiveKit video track for this media. */ @@ -88,7 +90,7 @@ export function createMemberMedia( encryptionSystem, ...inputs }: MemberMediaInputs, -): MemberMediaViewModel { +): BaseMemberMediaViewModel { const trackBehavior$ = ( source: Track.Source, ): Behavior => @@ -270,3 +272,8 @@ function observeRemoteTrackReceivingOkay$( startWith(undefined), ); } + +/** + * Media belonging to an active member of the call. + */ +export type MemberMediaViewModel = UserMediaViewModel | ScreenShareViewModel; diff --git a/src/state/media/RingingMediaViewModel.ts b/src/state/media/RingingMediaViewModel.ts new file mode 100644 index 00000000..23291723 --- /dev/null +++ b/src/state/media/RingingMediaViewModel.ts @@ -0,0 +1,51 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type Behavior } from "../Behavior"; +import { type MuteStates } from "../MuteStates"; +import { + type BaseMediaInputs, + type BaseMediaViewModel, + createBaseMedia, +} from "./MediaViewModel"; + +/** + * Media representing a user who is not yet part of the call — one that we are + * *ringing*. + */ +export interface RingingMediaViewModel extends BaseMediaViewModel { + type: "ringing"; + pickupState$: Behavior<"ringing" | "timeout" | "decline">; + /** + * Whether this media would be expected to have video, were it not simply a + * placeholder. + */ + videoEnabled$: Behavior; +} + +export interface RingingMediaInputs extends BaseMediaInputs { + pickupState$: Behavior<"ringing" | "timeout" | "decline">; + /** + * The local user's own mute states. + */ + muteStates: MuteStates; +} + +export function createRingingMedia({ + pickupState$, + muteStates, + ...inputs +}: RingingMediaInputs): RingingMediaViewModel { + return { + ...createBaseMedia(inputs), + type: "ringing", + pickupState$, + // If our own video is enabled, then this is a video call and we would + // expect remote media to have video as well + videoEnabled$: muteStates.video.enabled$, + }; +} diff --git a/src/state/media/ScreenShareViewModel.ts b/src/state/media/ScreenShareViewModel.ts index 36cd9440..8336f0a6 100644 --- a/src/state/media/ScreenShareViewModel.ts +++ b/src/state/media/ScreenShareViewModel.ts @@ -13,7 +13,7 @@ import { type LocalScreenShareViewModel } from "./LocalScreenShareViewModel"; import { createMemberMedia, type MemberMediaInputs, - type MemberMediaViewModel, + type BaseMemberMediaViewModel, } from "./MemberMediaViewModel"; import { type RemoteScreenShareViewModel } from "./RemoteScreenShareViewModel"; @@ -27,7 +27,7 @@ export type ScreenShareViewModel = /** * Properties which are common to all ScreenShareViewModels. */ -export interface BaseScreenShareViewModel extends MemberMediaViewModel { +export interface BaseScreenShareViewModel extends BaseMemberMediaViewModel { type: "screen share"; } diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts index 16af7f26..a20c489e 100644 --- a/src/state/media/UserMediaViewModel.ts +++ b/src/state/media/UserMediaViewModel.ts @@ -27,7 +27,7 @@ import { type LocalUserMediaViewModel } from "./LocalUserMediaViewModel"; import { createMemberMedia, type MemberMediaInputs, - type MemberMediaViewModel, + type BaseMemberMediaViewModel, } from "./MemberMediaViewModel"; import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel"; import { type ObservableScope } from "../ObservableScope"; @@ -42,7 +42,7 @@ export type UserMediaViewModel = | LocalUserMediaViewModel | RemoteUserMediaViewModel; -export interface BaseUserMediaViewModel extends MemberMediaViewModel { +export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel { type: "user"; speaking$: Behavior; audioEnabled$: Behavior; diff --git a/src/state/media/MediaItem.ts b/src/state/media/WrappedUserMediaViewModel.ts similarity index 98% rename from src/state/media/MediaItem.ts rename to src/state/media/WrappedUserMediaViewModel.ts index 6cd80045..e9575d0c 100644 --- a/src/state/media/MediaItem.ts +++ b/src/state/media/WrappedUserMediaViewModel.ts @@ -194,5 +194,3 @@ export function createWrappedUserMedia( ), }; } - -export type MediaItem = WrappedUserMediaViewModel | ScreenShareViewModel; diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 02f09a17..501f440c 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -7,9 +7,10 @@ Please see LICENSE in the repository root for full details. import { type RemoteTrackPublication } from "livekit-client"; import { test, expect } from "vitest"; -import { render, screen } from "@testing-library/react"; +import { act, render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject } from "rxjs"; import { GridTile } from "./GridTile"; import { @@ -21,6 +22,11 @@ import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsSenderProvider } from "../reactions/useReactionsSender"; import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; import { constant } from "../state/Behavior"; +import { + createRingingMedia, + type RingingMediaViewModel, +} from "../state/media/RingingMediaViewModel"; +import { type MuteStates } from "../state/MuteStates"; global.IntersectionObserver = class MockIntersectionObserver { public observe(): void {} @@ -28,6 +34,27 @@ global.IntersectionObserver = class MockIntersectionObserver { public disconnect(): void {} } as unknown as typeof IntersectionObserver; +const fakeRtcSession = { + on: () => {}, + off: () => {}, + room: { + on: () => {}, + off: () => {}, + client: { + getUserId: () => null, + getDeviceId: () => null, + on: () => {}, + off: () => {}, + }, + }, + memberships: [], +} as unknown as MatrixRTCSession; + +const callVm = { + reactions$: constant({}), + handsRaised$: constant({}), +} as Partial as CallViewModel; + test("GridTile is accessible", async () => { const vm = mockRemoteMedia( mockRtcMembership("@alice:example.org", "AAAA"), @@ -42,34 +69,15 @@ test("GridTile is accessible", async () => { }), ); - const fakeRtcSession = { - on: () => {}, - off: () => {}, - room: { - on: () => {}, - off: () => {}, - client: { - getUserId: () => null, - getDeviceId: () => null, - on: () => {}, - off: () => {}, - }, - }, - memberships: [], - } as unknown as MatrixRTCSession; - const cVm = { - reactions$: constant({}), - handsRaised$: constant({}), - } as Partial as CallViewModel; const { container } = render( - + {}} targetWidth={300} targetHeight={200} showSpeakingIndicators - focusable={true} + focusable /> , ); @@ -77,3 +85,40 @@ test("GridTile is accessible", async () => { // Name should be visible screen.getByText("Alice"); }); + +test("GridTile displays ringing media", async () => { + const pickupState$ = new BehaviorSubject< + RingingMediaViewModel["pickupState$"]["value"] + >("ringing"); + const vm = createRingingMedia({ + pickupState$, + muteStates: { + video: { enabled$: constant(false) }, + } as unknown as MuteStates, + id: "test", + userId: "@alice:example.org", + displayName$: constant("Alice"), + mxcAvatarUrl$: constant(undefined), + }); + + const { container } = render( + + {}} + targetWidth={300} + targetHeight={200} + showSpeakingIndicators + focusable + /> + , + ); + expect(await axe(container)).toHaveNoViolations(); + // Name and status should be visible + screen.getByText("Alice"); + screen.getByText("Calling…"); + + // Alice declines the call + act(() => pickupState$.next("decline")); + screen.getByText("Call ended"); +}); diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index c8052a65..13cf677f 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -29,6 +29,9 @@ import { UserProfileIcon, VolumeOffSolidIcon, SwitchCameraSolidIcon, + VideoCallSolidIcon, + VoiceCallSolidIcon, + EndCallIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import { ContextMenu, @@ -49,6 +52,7 @@ import { useBehavior } from "../useBehavior"; import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel"; import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel"; import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; +import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; interface TileProps { ref?: Ref; @@ -56,21 +60,56 @@ interface TileProps { style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; - focusUrl: string | undefined; displayName: string; mxcAvatarUrl: string | undefined; - showSpeakingIndicators: boolean; focusable: boolean; } +interface RingingMediaTileProps extends TileProps { + vm: RingingMediaViewModel; +} + +const RingingMediaTile: FC = ({ + vm, + className, + ...props +}) => { + const { t } = useTranslation(); + const pickupState = useBehavior(vm.pickupState$); + const videoEnabled = useBehavior(vm.videoEnabled$); + + return ( + + ); +}; + interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; + showSpeakingIndicators: boolean; mirror: boolean; playbackMuted: boolean; waitingForMedia?: boolean; primaryButton?: ReactNode; menuStart?: ReactNode; menuEnd?: ReactNode; + focusUrl: string | undefined; } const UserMediaTile: FC = ({ @@ -95,7 +134,6 @@ const UserMediaTile: FC = ({ const { t } = useTranslation(); const video = useBehavior(vm.video$); const unencryptedWarning = useBehavior(vm.unencryptedWarning$); - const encryptionStatus = useBehavior(vm.encryptionStatus$); const audioStreamStats = useObservableEagerState< RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined >(vm.audioStreamStats$); @@ -153,7 +191,6 @@ const UserMediaTile: FC = ({ video={video} userId={vm.userId} unencryptedWarning={unencryptedWarning} - encryptionStatus={encryptionStatus} videoEnabled={videoEnabled} videoFit={videoFit} className={classNames(className, styles.tile, { @@ -218,6 +255,7 @@ UserMediaTile.displayName = "UserMediaTile"; interface LocalUserMediaTileProps extends TileProps { vm: LocalUserMediaViewModel; + showSpeakingIndicators: boolean; onOpenProfile: (() => void) | null; } @@ -232,6 +270,7 @@ const LocalUserMediaTile: FC = ({ const mirror = useBehavior(vm.mirror$); const alwaysShow = useBehavior(vm.alwaysShow$); const switchCamera = useBehavior(vm.switchCamera$); + const focusUrl = useBehavior(vm.focusUrl$); const latestAlwaysShow = useLatest(alwaysShow); const onSelectAlwaysShow = useCallback( @@ -278,6 +317,7 @@ const LocalUserMediaTile: FC = ({ ) } focusable={focusable} + focusUrl={focusUrl} {...props} /> ); @@ -287,6 +327,7 @@ LocalUserMediaTile.displayName = "LocalUserMediaTile"; interface RemoteUserMediaTileProps extends TileProps { vm: RemoteUserMediaViewModel; + showSpeakingIndicators: boolean; } const RemoteUserMediaTile: FC = ({ @@ -298,6 +339,8 @@ const RemoteUserMediaTile: FC = ({ const waitingForMedia = useBehavior(vm.waitingForMedia$); const playbackMuted = useBehavior(vm.playbackMuted$); const playbackVolume = useBehavior(vm.playbackVolume$); + const focusUrl = useBehavior(vm.focusUrl$); + const onSelectMute = useCallback( (e: Event) => { e.preventDefault(); @@ -338,6 +381,7 @@ const RemoteUserMediaTile: FC = ({ } + focusUrl={focusUrl} {...props} /> ); @@ -360,23 +404,33 @@ interface GridTileProps { export const GridTile: FC = ({ ref: theirRef, vm, + showSpeakingIndicators, onOpenProfile, ...props }) => { const ourRef = useRef(null); const ref = useMergedRefs(ourRef, theirRef); const media = useBehavior(vm.media$); - const focusUrl = useBehavior(media.focusUrl$); const displayName = useBehavior(media.displayName$); const mxcAvatarUrl = useBehavior(media.mxcAvatarUrl$); - if (media.local) { + if (media.type === "ringing") { + return ( + + ); + } else if (media.local) { return ( = ({ 0) { @@ -71,14 +76,15 @@ unconditionally select the container so we can use cqmin units */ .fg { position: absolute; - inset: var( + --fg-inset: var( --media-view-fg-inset, calc(var(--media-view-border-radius) - var(--cpd-space-3x)) ); + inset: var(--fg-inset); display: grid; grid-template-columns: 30px 1fr 30px; grid-template-rows: 1fr auto; - grid-template-areas: "reactions status ." "nameTag nameTag button"; + grid-template-areas: "status status reactions" "nameTag nameTag button"; gap: var(--cpd-space-1x); place-items: start; } @@ -102,21 +108,19 @@ unconditionally select the container so we can use cqmin units */ .status { grid-area: status; - justify-self: center; - align-self: start; - padding: var(--cpd-space-2x); - padding-block: var(--cpd-space-2x); color: var(--cpd-color-text-primary); - background-color: var(--cpd-color-bg-canvas-default); display: flex; + flex-wrap: none; align-items: center; - border-radius: var(--cpd-radius-pill-effect); + gap: 3px; user-select: none; overflow: hidden; - box-shadow: var(--small-drop-shadow); - box-sizing: border-box; - max-inline-size: 100%; - text-align: center; + margin-block-start: calc(var(--cpd-space-3x) - var(--fg-inset)); + margin-inline-start: calc(var(--cpd-space-4x) - var(--fg-inset)); +} + +.status svg { + color: var(--cpd-color-icon-tertiary); } .reactions { diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index a509d3a5..6ef5eb7e 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -18,7 +18,6 @@ import { TrackInfo } from "@livekit/protocol"; import { type ComponentProps } from "react"; import { MediaView } from "./MediaView"; -import { EncryptionStatus } from "../state/media/MemberMediaViewModel"; import { mockLocalParticipant } from "../utils/test"; describe("MediaView", () => { @@ -41,7 +40,6 @@ describe("MediaView", () => { videoFit: "contain", targetWidth: 300, targetHeight: 200, - encryptionStatus: EncryptionStatus.Connecting, mirror: false, unencryptedWarning: false, video: trackReference, diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index f912c069..eb6cc6b4 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -7,7 +7,13 @@ Please see LICENSE in the repository root for full details. import { type TrackReferenceOrPlaceholder } from "@livekit/components-core"; import { animated } from "@react-spring/web"; -import { type FC, type ComponentProps, type ReactNode } from "react"; +import { + type FC, + type ComponentProps, + type ReactNode, + type ComponentType, + type SVGAttributes, +} from "react"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; import { VideoTrack } from "@livekit/components-react"; @@ -16,7 +22,6 @@ import { ErrorSolidIcon } from "@vector-im/compound-design-tokens/assets/web/ico import styles from "./MediaView.module.css"; import { Avatar } from "../Avatar"; -import { type EncryptionStatus } from "../state/media/MemberMediaViewModel"; import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator"; import { showConnectionStats as showConnectionStatsSetting, @@ -38,7 +43,7 @@ interface Props extends ComponentProps { userId: string; videoEnabled: boolean; unencryptedWarning: boolean; - encryptionStatus: EncryptionStatus; + status?: { text: string; Icon: ComponentType> }; nameTagLeadingIcon?: ReactNode; displayName: string; mxcAvatarUrl: string | undefined; @@ -72,7 +77,7 @@ export const MediaView: FC = ({ mxcAvatarUrl, focusable, primaryButton, - encryptionStatus, + status, raisedHandTime, currentReaction, raisedHandOnClick, @@ -106,7 +111,11 @@ export const MediaView: FC = ({ name={displayName} size={avatarSize} src={mxcAvatarUrl} - className={styles.avatar} + className={classNames(styles.avatar, { + // When the avatar is overlaid with a status, make it translucent + // for readability + [styles.translucent]: status, + })} style={{ display: video && videoEnabled ? "none" : "initial" }} /> {video?.publication !== undefined && ( @@ -152,6 +161,14 @@ export const MediaView: FC = ({ /> )} + {status && ( +
+ + + {status.text} + +
+ )} {/* TODO: Bring this back once encryption status is less broken */} {/*encryptionStatus !== EncryptionStatus.Okay && (
diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index aac81b9c..533c3b2f 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -6,10 +6,11 @@ Please see LICENSE in the repository root for full details. */ import { test, expect, vi } from "vitest"; -import { isInaccessible, render, screen } from "@testing-library/react"; +import { act, isInaccessible, render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import userEvent from "@testing-library/user-event"; import { TooltipProvider } from "@vector-im/compound-web"; +import { BehaviorSubject } from "rxjs"; import { SpotlightTile } from "./SpotlightTile"; import { @@ -23,6 +24,11 @@ import { } from "../utils/test"; import { SpotlightTileViewModel } from "../state/TileViewModel"; import { constant } from "../state/Behavior"; +import { + createRingingMedia, + type RingingMediaViewModel, +} from "../state/media/RingingMediaViewModel"; +import { type MuteStates } from "../state/MuteStates"; global.IntersectionObserver = class MockIntersectionObserver { public observe(): void {} @@ -140,3 +146,41 @@ test("Screen share volume UI is hidden when screen share has no audio", async () screen.queryByRole("button", { name: /volume/i }), ).not.toBeInTheDocument(); }); + +test("SpotlightTile displays ringing media", async () => { + const pickupState$ = new BehaviorSubject< + RingingMediaViewModel["pickupState$"]["value"] + >("ringing"); + const vm = createRingingMedia({ + pickupState$, + muteStates: { + video: { enabled$: constant(false) }, + } as unknown as MuteStates, + id: "test", + userId: "@alice:example.org", + displayName$: constant("Alice"), + mxcAvatarUrl$: constant(undefined), + }); + + const toggleExpanded = vi.fn(); + const { container } = render( + , + ); + + expect(await axe(container)).toHaveNoViolations(); + // Alice should be in the spotlight with the right status + screen.getByText("Alice"); + screen.getByText("Calling…"); + + // Now we time out ringing to Alice + act(() => pickupState$.next("timeout")); + screen.getByText("Call ended"); +}); diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index aa66d6b6..c5faba40 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -24,6 +24,9 @@ import { VolumeOnIcon, VolumeOffSolidIcon, VolumeOnSolidIcon, + VideoCallSolidIcon, + VoiceCallSolidIcon, + EndCallIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import { animated } from "@react-spring/web"; import { type Observable, map } from "rxjs"; @@ -43,7 +46,7 @@ import { useReactiveState } from "../useReactiveState"; import { useLatest } from "../useLatest"; import { type SpotlightTileViewModel } from "../state/TileViewModel"; import { useBehavior } from "../useBehavior"; -import { type EncryptionStatus } from "../state/media/MemberMediaViewModel"; +import { type MemberMediaViewModel } from "../state/media/MemberMediaViewModel"; import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel"; import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel"; import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; @@ -52,6 +55,7 @@ import { type RemoteScreenShareViewModel } from "../state/media/RemoteScreenShar import { type MediaViewModel } from "../state/media/MediaViewModel"; import { Slider } from "../Slider"; import { platform } from "../Platform"; +import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; interface SpotlightItemBaseProps { ref?: Ref; @@ -59,18 +63,20 @@ interface SpotlightItemBaseProps { "data-id": string; targetWidth: number; targetHeight: number; - video: TrackReferenceOrPlaceholder | undefined; userId: string; - unencryptedWarning: boolean; - encryptionStatus: EncryptionStatus; - focusUrl: string | undefined; displayName: string; mxcAvatarUrl: string | undefined; focusable: boolean; "aria-hidden"?: boolean; } -interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps { +interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps { + video: TrackReferenceOrPlaceholder | undefined; + unencryptedWarning: boolean; + focusUrl: string | undefined; +} + +interface SpotlightUserMediaItemBaseProps extends SpotlightMemberMediaItemBaseProps { videoFit: "contain" | "cover"; videoEnabled: boolean; } @@ -103,21 +109,32 @@ const SpotlightRemoteUserMediaItem: FC = ({ ); }; -interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps { +interface SpotlightUserMediaItemProps extends SpotlightMemberMediaItemBaseProps { vm: UserMediaViewModel; } const SpotlightUserMediaItem: FC = ({ vm, + targetWidth, + targetHeight, ...props }) => { const videoFit = useBehavior(vm.videoFit$); const videoEnabled = useBehavior(vm.videoEnabled$); + // Whenever target bounds change, inform the viewModel + useEffect(() => { + if (targetWidth > 0 && targetHeight > 0) { + vm.setTargetDimensions(targetWidth, targetHeight); + } + }, [targetWidth, targetHeight, vm]); + const baseProps: SpotlightUserMediaItemBaseProps & RefAttributes = { videoFit, videoEnabled, + targetWidth, + targetHeight, ...props, }; @@ -130,7 +147,7 @@ const SpotlightUserMediaItem: FC = ({ SpotlightUserMediaItem.displayName = "SpotlightUserMediaItem"; -interface SpotlightScreenShareItemProps extends SpotlightItemBaseProps { +interface SpotlightScreenShareItemProps extends SpotlightMemberMediaItemBaseProps { vm: ScreenShareViewModel; videoEnabled: boolean; } @@ -142,7 +159,7 @@ const SpotlightScreenShareItem: FC = ({ return ; }; -interface SpotlightRemoteScreenShareItemProps extends SpotlightItemBaseProps { +interface SpotlightRemoteScreenShareItemProps extends SpotlightMemberMediaItemBaseProps { vm: RemoteScreenShareViewModel; } @@ -155,6 +172,67 @@ const SpotlightRemoteScreenShareItem: FC< ); }; +interface SpotlightMemberMediaItemProps extends SpotlightItemBaseProps { + vm: MemberMediaViewModel; +} + +const SpotlightMemberMediaItem: FC = ({ + vm, + ...props +}) => { + const video = useBehavior(vm.video$); + const unencryptedWarning = useBehavior(vm.unencryptedWarning$); + const focusUrl = useBehavior(vm.focusUrl$); + + const baseProps: SpotlightMemberMediaItemBaseProps & + RefAttributes = { + video: video ?? undefined, + unencryptedWarning, + focusUrl, + ...props, + }; + + if (vm.type === "user") + return ; + return vm.local ? ( + + ) : ( + + ); +}; + +interface SpotlightRingingMediaItemProps extends SpotlightItemBaseProps { + vm: RingingMediaViewModel; +} + +const SpotlightRingingMediaItem: FC = ({ + vm, + ...props +}) => { + const { t } = useTranslation(); + const pickupState = useBehavior(vm.pickupState$); + const videoEnabled = useBehavior(vm.videoEnabled$); + + return ( + + ); +}; + interface SpotlightItemProps { ref?: Ref; vm: MediaViewModel; @@ -187,22 +265,9 @@ const SpotlightItem: FC = ({ }) => { const ourRef = useRef(null); - // Whenever target bounds change, inform the viewModel - useEffect(() => { - if (targetWidth > 0 && targetHeight > 0) { - if (vm.type != "screen share") { - vm.setTargetDimensions(targetWidth, targetHeight); - } - } - }, [targetWidth, targetHeight, vm]); - const ref = useMergedRefs(ourRef, theirRef); - const focusUrl = useBehavior(vm.focusUrl$); const displayName = useBehavior(vm.displayName$); const mxcAvatarUrl = useBehavior(vm.mxcAvatarUrl$); - const video = useBehavior(vm.video$); - const unencryptedWarning = useBehavior(vm.unencryptedWarning$); - const encryptionStatus = useBehavior(vm.encryptionStatus$); // Hook this item up to the intersection observer useEffect(() => { @@ -225,23 +290,17 @@ const SpotlightItem: FC = ({ className: classNames(styles.item, { [styles.snap]: snap }), targetWidth, targetHeight, - video: video ?? undefined, userId: vm.userId, - unencryptedWarning, - focusUrl, displayName, mxcAvatarUrl, focusable, - encryptionStatus, "aria-hidden": ariaHidden, }; - if (vm.type === "user") - return ; - return vm.local ? ( - + return vm.type === "ringing" ? ( + ) : ( - + ); };