New ringing UI

This implements the new ringing UI by showing a placeholder tile for the participant being dialed, rather than an overlay.
This commit is contained in:
Robin
2026-03-16 13:12:49 +01:00
parent 6d14f1d06f
commit 9dfade68ee
27 changed files with 703 additions and 478 deletions

View File

@@ -249,6 +249,8 @@
"version": "{{productName}} version: {{version}}",
"video_tile": {
"always_show": "Always show",
"call_ended": "Call ended",
"calling": "Calling…",
"camera_starting": "Video loading...",
"collapse": "Collapse",
"expand": "Expand",

View File

@@ -34,9 +34,12 @@ widgetTest(
.locator('iframe[title="Element Call"]')
.contentFrame();
// We should show a ringing overlay, let's check for that
// We should show a ringing tile, let's check for that
await expect(
brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`),
brooksFrame
.getByTestId("videoTile")
.filter({ has: brooksFrame.getByText(whistler.displayName) })
.filter({ has: brooksFrame.getByText("Calling…") }),
).toBeVisible();
await expect(whistler.page.getByText("Incoming voice call")).toBeVisible();
@@ -125,9 +128,12 @@ widgetTest(
.locator('iframe[title="Element Call"]')
.contentFrame();
// We should show a ringing overlay, let's check for that
// We should show a ringing tile, let's check for that
await expect(
brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`),
brooksFrame
.getByTestId("videoTile")
.filter({ has: brooksFrame.getByText(whistler.displayName) })
.filter({ has: brooksFrame.getByText("Calling…") }),
).toBeVisible();
await expect(whistler.page.getByText("Incoming video call")).toBeVisible();
@@ -216,9 +222,12 @@ widgetTest(
.locator('iframe[title="Element Call"]')
.contentFrame();
// We should show a ringing overlay, let's check for that
// We should show a ringing tile, let's check for that
await expect(
brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`),
brooksFrame
.getByTestId("videoTile")
.filter({ has: brooksFrame.getByText(whistler.displayName) })
.filter({ has: brooksFrame.getByText("Calling…") }),
).toBeVisible();
await expect(whistler.page.getByText("Incoming video call")).toBeVisible();

View File

@@ -51,15 +51,15 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
return (
<div ref={ref} className={styles.layer}>
<Slot
id={model.remote.id}
model={model.remote}
id={model.spotlight.id}
model={model.spotlight}
className={styles.container}
style={{ width: tileWidth, height: tileHeight }}
>
<Slot
className={classNames(styles.slot, styles.local)}
id={model.local.id}
model={model.local}
id={model.pip.id}
model={model.pip}
onDrag={onDragLocalTile}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { IconButton, Text, Tooltip } from "@vector-im/compound-web";
import { IconButton, Tooltip } from "@vector-im/compound-web";
import { type MatrixClient, type Room as MatrixRoom } from "matrix-js-sdk";
import {
type FC,
@@ -98,8 +98,6 @@ import { useAppBarHidden, useAppBarSecondaryButton } from "../AppBar.tsx";
import { useBehavior } from "../useBehavior.ts";
import { Toast } from "../Toast.tsx";
import overlayStyles from "../Overlay.module.css";
import { Avatar, Size as AvatarSize } from "../Avatar";
import waitingStyles from "./WaitingForJoin.module.css";
import { prefetchSounds } from "../soundUtils";
import { useAudioContext } from "../useAudioContext";
import ringtoneMp3 from "../sound/ringtone.mp3?url";
@@ -107,6 +105,7 @@ import ringtoneOgg from "../sound/ringtone.ogg?url";
import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx";
import { type Layout } from "../state/layout-types.ts";
import { ObservableScope } from "../state/ObservableScope.ts";
import { useLatest } from "../useLatest.ts";
const logger = rootLogger.getChild("[InCallView]");
@@ -224,8 +223,6 @@ export const InCallView: FC<InCallViewProps> = ({
const { showControls } = useUrlParams();
const muteAllAudio = useBehavior(muteAllAudio$);
// Call pickup state and display names are needed for waiting overlay/sounds
const callPickupState = useBehavior(vm.callPickupState$);
// Preload a waiting and decline sounds
const pickupPhaseSoundCache = useInitial(async () => {
@@ -239,6 +236,7 @@ export const InCallView: FC<InCallViewProps> = ({
latencyHint: "interactive",
muted: muteAllAudio,
});
const latestPickupPhaseAudio = useLatest(pickupPhaseAudio);
const audioEnabled = useBehavior(muteStates.audio.enabled$);
const videoEnabled = useBehavior(muteStates.video.enabled$);
@@ -257,6 +255,7 @@ export const InCallView: FC<InCallViewProps> = ({
() => void toggleRaisedHand(),
);
const ringing = useBehavior(vm.ringing$);
const audioParticipants = useBehavior(vm.livekitRoomItems$);
const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$);
@@ -271,7 +270,6 @@ export const InCallView: FC<InCallViewProps> = ({
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
const sharingScreen = useBehavior(vm.sharingScreen$);
const ringOverlay = useBehavior(vm.ringOverlay$);
const fatalCallError = useBehavior(vm.fatalError$);
// Stop the rendering and throw for the error boundary
if (fatalCallError) {
@@ -279,58 +277,21 @@ export const InCallView: FC<InCallViewProps> = ({
throw fatalCallError;
}
// We need to set the proper timings on the animation based upon the sound length.
const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1;
useEffect((): (() => void) => {
// The CSS animation includes the delay, so we must double the length of the sound.
window.document.body.style.setProperty(
"--call-ring-duration-s",
`${ringDuration * 2}s`,
);
window.document.body.style.setProperty(
"--call-ring-delay-s",
`${ringDuration}s`,
);
// Remove properties when we unload.
return () => {
window.document.body.style.removeProperty("--call-ring-duration-s");
window.document.body.style.removeProperty("--call-ring-delay-s");
};
}, [pickupPhaseAudio?.soundDuration, ringDuration]);
// When waiting for pickup, loop a waiting sound
// While ringing, loop the ringtone
useEffect((): void | (() => void) => {
if (callPickupState !== "ringing" || !pickupPhaseAudio) return;
const endSound = pickupPhaseAudio.playSoundLooping("waiting", ringDuration);
return () => {
void endSound().catch((e) => {
logger.error("Failed to stop ringing sound", e);
});
};
}, [callPickupState, pickupPhaseAudio, ringDuration]);
// Waiting UI overlay
const waitingOverlay: JSX.Element | null = useMemo(() => {
return ringOverlay ? (
<div className={classNames(overlayStyles.bg, waitingStyles.overlay)}>
<div
className={classNames(overlayStyles.content, waitingStyles.content)}
>
<div className={waitingStyles.pulse}>
<Avatar
id={ringOverlay.idForAvatar}
name={ringOverlay.name}
src={ringOverlay.avatarMxc}
size={AvatarSize.XL}
/>
</div>
<Text size="md" className={waitingStyles.text}>
{ringOverlay.text}
</Text>
</div>
</div>
) : null;
}, [ringOverlay]);
const audio = latestPickupPhaseAudio.current;
if (ringing && audio) {
const endSound = audio.playSoundLooping(
"waiting",
audio.soundDuration["waiting"] ?? 1,
);
return () => {
void endSound().catch((e) => {
logger.error("Failed to stop ringing sound", e);
});
};
}
}, [ringing, latestPickupPhaseAudio]);
const onViewClick = useCallback(
(e: ReactMouseEvent) => {
@@ -764,7 +725,6 @@ export const InCallView: FC<InCallViewProps> = ({
{reconnectingToast}
{earpieceOverlay}
<ReactionsOverlay vm={vm} />
{waitingOverlay}
{footer}
{layout.type !== "pip" && (
<>

View File

@@ -1,61 +0,0 @@
.overlay {
position: absolute;
inset: 0;
display: flex;
align-items: center;
justify-content: center;
pointer-events: none;
}
.content {
display: flex;
flex-direction: column;
align-items: center;
gap: 16px;
}
.pulse {
position: relative;
height: 90px;
}
.pulse::before {
content: "";
position: absolute;
inset: -12px;
border-radius: 9999px;
border: 12px solid rgba(255, 255, 255, 0.6);
animation: pulse var(--call-ring-duration-s) ease-out infinite;
animation-delay: 1s;
opacity: 0;
}
.text {
color: var(--cpd-color-text-on-solid-primary);
}
@keyframes pulse {
0% {
transform: scale(0.95);
opacity: 0.7;
transform: scale(0);
opacity: 1;
}
35% {
transform: scale(1.15);
opacity: 0.15;
}
50% {
transform: scale(1.2);
opacity: 0;
}
50.01% {
transform: scale(0);
}
85% {
transform: scale(0);
}
100% {
transform: scale(0);
}
}

View File

@@ -89,7 +89,6 @@ export interface Props {
* `callPickupState$` The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.
* This may also be set if we are disconnected.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
* The call failed. If desired this can be used as a trigger to exit the call.
@@ -131,15 +130,9 @@ export function createCallNotificationLifecycle$({
) as Behavior<Epoch<boolean>>;
/**
* Whenever the RTC session tells us that it intends to ring the remote
* participant's devices, this emits an Observable tracking the current state of
* that ringing process.
* The state of the current ringing attempt, if the RTC session is indeed
* ringing the remote participant's devices. Otherwise `null`.
*/
// This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$`
// has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`.
// A behavior will emit the latest observable with the running timer to new subscribers.
// see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if
// `ring$` would not be a behavior.
const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> =
scope.behavior(
sentCallNotification$.pipe(

View File

@@ -46,9 +46,11 @@ import {
} from "../../utils/test.ts";
import { E2eeType } from "../../e2ee/e2eeType.ts";
import {
alice,
aliceId,
aliceParticipant,
aliceRtcMember,
aliceUserId,
bobId,
bobRtcMember,
local,
@@ -140,8 +142,8 @@ export interface SpotlightExpandedLayoutSummary {
export interface OneOnOneLayoutSummary {
type: "one-on-one";
local: string;
remote: string;
spotlight: string;
pip: string;
}
export interface PipLayoutSummary {
@@ -194,11 +196,11 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
);
case "one-on-one":
return combineLatest(
[l.local.media$, l.remote.media$],
(local, remote) => ({
[l.spotlight.media$, l.pip.media$],
(spotlight, pip) => ({
type: l.type,
local: local.id,
remote: remote.id,
spotlight: spotlight.id,
pip: pip.id,
}),
);
case "pip":
@@ -537,8 +539,8 @@ describe.each([
b: {
// In a larger window, expect the normal one-on-one layout
type: "one-on-one",
local: `${localId}:0`,
remote: `${aliceId}:0`,
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
c: {
// In a PiP-sized window, we of course expect a PiP layout
@@ -840,8 +842,8 @@ describe.each([
},
b: {
type: "one-on-one",
local: `${localId}:0`,
remote: `${aliceId}:0`,
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
c: {
type: "grid",
@@ -883,8 +885,8 @@ describe.each([
},
b: {
type: "one-on-one",
local: `${localId}:0`,
remote: `${aliceId}:0`,
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
c: {
type: "grid",
@@ -893,8 +895,8 @@ describe.each([
},
d: {
type: "one-on-one",
local: `${localId}:0`,
remote: `${daveId}:0`,
pip: `${localId}:0`,
spotlight: `${daveId}:0`,
},
},
);
@@ -1087,83 +1089,81 @@ describe.each([
});
});
describe("waitForCallPickup$", () => {
it.skip("regression test: does stop ringing in case livekitConnectionState$ emits after didSendCallNotification$ has already emitted", () => {
withTestScheduler(({ schedule, expectObservable, behavior }) => {
withCallViewModel(
{
livekitConnectionState$: behavior("d 9ms c", {
d: ConnectionState.Disconnected,
c: ConnectionState.Connected,
}),
},
(vm, rtcSession) => {
// Fire a call notification IMMEDIATELY (its important for this test, that this happens before the livekitConnectionState$ emits)
schedule("n", {
n: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
mockRingEvent("$notif1", 30),
);
},
});
test("recipient has placeholder tile while ringing or timed out", () => {
withTestScheduler(({ schedule, expectObservable }) => {
withCallViewModel(
{
roomMembers: [alice, local], // Simulate a DM
},
(vm, rtcSession) => {
// Fire a ringing notification
schedule("n", {
n: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
mockRingEvent("$notif1", 30),
);
},
});
expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", {
a: "unknown",
b: "ringing",
c: "timeout",
});
},
{
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
// Should ring for 30ms and then time out
expectObservable(vm.ringing$).toBe("(ny) 26ms n", yesNo);
// Layout should show placeholder media for the participant we're
// ringing the entire time (even once timed out)
expectObservable(summarizeLayout$(vm.layout$)).toBe("a", {
a: {
type: "one-on-one",
spotlight: `${localId}:0`,
pip: `ringing:${aliceUserId}`,
},
});
},
{ waitForCallPickup: true },
);
});
});
it.skip("ringing -> unknown if we get disconnected", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
const connectionState$ = new BehaviorSubject(ConnectionState.Connected);
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
withCallViewModel(
{
remoteParticipants$: behavior("a 19ms b", {
a: [],
b: [aliceParticipant],
}),
rtcMembers$: behavior("a 19ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
livekitConnectionState$: connectionState$,
},
(vm, rtcSession) => {
// Notify at 5ms so we enter ringing, then get disconnected 5ms later
schedule(" 5ms r 5ms d", {
r: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
mockRingEvent("$notif2", 100),
);
},
d: () => {
connectionState$.next(ConnectionState.Disconnected);
},
});
test("recipient's placeholder tile is replaced by their real tile once they answer", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
withCallViewModel(
{
// Alice answers after 20ms
rtcMembers$: behavior("a 20ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
roomMembers: [alice, local], // Simulate a DM
},
(vm, rtcSession) => {
// Fire a ringing notification
schedule("n", {
n: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
mockRingEvent("$notif1", 30),
);
},
});
expectObservable(vm.callPickupState$).toBe("a 4ms b 5ms c", {
a: "unknown",
b: "ringing",
c: "unknown",
});
},
{
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
// Should ring until Alice joins
expectObservable(vm.ringing$).toBe("(ny) 17ms n", yesNo);
// Layout should show placeholder media for the participant we're
// ringing the entire time
expectObservable(summarizeLayout$(vm.layout$)).toBe("a 20ms b", {
a: {
type: "one-on-one",
spotlight: `${localId}:0`,
pip: `ringing:${aliceUserId}`,
},
b: {
type: "one-on-one",
spotlight: `${aliceId}:0`,
pip: `${localId}:0`,
},
});
},
{ waitForCallPickup: true },
);
});
});

View File

@@ -128,7 +128,6 @@ import {
createSentCallNotification$,
} from "./CallNotificationLifecycle.ts";
import {
createDMMember$,
createMatrixMemberMetadata$,
createRoomMembers$,
} from "./remoteMembers/MatrixMemberMetadata.ts";
@@ -137,12 +136,17 @@ import { type Connection } from "./remoteMembers/Connection.ts";
import { createLayoutModeSwitch } from "./LayoutSwitch.ts";
import {
createWrappedUserMedia,
type MediaItem,
type WrappedUserMediaViewModel,
} from "../media/MediaItem.ts";
} from "../media/WrappedUserMediaViewModel.ts";
import { type ScreenShareViewModel } from "../media/ScreenShareViewModel.ts";
import { type UserMediaViewModel } from "../media/UserMediaViewModel.ts";
import { type MediaViewModel } from "../media/MediaViewModel.ts";
import { type LocalUserMediaViewModel } from "../media/LocalUserMediaViewModel.ts";
import { type RemoteUserMediaViewModel } from "../media/RemoteUserMediaViewModel.ts";
import {
createRingingMedia,
type RingingMediaViewModel,
} from "../media/RingingMediaViewModel.ts";
const logger = rootLogger.getChild("[CallViewModel]");
//TODO
@@ -210,11 +214,10 @@ export type LivekitRoomItem = {
export interface CallViewModel {
// lifecycle
autoLeave$: Observable<AutoLeaveReason>;
// TODO if we are in "unknown" state we need a loading rendering (or empty screen)
// Otherwise it looks like we already connected and only than the ringing starts which is weird.
callPickupState$: Behavior<
"unknown" | "ringing" | "timeout" | "decline" | "success" | null
>;
/**
* Whether we are ringing a call recipient.
*/
ringing$: Behavior<boolean>;
/** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is
* - by ending the scope
@@ -289,13 +292,6 @@ export interface CallViewModel {
/** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
reactions$: Behavior<Record<string, ReactionOption>>;
ringOverlay$: Behavior<null | {
name: string;
/** roomId or userId for the avatar generation. */
idForAvatar: string;
text: string;
avatarMxc?: string;
}>;
// sounds and events
joinSoundEffect$: Observable<void>;
leaveSoundEffect$: Observable<void>;
@@ -611,40 +607,6 @@ export function createCallViewModel$(
matrixRoomMembers$,
);
const dmMember$ = createDMMember$(scope, matrixRoomMembers$, matrixRoom);
const noUserToCallInRoom$ = scope.behavior(
matrixRoomMembers$.pipe(
map(
(roomMembersMap) =>
roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined,
),
),
);
const ringOverlay$ = scope.behavior(
combineLatest([noUserToCallInRoom$, dmMember$, callPickupState$]).pipe(
map(([noUserToCallInRoom, dmMember, callPickupState]) => {
// No overlay if not in ringing state
if (callPickupState !== "ringing" || noUserToCallInRoom) return null;
const name = dmMember ? dmMember.rawDisplayName : matrixRoom.name;
const id = dmMember ? dmMember.userId : matrixRoom.roomId;
const text = dmMember
? `Waiting for ${name} to join…`
: "Waiting for other participants…";
const avatarMxc = dmMember
? (dmMember.getMxcAvatarUrl?.() ?? undefined)
: (matrixRoom.getMxcAvatarUrl() ?? undefined);
return {
name: name ?? id,
idForAvatar: id,
text,
avatarMxc,
};
}),
),
);
const allConnections$ = scope.behavior(
connectionManager.connectionManagerData$.pipe(map((d) => d.value)),
);
@@ -720,7 +682,7 @@ export function createCallViewModel$(
matrixLivekitMembers$,
duplicateTiles.value$,
]).pipe(
// Generate a collection of MediaItems from the list of expected (whether
// Generate a collection of user media from the list of expected (whether
// present or missing) LiveKit participants.
generateItems(
"CallViewModel userMedia$",
@@ -793,32 +755,67 @@ export function createCallViewModel$(
),
);
const ringingMedia$ = scope.behavior<RingingMediaViewModel[]>(
combineLatest([userMedia$, matrixRoomMembers$, callPickupState$]).pipe(
generateItems(
"CallViewModel ringingMedia$",
function* ([userMedia, roomMembers, callPickupState]) {
if (
callPickupState === "ringing" ||
callPickupState === "timeout" ||
callPickupState === "decline"
) {
for (const member of roomMembers.values()) {
if (!userMedia.some((vm) => vm.userId === member.userId))
yield {
keys: [member.userId],
data: callPickupState,
};
}
}
},
(scope, pickupState$, userId) =>
createRingingMedia({
id: `ringing:${userId}`,
userId,
displayName$: scope.behavior(
matrixRoomMembers$.pipe(
map((members) => members.get(userId)?.rawDisplayName || userId),
),
),
mxcAvatarUrl$:
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
pickupState$,
muteStates,
}),
),
distinctUntilChanged(shallowEquals),
tap((ringingMedia) => {
if (ringingMedia.length > 1)
// Warn that UI may do something unexpected in this case
logger.warn(
`Ringing more than one participant is not supported (ringing ${ringingMedia.map((vm) => vm.userId).join(", ")})`,
);
}),
),
);
/**
* List of all media items (user media and screen share media) that we want
* tiles for.
* All screen share media that we want to display.
*/
const mediaItems$ = scope.behavior<MediaItem[]>(
const screenShares$ = scope.behavior<ScreenShareViewModel[]>(
userMedia$.pipe(
switchMap((userMedia) =>
userMedia.length === 0
? of([])
: combineLatest(
userMedia.map((m) => m.screenShares$),
(...screenShares) => [...userMedia, ...screenShares.flat(1)],
(...screenShares) => screenShares.flat(1),
),
),
),
);
/**
* List of MediaItems that we want to display, that are of type ScreenShare
*/
const screenShares$ = scope.behavior<ScreenShareViewModel[]>(
mediaItems$.pipe(
map((mediaItems) => mediaItems.filter((m) => m.type === "screen share")),
),
);
const joinSoundEffect$ = userMedia$.pipe(
pairwise(),
filter(
@@ -931,40 +928,20 @@ export function createCallViewModel$(
),
);
const spotlight$ = scope.behavior<MediaViewModel[]>(
screenShares$.pipe(
switchMap((screenShares) => {
if (screenShares.length > 0) return of(screenShares);
return spotlightSpeaker$.pipe(
map((speaker) => (speaker ? [speaker] : [])),
/**
* Local user media suitable for displaying in a PiP (undefined if not found
* or if user prefers to not see themselves).
*/
const localUserMediaForPip$ = scope.behavior<
LocalUserMediaViewModel | undefined
>(
userMedia$.pipe(
switchMap((userMedia) => {
const localUserMedia = userMedia.find(
(m): m is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
m.type === "user" && m.local,
);
}),
distinctUntilChanged<MediaViewModel[]>(shallowEquals),
),
);
const pip$ = scope.behavior<UserMediaViewModel | undefined>(
combineLatest([
// TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits
screenShares$,
spotlightSpeaker$,
mediaItems$,
]).pipe(
switchMap(([screenShares, spotlight, mediaItems]) => {
if (screenShares.length > 0) {
return spotlightSpeaker$;
}
if (!spotlight || spotlight.local) {
return of(undefined);
}
const localUserMedia = mediaItems.find(
(m) => m.type === "user" && m.local,
);
if (!localUserMedia) {
return of(undefined);
}
if (!localUserMedia) return of(undefined);
return localUserMedia.alwaysShow$.pipe(
map((alwaysShow) => (alwaysShow ? localUserMedia : undefined)),
);
@@ -972,6 +949,39 @@ export function createCallViewModel$(
),
);
const spotlightAndPip$ = scope.behavior<{
spotlight: MediaViewModel[];
pip$: Behavior<UserMediaViewModel | undefined>;
}>(
ringingMedia$.pipe(
switchMap((ringingMedia) => {
if (ringingMedia.length > 0)
return of({ spotlight: ringingMedia, pip$: localUserMediaForPip$ });
return screenShares$.pipe(
switchMap((screenShares) => {
if (screenShares.length > 0)
return of({ spotlight: screenShares, pip$: spotlightSpeaker$ });
return spotlightSpeaker$.pipe(
map((speaker) => ({
spotlight: speaker ? [speaker] : [],
pip$: localUserMediaForPip$,
})),
);
}),
);
}),
),
);
const spotlight$ = scope.behavior<MediaViewModel[]>(
spotlightAndPip$.pipe(
map(({ spotlight }) => spotlight),
distinctUntilChanged<MediaViewModel[]>(shallowEquals),
),
);
const hasRemoteScreenShares$ = scope.behavior<boolean>(
spotlight$.pipe(
map((spotlight) =>
@@ -1054,24 +1064,61 @@ export function createCallViewModel$(
}));
const spotlightExpandedLayoutMedia$: Observable<SpotlightExpandedLayoutMedia> =
combineLatest([spotlight$, pip$], (spotlight, pip) => ({
type: "spotlight-expanded",
spotlight,
pip: pip ?? undefined,
}));
spotlightAndPip$.pipe(
switchMap(({ spotlight, pip$ }) =>
pip$.pipe(
map((pip) => ({
type: "spotlight-expanded" as const,
spotlight,
pip: pip ?? undefined,
})),
),
),
);
const oneOnOneLayoutMedia$: Observable<OneOnOneLayoutMedia | null> =
mediaItems$.pipe(
map((mediaItems) => {
if (mediaItems.length !== 2) return null;
const local = mediaItems.find((vm) => vm.type === "user" && vm.local);
const remote = mediaItems.find((vm) => vm.type === "user" && !vm.local);
// There might not be a remote tile if there are screen shares, or if
// only the local user is in the call and they're using the duplicate
// tiles option
if (!remote || !local) return null;
userMedia$.pipe(
switchMap((userMedia) => {
if (userMedia.length <= 2) {
const local = userMedia.find(
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
vm.type === "user" && vm.local,
);
return { type: "one-on-one", local, remote };
if (local !== undefined) {
const remote = userMedia.find(
(
vm,
): vm is WrappedUserMediaViewModel & RemoteUserMediaViewModel =>
vm.type === "user" && !vm.local,
);
if (remote !== undefined)
return of({
type: "one-on-one" as const,
spotlight: remote,
pip: local,
});
// If there's no other user media in the call (could still happen in
// this branch due to the duplicate tiles option), we could possibly
// show ringing media instead
if (userMedia.length === 1)
return ringingMedia$.pipe(
map((ringingMedia) => {
return ringingMedia.length === 1
? {
type: "one-on-one" as const,
spotlight: local,
pip: ringingMedia[0],
}
: null;
}),
);
}
}
return of(null);
}),
);
@@ -1482,8 +1529,9 @@ export function createCallViewModel$(
return {
autoLeave$: autoLeave$,
callPickupState$: callPickupState$,
ringOverlay$: ringOverlay$,
ringing$: scope.behavior(
callPickupState$.pipe(map((state) => state === "ringing")),
),
leave$: leave$,
hangup: (): void => userHangup$.next(),
join: localMembership.requestJoinAndPublish,

View File

@@ -17,7 +17,7 @@ import {
import { SyncState } from "matrix-js-sdk/lib/sync";
import { BehaviorSubject, type Observable, map, of } from "rxjs";
import { onTestFinished, vi } from "vitest";
import { ClientEvent, type MatrixClient } from "matrix-js-sdk";
import { ClientEvent, type RoomMember, type MatrixClient } from "matrix-js-sdk";
import EventEmitter from "events";
import * as ComponentsCore from "@livekit/components-core";
@@ -63,15 +63,10 @@ const carol = local;
const dave = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "Dave" });
const roomMembers = new Map(
[alice, aliceDoppelganger, bob, bobZeroWidthSpace, carol, dave, daveRTL].map(
(p) => [p.userId, p],
),
);
export interface CallViewModelInputs {
remoteParticipants$: Behavior<RemoteParticipant[]>;
rtcMembers$: Behavior<Partial<CallMembership>[]>;
roomMembers: RoomMember[];
livekitConnectionState$: Behavior<ConnectionState>;
speaking: Map<Participant, Observable<boolean>>;
mediaDevices: MediaDevices;
@@ -86,6 +81,15 @@ export function withCallViewModel(mode: MatrixRTCMode) {
{
remoteParticipants$ = constant([]),
rtcMembers$ = constant([localRtcMember]),
roomMembers = [
alice,
aliceDoppelganger,
bob,
bobZeroWidthSpace,
carol,
dave,
daveRTL,
],
livekitConnectionState$: connectionState$ = constant(
ConnectionState.Connected,
),
@@ -128,8 +132,8 @@ export function withCallViewModel(mode: MatrixRTCMode) {
return syncState;
}
})() as Partial<MatrixClient> as MatrixClient,
getMembers: () => Array.from(roomMembers.values()),
getMembersWithMembership: () => Array.from(roomMembers.values()),
getMembers: () => roomMembers,
getMembersWithMembership: () => roomMembers,
});
const rtcSession = new MockRTCSession(room, []).withMemberships(
rtcMembers$,

View File

@@ -54,31 +54,6 @@ export function createRoomMembers$(
);
}
/**
* creates the member that this DM is with in case it is a DM (two members) otherwise null
*/
export function createDMMember$(
scope: ObservableScope,
roomMembers$: Behavior<RoomMemberMap>,
matrixRoom: MatrixRoom,
): Behavior<Pick<
RoomMember,
"userId" | "getMxcAvatarUrl" | "rawDisplayName"
> | null> {
// We cannot use the normal direct check from matrix since we do not have access to the account data.
// use primitive member count === 2 check instead.
return scope.behavior(
roomMembers$.pipe(
map((membersMap) => {
// primitive appraoch do to no access to account data.
const isDM = membersMap.size === 2;
if (!isDM) return null;
return matrixRoom.getMember(matrixRoom.guessDMUserId());
}),
),
);
}
/**
* Displayname for each member of the call. This will disambiguate
* any displayname that clashes with another member. Only members

View File

@@ -16,14 +16,14 @@ export function oneOnOneLayout(
prevTiles: TileStore,
): [OneOnOneLayout, TileStore] {
const update = prevTiles.from(2);
update.registerGridTile(media.local);
update.registerGridTile(media.remote);
update.registerGridTile(media.pip);
update.registerGridTile(media.spotlight);
const tiles = update.build();
return [
{
type: media.type,
local: tiles.gridTilesByMedia.get(media.local)!,
remote: tiles.gridTilesByMedia.get(media.remote)!,
spotlight: tiles.gridTilesByMedia.get(media.spotlight)!,
pip: tiles.gridTilesByMedia.get(media.pip)!,
},
tiles,
];

View File

@@ -13,6 +13,7 @@ import { fillGaps } from "../utils/iter";
import { debugTileLayout } from "../settings/settings";
import { type MediaViewModel } from "./media/MediaViewModel";
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
function debugEntries(entries: GridTileData[]): string[] {
return entries.map((e) => e.media.displayName$.value);
@@ -48,8 +49,10 @@ class SpotlightTileData {
}
class GridTileData {
private readonly media$: BehaviorSubject<UserMediaViewModel>;
public get media(): UserMediaViewModel {
private readonly media$: BehaviorSubject<
UserMediaViewModel | RingingMediaViewModel
>;
public get media(): UserMediaViewModel | RingingMediaViewModel {
return this.media$.value;
}
public set media(value: UserMediaViewModel) {
@@ -58,7 +61,7 @@ class GridTileData {
public readonly vm: GridTileViewModel;
public constructor(media: UserMediaViewModel) {
public constructor(media: UserMediaViewModel | RingingMediaViewModel) {
this.media$ = new BehaviorSubject(media);
this.vm = new GridTileViewModel(this.media$);
}
@@ -178,7 +181,9 @@ export class TileStoreBuilder {
* Sets up a grid tile for the given media. If this is never called for some
* media, then that media will have no grid tile.
*/
public registerGridTile(media: UserMediaViewModel): void {
public registerGridTile(
media: UserMediaViewModel | RingingMediaViewModel,
): void {
if (DEBUG_ENABLED)
logger.debug(
`[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`,
@@ -187,7 +192,11 @@ export class TileStoreBuilder {
if (this.spotlight !== null) {
// We actually *don't* want spotlight speakers to appear in both the
// spotlight and the grid, so they're filtered out here
if (!media.local && this.spotlight.media.includes(media)) return;
if (
!(media.type === "user" && media.local) &&
this.spotlight.media.includes(media)
)
return;
// When the spotlight speaker changes, we would see one grid tile appear
// and another grid tile disappear. This would be an undesirable layout
// shift, so instead what we do is take the speaker's grid tile and swap

View File

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
import { type Behavior } from "./Behavior";
import { type MediaViewModel } from "./media/MediaViewModel";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
let nextId = 0;
@@ -17,7 +18,11 @@ function createId(): string {
export class GridTileViewModel {
public readonly id = createId();
public constructor(public readonly media$: Behavior<UserMediaViewModel>) {}
public constructor(
public readonly media$: Behavior<
UserMediaViewModel | RingingMediaViewModel
>,
) {}
}
export class SpotlightTileViewModel {

View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts";
import { type MediaViewModel } from "./media/MediaViewModel.ts";
import { type RemoteUserMediaViewModel } from "./media/RemoteUserMediaViewModel.ts";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel.ts";
import { type UserMediaViewModel } from "./media/UserMediaViewModel.ts";
import {
type GridTileViewModel,
@@ -40,8 +40,8 @@ export interface SpotlightExpandedLayoutMedia {
export interface OneOnOneLayoutMedia {
type: "one-on-one";
local: LocalUserMediaViewModel;
remote: RemoteUserMediaViewModel;
spotlight: UserMediaViewModel;
pip: LocalUserMediaViewModel | RingingMediaViewModel;
}
export interface PipLayoutMedia {
@@ -86,8 +86,8 @@ export interface SpotlightExpandedLayout {
export interface OneOnOneLayout {
type: "one-on-one";
local: GridTileViewModel;
remote: GridTileViewModel;
spotlight: GridTileViewModel;
pip: GridTileViewModel;
}
export interface PipLayout {

View File

@@ -7,13 +7,17 @@ Please see LICENSE in the repository root for full details.
*/
import { type Behavior } from "../Behavior";
import { type RingingMediaViewModel } from "./RingingMediaViewModel";
import { type ScreenShareViewModel } from "./ScreenShareViewModel";
import { type UserMediaViewModel } from "./UserMediaViewModel";
/**
* A participant's media.
*/
export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel;
export type MediaViewModel =
| UserMediaViewModel
| ScreenShareViewModel
| RingingMediaViewModel;
/**
* Properties which are common to all MediaViewModels.

View File

@@ -38,6 +38,8 @@ import { type ObservableScope } from "../ObservableScope";
import { observeTrackReference$ } from "../observeTrackReference";
import { E2eeType } from "../../e2ee/e2eeType";
import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats";
import { type UserMediaViewModel } from "./UserMediaViewModel";
import { type ScreenShareViewModel } from "./ScreenShareViewModel";
// TODO: Encryption status is kinda broken and thus unused right now. Remove?
export enum EncryptionStatus {
@@ -49,9 +51,9 @@ export enum EncryptionStatus {
}
/**
* Media belonging to an active member of the RTC session.
* Properties common to all MemberMediaViewModels.
*/
export interface MemberMediaViewModel extends BaseMediaViewModel {
export interface BaseMemberMediaViewModel extends BaseMediaViewModel {
/**
* The LiveKit video track for this media.
*/
@@ -88,7 +90,7 @@ export function createMemberMedia(
encryptionSystem,
...inputs
}: MemberMediaInputs,
): MemberMediaViewModel {
): BaseMemberMediaViewModel {
const trackBehavior$ = (
source: Track.Source,
): Behavior<TrackReference | undefined> =>
@@ -270,3 +272,8 @@ function observeRemoteTrackReceivingOkay$(
startWith(undefined),
);
}
/**
* Media belonging to an active member of the call.
*/
export type MemberMediaViewModel = UserMediaViewModel | ScreenShareViewModel;

View File

@@ -0,0 +1,51 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Behavior } from "../Behavior";
import { type MuteStates } from "../MuteStates";
import {
type BaseMediaInputs,
type BaseMediaViewModel,
createBaseMedia,
} from "./MediaViewModel";
/**
* Media representing a user who is not yet part of the call — one that we are
* *ringing*.
*/
export interface RingingMediaViewModel extends BaseMediaViewModel {
type: "ringing";
pickupState$: Behavior<"ringing" | "timeout" | "decline">;
/**
* Whether this media would be expected to have video, were it not simply a
* placeholder.
*/
videoEnabled$: Behavior<boolean>;
}
export interface RingingMediaInputs extends BaseMediaInputs {
pickupState$: Behavior<"ringing" | "timeout" | "decline">;
/**
* The local user's own mute states.
*/
muteStates: MuteStates;
}
export function createRingingMedia({
pickupState$,
muteStates,
...inputs
}: RingingMediaInputs): RingingMediaViewModel {
return {
...createBaseMedia(inputs),
type: "ringing",
pickupState$,
// If our own video is enabled, then this is a video call and we would
// expect remote media to have video as well
videoEnabled$: muteStates.video.enabled$,
};
}

View File

@@ -13,7 +13,7 @@ import { type LocalScreenShareViewModel } from "./LocalScreenShareViewModel";
import {
createMemberMedia,
type MemberMediaInputs,
type MemberMediaViewModel,
type BaseMemberMediaViewModel,
} from "./MemberMediaViewModel";
import { type RemoteScreenShareViewModel } from "./RemoteScreenShareViewModel";
@@ -27,7 +27,7 @@ export type ScreenShareViewModel =
/**
* Properties which are common to all ScreenShareViewModels.
*/
export interface BaseScreenShareViewModel extends MemberMediaViewModel {
export interface BaseScreenShareViewModel extends BaseMemberMediaViewModel {
type: "screen share";
}

View File

@@ -27,7 +27,7 @@ import { type LocalUserMediaViewModel } from "./LocalUserMediaViewModel";
import {
createMemberMedia,
type MemberMediaInputs,
type MemberMediaViewModel,
type BaseMemberMediaViewModel,
} from "./MemberMediaViewModel";
import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel";
import { type ObservableScope } from "../ObservableScope";
@@ -42,7 +42,7 @@ export type UserMediaViewModel =
| LocalUserMediaViewModel
| RemoteUserMediaViewModel;
export interface BaseUserMediaViewModel extends MemberMediaViewModel {
export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel {
type: "user";
speaking$: Behavior<boolean>;
audioEnabled$: Behavior<boolean>;

View File

@@ -194,5 +194,3 @@ export function createWrappedUserMedia(
),
};
}
export type MediaItem = WrappedUserMediaViewModel | ScreenShareViewModel;

View File

@@ -7,9 +7,10 @@ Please see LICENSE in the repository root for full details.
import { type RemoteTrackPublication } from "livekit-client";
import { test, expect } from "vitest";
import { render, screen } from "@testing-library/react";
import { act, render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject } from "rxjs";
import { GridTile } from "./GridTile";
import {
@@ -21,6 +22,11 @@ import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
import { constant } from "../state/Behavior";
import {
createRingingMedia,
type RingingMediaViewModel,
} from "../state/media/RingingMediaViewModel";
import { type MuteStates } from "../state/MuteStates";
global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {}
@@ -28,6 +34,27 @@ global.IntersectionObserver = class MockIntersectionObserver {
public disconnect(): void {}
} as unknown as typeof IntersectionObserver;
const fakeRtcSession = {
on: () => {},
off: () => {},
room: {
on: () => {},
off: () => {},
client: {
getUserId: () => null,
getDeviceId: () => null,
on: () => {},
off: () => {},
},
},
memberships: [],
} as unknown as MatrixRTCSession;
const callVm = {
reactions$: constant({}),
handsRaised$: constant({}),
} as Partial<CallViewModel> as CallViewModel;
test("GridTile is accessible", async () => {
const vm = mockRemoteMedia(
mockRtcMembership("@alice:example.org", "AAAA"),
@@ -42,34 +69,15 @@ test("GridTile is accessible", async () => {
}),
);
const fakeRtcSession = {
on: () => {},
off: () => {},
room: {
on: () => {},
off: () => {},
client: {
getUserId: () => null,
getDeviceId: () => null,
on: () => {},
off: () => {},
},
},
memberships: [],
} as unknown as MatrixRTCSession;
const cVm = {
reactions$: constant({}),
handsRaised$: constant({}),
} as Partial<CallViewModel> as CallViewModel;
const { container } = render(
<ReactionsSenderProvider vm={cVm} rtcSession={fakeRtcSession}>
<ReactionsSenderProvider vm={callVm} rtcSession={fakeRtcSession}>
<GridTile
vm={new GridTileViewModel(constant(vm))}
onOpenProfile={() => {}}
targetWidth={300}
targetHeight={200}
showSpeakingIndicators
focusable={true}
focusable
/>
</ReactionsSenderProvider>,
);
@@ -77,3 +85,40 @@ test("GridTile is accessible", async () => {
// Name should be visible
screen.getByText("Alice");
});
test("GridTile displays ringing media", async () => {
const pickupState$ = new BehaviorSubject<
RingingMediaViewModel["pickupState$"]["value"]
>("ringing");
const vm = createRingingMedia({
pickupState$,
muteStates: {
video: { enabled$: constant(false) },
} as unknown as MuteStates,
id: "test",
userId: "@alice:example.org",
displayName$: constant("Alice"),
mxcAvatarUrl$: constant(undefined),
});
const { container } = render(
<ReactionsSenderProvider vm={callVm} rtcSession={fakeRtcSession}>
<GridTile
vm={new GridTileViewModel(constant(vm))}
onOpenProfile={() => {}}
targetWidth={300}
targetHeight={200}
showSpeakingIndicators
focusable
/>
</ReactionsSenderProvider>,
);
expect(await axe(container)).toHaveNoViolations();
// Name and status should be visible
screen.getByText("Alice");
screen.getByText("Calling…");
// Alice declines the call
act(() => pickupState$.next("decline"));
screen.getByText("Call ended");
});

View File

@@ -29,6 +29,9 @@ import {
UserProfileIcon,
VolumeOffSolidIcon,
SwitchCameraSolidIcon,
VideoCallSolidIcon,
VoiceCallSolidIcon,
EndCallIcon,
} from "@vector-im/compound-design-tokens/assets/web/icons";
import {
ContextMenu,
@@ -49,6 +52,7 @@ import { useBehavior } from "../useBehavior";
import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel";
import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel";
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel";
interface TileProps {
ref?: Ref<HTMLDivElement>;
@@ -56,21 +60,56 @@ interface TileProps {
style?: ComponentProps<typeof animated.div>["style"];
targetWidth: number;
targetHeight: number;
focusUrl: string | undefined;
displayName: string;
mxcAvatarUrl: string | undefined;
showSpeakingIndicators: boolean;
focusable: boolean;
}
interface RingingMediaTileProps extends TileProps {
vm: RingingMediaViewModel;
}
const RingingMediaTile: FC<RingingMediaTileProps> = ({
vm,
className,
...props
}) => {
const { t } = useTranslation();
const pickupState = useBehavior(vm.pickupState$);
const videoEnabled = useBehavior(vm.videoEnabled$);
return (
<MediaView
className={classNames(className, styles.tile)}
video={undefined}
userId={vm.userId}
unencryptedWarning={false}
status={
pickupState === "ringing"
? {
text: t("video_tile.calling"),
Icon: videoEnabled ? VideoCallSolidIcon : VoiceCallSolidIcon,
}
: { text: t("video_tile.call_ended"), Icon: EndCallIcon }
}
videoEnabled={videoEnabled}
videoFit="cover"
mirror={false}
{...props}
/>
);
};
interface UserMediaTileProps extends TileProps {
vm: UserMediaViewModel;
showSpeakingIndicators: boolean;
mirror: boolean;
playbackMuted: boolean;
waitingForMedia?: boolean;
primaryButton?: ReactNode;
menuStart?: ReactNode;
menuEnd?: ReactNode;
focusUrl: string | undefined;
}
const UserMediaTile: FC<UserMediaTileProps> = ({
@@ -95,7 +134,6 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
const { t } = useTranslation();
const video = useBehavior(vm.video$);
const unencryptedWarning = useBehavior(vm.unencryptedWarning$);
const encryptionStatus = useBehavior(vm.encryptionStatus$);
const audioStreamStats = useObservableEagerState<
RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
>(vm.audioStreamStats$);
@@ -153,7 +191,6 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
video={video}
userId={vm.userId}
unencryptedWarning={unencryptedWarning}
encryptionStatus={encryptionStatus}
videoEnabled={videoEnabled}
videoFit={videoFit}
className={classNames(className, styles.tile, {
@@ -218,6 +255,7 @@ UserMediaTile.displayName = "UserMediaTile";
interface LocalUserMediaTileProps extends TileProps {
vm: LocalUserMediaViewModel;
showSpeakingIndicators: boolean;
onOpenProfile: (() => void) | null;
}
@@ -232,6 +270,7 @@ const LocalUserMediaTile: FC<LocalUserMediaTileProps> = ({
const mirror = useBehavior(vm.mirror$);
const alwaysShow = useBehavior(vm.alwaysShow$);
const switchCamera = useBehavior(vm.switchCamera$);
const focusUrl = useBehavior(vm.focusUrl$);
const latestAlwaysShow = useLatest(alwaysShow);
const onSelectAlwaysShow = useCallback(
@@ -278,6 +317,7 @@ const LocalUserMediaTile: FC<LocalUserMediaTileProps> = ({
)
}
focusable={focusable}
focusUrl={focusUrl}
{...props}
/>
);
@@ -287,6 +327,7 @@ LocalUserMediaTile.displayName = "LocalUserMediaTile";
interface RemoteUserMediaTileProps extends TileProps {
vm: RemoteUserMediaViewModel;
showSpeakingIndicators: boolean;
}
const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
@@ -298,6 +339,8 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
const waitingForMedia = useBehavior(vm.waitingForMedia$);
const playbackMuted = useBehavior(vm.playbackMuted$);
const playbackVolume = useBehavior(vm.playbackVolume$);
const focusUrl = useBehavior(vm.focusUrl$);
const onSelectMute = useCallback(
(e: Event) => {
e.preventDefault();
@@ -338,6 +381,7 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
</MenuItem>
</>
}
focusUrl={focusUrl}
{...props}
/>
);
@@ -360,23 +404,33 @@ interface GridTileProps {
export const GridTile: FC<GridTileProps> = ({
ref: theirRef,
vm,
showSpeakingIndicators,
onOpenProfile,
...props
}) => {
const ourRef = useRef<HTMLDivElement | null>(null);
const ref = useMergedRefs(ourRef, theirRef);
const media = useBehavior(vm.media$);
const focusUrl = useBehavior(media.focusUrl$);
const displayName = useBehavior(media.displayName$);
const mxcAvatarUrl = useBehavior(media.mxcAvatarUrl$);
if (media.local) {
if (media.type === "ringing") {
return (
<RingingMediaTile
ref={ref}
vm={media}
{...props}
displayName={displayName}
mxcAvatarUrl={mxcAvatarUrl}
/>
);
} else if (media.local) {
return (
<LocalUserMediaTile
ref={ref}
vm={media}
showSpeakingIndicators={showSpeakingIndicators}
onOpenProfile={onOpenProfile}
focusUrl={focusUrl}
displayName={displayName}
mxcAvatarUrl={mxcAvatarUrl}
{...props}
@@ -387,7 +441,7 @@ export const GridTile: FC<GridTileProps> = ({
<RemoteUserMediaTile
ref={ref}
vm={media}
focusUrl={focusUrl}
showSpeakingIndicators={showSpeakingIndicators}
displayName={displayName}
mxcAvatarUrl={mxcAvatarUrl}
{...props}

View File

@@ -52,6 +52,11 @@ Please see LICENSE in the repository root for full details.
pointer-events: none;
}
.translucent {
opacity: 50%;
mix-blend-mode: multiply;
}
/* CSS makes us put a condition here, even though all we want to do is
unconditionally select the container so we can use cqmin units */
@container mediaView (width > 0) {
@@ -71,14 +76,15 @@ unconditionally select the container so we can use cqmin units */
.fg {
position: absolute;
inset: var(
--fg-inset: var(
--media-view-fg-inset,
calc(var(--media-view-border-radius) - var(--cpd-space-3x))
);
inset: var(--fg-inset);
display: grid;
grid-template-columns: 30px 1fr 30px;
grid-template-rows: 1fr auto;
grid-template-areas: "reactions status ." "nameTag nameTag button";
grid-template-areas: "status status reactions" "nameTag nameTag button";
gap: var(--cpd-space-1x);
place-items: start;
}
@@ -102,21 +108,19 @@ unconditionally select the container so we can use cqmin units */
.status {
grid-area: status;
justify-self: center;
align-self: start;
padding: var(--cpd-space-2x);
padding-block: var(--cpd-space-2x);
color: var(--cpd-color-text-primary);
background-color: var(--cpd-color-bg-canvas-default);
display: flex;
flex-wrap: none;
align-items: center;
border-radius: var(--cpd-radius-pill-effect);
gap: 3px;
user-select: none;
overflow: hidden;
box-shadow: var(--small-drop-shadow);
box-sizing: border-box;
max-inline-size: 100%;
text-align: center;
margin-block-start: calc(var(--cpd-space-3x) - var(--fg-inset));
margin-inline-start: calc(var(--cpd-space-4x) - var(--fg-inset));
}
.status svg {
color: var(--cpd-color-icon-tertiary);
}
.reactions {

View File

@@ -18,7 +18,6 @@ import { TrackInfo } from "@livekit/protocol";
import { type ComponentProps } from "react";
import { MediaView } from "./MediaView";
import { EncryptionStatus } from "../state/media/MemberMediaViewModel";
import { mockLocalParticipant } from "../utils/test";
describe("MediaView", () => {
@@ -41,7 +40,6 @@ describe("MediaView", () => {
videoFit: "contain",
targetWidth: 300,
targetHeight: 200,
encryptionStatus: EncryptionStatus.Connecting,
mirror: false,
unencryptedWarning: false,
video: trackReference,

View File

@@ -7,7 +7,13 @@ Please see LICENSE in the repository root for full details.
import { type TrackReferenceOrPlaceholder } from "@livekit/components-core";
import { animated } from "@react-spring/web";
import { type FC, type ComponentProps, type ReactNode } from "react";
import {
type FC,
type ComponentProps,
type ReactNode,
type ComponentType,
type SVGAttributes,
} from "react";
import { useTranslation } from "react-i18next";
import classNames from "classnames";
import { VideoTrack } from "@livekit/components-react";
@@ -16,7 +22,6 @@ import { ErrorSolidIcon } from "@vector-im/compound-design-tokens/assets/web/ico
import styles from "./MediaView.module.css";
import { Avatar } from "../Avatar";
import { type EncryptionStatus } from "../state/media/MemberMediaViewModel";
import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
import {
showConnectionStats as showConnectionStatsSetting,
@@ -38,7 +43,7 @@ interface Props extends ComponentProps<typeof animated.div> {
userId: string;
videoEnabled: boolean;
unencryptedWarning: boolean;
encryptionStatus: EncryptionStatus;
status?: { text: string; Icon: ComponentType<SVGAttributes<SVGElement>> };
nameTagLeadingIcon?: ReactNode;
displayName: string;
mxcAvatarUrl: string | undefined;
@@ -72,7 +77,7 @@ export const MediaView: FC<Props> = ({
mxcAvatarUrl,
focusable,
primaryButton,
encryptionStatus,
status,
raisedHandTime,
currentReaction,
raisedHandOnClick,
@@ -106,7 +111,11 @@ export const MediaView: FC<Props> = ({
name={displayName}
size={avatarSize}
src={mxcAvatarUrl}
className={styles.avatar}
className={classNames(styles.avatar, {
// When the avatar is overlaid with a status, make it translucent
// for readability
[styles.translucent]: status,
})}
style={{ display: video && videoEnabled ? "none" : "initial" }}
/>
{video?.publication !== undefined && (
@@ -152,6 +161,14 @@ export const MediaView: FC<Props> = ({
/>
</>
)}
{status && (
<div className={styles.status}>
<status.Icon width={16} height={16} aria-hidden />
<Text as="span" size="sm" weight="medium">
{status.text}
</Text>
</div>
)}
{/* TODO: Bring this back once encryption status is less broken */}
{/*encryptionStatus !== EncryptionStatus.Okay && (
<div className={styles.status}>

View File

@@ -6,10 +6,11 @@ Please see LICENSE in the repository root for full details.
*/
import { test, expect, vi } from "vitest";
import { isInaccessible, render, screen } from "@testing-library/react";
import { act, isInaccessible, render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import userEvent from "@testing-library/user-event";
import { TooltipProvider } from "@vector-im/compound-web";
import { BehaviorSubject } from "rxjs";
import { SpotlightTile } from "./SpotlightTile";
import {
@@ -23,6 +24,11 @@ import {
} from "../utils/test";
import { SpotlightTileViewModel } from "../state/TileViewModel";
import { constant } from "../state/Behavior";
import {
createRingingMedia,
type RingingMediaViewModel,
} from "../state/media/RingingMediaViewModel";
import { type MuteStates } from "../state/MuteStates";
global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {}
@@ -140,3 +146,41 @@ test("Screen share volume UI is hidden when screen share has no audio", async ()
screen.queryByRole("button", { name: /volume/i }),
).not.toBeInTheDocument();
});
test("SpotlightTile displays ringing media", async () => {
const pickupState$ = new BehaviorSubject<
RingingMediaViewModel["pickupState$"]["value"]
>("ringing");
const vm = createRingingMedia({
pickupState$,
muteStates: {
video: { enabled$: constant(false) },
} as unknown as MuteStates,
id: "test",
userId: "@alice:example.org",
displayName$: constant("Alice"),
mxcAvatarUrl$: constant(undefined),
});
const toggleExpanded = vi.fn();
const { container } = render(
<SpotlightTile
vm={new SpotlightTileViewModel(constant([vm]), constant(false))}
targetWidth={300}
targetHeight={200}
expanded={false}
onToggleExpanded={toggleExpanded}
showIndicators
focusable={true}
/>,
);
expect(await axe(container)).toHaveNoViolations();
// Alice should be in the spotlight with the right status
screen.getByText("Alice");
screen.getByText("Calling…");
// Now we time out ringing to Alice
act(() => pickupState$.next("timeout"));
screen.getByText("Call ended");
});

View File

@@ -24,6 +24,9 @@ import {
VolumeOnIcon,
VolumeOffSolidIcon,
VolumeOnSolidIcon,
VideoCallSolidIcon,
VoiceCallSolidIcon,
EndCallIcon,
} from "@vector-im/compound-design-tokens/assets/web/icons";
import { animated } from "@react-spring/web";
import { type Observable, map } from "rxjs";
@@ -43,7 +46,7 @@ import { useReactiveState } from "../useReactiveState";
import { useLatest } from "../useLatest";
import { type SpotlightTileViewModel } from "../state/TileViewModel";
import { useBehavior } from "../useBehavior";
import { type EncryptionStatus } from "../state/media/MemberMediaViewModel";
import { type MemberMediaViewModel } from "../state/media/MemberMediaViewModel";
import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel";
import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel";
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
@@ -52,6 +55,7 @@ import { type RemoteScreenShareViewModel } from "../state/media/RemoteScreenShar
import { type MediaViewModel } from "../state/media/MediaViewModel";
import { Slider } from "../Slider";
import { platform } from "../Platform";
import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel";
interface SpotlightItemBaseProps {
ref?: Ref<HTMLDivElement>;
@@ -59,18 +63,20 @@ interface SpotlightItemBaseProps {
"data-id": string;
targetWidth: number;
targetHeight: number;
video: TrackReferenceOrPlaceholder | undefined;
userId: string;
unencryptedWarning: boolean;
encryptionStatus: EncryptionStatus;
focusUrl: string | undefined;
displayName: string;
mxcAvatarUrl: string | undefined;
focusable: boolean;
"aria-hidden"?: boolean;
}
interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps {
interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps {
video: TrackReferenceOrPlaceholder | undefined;
unencryptedWarning: boolean;
focusUrl: string | undefined;
}
interface SpotlightUserMediaItemBaseProps extends SpotlightMemberMediaItemBaseProps {
videoFit: "contain" | "cover";
videoEnabled: boolean;
}
@@ -103,21 +109,32 @@ const SpotlightRemoteUserMediaItem: FC<SpotlightRemoteUserMediaItemProps> = ({
);
};
interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps {
interface SpotlightUserMediaItemProps extends SpotlightMemberMediaItemBaseProps {
vm: UserMediaViewModel;
}
const SpotlightUserMediaItem: FC<SpotlightUserMediaItemProps> = ({
vm,
targetWidth,
targetHeight,
...props
}) => {
const videoFit = useBehavior(vm.videoFit$);
const videoEnabled = useBehavior(vm.videoEnabled$);
// Whenever target bounds change, inform the viewModel
useEffect(() => {
if (targetWidth > 0 && targetHeight > 0) {
vm.setTargetDimensions(targetWidth, targetHeight);
}
}, [targetWidth, targetHeight, vm]);
const baseProps: SpotlightUserMediaItemBaseProps &
RefAttributes<HTMLDivElement> = {
videoFit,
videoEnabled,
targetWidth,
targetHeight,
...props,
};
@@ -130,7 +147,7 @@ const SpotlightUserMediaItem: FC<SpotlightUserMediaItemProps> = ({
SpotlightUserMediaItem.displayName = "SpotlightUserMediaItem";
interface SpotlightScreenShareItemProps extends SpotlightItemBaseProps {
interface SpotlightScreenShareItemProps extends SpotlightMemberMediaItemBaseProps {
vm: ScreenShareViewModel;
videoEnabled: boolean;
}
@@ -142,7 +159,7 @@ const SpotlightScreenShareItem: FC<SpotlightScreenShareItemProps> = ({
return <MediaView videoFit="contain" mirror={false} {...props} />;
};
interface SpotlightRemoteScreenShareItemProps extends SpotlightItemBaseProps {
interface SpotlightRemoteScreenShareItemProps extends SpotlightMemberMediaItemBaseProps {
vm: RemoteScreenShareViewModel;
}
@@ -155,6 +172,67 @@ const SpotlightRemoteScreenShareItem: FC<
);
};
interface SpotlightMemberMediaItemProps extends SpotlightItemBaseProps {
vm: MemberMediaViewModel;
}
const SpotlightMemberMediaItem: FC<SpotlightMemberMediaItemProps> = ({
vm,
...props
}) => {
const video = useBehavior(vm.video$);
const unencryptedWarning = useBehavior(vm.unencryptedWarning$);
const focusUrl = useBehavior(vm.focusUrl$);
const baseProps: SpotlightMemberMediaItemBaseProps &
RefAttributes<HTMLDivElement> = {
video: video ?? undefined,
unencryptedWarning,
focusUrl,
...props,
};
if (vm.type === "user")
return <SpotlightUserMediaItem vm={vm} {...baseProps} />;
return vm.local ? (
<SpotlightScreenShareItem vm={vm} videoEnabled {...baseProps} />
) : (
<SpotlightRemoteScreenShareItem vm={vm} {...baseProps} />
);
};
interface SpotlightRingingMediaItemProps extends SpotlightItemBaseProps {
vm: RingingMediaViewModel;
}
const SpotlightRingingMediaItem: FC<SpotlightRingingMediaItemProps> = ({
vm,
...props
}) => {
const { t } = useTranslation();
const pickupState = useBehavior(vm.pickupState$);
const videoEnabled = useBehavior(vm.videoEnabled$);
return (
<MediaView
video={undefined}
unencryptedWarning={false}
status={
pickupState === "ringing"
? {
text: t("video_tile.calling"),
Icon: videoEnabled ? VideoCallSolidIcon : VoiceCallSolidIcon,
}
: { text: t("video_tile.call_ended"), Icon: EndCallIcon }
}
videoEnabled={false}
videoFit="cover"
mirror={false}
{...props}
/>
);
};
interface SpotlightItemProps {
ref?: Ref<HTMLDivElement>;
vm: MediaViewModel;
@@ -187,22 +265,9 @@ const SpotlightItem: FC<SpotlightItemProps> = ({
}) => {
const ourRef = useRef<HTMLDivElement | null>(null);
// Whenever target bounds change, inform the viewModel
useEffect(() => {
if (targetWidth > 0 && targetHeight > 0) {
if (vm.type != "screen share") {
vm.setTargetDimensions(targetWidth, targetHeight);
}
}
}, [targetWidth, targetHeight, vm]);
const ref = useMergedRefs(ourRef, theirRef);
const focusUrl = useBehavior(vm.focusUrl$);
const displayName = useBehavior(vm.displayName$);
const mxcAvatarUrl = useBehavior(vm.mxcAvatarUrl$);
const video = useBehavior(vm.video$);
const unencryptedWarning = useBehavior(vm.unencryptedWarning$);
const encryptionStatus = useBehavior(vm.encryptionStatus$);
// Hook this item up to the intersection observer
useEffect(() => {
@@ -225,23 +290,17 @@ const SpotlightItem: FC<SpotlightItemProps> = ({
className: classNames(styles.item, { [styles.snap]: snap }),
targetWidth,
targetHeight,
video: video ?? undefined,
userId: vm.userId,
unencryptedWarning,
focusUrl,
displayName,
mxcAvatarUrl,
focusable,
encryptionStatus,
"aria-hidden": ariaHidden,
};
if (vm.type === "user")
return <SpotlightUserMediaItem vm={vm} {...baseProps} />;
return vm.local ? (
<SpotlightScreenShareItem vm={vm} videoEnabled {...baseProps} />
return vm.type === "ringing" ? (
<SpotlightRingingMediaItem vm={vm} {...baseProps} />
) : (
<SpotlightRemoteScreenShareItem vm={vm} {...baseProps} />
<SpotlightMemberMediaItem vm={vm} {...baseProps} />
);
};