Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-04-03 07:10:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

New ringing UI

Browse Source 
This implements the new ringing UI by showing a placeholder tile for the participant being dialed, rather than an overlay.
This commit is contained in:
Robin
2026-03-16 13:12:49 +01:00
parent 6d14f1d06f
commit 9dfade68ee
27 changed files with 703 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/state/CallViewModel/CallNotificationLifecycle.ts
View File

@@ -89,7 +89,6 @@ export interface Props {
* `callPickupState$` The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.
* This may also be set if we are disconnected.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
* The call failed. If desired this can be used as a trigger to exit the call.
@@ -131,15 +130,9 @@ export function createCallNotificationLifecycle$({
) as Behavior<Epoch<boolean>>;
/**
* Whenever the RTC session tells us that it intends to ring the remote
* participant's devices, this emits an Observable tracking the current state of
* that ringing process.
* The state of the current ringing attempt, if the RTC session is indeed
* ringing the remote participant's devices. Otherwise `null`.
*/
// This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$`
// has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`.
// A behavior will emit the latest observable with the running timer to new subscribers.
// see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if
// `ring$` would not be a behavior.
const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> =
scope.behavior(
sentCallNotification$.pipe(

174
src/state/CallViewModel/CallViewModel.test.ts
View File

@@ -46,9 +46,11 @@ import {
} from "../../utils/test.ts";
import { E2eeType } from "../../e2ee/e2eeType.ts";
import {
alice,
aliceId,
aliceParticipant,
aliceRtcMember,
aliceUserId,
bobId,
bobRtcMember,
local,
@@ -140,8 +142,8 @@ export interface SpotlightExpandedLayoutSummary {
export interface OneOnOneLayoutSummary {
type: "one-on-one";
local: string;
remote: string;
spotlight: string;
pip: string;
}
export interface PipLayoutSummary {
@@ -194,11 +196,11 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
);
case "one-on-one":
return combineLatest(
[l.local.media$, l.remote.media$],
(local, remote) => ({
[l.spotlight.media$, l.pip.media$],
(spotlight, pip) => ({
type: l.type,
local: local.id,
remote: remote.id,
spotlight: spotlight.id,
pip: pip.id,
}),
);
case "pip":
@@ -537,8 +539,8 @@ describe.each([
b: {
// In a larger window, expect the normal one-on-one layout
type: "one-on-one",
local: `${localId}:0`,
remote: `${aliceId}:0`,
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
c: {
// In a PiP-sized window, we of course expect a PiP layout
@@ -840,8 +842,8 @@ describe.each([
},
b: {
type: "one-on-one",
local: `${localId}:0`,
remote: `${aliceId}:0`,
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
c: {
type: "grid",
@@ -883,8 +885,8 @@ describe.each([
},
b: {
type: "one-on-one",
local: `${localId}:0`,
remote: `${aliceId}:0`,
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
c: {
type: "grid",
@@ -893,8 +895,8 @@ describe.each([
},
d: {
type: "one-on-one",
local: `${localId}:0`,
remote: `${daveId}:0`,
pip: `${localId}:0`,
spotlight: `${daveId}:0`,
},
},
);
@@ -1087,83 +1089,81 @@ describe.each([
});
});
describe("waitForCallPickup$", () => {
it.skip("regression test: does stop ringing in case livekitConnectionState$ emits after didSendCallNotification$ has already emitted", () => {
withTestScheduler(({ schedule, expectObservable, behavior }) => {
withCallViewModel(
{
livekitConnectionState$: behavior("d 9ms c", {
d: ConnectionState.Disconnected,
c: ConnectionState.Connected,
}),
},
(vm, rtcSession) => {
// Fire a call notification IMMEDIATELY (its important for this test, that this happens before the livekitConnectionState$ emits)
schedule("n", {
n: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
mockRingEvent("$notif1", 30),
);
},
});
test("recipient has placeholder tile while ringing or timed out", () => {
withTestScheduler(({ schedule, expectObservable }) => {
withCallViewModel(
{
roomMembers: [alice, local], // Simulate a DM
},
(vm, rtcSession) => {
// Fire a ringing notification
schedule("n", {
n: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
mockRingEvent("$notif1", 30),
);
},
});
expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", {
a: "unknown",
b: "ringing",
c: "timeout",
});
},
{
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
// Should ring for 30ms and then time out
expectObservable(vm.ringing$).toBe("(ny) 26ms n", yesNo);
// Layout should show placeholder media for the participant we're
// ringing the entire time (even once timed out)
expectObservable(summarizeLayout$(vm.layout$)).toBe("a", {
a: {
type: "one-on-one",
spotlight: `${localId}:0`,
pip: `ringing:${aliceUserId}`,
},
});
},
{ waitForCallPickup: true },
);
});
});
it.skip("ringing -> unknown if we get disconnected", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
const connectionState$ = new BehaviorSubject(ConnectionState.Connected);
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
withCallViewModel(
{
remoteParticipants$: behavior("a 19ms b", {
a: [],
b: [aliceParticipant],
}),
rtcMembers$: behavior("a 19ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
livekitConnectionState$: connectionState$,
},
(vm, rtcSession) => {
// Notify at 5ms so we enter ringing, then get disconnected 5ms later
schedule(" 5ms r 5ms d", {
r: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
mockRingEvent("$notif2", 100),
);
},
d: () => {
connectionState$.next(ConnectionState.Disconnected);
},
});
test("recipient's placeholder tile is replaced by their real tile once they answer", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
withCallViewModel(
{
// Alice answers after 20ms
rtcMembers$: behavior("a 20ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
roomMembers: [alice, local], // Simulate a DM
},
(vm, rtcSession) => {
// Fire a ringing notification
schedule("n", {
n: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
mockRingEvent("$notif1", 30),
);
},
});
expectObservable(vm.callPickupState$).toBe("a 4ms b 5ms c", {
a: "unknown",
b: "ringing",
c: "unknown",
});
},
{
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
// Should ring until Alice joins
expectObservable(vm.ringing$).toBe("(ny) 17ms n", yesNo);
// Layout should show placeholder media for the participant we're
// ringing the entire time
expectObservable(summarizeLayout$(vm.layout$)).toBe("a 20ms b", {
a: {
type: "one-on-one",
spotlight: `${localId}:0`,
pip: `ringing:${aliceUserId}`,
},
b: {
type: "one-on-one",
spotlight: `${aliceId}:0`,
pip: `${localId}:0`,
},
});
},
{ waitForCallPickup: true },
);
});
});

274
src/state/CallViewModel/CallViewModel.ts
View File

@@ -128,7 +128,6 @@ import {
createSentCallNotification$,
} from "./CallNotificationLifecycle.ts";
import {
createDMMember$,
createMatrixMemberMetadata$,
createRoomMembers$,
} from "./remoteMembers/MatrixMemberMetadata.ts";
@@ -137,12 +136,17 @@ import { type Connection } from "./remoteMembers/Connection.ts";
import { createLayoutModeSwitch } from "./LayoutSwitch.ts";
import {
createWrappedUserMedia,
type MediaItem,
type WrappedUserMediaViewModel,
} from "../media/MediaItem.ts";
} from "../media/WrappedUserMediaViewModel.ts";
import { type ScreenShareViewModel } from "../media/ScreenShareViewModel.ts";
import { type UserMediaViewModel } from "../media/UserMediaViewModel.ts";
import { type MediaViewModel } from "../media/MediaViewModel.ts";
import { type LocalUserMediaViewModel } from "../media/LocalUserMediaViewModel.ts";
import { type RemoteUserMediaViewModel } from "../media/RemoteUserMediaViewModel.ts";
import {
createRingingMedia,
type RingingMediaViewModel,
} from "../media/RingingMediaViewModel.ts";
const logger = rootLogger.getChild("[CallViewModel]");
//TODO
@@ -210,11 +214,10 @@ export type LivekitRoomItem = {
export interface CallViewModel {
// lifecycle
autoLeave$: Observable<AutoLeaveReason>;
// TODO if we are in "unknown" state we need a loading rendering (or empty screen)
// Otherwise it looks like we already connected and only than the ringing starts which is weird.
callPickupState$: Behavior<
"unknown" | "ringing" | "timeout" | "decline" | "success" | null
>;
/**
* Whether we are ringing a call recipient.
*/
ringing$: Behavior<boolean>;
/** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is
* - by ending the scope
@@ -289,13 +292,6 @@ export interface CallViewModel {
/** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
reactions$: Behavior<Record<string, ReactionOption>>;
ringOverlay$: Behavior<null | {
name: string;
/** roomId or userId for the avatar generation. */
idForAvatar: string;
text: string;
avatarMxc?: string;
}>;
// sounds and events
joinSoundEffect$: Observable<void>;
leaveSoundEffect$: Observable<void>;
@@ -611,40 +607,6 @@ export function createCallViewModel$(
matrixRoomMembers$,
);
const dmMember$ = createDMMember$(scope, matrixRoomMembers$, matrixRoom);
const noUserToCallInRoom$ = scope.behavior(
matrixRoomMembers$.pipe(
map(
(roomMembersMap) =>
roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined,
),
),
);
const ringOverlay$ = scope.behavior(
combineLatest([noUserToCallInRoom$, dmMember$, callPickupState$]).pipe(
map(([noUserToCallInRoom, dmMember, callPickupState]) => {
// No overlay if not in ringing state
if (callPickupState !== "ringing" || noUserToCallInRoom) return null;
const name = dmMember ? dmMember.rawDisplayName : matrixRoom.name;
const id = dmMember ? dmMember.userId : matrixRoom.roomId;
const text = dmMember
? `Waiting for ${name} to join…`
: "Waiting for other participants…";
const avatarMxc = dmMember
? (dmMember.getMxcAvatarUrl?.() ?? undefined)
: (matrixRoom.getMxcAvatarUrl() ?? undefined);
return {
name: name ?? id,
idForAvatar: id,
text,
avatarMxc,
};
}),
),
);
const allConnections$ = scope.behavior(
connectionManager.connectionManagerData$.pipe(map((d) => d.value)),
);
@@ -720,7 +682,7 @@ export function createCallViewModel$(
matrixLivekitMembers$,
duplicateTiles.value$,
]).pipe(
// Generate a collection of MediaItems from the list of expected (whether
// Generate a collection of user media from the list of expected (whether
// present or missing) LiveKit participants.
generateItems(
"CallViewModel userMedia$",
@@ -793,32 +755,67 @@ export function createCallViewModel$(
),
);
const ringingMedia$ = scope.behavior<RingingMediaViewModel[]>(
combineLatest([userMedia$, matrixRoomMembers$, callPickupState$]).pipe(
generateItems(
"CallViewModel ringingMedia$",
function* ([userMedia, roomMembers, callPickupState]) {
if (
callPickupState === "ringing" ||
callPickupState === "timeout" ||
callPickupState === "decline"
) {
for (const member of roomMembers.values()) {
if (!userMedia.some((vm) => vm.userId === member.userId))
yield {
keys: [member.userId],
data: callPickupState,
};
}
}
},
(scope, pickupState$, userId) =>
createRingingMedia({
id: `ringing:${userId}`,
userId,
displayName$: scope.behavior(
matrixRoomMembers$.pipe(
map((members) => members.get(userId)?.rawDisplayName || userId),
),
),
mxcAvatarUrl$:
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
pickupState$,
muteStates,
}),
),
distinctUntilChanged(shallowEquals),
tap((ringingMedia) => {
if (ringingMedia.length > 1)
// Warn that UI may do something unexpected in this case
logger.warn(
`Ringing more than one participant is not supported (ringing ${ringingMedia.map((vm) => vm.userId).join(", ")})`,
);
}),
),
);
/**
* List of all media items (user media and screen share media) that we want
* tiles for.
* All screen share media that we want to display.
*/
const mediaItems$ = scope.behavior<MediaItem[]>(
const screenShares$ = scope.behavior<ScreenShareViewModel[]>(
userMedia$.pipe(
switchMap((userMedia) =>
userMedia.length === 0
? of([])
: combineLatest(
userMedia.map((m) => m.screenShares$),
(...screenShares) => [...userMedia, ...screenShares.flat(1)],
(...screenShares) => screenShares.flat(1),
),
),
),
);
/**
* List of MediaItems that we want to display, that are of type ScreenShare
*/
const screenShares$ = scope.behavior<ScreenShareViewModel[]>(
mediaItems$.pipe(
map((mediaItems) => mediaItems.filter((m) => m.type === "screen share")),
),
);
const joinSoundEffect$ = userMedia$.pipe(
pairwise(),
filter(
@@ -931,40 +928,20 @@ export function createCallViewModel$(
),
);
const spotlight$ = scope.behavior<MediaViewModel[]>(
screenShares$.pipe(
switchMap((screenShares) => {
if (screenShares.length > 0) return of(screenShares);
return spotlightSpeaker$.pipe(
map((speaker) => (speaker ? [speaker] : [])),
/**
* Local user media suitable for displaying in a PiP (undefined if not found
* or if user prefers to not see themselves).
*/
const localUserMediaForPip$ = scope.behavior<
LocalUserMediaViewModel | undefined
>(
userMedia$.pipe(
switchMap((userMedia) => {
const localUserMedia = userMedia.find(
(m): m is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
m.type === "user" && m.local,
);
}),
distinctUntilChanged<MediaViewModel[]>(shallowEquals),
),
);
const pip$ = scope.behavior<UserMediaViewModel | undefined>(
combineLatest([
// TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits
screenShares$,
spotlightSpeaker$,
mediaItems$,
]).pipe(
switchMap(([screenShares, spotlight, mediaItems]) => {
if (screenShares.length > 0) {
return spotlightSpeaker$;
}
if (!spotlight || spotlight.local) {
return of(undefined);
}
const localUserMedia = mediaItems.find(
(m) => m.type === "user" && m.local,
);
if (!localUserMedia) {
return of(undefined);
}
if (!localUserMedia) return of(undefined);
return localUserMedia.alwaysShow$.pipe(
map((alwaysShow) => (alwaysShow ? localUserMedia : undefined)),
);
@@ -972,6 +949,39 @@ export function createCallViewModel$(
),
);
const spotlightAndPip$ = scope.behavior<{
spotlight: MediaViewModel[];
pip$: Behavior<UserMediaViewModel | undefined>;
}>(
ringingMedia$.pipe(
switchMap((ringingMedia) => {
if (ringingMedia.length > 0)
return of({ spotlight: ringingMedia, pip$: localUserMediaForPip$ });
return screenShares$.pipe(
switchMap((screenShares) => {
if (screenShares.length > 0)
return of({ spotlight: screenShares, pip$: spotlightSpeaker$ });
return spotlightSpeaker$.pipe(
map((speaker) => ({
spotlight: speaker ? [speaker] : [],
pip$: localUserMediaForPip$,
})),
);
}),
);
}),
),
);
const spotlight$ = scope.behavior<MediaViewModel[]>(
spotlightAndPip$.pipe(
map(({ spotlight }) => spotlight),
distinctUntilChanged<MediaViewModel[]>(shallowEquals),
),
);
const hasRemoteScreenShares$ = scope.behavior<boolean>(
spotlight$.pipe(
map((spotlight) =>
@@ -1054,24 +1064,61 @@ export function createCallViewModel$(
}));
const spotlightExpandedLayoutMedia$: Observable<SpotlightExpandedLayoutMedia> =
combineLatest([spotlight$, pip$], (spotlight, pip) => ({
type: "spotlight-expanded",
spotlight,
pip: pip ?? undefined,
}));
spotlightAndPip$.pipe(
switchMap(({ spotlight, pip$ }) =>
pip$.pipe(
map((pip) => ({
type: "spotlight-expanded" as const,
spotlight,
pip: pip ?? undefined,
})),
),
),
);
const oneOnOneLayoutMedia$: Observable<OneOnOneLayoutMedia | null> =
mediaItems$.pipe(
map((mediaItems) => {
if (mediaItems.length !== 2) return null;
const local = mediaItems.find((vm) => vm.type === "user" && vm.local);
const remote = mediaItems.find((vm) => vm.type === "user" && !vm.local);
// There might not be a remote tile if there are screen shares, or if
// only the local user is in the call and they're using the duplicate
// tiles option
if (!remote || !local) return null;
userMedia$.pipe(
switchMap((userMedia) => {
if (userMedia.length <= 2) {
const local = userMedia.find(
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
vm.type === "user" && vm.local,
);
return { type: "one-on-one", local, remote };
if (local !== undefined) {
const remote = userMedia.find(
(
vm,
): vm is WrappedUserMediaViewModel & RemoteUserMediaViewModel =>
vm.type === "user" && !vm.local,
);
if (remote !== undefined)
return of({
type: "one-on-one" as const,
spotlight: remote,
pip: local,
});
// If there's no other user media in the call (could still happen in
// this branch due to the duplicate tiles option), we could possibly
// show ringing media instead
if (userMedia.length === 1)
return ringingMedia$.pipe(
map((ringingMedia) => {
return ringingMedia.length === 1
? {
type: "one-on-one" as const,
spotlight: local,
pip: ringingMedia[0],
}
: null;
}),
);
}
}
return of(null);
}),
);
@@ -1482,8 +1529,9 @@ export function createCallViewModel$(
return {
autoLeave$: autoLeave$,
callPickupState$: callPickupState$,
ringOverlay$: ringOverlay$,
ringing$: scope.behavior(
callPickupState$.pipe(map((state) => state === "ringing")),
),
leave$: leave$,
hangup: (): void => userHangup$.next(),
join: localMembership.requestJoinAndPublish,

22
src/state/CallViewModel/CallViewModelTestUtils.ts
View File

@@ -17,7 +17,7 @@ import {
import { SyncState } from "matrix-js-sdk/lib/sync";
import { BehaviorSubject, type Observable, map, of } from "rxjs";
import { onTestFinished, vi } from "vitest";
import { ClientEvent, type MatrixClient } from "matrix-js-sdk";
import { ClientEvent, type RoomMember, type MatrixClient } from "matrix-js-sdk";
import EventEmitter from "events";
import * as ComponentsCore from "@livekit/components-core";
@@ -63,15 +63,10 @@ const carol = local;
const dave = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "Dave" });
const roomMembers = new Map(
[alice, aliceDoppelganger, bob, bobZeroWidthSpace, carol, dave, daveRTL].map(
(p) => [p.userId, p],
),
);
export interface CallViewModelInputs {
remoteParticipants$: Behavior<RemoteParticipant[]>;
rtcMembers$: Behavior<Partial<CallMembership>[]>;
roomMembers: RoomMember[];
livekitConnectionState$: Behavior<ConnectionState>;
speaking: Map<Participant, Observable<boolean>>;
mediaDevices: MediaDevices;
@@ -86,6 +81,15 @@ export function withCallViewModel(mode: MatrixRTCMode) {
{
remoteParticipants$ = constant([]),
rtcMembers$ = constant([localRtcMember]),
roomMembers = [
alice,
aliceDoppelganger,
bob,
bobZeroWidthSpace,
carol,
dave,
daveRTL,
],
livekitConnectionState$: connectionState$ = constant(
ConnectionState.Connected,
),
@@ -128,8 +132,8 @@ export function withCallViewModel(mode: MatrixRTCMode) {
return syncState;
}
})() as Partial<MatrixClient> as MatrixClient,
getMembers: () => Array.from(roomMembers.values()),
getMembersWithMembership: () => Array.from(roomMembers.values()),
getMembers: () => roomMembers,
getMembersWithMembership: () => roomMembers,
});
const rtcSession = new MockRTCSession(room, []).withMemberships(
rtcMembers$,

25
src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts
View File

@@ -54,31 +54,6 @@ export function createRoomMembers$(
);
}
/**
* creates the member that this DM is with in case it is a DM (two members) otherwise null
*/
export function createDMMember$(
scope: ObservableScope,
roomMembers$: Behavior<RoomMemberMap>,
matrixRoom: MatrixRoom,
): Behavior<Pick<
RoomMember,
"userId" | "getMxcAvatarUrl" | "rawDisplayName"
> | null> {
// We cannot use the normal direct check from matrix since we do not have access to the account data.
// use primitive member count === 2 check instead.
return scope.behavior(
roomMembers$.pipe(
map((membersMap) => {
// primitive appraoch do to no access to account data.
const isDM = membersMap.size === 2;
if (!isDM) return null;
return matrixRoom.getMember(matrixRoom.guessDMUserId());
}),
),
);
}
/**
* Displayname for each member of the call. This will disambiguate
* any displayname that clashes with another member. Only members