Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-03-04 05:37:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'toger5/waitForNotificationAnswer' into toger5/call-pickup-state-decline-event

Browse Source
This commit is contained in:
Robin
2025-09-03 17:59:16 +02:00
parent 7724cbf9be 1e8b87408c
commit e9c43856d0
8 changed files with 266 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/url-params.md
View File

@@ -70,6 +70,8 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st
| `theme` | One of: `light`, `dark`, `light-high-contrast`, `dark-high-contrast` | No, defaults to `dark` | No, defaults to `dark` | UI theme to use. |
| `viaServers` | Comma separated list of [Matrix Server Names](https://spec.matrix.org/v1.12/appendices/#server-name) | Not applicable | No | Homeserver for joining a room, non-empty value required for rooms not on the user’s default homeserver. |
| `sendNotificationType` | `ring` or `notification` | No | No | Will send a "ring" or "notification" `m.rtc.notification` event if the user is the first one in the call. |
| `autoLeaveWhenOthersLeft` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Whether the app should automatically leave the call when there is no one left in the call. |
| `waitForCallPickup` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | When sending a notification, show UI that the app is awaiting an answer, play a dial tone, and (in widget mode) auto-close the widget once the notification expires. |
### Widget-only parameters

10
src/UrlParams.ts
View File

@@ -226,7 +226,7 @@ export interface UrlConfiguration {
* - play a sound that indicates that it is awaiting an answer
* - auto-dismiss the call widget once the notification lifetime expires on the receivers side.
*/
shouldWaitForCallPickup: boolean;
waitForCallPickup: boolean;
}
// If you need to add a new flag to this interface, prefer a name that describes
@@ -358,7 +358,7 @@ export const getUrlParams = (
returnToLobby: false,
sendNotificationType: "notification" as RTCNotificationType,
autoLeaveWhenOthersLeft: false,
shouldWaitForCallPickup: false,
waitForCallPickup: false,
};
switch (intent) {
case UserIntent.StartNewCall:
@@ -378,7 +378,7 @@ export const getUrlParams = (
...inAppDefault,
skipLobby: true,
autoLeaveWhenOthersLeft: true,
shouldWaitForCallPickup: true,
waitForCallPickup: true,
};
break;
case UserIntent.JoinExistingCallDM:
@@ -404,7 +404,7 @@ export const getUrlParams = (
returnToLobby: false,
sendNotificationType: undefined,
autoLeaveWhenOthersLeft: false,
shouldWaitForCallPickup: false,
waitForCallPickup: false,
};
}
@@ -456,7 +456,7 @@ export const getUrlParams = (
"ring",
"notification",
]),
shouldWaitForCallPickup: parser.getFlag("shouldWaitForCallPickup"),
waitForCallPickup: parser.getFlag("waitForCallPickup"),
autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
};

10
src/room/CallEventAudioRenderer.test.tsx
View File

@@ -19,10 +19,7 @@ import { act } from "react";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { mockRtcMembership } from "../utils/test";
import {
CallEventAudioRenderer,
MAX_PARTICIPANT_COUNT_FOR_SOUND,
} from "./CallEventAudioRenderer";
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
@@ -33,6 +30,7 @@ import {
local,
localRtcMember,
} from "../utils/test-fixtures";
import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel";
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
@@ -172,7 +170,7 @@ test("should not play a sound when a hand raise is retracted", () => {
},
});
});
expect(playSound).toHaveBeenCalledTimes(2);
expect(playSound).toHaveBeenCalledTimes(1);
act(() => {
handRaisedSubject$.next({
["foo"]: {
@@ -182,5 +180,5 @@ test("should not play a sound when a hand raise is retracted", () => {
},
});
});
expect(playSound).toHaveBeenCalledTimes(2);
expect(playSound).toHaveBeenCalledTimes(1);
});

49
src/room/CallEventAudioRenderer.tsx
View File

@@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details.
*/
import { type ReactNode, useEffect } from "react";
import { filter, interval, throttle } from "rxjs";
import { type CallViewModel } from "../state/CallViewModel";
import joinCallSoundMp3 from "../sound/join_call.mp3";
@@ -21,11 +20,6 @@ import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useLatest } from "../useLatest";
// Do not play any sounds if the participant count has exceeded this
// number.
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const THROTTLE_SOUND_EFFECT_MS = 500;
export const callEventAudioSounds = prefetchSounds({
join: {
mp3: joinCallSoundMp3,
@@ -60,37 +54,18 @@ export function CallEventAudioRenderer({
const audioEngineRef = useLatest(audioEngineCtx);
useEffect(() => {
const joinSub = vm.participantChanges$
.pipe(
filter(
({ joined, ids }) =>
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0,
),
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(() => {
void audioEngineRef.current?.playSound("join");
});
const leftSub = vm.participantChanges$
.pipe(
filter(
({ ids, left }) =>
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
),
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(() => {
void audioEngineRef.current?.playSound("left");
});
const handRaisedSub = vm.newHandRaised$.subscribe(() => {
void audioEngineRef.current?.playSound("raiseHand");
});
const screenshareSub = vm.newScreenShare$.subscribe(() => {
void audioEngineRef.current?.playSound("screenshareStarted");
});
const joinSub = vm.joinSoundEffect$.subscribe(
() => void audioEngineRef.current?.playSound("join"),
);
const leftSub = vm.leaveSoundEffect$.subscribe(
() => void audioEngineRef.current?.playSound("left"),
);
const handRaisedSub = vm.newHandRaised$.subscribe(
() => void audioEngineRef.current?.playSound("raiseHand"),
);
const screenshareSub = vm.newScreenShare$.subscribe(
() => void audioEngineRef.current?.playSound("screenshareStarted"),
);
return (): void => {
joinSub.unsubscribe();

2
src/room/InCallView.tsx
View File

@@ -321,7 +321,7 @@ export const InCallView: FC<InCallViewProps> = ({
const showFooter = useBehavior(vm.showFooter$);
const earpieceMode = useBehavior(vm.earpieceMode$);
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
useSubscription(vm.autoLeaveWhenOthersLeft$, onLeave);
useSubscription(vm.autoLeave$, onLeave);
// Ideally we could detect taps by listening for click events and checking
// that the pointerType of the event is "touch", but this isn't yet supported

259
src/state/CallViewModel.test.ts
View File

@@ -25,6 +25,7 @@ import {
RoomEvent as MatrixRoomEvent,
MatrixEvent,
type IRoomTimelineData,
EventType,
} from "matrix-js-sdk";
import {
ConnectionState,
@@ -318,7 +319,7 @@ function withCallViewModel(
const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((room, eventType) => of());
.mockImplementation((_room, _eventType) => of());
const livekitRoom = mockLivekitRoom(
{ localParticipant },
@@ -1078,9 +1079,9 @@ it("should rank raised hands above video feeds and below speakers and presenters
});
function nooneEverThere$<T>(
hot: (marbles: string, values: Record<string, T[]>) => Observable<T[]>,
): Observable<T[]> {
return hot("a-b-c-d", {
behavior: (marbles: string, values: Record<string, T[]>) => Behavior<T[]>,
): Behavior<T[]> {
return behavior("a-b-c-d", {
a: [], // Start empty
b: [], // Alice joins
c: [], // Alice still there
@@ -1089,12 +1090,12 @@ function nooneEverThere$<T>(
}
function participantJoinLeave$(
hot: (
behavior: (
marbles: string,
values: Record<string, RemoteParticipant[]>,
) => Observable<RemoteParticipant[]>,
): Observable<RemoteParticipant[]> {
return hot("a-b-c-d", {
) => Behavior<RemoteParticipant[]>,
): Behavior<RemoteParticipant[]> {
return behavior("a-b-c-d", {
a: [], // Start empty
b: [aliceParticipant], // Alice joins
c: [aliceParticipant], // Alice still there
@@ -1103,12 +1104,12 @@ function participantJoinLeave$(
}
function rtcMemberJoinLeave$(
hot: (
behavior: (
marbles: string,
values: Record<string, CallMembership[]>,
) => Observable<CallMembership[]>,
): Observable<CallMembership[]> {
return hot("a-b-c-d", {
) => Behavior<CallMembership[]>,
): Behavior<CallMembership[]> {
return behavior("a-b-c-d", {
a: [localRtcMember], // Start empty
b: [localRtcMember, aliceRtcMember], // Alice joins
c: [localRtcMember, aliceRtcMember], // Alice still there
@@ -1116,47 +1117,15 @@ function rtcMemberJoinLeave$(
});
}
test("allOthersLeft$ emits only when someone joined and then all others left", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
// Test scenario 1: No one ever joins - should only emit initial false and never emit again
withCallViewModel(
{ remoteParticipants$: scope.behavior(nooneEverThere$(hot), []) },
(vm) => {
expectObservable(vm.allOthersLeft$).toBe("n------", { n: false });
},
);
});
});
test("allOthersLeft$ emits true when someone joined and then all others left", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
test("autoLeave$ emits only when autoLeaveWhenOthersLeft option is enabled", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []),
rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []),
remoteParticipants$: participantJoinLeave$(behavior),
rtcMembers$: rtcMemberJoinLeave$(behavior),
},
(vm) => {
expectObservable(vm.allOthersLeft$).toBe(
"n-----u", // false initially, then at frame 6: true then false emissions in same frame
{ n: false, u: true }, // map(() => {})
);
},
);
});
});
test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is enabled", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []),
rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []),
},
(vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe(
"------e", // false initially, then at frame 6: true then false emissions in same frame
{ e: undefined },
);
expectObservable(vm.autoLeave$).toBe("------(e|)", { e: undefined });
},
{
autoLeaveWhenOthersLeft: true,
@@ -1166,15 +1135,15 @@ test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is
});
});
test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
test("autoLeave$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(nooneEverThere$(hot), []),
rtcMembers$: scope.behavior(nooneEverThere$(hot), []),
remoteParticipants$: nooneEverThere$(behavior),
rtcMembers$: nooneEverThere$(behavior),
},
(vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------");
expectObservable(vm.autoLeave$).toBe("-");
},
{
autoLeaveWhenOthersLeft: true,
@@ -1184,15 +1153,15 @@ test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is ena
});
});
test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
test("autoLeave$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []),
rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []),
remoteParticipants$: participantJoinLeave$(behavior),
rtcMembers$: rtcMemberJoinLeave$(behavior),
},
(vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------");
expectObservable(vm.autoLeave$).toBe("-");
},
{
autoLeaveWhenOthersLeft: false,
@@ -1202,31 +1171,25 @@ test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option
});
});
test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => {
withTestScheduler(({ hot, expectObservable, scope }) => {
test("autoLeave$ emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(
hot("a-b-c-d", {
a: [], // Alone
b: [aliceParticipant], // Alice joins
c: [aliceParticipant],
d: [], // Local joins with a second device
}),
[], //Alice leaves
),
rtcMembers$: scope.behavior(
hot("a-b-c-d", {
a: [localRtcMember], // Start empty
b: [localRtcMember, aliceRtcMember], // Alice joins
c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there
d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves
}),
[],
),
remoteParticipants$: behavior("a-b-c-d", {
a: [], // Alone
b: [aliceParticipant], // Alice joins
c: [aliceParticipant],
d: [], // Local joins with a second device
}),
rtcMembers$: behavior("a-b-c-d", {
a: [localRtcMember], // Start empty
b: [localRtcMember, aliceRtcMember], // Alice joins
c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there
d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves
}),
},
(vm) => {
expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("------e", {
expectObservable(vm.autoLeave$).toBe("------(e|)", {
e: undefined,
});
},
@@ -1238,18 +1201,12 @@ test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option
});
});
describe("shouldWaitForCallPickup$", () => {
describe("waitForCallPickup$", () => {
test("unknown -> ringing -> timeout when notified and nobody joins", () => {
withTestScheduler(({ hot, schedule, expectObservable, scope }) => {
withTestScheduler(({ schedule, expectObservable }) => {
// No one ever joins (only local user)
withCallViewModel(
{
remoteParticipants$: scope.behavior(hot("a", { a: [] }), []),
rtcMembers$: scope.behavior(hot("a", { a: [localRtcMember] }), []),
connectionState$: of(ConnectionState.Connected),
speaking: new Map(),
mediaDevices: mockMediaDevices({}),
},
{ remoteParticipants$: constant([]) },
(vm, rtcSession) => {
// Fire a call notification at 10ms with lifetime 30ms
schedule(" 10ms r", {
@@ -1273,7 +1230,7 @@ describe("shouldWaitForCallPickup$", () => {
});
},
{
shouldWaitForCallPickup: true,
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
@@ -1281,25 +1238,18 @@ describe("shouldWaitForCallPickup$", () => {
});
test("ringing -> success if someone joins before timeout", () => {
withTestScheduler(({ hot, schedule, expectObservable, scope }) => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const remote$ = scope.behavior(
hot("a--b", { a: [], b: [aliceParticipant] }),
[],
);
const rtc$ = scope.behavior(
hot("a--b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
[],
);
withCallViewModel(
{
remoteParticipants$: remote$,
rtcMembers$: rtc$,
connectionState$: of(ConnectionState.Connected),
remoteParticipants$: behavior("a 19ms b", {
a: [],
b: [aliceParticipant],
}),
rtcMembers$: behavior("a 19ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
},
(vm, rtcSession) => {
// Notify at 5ms so we enter ringing, then success at 20ms
@@ -1320,13 +1270,14 @@ describe("shouldWaitForCallPickup$", () => {
},
});
expectObservable(vm.callPickupState$).toBe("a 2ms c", {
expectObservable(vm.callPickupState$).toBe("a 4ms b 14ms c", {
a: "unknown",
b: "ringing",
c: "success",
});
},
{
shouldWaitForCallPickup: true,
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
@@ -1334,27 +1285,18 @@ describe("shouldWaitForCallPickup$", () => {
});
test("success when someone joins before we notify", () => {
withTestScheduler(({ hot, schedule, expectObservable, scope }) => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Join at 10ms, notify later at 20ms (state should stay success)
const remote$ = scope.behavior(
hot("a-b", { a: [], b: [aliceParticipant] }),
[],
);
const rtc$ = scope.behavior(
hot("a-b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
[],
);
withCallViewModel(
{
remoteParticipants$: remote$,
rtcMembers$: rtc$,
connectionState$: of(ConnectionState.Connected),
speaking: new Map(),
mediaDevices: mockMediaDevices({}),
remoteParticipants$: behavior("a 9ms b", {
a: [],
b: [aliceParticipant],
}),
rtcMembers$: behavior("a 9ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
},
(vm, rtcSession) => {
schedule(" 20ms r", {
@@ -1370,13 +1312,13 @@ describe("shouldWaitForCallPickup$", () => {
);
},
});
expectObservable(vm.callPickupState$).toBe("a 1ms b", {
expectObservable(vm.callPickupState$).toBe("a 9ms b", {
a: "unknown",
b: "success",
});
},
{
shouldWaitForCallPickup: true,
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
@@ -1384,21 +1326,15 @@ describe("shouldWaitForCallPickup$", () => {
});
test("notify without lifetime -> immediate timeout", () => {
withTestScheduler(({ hot, schedule, expectObservable, scope }) => {
withTestScheduler(({ schedule, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: scope.behavior(hot("a", { a: [] }), []),
rtcMembers$: scope.behavior(hot("a", { a: [localRtcMember] }), []),
connectionState$: of(ConnectionState.Connected),
speaking: new Map(),
mediaDevices: mockMediaDevices({}),
},
{},
(vm, rtcSession) => {
schedule(" 10ms r", {
r: () => {
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
{ event_id: "$notif4", lifetime: 0 } as unknown as {
{ event_id: "$notif4" } as unknown as {
event_id: string;
} & IRTCNotificationContent, // no lifetime
{ event_id: "$notif4" } as unknown as {
@@ -1413,34 +1349,25 @@ describe("shouldWaitForCallPickup$", () => {
});
},
{
shouldWaitForCallPickup: true,
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
});
});
test("stays null when shouldWaitForCallPickup=false", () => {
withTestScheduler(({ hot, schedule, expectObservable, scope }) => {
const remote$ = scope.behavior(
hot("a--b", { a: [], b: [aliceParticipant] }),
[],
);
const rtc$ = scope.behavior(
hot("a--b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
[],
);
test("stays null when waitForCallPickup=false", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: remote$,
rtcMembers$: rtc$,
connectionState$: of(ConnectionState.Connected),
speaking: new Map(),
mediaDevices: mockMediaDevices({}),
remoteParticipants$: behavior("a--b", {
a: [],
b: [aliceParticipant],
}),
rtcMembers$: behavior("a--b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
},
(vm, rtcSession) => {
schedule(" 5ms r", {
@@ -1461,7 +1388,7 @@ describe("shouldWaitForCallPickup$", () => {
});
},
{
shouldWaitForCallPickup: false,
waitForCallPickup: false,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
@@ -1494,7 +1421,15 @@ describe("shouldWaitForCallPickup$", () => {
// Emit decline timeline event with id matching the notification
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({ event_id: "$decl1", type: "m.rtc.decline" }),
new MatrixEvent({
type: EventType.RTCDecline,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$decl1",
},
},
}),
rtcSession.room,
undefined,
false,
@@ -1509,7 +1444,7 @@ describe("shouldWaitForCallPickup$", () => {
});
},
{
shouldWaitForCallPickup: true,
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
@@ -1557,7 +1492,7 @@ describe("shouldWaitForCallPickup$", () => {
});
},
{
shouldWaitForCallPickup: true,
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
@@ -1604,7 +1539,7 @@ describe("shouldWaitForCallPickup$", () => {
});
},
{
shouldWaitForCallPickup: true,
waitForCallPickup: true,
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);

343
src/state/CallViewModel.ts
View File

@@ -30,18 +30,22 @@ import {
import {
BehaviorSubject,
EMPTY,
NEVER,
type Observable,
Subject,
combineLatest,
concat,
distinctUntilChanged,
endWith,
filter,
forkJoin,
fromEvent,
ignoreElements,
map,
merge,
mergeMap,
of,
pairwise,
race,
scan,
skip,
@@ -50,6 +54,8 @@ import {
switchMap,
switchScan,
take,
takeUntil,
throttleTime,
timer,
withLatestFrom,
} from "rxjs";
@@ -109,7 +115,7 @@ import { observeSpeaker$ } from "./observeSpeaker";
import { shallowEquals } from "../utils/array";
import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname";
import { type MediaDevices } from "./MediaDevices";
import { type Behavior } from "./Behavior";
import { constant, type Behavior } from "./Behavior";
export interface CallViewModelOptions {
encryptionSystem: EncryptionSystem;
@@ -118,13 +124,18 @@ export interface CallViewModelOptions {
* If the call is started in a way where we want it to behave like a telephone usecase
* If we sent a notification event, we want the ui to show a ringing state
*/
shouldWaitForCallPickup?: boolean;
waitForCallPickup?: boolean;
}
// How long we wait after a focus switch before showing the real participant
// list again
const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
// Do not play any sounds if the participant count has exceeded this
// number.
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const THROTTLE_SOUND_EFFECT_MS = 500;
// This is the number of participants that we think constitutes a "small" call
// on mobile. No spotlight tile should be shown below this threshold.
const smallMobileCallThreshold = 3;
@@ -437,14 +448,7 @@ export class CallViewModel extends ViewModel {
MembershipManagerEvent.StatusChanged,
).pipe(
startWith(null),
map(
() =>
(
this.matrixRTCSession as unknown as {
membershipStatus?: Status;
}
).membershipStatus === Status.Connected,
),
map(() => this.matrixRTCSession.membershipStatus === Status.Connected),
),
// Also watch out for warnings that we've likely hit a timeout and our
// delayed leave event is being sent (this condition is here because it
@@ -455,11 +459,7 @@ export class CallViewModel extends ViewModel {
MembershipManagerEvent.ProbablyLeft,
).pipe(
startWith(null),
map(
() =>
(this.matrixRTCSession as unknown as { probablyLeft?: boolean })
.probablyLeft !== true,
),
map(() => this.matrixRTCSession.probablyLeft !== true),
),
),
);
@@ -576,6 +576,17 @@ export class CallViewModel extends ViewModel {
)
.pipe(pauseWhen(this.pretendToBeDisconnected$));
private readonly memberships$ = this.scope.behavior(
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
).pipe(
startWith(null),
pauseWhen(this.pretendToBeDisconnected$),
map(() => this.matrixRTCSession.memberships),
),
);
/**
* Displaynames for each member of the call. This will disambiguate
* any displaynames that clashes with another member. Only members
@@ -585,18 +596,17 @@ export class CallViewModel extends ViewModel {
// than on Chrome/Firefox). This means it is important that we multicast the result so that we
// don't do this work more times than we need to. This is achieved by converting to a behavior:
public readonly memberDisplaynames$ = this.scope.behavior(
merge(
// Handle call membership changes.
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
),
// Handle room membership changes (and displayname updates)
fromEvent(this.matrixRoom, RoomStateEvent.Members),
).pipe(
startWith(null),
map(() => {
const memberships = this.matrixRTCSession.memberships;
// React to call memberships and also display name updates
// (calculateDisplayName implicitly depends on the room member data)
combineLatest(
[
this.memberships$,
fromEvent(this.matrixRoom, RoomStateEvent.Members).pipe(
startWith(null),
pauseWhen(this.pretendToBeDisconnected$),
),
],
(memberships, _members) => {
const displaynameMap = new Map<string, string>();
const room = this.matrixRoom;
@@ -618,8 +628,7 @@ export class CallViewModel extends ViewModel {
);
}
return displaynameMap;
}),
pauseWhen(this.pretendToBeDisconnected$),
},
),
);
@@ -649,13 +658,7 @@ export class CallViewModel extends ViewModel {
this.remoteParticipants$,
observeParticipantMedia(this.livekitRoom.localParticipant),
duplicateTiles.value$,
// Also react to changes in the MatrixRTC session list.
// The session list will also be update if a room membership changes.
// No additional RoomState event listener needs to be set up.
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
).pipe(startWith(null), pauseWhen(this.pretendToBeDisconnected$)),
this.memberships$,
showNonMemberTiles.value$,
]).pipe(
scan(
@@ -665,7 +668,7 @@ export class CallViewModel extends ViewModel {
remoteParticipants,
{ participant: localParticipant },
duplicateTiles,
_membershipsChanged,
memberships,
showNonMemberTiles,
],
) => {
@@ -673,7 +676,7 @@ export class CallViewModel extends ViewModel {
function* (this: CallViewModel): Iterable<[string, MediaItem]> {
const room = this.matrixRoom;
// m.rtc.members are the basis for calculating what is visible in the call
for (const rtcMember of this.matrixRTCSession.memberships) {
for (const rtcMember of memberships) {
const { member, id: livekitParticipantId } =
getRoomMemberFromRtcMember(rtcMember, room);
const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`;
@@ -839,205 +842,143 @@ export class CallViewModel extends ViewModel {
),
);
/**
* This observable tracks the currently connected participants.
*
* - Each participant has one livekit connection
* - Each participant has a corresponding MatrixRTC membership state event
* - There can be multiple participants for one matrix user.
*/
public readonly participantChanges$ = this.scope.behavior(
this.userMedia$.pipe(
map((mediaItems) => mediaItems.map((m) => m.id)),
scan<string[], { ids: string[]; joined: string[]; left: string[] }>(
(prev, ids) => {
const left = prev.ids.filter((id) => !ids.includes(id));
const joined = ids.filter((id) => !prev.ids.includes(id));
return { ids, joined, left };
},
{ ids: [], joined: [], left: [] },
),
public readonly joinSoundEffect$ = this.userMedia$.pipe(
pairwise(),
filter(
([prev, current]) =>
current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND &&
current.length > prev.length,
),
map(() => {}),
throttleTime(THROTTLE_SOUND_EFFECT_MS),
);
public readonly leaveSoundEffect$ = this.userMedia$.pipe(
pairwise(),
filter(
([prev, current]) =>
current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND &&
current.length < prev.length,
),
map(() => {}),
throttleTime(THROTTLE_SOUND_EFFECT_MS),
);
/**
* The number of participants currently in the call.
*
* - Each participant has one livekit connection
* - Each participant has a corresponding MatrixRTC membership state event
* - There can be multiple participants for one matrix user.
* - There can be multiple participants for one Matrix user if they join from
* multiple devices.
*/
public readonly participantCount$ = this.scope.behavior(
this.participantChanges$.pipe(map(({ ids }) => ids.length)),
this.memberships$.pipe(map((ms) => ms.length)),
);
/**
* This observable tracks the matrix users that are currently in the call.
* There can be just one matrix user with multiple participants (see also participantChanges$)
*/
public readonly matrixUserChanges$ = this.scope.behavior(
this.userMedia$.pipe(
map(
(mediaItems) =>
new Set(
mediaItems
.map((m) => m.vm.member?.userId)
.filter((id) => id !== undefined),
),
),
scan<
Set<string>,
{
userIds: Set<string>;
joinedUserIds: Set<string>;
leftUserIds: Set<string>;
}
>(
(prevState, userIds) => {
const left = new Set(
[...prevState.userIds].filter((id) => !userIds.has(id)),
);
const joined = new Set(
[...userIds].filter((id) => !prevState.userIds.has(id)),
);
return { userIds: userIds, joinedUserIds: joined, leftUserIds: left };
},
{
userIds: new Set(),
joinedUserIds: new Set(),
leftUserIds: new Set(),
},
),
private readonly allOthersLeft$ = this.memberships$.pipe(
pairwise(),
filter(
([prev, current]) =>
current.every((m) => m.sender === this.userId) &&
prev.some((m) => m.sender !== this.userId),
),
);
public readonly allOthersLeft$ = this.matrixUserChanges$.pipe(
map(({ userIds, leftUserIds }) => {
if (!this.userId) {
logger.warn("Could not access user ID to compute allOthersLeft");
return false;
}
return (
userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0
);
}),
startWith(false),
distinctUntilChanged(),
);
public readonly autoLeaveWhenOthersLeft$ = this.allOthersLeft$.pipe(
distinctUntilChanged(),
filter((leave) => (leave && this.options.autoLeaveWhenOthersLeft) ?? false),
map(() => {}),
take(1),
);
public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft
? this.allOthersLeft$
: NEVER;
/**
* "unknown": We don't know if the RTC session decides to send a notify event yet.
* It will only be known once we sent our own membership and know we were the first one to join.
* "ringing": The notification event was sent.
* "ringEnded": The notification events lifetime has timed out -> ringing stopped on all receiving clients.
* Whenever the RTC session tells us that it intends to ring the remote
* participant's devices, this emits an Observable tracking the current state of
* that ringing process.
*/
private readonly rtcNotificationEventState$: Observable<
{ state: "unknown" | "ringEnded" } | { state: "ringing"; event_id: string }
> = fromEvent<
Parameters<
MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification]
private readonly ring$: Observable<
Observable<"ringing" | "timeout" | "decline">
> = (
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.DidSendCallNotification,
) as Observable<
Parameters<
MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification]
>
>
>(this.matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification).pipe(
switchMap(([notificationEvent]) => {
).pipe(
map(([notificationEvent]) => {
// event.lifetime is expected to be in ms
const lifetime = notificationEvent?.lifetime ?? 0;
if (lifetime > 0) {
// Emit true immediately, then false after lifetime ms
return concat(
of({
state: "ringing",
event_id: notificationEvent.event_id,
} as {
state: "ringing";
event_id: string;
}),
timer(lifetime).pipe(
map(() => ({ state: "ringEnded" }) as { state: "ringEnded" }),
return concat(
lifetime === 0
? // If no lifetime, skip the ring state
EMPTY
: // Ring until lifetime ms have passed
timer(lifetime).pipe(
ignoreElements(),
startWith("ringing" as const),
),
// The notification lifetime has timed out, meaning ringing has likely
// stopped on all receiving clients.
of("timeout" as const),
NEVER,
).pipe(
takeUntil(
(
fromEvent(this.matrixRoom, RoomEvent.Timeline) as Observable<
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
>
).pipe(
filter(
([event]) =>
event.getType() === EventType.RTCDecline &&
event.getRelation()?.rel_type === "m.reference" &&
event.getRelation()?.event_id === notificationEvent.event_id,
),
),
);
}
// If no lifetime, the notify event is basically invalid and we enter ringEnded immediately.
return of({ state: "ringEnded" } as { state: "ringEnded" });
),
endWith("decline" as const),
);
}),
startWith({ state: "unknown" } as { state: "unknown" }),
);
/**
* If some other matrix user has joined the call. It can start with true if there are already multiple matrix users.
* Whether some Matrix user other than ourself is joined to the call.
*/
private readonly someoneElseJoined$ = this.matrixUserChanges$.pipe(
scan(
(someoneJoined, { joinedUserIds }) =>
someoneJoined || [...joinedUserIds].some((id) => id !== this.userId),
false,
),
startWith(this.matrixUserChanges$.value.userIds.size > 1),
private readonly someoneElseJoined$ = this.memberships$.pipe(
map((ms) => ms.some((m) => m.sender !== this.userId)),
);
/**
* The current call pickup state of the call.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
* The call failed. If desired this can be used as a trigger to exit the call.
* - "success": Someone else joined. The call is in a normal state. Stop audiovisual feedback.
* - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
* - null: EC is configured to never show any waiting for answer state.
*/
public readonly callPickupState$: Behavior<
"unknown" | "ringing" | "timeout" | "success" | "decline" | null
> = this.scope.behavior(
combineLatest([
this.rtcNotificationEventState$,
this.someoneElseJoined$,
fromEvent<Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>>(
this.matrixRoom,
RoomEvent.Timeline,
).pipe(
map(([event]) => {
if (event.getType() === EventType.RTCDecline) return event;
else return null;
}),
startWith(null),
),
]).pipe(
map(([notificationEventState, someoneJoined, declineEvent]) => {
// Never enter waiting for answer state if the app is not configured with waitingForAnswer.
if (!this.options.shouldWaitForCallPickup) return null;
// As soon as someone joins, we can consider the call "wait for answer" successful
if (someoneJoined) return "success";
switch (notificationEventState?.state) {
case "unknown":
return "unknown";
case "ringing":
// Check if the decline event corresponds to the current notification event
if (declineEvent?.getId() === notificationEventState.event_id) {
return "decline";
}
return "ringing";
case "ringEnded":
return "timeout";
default:
return "timeout";
}
}),
// Once we reach a terminal state, keep it
scan((prev, next) => {
if (prev === "decline" || prev === "timeout" || prev === "success") {
return prev;
}
return next;
}),
distinctUntilChanged(),
),
);
public readonly callPickupState$ = this.options.waitForCallPickup
? this.scope.behavior<
"unknown" | "ringing" | "timeout" | "decline" | "success"
>(
this.someoneElseJoined$.pipe(
switchMap((someoneElseJoined) =>
someoneElseJoined
? of("success" as const)
: // Show the ringing state of the most recent ringing attempt.
this.ring$.pipe(switchAll()),
),
// The state starts as 'unknown' because we don't know if the RTC
// session will actually send a notify event yet. It will only be
// known once we send our own membership and see that we were the
// first one to join.
startWith("unknown" as const),
),
)
: constant(null);
/**
* List of MediaItems that we want to display, that are of type ScreenShare

6
yarn.lock
View File

@@ -10046,8 +10046,8 @@ __metadata:
linkType: hard
"livekit-client@npm:^2.13.0":
version: 2.15.5
resolution: "livekit-client@npm:2.15.5"
version: 2.15.6
resolution: "livekit-client@npm:2.15.6"
dependencies:
"@livekit/mutex": "npm:1.1.1"
"@livekit/protocol": "npm:1.39.3"
@@ -10060,7 +10060,7 @@ __metadata:
webrtc-adapter: "npm:^9.0.1"
peerDependencies:
"@types/dom-mediacapture-record": ^1
checksum: 10c0/52a70bdd39d802737ed7c25ae5d06daf9921156c4fc74f918009e86204430b2d200b66c55cefab949be4e5411cbc4d25eac92976f62f96b7226057a5b0706baa
checksum: 10c0/f1ab6cdf2b85647036e9de906734c1394dac497da0bd879a29d0c587c437ada262021478fcef24df99b3489a39d97fe67ab33de0785ed0a63335da2fef577192
languageName: node
linkType: hard