Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-06-30 18:02:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve internal model of ringing, expose ringing intent to call UI

Browse Source 
I found our code's internal model of ringing a little overgrown (it had superfluous states like 'unknown') and difficult to extend with metadata or callbacks relating to ring attempts. By modeling ringing instead as a stream of ring attempts, where each attempt has an intent, a recipient, and an eventual outcome (accept/decline/timeout), I find it more natural to work with.

This makes room for a future 'try again' callback to allow ringing someone again after a timeout, and also forced me to look for a simpler solution to the duplicate leave sound effects. I exposed the intent of the ringing attempt to the call UI so I can later use it in the header.
This commit is contained in:
Robin
2026-06-10 15:49:00 +02:00
parent 3a824dfff0
commit 2ac6cdeb46
13 changed files with 468 additions and 577 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/room/GroupCallView.test.tsx
View File

@@ -251,7 +251,7 @@ test.skip("GroupCallView plays a leave sound synchronously in widget mode", asyn
expect(leaveRTCSession).toHaveBeenCalledOnce();
});
test("Should close widget when all other left and have time to play a sound", async () => {
test("Should close widget when all other left and play a sound", async () => {
const user = userEvent.setup();
let widgetClosedCalled = false;
const { promise: widgetClosedPromise, resolve: widgetClosedResolver } =
@@ -289,46 +289,13 @@ test("Should close widget when all other left and have time to play a sound", as
expect(widgetClosedCalled).toBeFalsy();
resolvePlaySound.resolve();
// Expect the leave sound to be played but silent (volumeOverwrite = 0)
// The allOthersLeft effect should already play a leave sound for the last user in the call.
expect(playSound).toHaveBeenCalledWith("left", 0);
expect(playSound).toHaveBeenCalledWith("left");
await widgetClosedPromise;
await flushPromises();
expect(widgetClosedCalled).toBeTruthy();
expect(widgetStopMock).toHaveBeenCalledOnce();
}, 80000);
test("Should close widget when all other left", async () => {
const user = userEvent.setup();
const widgetClosedCalled = Promise.withResolvers<void>();
const widgetSendMock = vi.fn().mockImplementation((action: string) => {
if (action === ElementWidgetActions.Close) {
widgetClosedCalled.resolve();
}
});
const widgetStopMock = vi.fn().mockResolvedValue(undefined);
const widget = {
api: {
setAlwaysOnScreen: vi.fn().mockResolvedValue(true),
transport: {
send: widgetSendMock,
reply: vi.fn().mockResolvedValue(undefined),
stop: widgetStopMock,
} as unknown as ITransport,
} as Partial<WidgetHelpers["api"]>,
lazyActions: new LazyEventEmitter(),
};
const { getByText } = createGroupCallView(widget as WidgetHelpers);
const leaveButton = getByText("SimulateOtherLeft");
await user.click(leaveButton);
await flushPromises();
await widgetClosedCalled.promise;
await flushPromises();
expect(widgetStopMock).toHaveBeenCalledOnce();
});
test("Should not close widget when auto leave due to error", async () => {
const user = userEvent.setup();

19
src/room/GroupCallView.tsx
View File

@@ -318,25 +318,12 @@ export const GroupCallView: FC<Props> = ({
(
reason: "timeout" | "user" | "allOthersLeft" | "decline" | "error",
): void => {
let audioPromise: Promise<void> | undefined = undefined;
switch (reason) {
case "allOthersLeft":
// When "allOthersLeft", the leaveSoundEffect$ in CallEventAudioRenderer
// already plays the "left" sound when the remote participant's media
// disappears. We play it here silenced (volumeOverwrite = 0) so we have the right duration in the audioPromise.
// (used to destory the widget)
audioPromise = leaveSoundContext.current?.playSound("left", 0);
break;
case "timeout":
case "decline":
audioPromise = leaveSoundContext.current?.playSound(reason);
break;
default:
audioPromise = leaveSoundContext.current?.playSound("left");
}
let playSound: CallEventSounds = "left";
if (reason === "timeout" || reason === "decline") playSound = reason;
setJoined(false);
setLeft(true);
const audioPromise = leaveSoundContext.current?.playSound(playSound);
// We need to wait until the callEnded event is tracked on PostHog,
// otherwise the iframe may get killed first.

6
src/room/InCallView.tsx
View File

@@ -266,7 +266,7 @@ export const InCallView: FC<InCallViewProps> = ({
() => void toggleRaisedHand(),
);
const ringing = useBehavior(vm.ringing$);
const ringingIntent = useBehavior(vm.ringingIntent$);
const audioParticipants = useBehavior(vm.livekitRoomItems$);
const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$);
@@ -289,7 +289,7 @@ export const InCallView: FC<InCallViewProps> = ({
// While ringing, loop the ringtone
useEffect((): void | (() => void) => {
const audio = latestPickupPhaseAudio.current;
if (ringing && audio) {
if (ringingIntent !== null && audio) {
const endSound = audio.playSoundLooping(
"waiting",
audio.soundDuration["waiting"] ?? 1,
@@ -300,7 +300,7 @@ export const InCallView: FC<InCallViewProps> = ({
});
};
}
}, [ringing, latestPickupPhaseAudio]);
}, [ringingIntent, latestPickupPhaseAudio]);
// iOS Safari doesn't reliably fire `click` on plain <div>s, so we listen
// for `pointerup` instead. Scrolls end in `pointercancel`, not `pointerup`,

576
src/state/CallViewModel/CallNotificationLifecycle.test.ts
View File

@@ -5,17 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, it } from "vitest";
import { test } from "vitest";
import {
EventType,
type IEvent,
type IRoomTimelineData,
MatrixEvent,
type Room,
} from "matrix-js-sdk";
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
import { map, mergeMap, NEVER, type Observable, startWith } from "rxjs";
import { withTestScheduler } from "../../utils/test";
import {
alice,
aliceRtcMember,
local,
localRtcMember,
@@ -23,9 +25,10 @@ import {
import {
type CallNotificationWrapper,
createCallNotificationLifecycle$,
type Props as CallNotificationLifecycleProps,
type RingAttempt,
} from "./CallNotificationLifecycle";
import { trackEpoch } from "../ObservableScope";
import { Epoch, trackEpoch } from "../ObservableScope";
import { constant } from "../Behavior";
function mockRingEvent(
eventId: string,
@@ -40,311 +43,272 @@ function mockRingEvent(
} as unknown as CallNotificationWrapper;
}
describe("waitForCallPickup$", () => {
it("unknown -> ringing -> timeout when notified and nobody joins", () => {
withTestScheduler(({ scope, expectObservable, behavior, hot }) => {
// No one ever joins (only local user)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", { a: [] }).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const defaultProps = {
memberships$: constant(new Epoch([])),
matrixRoomMembers$: constant(new Map([[alice.userId, alice]])),
receivedDecline$: NEVER,
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
function summarizeRingAttempts$(
ringAttempts$: Observable<RingAttempt>,
): Observable<
| { intent: RTCCallIntent; recipient: string }
| { outcome: "accept" | "decline" | "timeout" }
> {
return ringAttempts$.pipe(
mergeMap(({ intent, recipient, outcome$ }) =>
outcome$.pipe(
map((outcome) => ({ outcome })),
startWith({ intent, recipient }),
),
),
);
}
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms c", {
a: "unknown",
b: "ringing",
c: "timeout",
});
test("no ring attempt when waitForCallPickup=false", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
options: { ...defaultProps.options, waitForCallPickup: false },
});
});
it("ringing -> success if someone joins before timeout is reached", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a 19ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("5ms a", {
a: mockRingEvent("$notif2", 100),
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 4ms b 14ms c", {
a: "unknown",
b: "ringing",
c: "success",
});
});
});
it("success when someone joins before we notify", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a 9ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("20ms a", {
a: mockRingEvent("$notif2", 50),
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", {
a: "unknown",
b: "success",
});
});
});
it("notify without lifetime -> immediate timeout", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$notif2", undefined),
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", {
a: "unknown",
b: "timeout",
});
});
});
it("stays null when waitForCallPickup=false", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const validProps: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a--b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$notif5", 30),
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const propsDeactivated = {
...validProps,
options: {
...validProps.options,
waitForCallPickup: false,
},
};
const lifecycle = createCallNotificationLifecycle$(propsDeactivated);
expectObservable(lifecycle.callPickupState$).toBe("n", {
n: null,
});
const lifecycleReference = createCallNotificationLifecycle$(validProps);
expectObservable(lifecycleReference.callPickupState$).toBe("u--s", {
u: "unknown",
s: "success",
});
});
});
it("decline before timeout window ends -> decline", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$decl1", 50),
}),
receivedDecline$: hot("40ms d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$decl1",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms e", {
a: "unknown",
b: "ringing",
e: "decline",
});
});
});
it("decline after timeout window ends -> stays timeout", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$decl", 20),
}),
receivedDecline$: hot("40ms d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$decl",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$, "50ms !").toBe(
"a 9ms b 19ms e",
{
a: "unknown",
b: "ringing",
e: "timeout",
},
);
});
});
//
function testStaysRinging(
declineEvent: Partial<IEvent>,
expectDecline: boolean,
): void {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$right", 50),
}),
receivedDecline$: hot("20ms d", {
d: [
new MatrixEvent(declineEvent),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
const marbles = expectDecline ? "a 9ms b 9ms d" : "a 9ms b";
expectObservable(lifecycle.callPickupState$, "21ms !").toBe(marbles, {
a: "unknown",
b: "ringing",
d: "decline",
});
});
}
const reference = (refId?: string, sender?: string): Partial<IEvent> => ({
event_id: "$decline",
type: EventType.RTCDecline,
sender: sender ?? "@other:example.org",
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: refId ?? "$right",
},
},
});
it("decline reference works", () => {
testStaysRinging(reference(), true);
});
it("decline with wrong id is ignored (stays ringing)", () => {
testStaysRinging(reference("$wrong"), false);
});
it("decline with wrong id is ignored (stays ringing)", () => {
testStaysRinging(reference(undefined, local.userId), false);
expectObservable(ringAttempts$).toBe("");
});
});
test("no ring attempt when notification type is not ring", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: {
...mockRingEvent("$notif1", 30),
notification_type: "notification",
},
}),
});
expectObservable(ringAttempts$).toBe("");
});
});
test("no ring attempt if lifetime is missing", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", undefined),
}),
});
expectObservable(ringAttempts$).toBe("");
});
});
test("ring attempt times out after nobody joins", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
// No one ever joins (only local user)
memberships$: constant(new Epoch([])),
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "timeout" },
});
});
});
test("ring attempt is accepted once recipient joins", () => {
withTestScheduler(({ scope, expectObservable, hot, behavior }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
memberships$: scope.behavior(
behavior("a-b", { a: [], b: [aliceRtcMember] }).pipe(trackEpoch()),
),
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-aA", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "accept" },
});
});
});
test("ring attempt is immediately accepted if recipient is already joined", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
memberships$: constant(new Epoch([aliceRtcMember])),
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-(aA)", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "accept" },
});
});
});
test("ring attempt can be declined", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot("--d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
sender: alice.userId,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$notif1",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-aA", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "decline" },
});
});
});
test("ring attempt times out if recipient declines too late", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot("100ms d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
sender: alice.userId,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$notif1",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "timeout" },
});
});
});
test("decline event relating to wrong event is ignored (times out)", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot("--d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
sender: alice.userId,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$other", // <---- WRONG
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "timeout" },
});
});
});
test("decline event from wrong sender is ignored (times out)", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot("--d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
sender: local.userId, // <---- WRONG
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$notif1",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "timeout" },
});
});
});

205
src/state/CallViewModel/CallNotificationLifecycle.ts
View File

@@ -10,24 +10,22 @@ import {
type IRTCNotificationContent,
type MatrixRTCSession,
MatrixRTCSessionEvent,
type RTCCallIntent,
} from "matrix-js-sdk/lib/matrixrtc";
import {
combineLatest,
concat,
endWith,
filter,
fromEvent,
ignoreElements,
map,
merge,
NEVER,
type Observable,
of,
pairwise,
startWith,
switchMap,
takeUntil,
timer,
EMPTY,
race,
take,
} from "rxjs";
import {
type EventTimelineSetHandlerMap,
@@ -35,18 +33,28 @@ import {
type Room as MatrixRoom,
RoomEvent,
} from "matrix-js-sdk";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../Behavior";
import { type Epoch, mapEpoch, type ObservableScope } from "../ObservableScope";
import { type Epoch, type ObservableScope } from "../ObservableScope";
import { type RoomMemberMap } from "./remoteMembers/MatrixMemberMetadata";
const logger = rootLogger.getChild("[CallNotificationLifecycle]");
export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline";
export type CallPickupState =
| "unknown"
| "ringing"
| "timeout"
| "decline"
| "success"
| null;
export interface RingAttempt {
intent: RTCCallIntent;
/**
* The user ID of the recipient being rung.
*/
recipient: string;
/**
* The eventual outcome of the ringing attempt. (Emits a single value.)
*/
// TODO: Include a callback for attempting ringing again in case of a timeout
outcome$: Observable<"accept" | "decline" | "timeout">;
}
export type CallNotificationWrapper = {
event_id: string;
@@ -76,6 +84,7 @@ export function createReceivedDecline$(
export interface Props {
scope: ObservableScope;
memberships$: Behavior<Epoch<CallMembership[]>>;
matrixRoomMembers$: Behavior<RoomMemberMap>;
sentCallNotification$: Observable<CallNotificationWrapper | null>;
receivedDecline$: Observable<
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
@@ -84,34 +93,81 @@ export interface Props {
localUser: { deviceId: string; userId: string };
}
/**
* @returns two observables:
* `callPickupState$` The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
* The call failed. If desired this can be used as a trigger to exit the call.
* - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
* - null: EC is configured to never show any waiting for answer state.
*
* `autoLeave$` An observable that emits (null) when the call should be automatically left.
* - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left.
* - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined.
* - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit.
*
*/
export function createCallNotificationLifecycle$({
scope,
memberships$,
matrixRoomMembers$,
sentCallNotification$,
receivedDecline$,
options,
localUser,
}: Props): {
callPickupState$: Behavior<CallPickupState>;
/**
* An observable of attempts to ring the remote participant's devices.
*/
ringAttempts$: Observable<RingAttempt>;
/**
* An observable that emits when the call should be automatically left.
* - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left.
* - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined.
* - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit.
*/
autoLeave$: Observable<AutoLeaveReason>;
} {
let ringAttempts$: Observable<RingAttempt> = NEVER;
if (options.waitForCallPickup)
ringAttempts$ = sentCallNotification$.pipe(
filter(
(
notificationEvent: CallNotificationWrapper | null,
): notificationEvent is CallNotificationWrapper =>
// only care about new events (legacy do not have decline pattern)
notificationEvent?.notification_type === "ring" &&
notificationEvent.lifetime > 0,
),
switchMap((notificationEvent) => {
// We assume that there is only one other user in the room when ringing
// TODO: Respect io.element.functional_members
const recipient = [...matrixRoomMembers$.value.keys()].find(
(userId) => userId !== localUser.userId,
);
if (recipient === undefined) {
logger.warn("No recipient for notification event; not ringing.");
return EMPTY;
}
// Ringing times out after lifetime ms have passed
const timeout$ = timer(notificationEvent.lifetime).pipe(
map(() => "timeout" as const),
);
// Call is accepted when someone else joins
const accept$ = memberships$.pipe(
filter((ms) => ms.value.some((m) => m.userId !== localUser.userId)),
map(() => "accept" as const),
);
// Call is declined when we receive a decline event
const decline$ = receivedDecline$.pipe(
filter(
([event]) =>
event.getRelation()?.rel_type === "m.reference" &&
event.getRelation()?.event_id === notificationEvent.event_id &&
event.getSender() === recipient,
),
map(() => "decline" as const),
);
return of({
intent: notificationEvent["m.call.intent"] ?? "audio",
recipient,
outcome$: race(timeout$, accept$, decline$).pipe(
take(1),
scope.share,
),
});
}),
scope.share,
);
const allOthersLeft$ = memberships$.pipe(
pairwise(),
filter(
@@ -122,87 +178,18 @@ export function createCallNotificationLifecycle$({
map(() => {}),
);
/**
* Whether some Matrix user other than ourself is joined to the call.
*/
const someoneElseJoined$ = memberships$.pipe(
mapEpoch((ms) => ms.some((m) => m.userId !== localUser.userId)),
) as Behavior<Epoch<boolean>>;
/**
* The state of the current ringing attempt, if the RTC session is indeed
* ringing the remote participant's devices. Otherwise `null`.
*/
const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> =
scope.behavior(
sentCallNotification$.pipe(
filter(
(notificationEventArgs: CallNotificationWrapper | null) =>
// only care about new events (legacy do not have decline pattern)
notificationEventArgs?.notification_type === "ring",
),
map((e) => e as CallNotificationWrapper),
switchMap((notificationEvent) => {
const lifetimeMs = notificationEvent?.lifetime ?? 0;
return concat(
lifetimeMs === 0
? // If no lifetime, skip the ring state
of(null)
: // Ring until lifetime ms have passed
timer(lifetimeMs).pipe(
ignoreElements(),
startWith("ringing" as const),
),
// The notification lifetime has timed out, meaning ringing has likely
// stopped on all receiving clients.
of("timeout" as const),
// This makes sure we will not drop into the `endWith("decline" as const)` state
NEVER,
).pipe(
takeUntil(
receivedDecline$.pipe(
filter(
([event]) =>
event.getRelation()?.rel_type === "m.reference" &&
event.getRelation()?.event_id ===
notificationEvent.event_id &&
event.getSender() !== localUser.userId &&
callPickupState$.value !== "timeout",
),
),
),
endWith("decline" as const),
);
}),
),
null,
);
const callPickupState$ = scope.behavior(
options.waitForCallPickup === true
? combineLatest(
[someoneElseJoined$, remoteRingState$],
(someoneElseJoined, ring) => {
if (someoneElseJoined.value === true) {
return "success" as const;
}
// Show the ringing state of the most recent ringing attempt.
// as long as we have not yet sent an RTC notification event or noone else joined,
// ring will be null -> callPickupState$ = unknown.
return ring ?? ("unknown" as const);
},
)
: NEVER,
null,
);
const autoLeave$ = merge(
options.autoLeaveWhenOthersLeft === true
? allOthersLeft$.pipe(map(() => "allOthersLeft" as const))
: NEVER,
callPickupState$.pipe(
filter((state) => state === "timeout" || state === "decline"),
ringAttempts$.pipe(
switchMap(({ outcome$ }) =>
outcome$.pipe(
filter((outcome) => outcome === "timeout" || outcome === "decline"),
),
),
),
);
return { autoLeave$, callPickupState$ };
return { ringAttempts$, autoLeave$ };
}

10
src/state/CallViewModel/CallViewModel.test.ts
View File

@@ -1386,7 +1386,10 @@ describe.each([
});
// Should ring for 30ms and then time out
expectObservable(vm.ringing$).toBe("(ny) 26ms n", yesNo);
expectObservable(vm.ringingIntent$).toBe("(ab) 26ms a", {
a: null,
b: "audio",
});
// Layout should show placeholder media for the participant we're
// ringing the entire time (even once timed out)
expectObservable(summarizeLayout$(vm.layout$)).toBe("a", {
@@ -1425,7 +1428,10 @@ describe.each([
});
// Should ring until Alice joins
expectObservable(vm.ringing$).toBe("(ny) 17ms n", yesNo);
expectObservable(vm.ringingIntent$).toBe("(ab) 17ms a", {
a: null,
b: "audio",
});
// Layout should show placeholder media for the participant we're
// ringing the entire time
expectObservable(summarizeLayout$(vm.layout$)).toBe("a 20ms b", {

146
src/state/CallViewModel/CallViewModel.ts
View File

@@ -29,7 +29,6 @@ import {
pairwise,
race,
scan,
skipWhile,
startWith,
Subject,
switchAll,
@@ -39,10 +38,13 @@ import {
tap,
throttleTime,
timer,
takeUntil,
concat,
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
MembershipManagerEvent,
type RTCCallIntent,
type LivekitTransportConfig,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
@@ -230,9 +232,9 @@ export interface CallViewModel {
// lifecycle
autoLeave$: Observable<AutoLeaveReason>;
/**
* Whether we are ringing a call recipient.
* Whether we are ringing a call recipient. Contains the ringing intent if so.
*/
ringing$: Behavior<boolean>;
ringingIntent$: Behavior<RTCCallIntent | null>;
/** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is
* - by ending the scope
@@ -610,20 +612,6 @@ export function createCallViewModel$(
),
);
// ------------------------------------------------------------------------
// callLifecycle
// TODO if we are in "unknown" state we need a loading rendering (or empty screen)
// Otherwise it looks like we already connected and only than the ringing starts which is weird.
const { callPickupState$, autoLeave$ } = createCallNotificationLifecycle$({
scope: scope,
memberships$: memberships$,
sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession),
receivedDecline$: createReceivedDecline$(matrixRoom),
options: options,
localUser: { userId: userId, deviceId: deviceId },
});
// ------------------------------------------------------------------------
// matrixMemberMetadataStore
@@ -634,6 +622,21 @@ export function createCallViewModel$(
matrixRoomMembers$,
);
// ------------------------------------------------------------------------
// callLifecycle
// TODO if we are in "unknown" state we need a loading rendering (or empty screen)
// Otherwise it looks like we already connected and only than the ringing starts which is weird.
const { ringAttempts$, autoLeave$ } = createCallNotificationLifecycle$({
scope,
memberships$,
matrixRoomMembers$,
sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession),
receivedDecline$: createReceivedDecline$(matrixRoom),
options,
localUser: { userId, deviceId },
});
const allConnections$ = scope.behavior(
connectionManager.connectionManagerData$.pipe(map((d) => d.value)),
);
@@ -782,49 +785,39 @@ export function createCallViewModel$(
),
);
const ringingMedia$ = scope.behavior<RingingMediaViewModel[]>(
combineLatest([userMedia$, matrixRoomMembers$, callPickupState$]).pipe(
generateItems(
"CallViewModel ringingMedia$",
function* ([userMedia, roomMembers, callPickupState]) {
if (
callPickupState === "ringing" ||
callPickupState === "timeout" ||
callPickupState === "decline"
) {
// TODO: Respect io.element.functional_members
for (const member of roomMembers.values()) {
if (!userMedia.some((vm) => vm.userId === member.userId))
yield {
keys: [member.userId],
data: callPickupState,
};
}
}
},
(scope, pickupState$, userId) =>
createRingingMedia({
id: `ringing:${userId}`,
userId,
displayName$: scope.behavior(
matrixRoomMembers$.pipe(
map((members) => members.get(userId)?.rawDisplayName || userId),
),
),
mxcAvatarUrl$:
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
pickupState$,
muteStates,
}),
const ringingMedia$ = scope.behavior<RingingMediaViewModel | null>(
ringAttempts$.pipe(
switchMap(({ intent, recipient, outcome$ }) =>
outcome$.pipe(
startWith("ringing" as const),
generateItems(
"CallViewModel ringingMedia$",
function* (pickupState) {
if (pickupState !== "accept")
yield { keys: [intent, recipient], data: pickupState };
},
(scope, pickupState$, intent, userId) =>
createRingingMedia({
id: `ringing:${userId}`,
userId,
displayName$: scope.behavior(
matrixRoomMembers$.pipe(
map(
(members) =>
members.get(userId)?.rawDisplayName || userId,
),
),
),
mxcAvatarUrl$:
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
pickupState$,
intent,
}),
),
map(([media]) => media ?? null),
),
),
distinctUntilChanged(shallowEquals),
tap((ringingMedia) => {
if (ringingMedia.length > 1)
// Warn that UI may do something unexpected in this case
logger.warn(
`Ringing more than one participant is not supported (ringing ${ringingMedia.map((vm) => vm.userId).join(", ")})`,
);
}),
startWith(null),
),
);
@@ -866,11 +859,7 @@ export function createCallViewModel$(
matrixLivekitMembers$.pipe(map((ms) => ms.value.length)),
);
const leaveSoundEffect$ = combineLatest([callPickupState$, userMedia$]).pipe(
// Until the call is successful, do not play a leave sound.
// If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound.
skipWhile(([c]) => c !== null && c !== "success"),
map(([, userMedia]) => userMedia),
const leaveSoundEffect$ = userMedia$.pipe(
pairwise(),
filter(
([prev, current]) =>
@@ -879,6 +868,9 @@ export function createCallViewModel$(
),
map(() => {}),
throttleTime(THROTTLE_SOUND_EFFECT_MS),
// Avoid doubling up on any auto-leave sounds (e.g. the decline sound),
// which are handled elsewhere
takeUntil(autoLeave$),
);
const userHangup$ = new Subject<void>();
@@ -983,8 +975,8 @@ export function createCallViewModel$(
}>(
ringingMedia$.pipe(
switchMap((ringingMedia) => {
if (ringingMedia.length > 0)
return of({ spotlight: ringingMedia, pip$: localUserMediaForPip$ });
if (ringingMedia !== null)
return of({ spotlight: [ringingMedia], pip$: localUserMediaForPip$ });
return screenShares$.pipe(
switchMap((screenShares) => {
@@ -1140,14 +1132,10 @@ export function createCallViewModel$(
// show ringing media instead
if (userMedia.length === 1)
return ringingMedia$.pipe(
map((ringingMedia) => {
return ringingMedia.length === 1
? {
local,
remote: ringingMedia[0],
}
: null;
}),
map(
(ringingMedia) =>
ringingMedia && { local, remote: ringingMedia },
),
);
}
}
@@ -1697,8 +1685,14 @@ export function createCallViewModel$(
return {
autoLeave$: autoLeave$,
ringing$: scope.behavior(
callPickupState$.pipe(map((state) => state === "ringing")),
ringingIntent$: scope.behavior(
ringAttempts$.pipe(
switchMap(({ intent, outcome$ }) =>
// Hold the intent as the value until the ring attempt completes
concat(of(intent), NEVER.pipe(takeUntil(outcome$)), of(null)),
),
startWith<RTCCallIntent | null>(null),
),
),
leave$: leave$,
hangup: (): void => userHangup$.next(),

20
src/state/media/RingingMediaViewModel.ts
View File

@@ -5,8 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
import { type Behavior } from "../Behavior";
import { type MuteStates } from "../MuteStates";
import {
type BaseMediaInputs,
type BaseMediaViewModel,
@@ -20,32 +21,23 @@ import {
export interface RingingMediaViewModel extends BaseMediaViewModel {
type: "ringing";
pickupState$: Behavior<"ringing" | "timeout" | "decline">;
/**
* Whether this media would be expected to have video, were it not simply a
* placeholder.
*/
videoEnabled$: Behavior<boolean>;
intent: RTCCallIntent;
}
export interface RingingMediaInputs extends BaseMediaInputs {
pickupState$: Behavior<"ringing" | "timeout" | "decline">;
/**
* The local user's own mute states.
*/
muteStates: MuteStates;
intent: RTCCallIntent;
}
export function createRingingMedia({
pickupState$,
muteStates,
intent,
...inputs
}: RingingMediaInputs): RingingMediaViewModel {
return {
...createBaseMedia(inputs),
type: "ringing",
pickupState$,
// If our own video is enabled, then this is a video call and we would
// expect remote media to have video as well
videoEnabled$: muteStates.video.enabled$,
intent,
};
}

5
src/tile/GridTile.test.tsx
View File

@@ -26,7 +26,6 @@ import {
createRingingMedia,
type RingingMediaViewModel,
} from "../state/media/RingingMediaViewModel";
import { type MuteStates } from "../state/MuteStates";
global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {}
@@ -93,10 +92,8 @@ test("GridTile displays ringing media", async () => {
>("ringing");
const vm = createRingingMedia({
pickupState$,
muteStates: {
video: { enabled$: constant(false) },
} as unknown as MuteStates,
id: "test",
intent: "audio",
userId: "@alice:example.org",
displayName$: constant("Alice"),
mxcAvatarUrl$: constant(undefined),

6
src/tile/GridTile.tsx
View File

@@ -77,7 +77,6 @@ const RingingMediaTile: FC<RingingMediaTileProps> = ({
}) => {
const { t } = useTranslation();
const pickupState = useBehavior(vm.pickupState$);
const videoEnabled = useBehavior(vm.videoEnabled$);
return (
<MediaView
@@ -89,11 +88,12 @@ const RingingMediaTile: FC<RingingMediaTileProps> = ({
pickupState === "ringing"
? {
text: t("video_tile.calling"),
Icon: videoEnabled ? VideoCallSolidIcon : VoiceCallSolidIcon,
Icon:
vm.intent === "video" ? VideoCallSolidIcon : VoiceCallSolidIcon,
}
: { text: t("video_tile.call_ended"), Icon: EndCallIcon }
}
videoEnabled={videoEnabled}
videoEnabled={false}
videoFit="cover"
mirror={false}
{...props}

5
src/tile/SpotlightTile.test.tsx
View File

@@ -28,7 +28,6 @@ import {
createRingingMedia,
type RingingMediaViewModel,
} from "../state/media/RingingMediaViewModel";
import { type MuteStates } from "../state/MuteStates";
global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {}
@@ -156,10 +155,8 @@ test("SpotlightTile displays ringing media", async () => {
>("ringing");
const vm = createRingingMedia({
pickupState$,
muteStates: {
video: { enabled$: constant(false) },
} as unknown as MuteStates,
id: "test",
intent: "audio",
userId: "@alice:example.org",
displayName$: constant("Alice"),
mxcAvatarUrl$: constant(undefined),

4
src/tile/SpotlightTile.tsx
View File

@@ -212,7 +212,6 @@ const SpotlightRingingMediaItem: FC<SpotlightRingingMediaItemProps> = ({
}) => {
const { t } = useTranslation();
const pickupState = useBehavior(vm.pickupState$);
const videoEnabled = useBehavior(vm.videoEnabled$);
return (
<MediaView
@@ -222,7 +221,8 @@ const SpotlightRingingMediaItem: FC<SpotlightRingingMediaItemProps> = ({
pickupState === "ringing"
? {
text: t("video_tile.calling"),
Icon: videoEnabled ? VideoCallSolidIcon : VoiceCallSolidIcon,
Icon:
vm.intent === "video" ? VideoCallSolidIcon : VoiceCallSolidIcon,
}
: { text: t("video_tile.call_ended"), Icon: EndCallIcon }
}

6
src/useAudioContext.tsx
View File

@@ -114,7 +114,7 @@ interface Props<S extends string> {
}
interface UseAudioContext<S extends string> {
playSound(soundName: S, volumeOverwrite?: number): Promise<void>;
playSound(soundName: S): Promise<void>;
playSoundLooping(soundName: S, delayS?: number): () => Promise<void>;
/**
* Map of sound name to duration in seconds.
@@ -195,7 +195,7 @@ export function useAudioContext<S extends string>(
}
return {
playSound: async (name, volumeOverwrite?: number): Promise<void> => {
playSound: async (name): Promise<void> => {
if (!audioBuffers[name]) {
logger.debug(`Tried to play a sound that wasn't buffered (${name})`);
return;
@@ -203,7 +203,7 @@ export function useAudioContext<S extends string>(
return playSound(
audioContext,
audioBuffers[name],
volumeOverwrite ?? soundEffectVolume * earpieceVolume,
soundEffectVolume * earpieceVolume,
earpiecePan,
);
},