diff --git a/docs/url-params.md b/docs/url-params.md index 3fac185a..b2af8416 100644 --- a/docs/url-params.md +++ b/docs/url-params.md @@ -70,6 +70,8 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st | `theme` | One of: `light`, `dark`, `light-high-contrast`, `dark-high-contrast` | No, defaults to `dark` | No, defaults to `dark` | UI theme to use. | | `viaServers` | Comma separated list of [Matrix Server Names](https://spec.matrix.org/v1.12/appendices/#server-name) | Not applicable | No | Homeserver for joining a room, non-empty value required for rooms not on the user’s default homeserver. | | `sendNotificationType` | `ring` or `notification` | No | No | Will send a "ring" or "notification" `m.rtc.notification` event if the user is the first one in the call. | +| `autoLeaveWhenOthersLeft` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Whether the app should automatically leave the call when there is no one left in the call. | +| `waitForCallPickup` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | When sending a notification, show UI that the app is awaiting an answer, play a dial tone, and (in widget mode) auto-close the widget once the notification expires. | ### Widget-only parameters diff --git a/src/UrlParams.ts b/src/UrlParams.ts index 30019d36..5f43b17f 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -216,6 +216,17 @@ export interface UrlConfiguration { * This is one part to make the call matrixRTC session behave like a telephone call. */ autoLeaveWhenOthersLeft: boolean; + + /** + * If the client should behave like it is awaiting an answer if a notification was sent (wait for call pick up). + * This is a no-op if not combined with sendNotificationType. + * + * This entails: + * - show ui that it is awaiting an answer + * - play a sound that indicates that it is awaiting an answer + * - auto-dismiss the call widget once the notification lifetime expires on the receivers side. + */ + waitForCallPickup: boolean; } // If you need to add a new flag to this interface, prefer a name that describes @@ -347,6 +358,7 @@ export const getUrlParams = ( returnToLobby: false, sendNotificationType: "notification" as RTCNotificationType, autoLeaveWhenOthersLeft: false, + waitForCallPickup: false, }; switch (intent) { case UserIntent.StartNewCall: @@ -366,6 +378,7 @@ export const getUrlParams = ( ...inAppDefault, skipLobby: true, autoLeaveWhenOthersLeft: true, + waitForCallPickup: true, }; break; case UserIntent.JoinExistingCallDM: @@ -391,6 +404,7 @@ export const getUrlParams = ( returnToLobby: false, sendNotificationType: undefined, autoLeaveWhenOthersLeft: false, + waitForCallPickup: false, }; } @@ -442,6 +456,7 @@ export const getUrlParams = ( "ring", "notification", ]), + waitForCallPickup: parser.getFlag("waitForCallPickup"), autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"), }; diff --git a/src/livekit/useECConnectionState.test.tsx b/src/livekit/useECConnectionState.test.tsx index 8abbf438..883cb68c 100644 --- a/src/livekit/useECConnectionState.test.tsx +++ b/src/livekit/useECConnectionState.test.tsx @@ -94,7 +94,7 @@ describe("Leaking connection prevention", () => { test("Should cancel pending connections when the component is unmounted", async () => { const connectCall = vi.fn(); const pendingConnection = Promise.withResolvers(); - // let pendingDisconnection = defer() + // let pendingDisconnection = Promise.withResolvers() const disconnectMock = vi.fn(); const mockRoom = { @@ -142,7 +142,7 @@ describe("Leaking connection prevention", () => { test("Should cancel about to open but not yet opened connection", async () => { const createTracksCall = vi.fn(); const pendingCreateTrack = Promise.withResolvers(); - // let pendingDisconnection = defer() + // let pendingDisconnection = Promise.withResolvers() const disconnectMock = vi.fn(); const connectMock = vi.fn(); diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 1c515175..894d1d24 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -19,10 +19,7 @@ import { act } from "react"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { mockRtcMembership } from "../utils/test"; -import { - CallEventAudioRenderer, - MAX_PARTICIPANT_COUNT_FOR_SOUND, -} from "./CallEventAudioRenderer"; +import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { useAudioContext } from "../useAudioContext"; import { prefetchSounds } from "../soundUtils"; import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel"; @@ -33,6 +30,7 @@ import { local, localRtcMember, } from "../utils/test-fixtures"; +import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel"; vitest.mock("../useAudioContext"); vitest.mock("../soundUtils"); @@ -158,6 +156,7 @@ test("should not play a sound when a hand raise is retracted", () => { ]); render(); + playSound.mockClear(); act(() => { handRaisedSubject$.next({ ["foo"]: { @@ -172,7 +171,7 @@ test("should not play a sound when a hand raise is retracted", () => { }, }); }); - expect(playSound).toHaveBeenCalledTimes(2); + expect(playSound).toHaveBeenCalledExactlyOnceWith("raiseHand"); act(() => { handRaisedSubject$.next({ ["foo"]: { @@ -182,5 +181,5 @@ test("should not play a sound when a hand raise is retracted", () => { }, }); }); - expect(playSound).toHaveBeenCalledTimes(2); + expect(playSound).toHaveBeenCalledExactlyOnceWith("raiseHand"); }); diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index a39da82a..797501b6 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details. */ import { type ReactNode, useEffect } from "react"; -import { filter, interval, throttle } from "rxjs"; import { type CallViewModel } from "../state/CallViewModel"; import joinCallSoundMp3 from "../sound/join_call.mp3"; @@ -21,11 +20,6 @@ import { useAudioContext } from "../useAudioContext"; import { prefetchSounds } from "../soundUtils"; import { useLatest } from "../useLatest"; -// Do not play any sounds if the participant count has exceeded this -// number. -export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; -export const THROTTLE_SOUND_EFFECT_MS = 500; - export const callEventAudioSounds = prefetchSounds({ join: { mp3: joinCallSoundMp3, @@ -60,37 +54,18 @@ export function CallEventAudioRenderer({ const audioEngineRef = useLatest(audioEngineCtx); useEffect(() => { - const joinSub = vm.participantChanges$ - .pipe( - filter( - ({ joined, ids }) => - ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0, - ), - throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)), - ) - .subscribe(() => { - void audioEngineRef.current?.playSound("join"); - }); - - const leftSub = vm.participantChanges$ - .pipe( - filter( - ({ ids, left }) => - ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0, - ), - throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)), - ) - .subscribe(() => { - void audioEngineRef.current?.playSound("left"); - }); - - const handRaisedSub = vm.newHandRaised$.subscribe(() => { - void audioEngineRef.current?.playSound("raiseHand"); - }); - - const screenshareSub = vm.newScreenShare$.subscribe(() => { - void audioEngineRef.current?.playSound("screenshareStarted"); - }); + const joinSub = vm.joinSoundEffect$.subscribe( + () => void audioEngineRef.current?.playSound("join"), + ); + const leftSub = vm.leaveSoundEffect$.subscribe( + () => void audioEngineRef.current?.playSound("left"), + ); + const handRaisedSub = vm.newHandRaised$.subscribe( + () => void audioEngineRef.current?.playSound("raiseHand"), + ); + const screenshareSub = vm.newScreenShare$.subscribe( + () => void audioEngineRef.current?.playSound("screenshareStarted"), + ); return (): void => { joinSub.unsubscribe(); diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index ea57bd10..63fc942f 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -453,7 +453,6 @@ export const GroupCallView: FC = ({ matrixInfo={matrixInfo} rtcSession={rtcSession as MatrixRTCSession} matrixRoom={room} - participantCount={participantCount} onLeave={onLeave} header={header} muteStates={muteStates} diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index ec057e94..eb192f66 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -177,7 +177,6 @@ function createInCallView(): RenderResult & { }} matrixRoom={room} livekitRoom={livekitRoom} - participantCount={0} onLeave={function (): void { throw new Error("Function not implemented."); }} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 4e3229a5..034be16d 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -216,7 +216,6 @@ export interface InCallViewProps { matrixRoom: MatrixRoom; livekitRoom: LivekitRoom; muteStates: MuteStates; - participantCount: number; /** Function to call when the user explicitly ends the call */ onLeave: () => void; header: HeaderStyle; @@ -233,7 +232,6 @@ export const InCallView: FC = ({ matrixRoom, livekitRoom, muteStates, - participantCount, onLeave, header: headerStyle, connState, @@ -312,6 +310,7 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); + const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); const windowMode = useBehavior(vm.windowMode$); const layout = useBehavior(vm.layout$); @@ -322,7 +321,7 @@ export const InCallView: FC = ({ const showFooter = useBehavior(vm.showFooter$); const earpieceMode = useBehavior(vm.earpieceMode$); const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); - useSubscription(vm.autoLeaveWhenOthersLeft$, onLeave); + useSubscription(vm.autoLeave$, onLeave); // Ideally we could detect taps by listening for click events and checking // that the pointerType of the event is "touch", but this isn't yet supported diff --git a/src/room/__snapshots__/InCallView.test.tsx.snap b/src/room/__snapshots__/InCallView.test.tsx.snap index 7d6ab966..bfbef499 100644 --- a/src/room/__snapshots__/InCallView.test.tsx.snap +++ b/src/room/__snapshots__/InCallView.test.tsx.snap @@ -49,6 +49,34 @@ exports[`InCallView > rendering > renders 1`] = ` +
+ + + + + + + 2 + +
of()); + .mockImplementation((_room, _eventType) => of()); const livekitRoom = mockLivekitRoom( { localParticipant }, @@ -1068,9 +1071,9 @@ it("should rank raised hands above video feeds and below speakers and presenters }); function nooneEverThere$( - hot: (marbles: string, values: Record) => Observable, -): Observable { - return hot("a-b-c-d", { + behavior: (marbles: string, values: Record) => Behavior, +): Behavior { + return behavior("a-b-c-d", { a: [], // Start empty b: [], // Alice joins c: [], // Alice still there @@ -1079,12 +1082,12 @@ function nooneEverThere$( } function participantJoinLeave$( - hot: ( + behavior: ( marbles: string, values: Record, - ) => Observable, -): Observable { - return hot("a-b-c-d", { + ) => Behavior, +): Behavior { + return behavior("a-b-c-d", { a: [], // Start empty b: [aliceParticipant], // Alice joins c: [aliceParticipant], // Alice still there @@ -1093,12 +1096,12 @@ function participantJoinLeave$( } function rtcMemberJoinLeave$( - hot: ( + behavior: ( marbles: string, values: Record, - ) => Observable, -): Observable { - return hot("a-b-c-d", { + ) => Behavior, +): Behavior { + return behavior("a-b-c-d", { a: [localRtcMember], // Start empty b: [localRtcMember, aliceRtcMember], // Alice joins c: [localRtcMember, aliceRtcMember], // Alice still there @@ -1106,47 +1109,15 @@ function rtcMemberJoinLeave$( }); } -test("allOthersLeft$ emits only when someone joined and then all others left", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { - // Test scenario 1: No one ever joins - should only emit initial false and never emit again - withCallViewModel( - { remoteParticipants$: scope.behavior(nooneEverThere$(hot), []) }, - (vm) => { - expectObservable(vm.allOthersLeft$).toBe("n------", { n: false }); - }, - ); - }); -}); - -test("allOthersLeft$ emits true when someone joined and then all others left", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { +test("autoLeave$ emits only when autoLeaveWhenOthersLeft option is enabled", () => { + withTestScheduler(({ behavior, expectObservable }) => { withCallViewModel( { - remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []), - rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []), + remoteParticipants$: participantJoinLeave$(behavior), + rtcMembers$: rtcMemberJoinLeave$(behavior), }, (vm) => { - expectObservable(vm.allOthersLeft$).toBe( - "n-----u", // false initially, then at frame 6: true then false emissions in same frame - { n: false, u: true }, // map(() => {}) - ); - }, - ); - }); -}); - -test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is enabled", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { - withCallViewModel( - { - remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []), - rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []), - }, - (vm) => { - expectObservable(vm.autoLeaveWhenOthersLeft$).toBe( - "------e", // false initially, then at frame 6: true then false emissions in same frame - { e: undefined }, - ); + expectObservable(vm.autoLeave$).toBe("------(e|)", { e: undefined }); }, { autoLeaveWhenOthersLeft: true, @@ -1156,15 +1127,15 @@ test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is }); }); -test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { +test("autoLeave$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => { + withTestScheduler(({ behavior, expectObservable }) => { withCallViewModel( { - remoteParticipants$: scope.behavior(nooneEverThere$(hot), []), - rtcMembers$: scope.behavior(nooneEverThere$(hot), []), + remoteParticipants$: nooneEverThere$(behavior), + rtcMembers$: nooneEverThere$(behavior), }, (vm) => { - expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------"); + expectObservable(vm.autoLeave$).toBe("-"); }, { autoLeaveWhenOthersLeft: true, @@ -1174,15 +1145,15 @@ test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is ena }); }); -test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { +test("autoLeave$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => { + withTestScheduler(({ behavior, expectObservable }) => { withCallViewModel( { - remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []), - rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []), + remoteParticipants$: participantJoinLeave$(behavior), + rtcMembers$: rtcMemberJoinLeave$(behavior), }, (vm) => { - expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------"); + expectObservable(vm.autoLeave$).toBe("-"); }, { autoLeaveWhenOthersLeft: false, @@ -1192,31 +1163,25 @@ test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option }); }); -test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { +test("autoLeave$ emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => { + withTestScheduler(({ behavior, expectObservable }) => { withCallViewModel( { - remoteParticipants$: scope.behavior( - hot("a-b-c-d", { - a: [], // Alone - b: [aliceParticipant], // Alice joins - c: [aliceParticipant], - d: [], // Local joins with a second device - }), - [], //Alice leaves - ), - rtcMembers$: scope.behavior( - hot("a-b-c-d", { - a: [localRtcMember], // Start empty - b: [localRtcMember, aliceRtcMember], // Alice joins - c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there - d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves - }), - [], - ), + remoteParticipants$: behavior("a-b-c-d", { + a: [], // Alone + b: [aliceParticipant], // Alice joins + c: [aliceParticipant], + d: [], // Local joins with a second device + }), + rtcMembers$: behavior("a-b-c-d", { + a: [localRtcMember], // Start empty + b: [localRtcMember, aliceRtcMember], // Alice joins + c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there + d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves + }), }, (vm) => { - expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("------e", { + expectObservable(vm.autoLeave$).toBe("------(e|)", { e: undefined, }); }, @@ -1228,6 +1193,196 @@ test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option }); }); +describe("waitForCallPickup$", () => { + test("unknown -> ringing -> timeout when notified and nobody joins", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // No one ever joins (only local user) + withCallViewModel( + { remoteParticipants$: constant([]) }, + (vm, rtcSession) => { + // Fire a call notification at 10ms with lifetime 30ms + schedule(" 10ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + { lifetime: 30 } as unknown as { + event_id: string; + } & IRTCNotificationContent, + {} as unknown as { event_id: string } & ICallNotifyContent, + ); + }, + }); + + expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms c", { + a: "unknown", + b: "ringing", + c: "timeout", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("ringing -> success if someone joins before timeout", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + withCallViewModel( + { + remoteParticipants$: behavior("a 19ms b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a 19ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + }, + (vm, rtcSession) => { + // Notify at 5ms so we enter ringing, then success at 20ms + schedule(" 5ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + { lifetime: 100 } as unknown as { + event_id: string; + } & IRTCNotificationContent, + {} as unknown as { + event_id: string; + } & ICallNotifyContent, + ); + }, + }); + + expectObservable(vm.callPickupState$).toBe("a 4ms b 14ms c", { + a: "unknown", + b: "ringing", + c: "success", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("success when someone joins before we notify", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + // Join at 10ms, notify later at 20ms (state should stay success) + withCallViewModel( + { + remoteParticipants$: behavior("a 9ms b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a 9ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + }, + (vm, rtcSession) => { + schedule(" 20ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + { lifetime: 50 } as unknown as { + event_id: string; + } & IRTCNotificationContent, + {} as unknown as { + event_id: string; + } & ICallNotifyContent, + ); + }, + }); + expectObservable(vm.callPickupState$).toBe("a 9ms b", { + a: "unknown", + b: "success", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("notify without lifetime -> immediate timeout", () => { + withTestScheduler(({ schedule, expectObservable }) => { + withCallViewModel( + {}, + (vm, rtcSession) => { + schedule(" 10ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + {} as unknown as { + event_id: string; + } & IRTCNotificationContent, // no lifetime + {} as unknown as { + event_id: string; + } & ICallNotifyContent, + ); + }, + }); + expectObservable(vm.callPickupState$).toBe("a 9ms b", { + a: "unknown", + b: "timeout", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("stays null when waitForCallPickup=false", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: behavior("a--b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a--b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + }, + (vm, rtcSession) => { + schedule(" 5ms r", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + { lifetime: 30 } as unknown as { + event_id: string; + } & IRTCNotificationContent, + {} as unknown as { + event_id: string; + } & ICallNotifyContent, + ); + }, + }); + expectObservable(vm.callPickupState$).toBe("(n)", { + n: null, + }); + }, + { + waitForCallPickup: false, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); +}); + test("audio output changes when toggling earpiece mode", () => { withTestScheduler(({ schedule, expectObservable }) => { getUrlParams.mockReturnValue({ controlledAudioDevices: true }); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index d7bf1812..9112954f 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -27,6 +27,7 @@ import { import { BehaviorSubject, EMPTY, + NEVER, type Observable, Subject, combineLatest, @@ -35,10 +36,12 @@ import { filter, forkJoin, fromEvent, + ignoreElements, map, merge, mergeMap, of, + pairwise, race, scan, skip, @@ -47,12 +50,15 @@ import { switchMap, switchScan, take, + takeUntil, + throttleTime, timer, withLatestFrom, } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, + type IRTCNotificationContent, type MatrixRTCSession, MatrixRTCSessionEvent, MembershipManagerEvent, @@ -105,16 +111,27 @@ import { observeSpeaker$ } from "./observeSpeaker"; import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; -import { type Behavior } from "./Behavior"; +import { constant, type Behavior } from "./Behavior"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; autoLeaveWhenOthersLeft?: boolean; + /** + * If the call is started in a way where we want it to behave like a telephone usecase + * If we sent a notification event, we want the ui to show a ringing state + */ + waitForCallPickup?: boolean; } + // How long we wait after a focus switch before showing the real participant // list again const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; +// Do not play any sounds if the participant count has exceeded this +// number. +export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; +export const THROTTLE_SOUND_EFFECT_MS = 500; + // This is the number of participants that we think constitutes a "small" call // on mobile. No spotlight tile should be shown below this threshold. const smallMobileCallThreshold = 3; @@ -555,6 +572,17 @@ export class CallViewModel extends ViewModel { ) .pipe(pauseWhen(this.pretendToBeDisconnected$)); + private readonly memberships$ = this.scope.behavior( + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe( + startWith(null), + pauseWhen(this.pretendToBeDisconnected$), + map(() => this.matrixRTCSession.memberships), + ), + ); + /** * Displaynames for each member of the call. This will disambiguate * any displaynames that clashes with another member. Only members @@ -564,18 +592,17 @@ export class CallViewModel extends ViewModel { // than on Chrome/Firefox). This means it is important that we multicast the result so that we // don't do this work more times than we need to. This is achieved by converting to a behavior: public readonly memberDisplaynames$ = this.scope.behavior( - merge( - // Handle call membership changes. - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ), - // Handle room membership changes (and displayname updates) - fromEvent(this.matrixRoom, RoomStateEvent.Members), - ).pipe( - startWith(null), - map(() => { - const memberships = this.matrixRTCSession.memberships; + // React to call memberships and also display name updates + // (calculateDisplayName implicitly depends on the room member data) + combineLatest( + [ + this.memberships$, + fromEvent(this.matrixRoom, RoomStateEvent.Members).pipe( + startWith(null), + pauseWhen(this.pretendToBeDisconnected$), + ), + ], + (memberships, _members) => { const displaynameMap = new Map(); const room = this.matrixRoom; @@ -597,8 +624,7 @@ export class CallViewModel extends ViewModel { ); } return displaynameMap; - }), - pauseWhen(this.pretendToBeDisconnected$), + }, ), ); @@ -628,13 +654,7 @@ export class CallViewModel extends ViewModel { this.remoteParticipants$, observeParticipantMedia(this.livekitRoom.localParticipant), duplicateTiles.value$, - // Also react to changes in the MatrixRTC session list. - // The session list will also be update if a room membership changes. - // No additional RoomState event listener needs to be set up. - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe(startWith(null), pauseWhen(this.pretendToBeDisconnected$)), + this.memberships$, showNonMemberTiles.value$, ]).pipe( scan( @@ -644,7 +664,7 @@ export class CallViewModel extends ViewModel { remoteParticipants, { participant: localParticipant }, duplicateTiles, - _membershipsChanged, + memberships, showNonMemberTiles, ], ) => { @@ -652,7 +672,7 @@ export class CallViewModel extends ViewModel { function* (this: CallViewModel): Iterable<[string, MediaItem]> { const room = this.matrixRoom; // m.rtc.members are the basis for calculating what is visible in the call - for (const rtcMember of this.matrixRTCSession.memberships) { + for (const rtcMember of memberships) { const { member, id: livekitParticipantId } = getRoomMemberFromRtcMember(rtcMember, room); const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`; @@ -818,79 +838,118 @@ export class CallViewModel extends ViewModel { ), ); - /** - * This observable tracks the currently connected participants. - * - * - Each participant has one livekit connection - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one matrix user. - */ - public readonly participantChanges$ = this.userMedia$.pipe( - map((mediaItems) => mediaItems.map((m) => m.id)), - scan( - (prev, ids) => { - const left = prev.ids.filter((id) => !ids.includes(id)); - const joined = ids.filter((id) => !prev.ids.includes(id)); - return { ids, joined, left }; - }, - { ids: [], joined: [], left: [] }, + public readonly joinSoundEffect$ = this.userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, ), - ); - - /** - * This observable tracks the matrix users that are currently in the call. - * There can be just one matrix user with multiple participants (see also participantChanges$) - */ - public readonly matrixUserChanges$ = this.userMedia$.pipe( - map( - (mediaItems) => - new Set( - mediaItems - .map((m) => m.vm.member?.userId) - .filter((id) => id !== undefined), - ), - ), - scan< - Set, - { - userIds: Set; - joinedUserIds: Set; - leftUserIds: Set; - } - >( - (prevState, userIds) => { - const left = new Set( - [...prevState.userIds].filter((id) => !userIds.has(id)), - ); - const joined = new Set( - [...userIds].filter((id) => !prevState.userIds.has(id)), - ); - return { userIds: userIds, joinedUserIds: joined, leftUserIds: left }; - }, - { userIds: new Set(), joinedUserIds: new Set(), leftUserIds: new Set() }, - ), - ); - - public readonly allOthersLeft$ = this.matrixUserChanges$.pipe( - map(({ userIds, leftUserIds }) => { - if (!this.userId) { - logger.warn("Could not access user ID to compute allOthersLeft"); - return false; - } - return ( - userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0 - ); - }), - startWith(false), - distinctUntilChanged(), - ); - - public readonly autoLeaveWhenOthersLeft$ = this.allOthersLeft$.pipe( - distinctUntilChanged(), - filter((leave) => (leave && this.options.autoLeaveWhenOthersLeft) ?? false), map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), ); + public readonly leaveSoundEffect$ = this.userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + public readonly participantCount$ = this.scope.behavior( + this.memberships$.pipe(map((ms) => ms.length)), + ); + + private readonly allOthersLeft$ = this.memberships$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.every((m) => m.sender === this.userId) && + prev.some((m) => m.sender !== this.userId), + ), + map(() => {}), + take(1), + ); + + public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft + ? this.allOthersLeft$ + : NEVER; + + /** + * Emits whenever the RTC session tells us that it intends to ring for a given + * duration. + */ + private readonly beginRingingForMs$ = ( + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.DidSendCallNotification, + ) as Observable<[IRTCNotificationContent]> + ) + // event.lifetime is expected to be in ms + .pipe(map(([notificationEvent]) => notificationEvent?.lifetime ?? 0)); + + /** + * Whether some Matrix user other than ourself is joined to the call. + */ + private readonly someoneElseJoined$ = this.memberships$.pipe( + map((ms) => ms.some((m) => m.sender !== this.userId)), + ); + + /** + * The current call pickup state of the call. + * - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership. + * Then we can conclude if we were the first one to join or not. + * - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening). + * - "timeout": No-one picked up in the defined time this call should be ringing on others devices. + * The call failed. If desired this can be used as a trigger to exit the call. + * - "success": Someone else joined. The call is in a normal state. No audiovisual feedback. + * - null: EC is configured to never show any waiting for answer state. + */ + public readonly callPickupState$ = this.options.waitForCallPickup + ? this.scope.behavior<"unknown" | "ringing" | "timeout" | "success">( + concat( + concat( + // We don't know if the RTC session decides to send a notify event + // yet. It will only be known once we sent our own membership and + // know we were the first one to join. + of("unknown" as const), + // Once we get the signal to begin ringing: + this.beginRingingForMs$.pipe( + take(1), + switchMap((lifetime) => + lifetime === 0 + ? // If no lifetime, skip the ring state + EMPTY + : // Ring until lifetime ms have passed + timer(lifetime).pipe( + ignoreElements(), + startWith("ringing" as const), + ), + ), + ), + // The notification lifetime has timed out, meaning ringing has + // likely stopped on all receiving clients. + of("timeout" as const), + NEVER, + ).pipe( + takeUntil(this.someoneElseJoined$.pipe(filter((joined) => joined))), + ), + of("success" as const), + ), + ) + : constant(null); + /** * List of MediaItems that we want to display, that are of type ScreenShare */ diff --git a/yarn.lock b/yarn.lock index 94f8feaf..94cb867d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10280,7 +10280,7 @@ __metadata: "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop": version: 37.13.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=3a33c658bbcb8ce8791ec066db899f2571f5c52f" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=c4c7f945141e142e6f846b243c33c4af97a9a44b" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.1.0" @@ -10296,7 +10296,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:11" - checksum: 10c0/1db0d39cfbe4f1c69c8acda0ea7580a4819fc47a7d4bff057382e33e72d9a610f8c03043a6c00bc647dfdc2815aa643c69d25022fb759342a92b77e1841524f1 + checksum: 10c0/caa4b8a6d924ac36a21773dc2c8be6cb6b658a9feaabccdb24426719c563ac2cfe4778abb86f0889854ae36fc7ba02a6ed39acdbc0b73fdc31ce9a9789e7f36a languageName: node linkType: hard