diff --git a/docs/url-params.md b/docs/url-params.md index 3fac185a..b2af8416 100644 --- a/docs/url-params.md +++ b/docs/url-params.md @@ -70,6 +70,8 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st | `theme` | One of: `light`, `dark`, `light-high-contrast`, `dark-high-contrast` | No, defaults to `dark` | No, defaults to `dark` | UI theme to use. | | `viaServers` | Comma separated list of [Matrix Server Names](https://spec.matrix.org/v1.12/appendices/#server-name) | Not applicable | No | Homeserver for joining a room, non-empty value required for rooms not on the user’s default homeserver. | | `sendNotificationType` | `ring` or `notification` | No | No | Will send a "ring" or "notification" `m.rtc.notification` event if the user is the first one in the call. | +| `autoLeaveWhenOthersLeft` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Whether the app should automatically leave the call when there is no one left in the call. | +| `waitForCallPickup` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | When sending a notification, show UI that the app is awaiting an answer, play a dial tone, and (in widget mode) auto-close the widget once the notification expires. | ### Widget-only parameters diff --git a/src/UrlParams.ts b/src/UrlParams.ts index a26045b5..5f43b17f 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -226,7 +226,7 @@ export interface UrlConfiguration { * - play a sound that indicates that it is awaiting an answer * - auto-dismiss the call widget once the notification lifetime expires on the receivers side. */ - shouldWaitForCallPickup: boolean; + waitForCallPickup: boolean; } // If you need to add a new flag to this interface, prefer a name that describes @@ -358,7 +358,7 @@ export const getUrlParams = ( returnToLobby: false, sendNotificationType: "notification" as RTCNotificationType, autoLeaveWhenOthersLeft: false, - shouldWaitForCallPickup: false, + waitForCallPickup: false, }; switch (intent) { case UserIntent.StartNewCall: @@ -378,7 +378,7 @@ export const getUrlParams = ( ...inAppDefault, skipLobby: true, autoLeaveWhenOthersLeft: true, - shouldWaitForCallPickup: true, + waitForCallPickup: true, }; break; case UserIntent.JoinExistingCallDM: @@ -404,7 +404,7 @@ export const getUrlParams = ( returnToLobby: false, sendNotificationType: undefined, autoLeaveWhenOthersLeft: false, - shouldWaitForCallPickup: false, + waitForCallPickup: false, }; } @@ -456,7 +456,7 @@ export const getUrlParams = ( "ring", "notification", ]), - shouldWaitForCallPickup: parser.getFlag("shouldWaitForCallPickup"), + waitForCallPickup: parser.getFlag("waitForCallPickup"), autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"), }; diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 1c515175..f51e602d 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -19,10 +19,7 @@ import { act } from "react"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { mockRtcMembership } from "../utils/test"; -import { - CallEventAudioRenderer, - MAX_PARTICIPANT_COUNT_FOR_SOUND, -} from "./CallEventAudioRenderer"; +import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { useAudioContext } from "../useAudioContext"; import { prefetchSounds } from "../soundUtils"; import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel"; @@ -33,6 +30,7 @@ import { local, localRtcMember, } from "../utils/test-fixtures"; +import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel"; vitest.mock("../useAudioContext"); vitest.mock("../soundUtils"); @@ -172,7 +170,7 @@ test("should not play a sound when a hand raise is retracted", () => { }, }); }); - expect(playSound).toHaveBeenCalledTimes(2); + expect(playSound).toHaveBeenCalledTimes(1); act(() => { handRaisedSubject$.next({ ["foo"]: { @@ -182,5 +180,5 @@ test("should not play a sound when a hand raise is retracted", () => { }, }); }); - expect(playSound).toHaveBeenCalledTimes(2); + expect(playSound).toHaveBeenCalledTimes(1); }); diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index a39da82a..797501b6 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details. */ import { type ReactNode, useEffect } from "react"; -import { filter, interval, throttle } from "rxjs"; import { type CallViewModel } from "../state/CallViewModel"; import joinCallSoundMp3 from "../sound/join_call.mp3"; @@ -21,11 +20,6 @@ import { useAudioContext } from "../useAudioContext"; import { prefetchSounds } from "../soundUtils"; import { useLatest } from "../useLatest"; -// Do not play any sounds if the participant count has exceeded this -// number. -export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; -export const THROTTLE_SOUND_EFFECT_MS = 500; - export const callEventAudioSounds = prefetchSounds({ join: { mp3: joinCallSoundMp3, @@ -60,37 +54,18 @@ export function CallEventAudioRenderer({ const audioEngineRef = useLatest(audioEngineCtx); useEffect(() => { - const joinSub = vm.participantChanges$ - .pipe( - filter( - ({ joined, ids }) => - ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0, - ), - throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)), - ) - .subscribe(() => { - void audioEngineRef.current?.playSound("join"); - }); - - const leftSub = vm.participantChanges$ - .pipe( - filter( - ({ ids, left }) => - ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0, - ), - throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)), - ) - .subscribe(() => { - void audioEngineRef.current?.playSound("left"); - }); - - const handRaisedSub = vm.newHandRaised$.subscribe(() => { - void audioEngineRef.current?.playSound("raiseHand"); - }); - - const screenshareSub = vm.newScreenShare$.subscribe(() => { - void audioEngineRef.current?.playSound("screenshareStarted"); - }); + const joinSub = vm.joinSoundEffect$.subscribe( + () => void audioEngineRef.current?.playSound("join"), + ); + const leftSub = vm.leaveSoundEffect$.subscribe( + () => void audioEngineRef.current?.playSound("left"), + ); + const handRaisedSub = vm.newHandRaised$.subscribe( + () => void audioEngineRef.current?.playSound("raiseHand"), + ); + const screenshareSub = vm.newScreenShare$.subscribe( + () => void audioEngineRef.current?.playSound("screenshareStarted"), + ); return (): void => { joinSub.unsubscribe(); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 16e03987..034be16d 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -321,7 +321,7 @@ export const InCallView: FC = ({ const showFooter = useBehavior(vm.showFooter$); const earpieceMode = useBehavior(vm.earpieceMode$); const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); - useSubscription(vm.autoLeaveWhenOthersLeft$, onLeave); + useSubscription(vm.autoLeave$, onLeave); // Ideally we could detect taps by listening for click events and checking // that the pointerType of the event is "touch", but this isn't yet supported diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index 559a5226..b7c2bd32 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -25,6 +25,7 @@ import { RoomEvent as MatrixRoomEvent, MatrixEvent, type IRoomTimelineData, + EventType, } from "matrix-js-sdk"; import { ConnectionState, @@ -318,7 +319,7 @@ function withCallViewModel( const roomEventSelectorSpy = vi .spyOn(ComponentsCore, "roomEventSelector") - .mockImplementation((room, eventType) => of()); + .mockImplementation((_room, _eventType) => of()); const livekitRoom = mockLivekitRoom( { localParticipant }, @@ -1078,9 +1079,9 @@ it("should rank raised hands above video feeds and below speakers and presenters }); function nooneEverThere$( - hot: (marbles: string, values: Record) => Observable, -): Observable { - return hot("a-b-c-d", { + behavior: (marbles: string, values: Record) => Behavior, +): Behavior { + return behavior("a-b-c-d", { a: [], // Start empty b: [], // Alice joins c: [], // Alice still there @@ -1089,12 +1090,12 @@ function nooneEverThere$( } function participantJoinLeave$( - hot: ( + behavior: ( marbles: string, values: Record, - ) => Observable, -): Observable { - return hot("a-b-c-d", { + ) => Behavior, +): Behavior { + return behavior("a-b-c-d", { a: [], // Start empty b: [aliceParticipant], // Alice joins c: [aliceParticipant], // Alice still there @@ -1103,12 +1104,12 @@ function participantJoinLeave$( } function rtcMemberJoinLeave$( - hot: ( + behavior: ( marbles: string, values: Record, - ) => Observable, -): Observable { - return hot("a-b-c-d", { + ) => Behavior, +): Behavior { + return behavior("a-b-c-d", { a: [localRtcMember], // Start empty b: [localRtcMember, aliceRtcMember], // Alice joins c: [localRtcMember, aliceRtcMember], // Alice still there @@ -1116,47 +1117,15 @@ function rtcMemberJoinLeave$( }); } -test("allOthersLeft$ emits only when someone joined and then all others left", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { - // Test scenario 1: No one ever joins - should only emit initial false and never emit again - withCallViewModel( - { remoteParticipants$: scope.behavior(nooneEverThere$(hot), []) }, - (vm) => { - expectObservable(vm.allOthersLeft$).toBe("n------", { n: false }); - }, - ); - }); -}); - -test("allOthersLeft$ emits true when someone joined and then all others left", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { +test("autoLeave$ emits only when autoLeaveWhenOthersLeft option is enabled", () => { + withTestScheduler(({ behavior, expectObservable }) => { withCallViewModel( { - remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []), - rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []), + remoteParticipants$: participantJoinLeave$(behavior), + rtcMembers$: rtcMemberJoinLeave$(behavior), }, (vm) => { - expectObservable(vm.allOthersLeft$).toBe( - "n-----u", // false initially, then at frame 6: true then false emissions in same frame - { n: false, u: true }, // map(() => {}) - ); - }, - ); - }); -}); - -test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is enabled", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { - withCallViewModel( - { - remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []), - rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []), - }, - (vm) => { - expectObservable(vm.autoLeaveWhenOthersLeft$).toBe( - "------e", // false initially, then at frame 6: true then false emissions in same frame - { e: undefined }, - ); + expectObservable(vm.autoLeave$).toBe("------(e|)", { e: undefined }); }, { autoLeaveWhenOthersLeft: true, @@ -1166,15 +1135,15 @@ test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is }); }); -test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { +test("autoLeave$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => { + withTestScheduler(({ behavior, expectObservable }) => { withCallViewModel( { - remoteParticipants$: scope.behavior(nooneEverThere$(hot), []), - rtcMembers$: scope.behavior(nooneEverThere$(hot), []), + remoteParticipants$: nooneEverThere$(behavior), + rtcMembers$: nooneEverThere$(behavior), }, (vm) => { - expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------"); + expectObservable(vm.autoLeave$).toBe("-"); }, { autoLeaveWhenOthersLeft: true, @@ -1184,15 +1153,15 @@ test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is ena }); }); -test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { +test("autoLeave$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => { + withTestScheduler(({ behavior, expectObservable }) => { withCallViewModel( { - remoteParticipants$: scope.behavior(participantJoinLeave$(hot), []), - rtcMembers$: scope.behavior(rtcMemberJoinLeave$(hot), []), + remoteParticipants$: participantJoinLeave$(behavior), + rtcMembers$: rtcMemberJoinLeave$(behavior), }, (vm) => { - expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------"); + expectObservable(vm.autoLeave$).toBe("-"); }, { autoLeaveWhenOthersLeft: false, @@ -1202,31 +1171,25 @@ test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option }); }); -test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => { - withTestScheduler(({ hot, expectObservable, scope }) => { +test("autoLeave$ emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => { + withTestScheduler(({ behavior, expectObservable }) => { withCallViewModel( { - remoteParticipants$: scope.behavior( - hot("a-b-c-d", { - a: [], // Alone - b: [aliceParticipant], // Alice joins - c: [aliceParticipant], - d: [], // Local joins with a second device - }), - [], //Alice leaves - ), - rtcMembers$: scope.behavior( - hot("a-b-c-d", { - a: [localRtcMember], // Start empty - b: [localRtcMember, aliceRtcMember], // Alice joins - c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there - d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves - }), - [], - ), + remoteParticipants$: behavior("a-b-c-d", { + a: [], // Alone + b: [aliceParticipant], // Alice joins + c: [aliceParticipant], + d: [], // Local joins with a second device + }), + rtcMembers$: behavior("a-b-c-d", { + a: [localRtcMember], // Start empty + b: [localRtcMember, aliceRtcMember], // Alice joins + c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there + d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves + }), }, (vm) => { - expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("------e", { + expectObservable(vm.autoLeave$).toBe("------(e|)", { e: undefined, }); }, @@ -1238,18 +1201,12 @@ test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option }); }); -describe("shouldWaitForCallPickup$", () => { +describe("waitForCallPickup$", () => { test("unknown -> ringing -> timeout when notified and nobody joins", () => { - withTestScheduler(({ hot, schedule, expectObservable, scope }) => { + withTestScheduler(({ schedule, expectObservable }) => { // No one ever joins (only local user) withCallViewModel( - { - remoteParticipants$: scope.behavior(hot("a", { a: [] }), []), - rtcMembers$: scope.behavior(hot("a", { a: [localRtcMember] }), []), - connectionState$: of(ConnectionState.Connected), - speaking: new Map(), - mediaDevices: mockMediaDevices({}), - }, + { remoteParticipants$: constant([]) }, (vm, rtcSession) => { // Fire a call notification at 10ms with lifetime 30ms schedule(" 10ms r", { @@ -1273,7 +1230,7 @@ describe("shouldWaitForCallPickup$", () => { }); }, { - shouldWaitForCallPickup: true, + waitForCallPickup: true, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, }, ); @@ -1281,25 +1238,18 @@ describe("shouldWaitForCallPickup$", () => { }); test("ringing -> success if someone joins before timeout", () => { - withTestScheduler(({ hot, schedule, expectObservable, scope }) => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) - const remote$ = scope.behavior( - hot("a--b", { a: [], b: [aliceParticipant] }), - [], - ); - const rtc$ = scope.behavior( - hot("a--b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - [], - ); - withCallViewModel( { - remoteParticipants$: remote$, - rtcMembers$: rtc$, - connectionState$: of(ConnectionState.Connected), + remoteParticipants$: behavior("a 19ms b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a 19ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), }, (vm, rtcSession) => { // Notify at 5ms so we enter ringing, then success at 20ms @@ -1320,13 +1270,14 @@ describe("shouldWaitForCallPickup$", () => { }, }); - expectObservable(vm.callPickupState$).toBe("a 2ms c", { + expectObservable(vm.callPickupState$).toBe("a 4ms b 14ms c", { a: "unknown", + b: "ringing", c: "success", }); }, { - shouldWaitForCallPickup: true, + waitForCallPickup: true, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, }, ); @@ -1334,27 +1285,18 @@ describe("shouldWaitForCallPickup$", () => { }); test("success when someone joins before we notify", () => { - withTestScheduler(({ hot, schedule, expectObservable, scope }) => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { // Join at 10ms, notify later at 20ms (state should stay success) - const remote$ = scope.behavior( - hot("a-b", { a: [], b: [aliceParticipant] }), - [], - ); - const rtc$ = scope.behavior( - hot("a-b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - [], - ); - withCallViewModel( { - remoteParticipants$: remote$, - rtcMembers$: rtc$, - connectionState$: of(ConnectionState.Connected), - speaking: new Map(), - mediaDevices: mockMediaDevices({}), + remoteParticipants$: behavior("a 9ms b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a 9ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), }, (vm, rtcSession) => { schedule(" 20ms r", { @@ -1370,13 +1312,13 @@ describe("shouldWaitForCallPickup$", () => { ); }, }); - expectObservable(vm.callPickupState$).toBe("a 1ms b", { + expectObservable(vm.callPickupState$).toBe("a 9ms b", { a: "unknown", b: "success", }); }, { - shouldWaitForCallPickup: true, + waitForCallPickup: true, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, }, ); @@ -1384,21 +1326,15 @@ describe("shouldWaitForCallPickup$", () => { }); test("notify without lifetime -> immediate timeout", () => { - withTestScheduler(({ hot, schedule, expectObservable, scope }) => { + withTestScheduler(({ schedule, expectObservable }) => { withCallViewModel( - { - remoteParticipants$: scope.behavior(hot("a", { a: [] }), []), - rtcMembers$: scope.behavior(hot("a", { a: [localRtcMember] }), []), - connectionState$: of(ConnectionState.Connected), - speaking: new Map(), - mediaDevices: mockMediaDevices({}), - }, + {}, (vm, rtcSession) => { schedule(" 10ms r", { r: () => { rtcSession.emit( MatrixRTCSessionEvent.DidSendCallNotification, - { event_id: "$notif4", lifetime: 0 } as unknown as { + { event_id: "$notif4" } as unknown as { event_id: string; } & IRTCNotificationContent, // no lifetime { event_id: "$notif4" } as unknown as { @@ -1413,34 +1349,25 @@ describe("shouldWaitForCallPickup$", () => { }); }, { - shouldWaitForCallPickup: true, + waitForCallPickup: true, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, }, ); }); }); - test("stays null when shouldWaitForCallPickup=false", () => { - withTestScheduler(({ hot, schedule, expectObservable, scope }) => { - const remote$ = scope.behavior( - hot("a--b", { a: [], b: [aliceParticipant] }), - [], - ); - const rtc$ = scope.behavior( - hot("a--b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - [], - ); - + test("stays null when waitForCallPickup=false", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { withCallViewModel( { - remoteParticipants$: remote$, - rtcMembers$: rtc$, - connectionState$: of(ConnectionState.Connected), - speaking: new Map(), - mediaDevices: mockMediaDevices({}), + remoteParticipants$: behavior("a--b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a--b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), }, (vm, rtcSession) => { schedule(" 5ms r", { @@ -1461,7 +1388,7 @@ describe("shouldWaitForCallPickup$", () => { }); }, { - shouldWaitForCallPickup: false, + waitForCallPickup: false, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, }, ); @@ -1494,7 +1421,15 @@ describe("shouldWaitForCallPickup$", () => { // Emit decline timeline event with id matching the notification rtcSession.room.emit( MatrixRoomEvent.Timeline, - new MatrixEvent({ event_id: "$decl1", type: "m.rtc.decline" }), + new MatrixEvent({ + type: EventType.RTCDecline, + content: { + "m.relates_to": { + rel_type: "m.reference", + event_id: "$decl1", + }, + }, + }), rtcSession.room, undefined, false, @@ -1509,7 +1444,7 @@ describe("shouldWaitForCallPickup$", () => { }); }, { - shouldWaitForCallPickup: true, + waitForCallPickup: true, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, }, ); @@ -1557,7 +1492,7 @@ describe("shouldWaitForCallPickup$", () => { }); }, { - shouldWaitForCallPickup: true, + waitForCallPickup: true, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, }, ); @@ -1604,7 +1539,7 @@ describe("shouldWaitForCallPickup$", () => { }); }, { - shouldWaitForCallPickup: true, + waitForCallPickup: true, encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, }, ); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 96d8740b..5b946f09 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -30,18 +30,22 @@ import { import { BehaviorSubject, EMPTY, + NEVER, type Observable, Subject, combineLatest, concat, distinctUntilChanged, + endWith, filter, forkJoin, fromEvent, + ignoreElements, map, merge, mergeMap, of, + pairwise, race, scan, skip, @@ -50,6 +54,8 @@ import { switchMap, switchScan, take, + takeUntil, + throttleTime, timer, withLatestFrom, } from "rxjs"; @@ -109,7 +115,7 @@ import { observeSpeaker$ } from "./observeSpeaker"; import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; -import { type Behavior } from "./Behavior"; +import { constant, type Behavior } from "./Behavior"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -118,13 +124,18 @@ export interface CallViewModelOptions { * If the call is started in a way where we want it to behave like a telephone usecase * If we sent a notification event, we want the ui to show a ringing state */ - shouldWaitForCallPickup?: boolean; + waitForCallPickup?: boolean; } // How long we wait after a focus switch before showing the real participant // list again const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; +// Do not play any sounds if the participant count has exceeded this +// number. +export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; +export const THROTTLE_SOUND_EFFECT_MS = 500; + // This is the number of participants that we think constitutes a "small" call // on mobile. No spotlight tile should be shown below this threshold. const smallMobileCallThreshold = 3; @@ -437,14 +448,7 @@ export class CallViewModel extends ViewModel { MembershipManagerEvent.StatusChanged, ).pipe( startWith(null), - map( - () => - ( - this.matrixRTCSession as unknown as { - membershipStatus?: Status; - } - ).membershipStatus === Status.Connected, - ), + map(() => this.matrixRTCSession.membershipStatus === Status.Connected), ), // Also watch out for warnings that we've likely hit a timeout and our // delayed leave event is being sent (this condition is here because it @@ -455,11 +459,7 @@ export class CallViewModel extends ViewModel { MembershipManagerEvent.ProbablyLeft, ).pipe( startWith(null), - map( - () => - (this.matrixRTCSession as unknown as { probablyLeft?: boolean }) - .probablyLeft !== true, - ), + map(() => this.matrixRTCSession.probablyLeft !== true), ), ), ); @@ -576,6 +576,17 @@ export class CallViewModel extends ViewModel { ) .pipe(pauseWhen(this.pretendToBeDisconnected$)); + private readonly memberships$ = this.scope.behavior( + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe( + startWith(null), + pauseWhen(this.pretendToBeDisconnected$), + map(() => this.matrixRTCSession.memberships), + ), + ); + /** * Displaynames for each member of the call. This will disambiguate * any displaynames that clashes with another member. Only members @@ -585,18 +596,17 @@ export class CallViewModel extends ViewModel { // than on Chrome/Firefox). This means it is important that we multicast the result so that we // don't do this work more times than we need to. This is achieved by converting to a behavior: public readonly memberDisplaynames$ = this.scope.behavior( - merge( - // Handle call membership changes. - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ), - // Handle room membership changes (and displayname updates) - fromEvent(this.matrixRoom, RoomStateEvent.Members), - ).pipe( - startWith(null), - map(() => { - const memberships = this.matrixRTCSession.memberships; + // React to call memberships and also display name updates + // (calculateDisplayName implicitly depends on the room member data) + combineLatest( + [ + this.memberships$, + fromEvent(this.matrixRoom, RoomStateEvent.Members).pipe( + startWith(null), + pauseWhen(this.pretendToBeDisconnected$), + ), + ], + (memberships, _members) => { const displaynameMap = new Map(); const room = this.matrixRoom; @@ -618,8 +628,7 @@ export class CallViewModel extends ViewModel { ); } return displaynameMap; - }), - pauseWhen(this.pretendToBeDisconnected$), + }, ), ); @@ -649,13 +658,7 @@ export class CallViewModel extends ViewModel { this.remoteParticipants$, observeParticipantMedia(this.livekitRoom.localParticipant), duplicateTiles.value$, - // Also react to changes in the MatrixRTC session list. - // The session list will also be update if a room membership changes. - // No additional RoomState event listener needs to be set up. - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe(startWith(null), pauseWhen(this.pretendToBeDisconnected$)), + this.memberships$, showNonMemberTiles.value$, ]).pipe( scan( @@ -665,7 +668,7 @@ export class CallViewModel extends ViewModel { remoteParticipants, { participant: localParticipant }, duplicateTiles, - _membershipsChanged, + memberships, showNonMemberTiles, ], ) => { @@ -673,7 +676,7 @@ export class CallViewModel extends ViewModel { function* (this: CallViewModel): Iterable<[string, MediaItem]> { const room = this.matrixRoom; // m.rtc.members are the basis for calculating what is visible in the call - for (const rtcMember of this.matrixRTCSession.memberships) { + for (const rtcMember of memberships) { const { member, id: livekitParticipantId } = getRoomMemberFromRtcMember(rtcMember, room); const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`; @@ -839,205 +842,143 @@ export class CallViewModel extends ViewModel { ), ); - /** - * This observable tracks the currently connected participants. - * - * - Each participant has one livekit connection - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one matrix user. - */ - public readonly participantChanges$ = this.scope.behavior( - this.userMedia$.pipe( - map((mediaItems) => mediaItems.map((m) => m.id)), - scan( - (prev, ids) => { - const left = prev.ids.filter((id) => !ids.includes(id)); - const joined = ids.filter((id) => !prev.ids.includes(id)); - return { ids, joined, left }; - }, - { ids: [], joined: [], left: [] }, - ), + public readonly joinSoundEffect$ = this.userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + public readonly leaveSoundEffect$ = this.userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), ); /** * The number of participants currently in the call. * - * - Each participant has one livekit connection * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one matrix user. + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. */ public readonly participantCount$ = this.scope.behavior( - this.participantChanges$.pipe(map(({ ids }) => ids.length)), + this.memberships$.pipe(map((ms) => ms.length)), ); - /** - * This observable tracks the matrix users that are currently in the call. - * There can be just one matrix user with multiple participants (see also participantChanges$) - */ - public readonly matrixUserChanges$ = this.scope.behavior( - this.userMedia$.pipe( - map( - (mediaItems) => - new Set( - mediaItems - .map((m) => m.vm.member?.userId) - .filter((id) => id !== undefined), - ), - ), - scan< - Set, - { - userIds: Set; - joinedUserIds: Set; - leftUserIds: Set; - } - >( - (prevState, userIds) => { - const left = new Set( - [...prevState.userIds].filter((id) => !userIds.has(id)), - ); - const joined = new Set( - [...userIds].filter((id) => !prevState.userIds.has(id)), - ); - return { userIds: userIds, joinedUserIds: joined, leftUserIds: left }; - }, - { - userIds: new Set(), - joinedUserIds: new Set(), - leftUserIds: new Set(), - }, - ), + private readonly allOthersLeft$ = this.memberships$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.every((m) => m.sender === this.userId) && + prev.some((m) => m.sender !== this.userId), ), - ); - - public readonly allOthersLeft$ = this.matrixUserChanges$.pipe( - map(({ userIds, leftUserIds }) => { - if (!this.userId) { - logger.warn("Could not access user ID to compute allOthersLeft"); - return false; - } - return ( - userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0 - ); - }), - startWith(false), - distinctUntilChanged(), - ); - - public readonly autoLeaveWhenOthersLeft$ = this.allOthersLeft$.pipe( - distinctUntilChanged(), - filter((leave) => (leave && this.options.autoLeaveWhenOthersLeft) ?? false), map(() => {}), + take(1), ); + public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft + ? this.allOthersLeft$ + : NEVER; + /** - * "unknown": We don't know if the RTC session decides to send a notify event yet. - * It will only be known once we sent our own membership and know we were the first one to join. - * "ringing": The notification event was sent. - * "ringEnded": The notification events lifetime has timed out -> ringing stopped on all receiving clients. + * Whenever the RTC session tells us that it intends to ring the remote + * participant's devices, this emits an Observable tracking the current state of + * that ringing process. */ - private readonly rtcNotificationEventState$: Observable< - { state: "unknown" | "ringEnded" } | { state: "ringing"; event_id: string } - > = fromEvent< - Parameters< - MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification] + private readonly ring$: Observable< + Observable<"ringing" | "timeout" | "decline"> + > = ( + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.DidSendCallNotification, + ) as Observable< + Parameters< + MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification] + > > - >(this.matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification).pipe( - switchMap(([notificationEvent]) => { + ).pipe( + map(([notificationEvent]) => { // event.lifetime is expected to be in ms const lifetime = notificationEvent?.lifetime ?? 0; - if (lifetime > 0) { - // Emit true immediately, then false after lifetime ms - return concat( - of({ - state: "ringing", - event_id: notificationEvent.event_id, - } as { - state: "ringing"; - event_id: string; - }), - timer(lifetime).pipe( - map(() => ({ state: "ringEnded" }) as { state: "ringEnded" }), + return concat( + lifetime === 0 + ? // If no lifetime, skip the ring state + EMPTY + : // Ring until lifetime ms have passed + timer(lifetime).pipe( + ignoreElements(), + startWith("ringing" as const), + ), + // The notification lifetime has timed out, meaning ringing has likely + // stopped on all receiving clients. + of("timeout" as const), + NEVER, + ).pipe( + takeUntil( + ( + fromEvent(this.matrixRoom, RoomEvent.Timeline) as Observable< + Parameters + > + ).pipe( + filter( + ([event]) => + event.getType() === EventType.RTCDecline && + event.getRelation()?.rel_type === "m.reference" && + event.getRelation()?.event_id === notificationEvent.event_id, + ), ), - ); - } - // If no lifetime, the notify event is basically invalid and we enter ringEnded immediately. - return of({ state: "ringEnded" } as { state: "ringEnded" }); + ), + endWith("decline" as const), + ); }), - startWith({ state: "unknown" } as { state: "unknown" }), ); /** - * If some other matrix user has joined the call. It can start with true if there are already multiple matrix users. + * Whether some Matrix user other than ourself is joined to the call. */ - private readonly someoneElseJoined$ = this.matrixUserChanges$.pipe( - scan( - (someoneJoined, { joinedUserIds }) => - someoneJoined || [...joinedUserIds].some((id) => id !== this.userId), - false, - ), - startWith(this.matrixUserChanges$.value.userIds.size > 1), + private readonly someoneElseJoined$ = this.memberships$.pipe( + map((ms) => ms.some((m) => m.sender !== this.userId)), ); /** * The current call pickup state of the call. - * - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening). * - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership. * Then we can conclude if we were the first one to join or not. + * - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening). * - "timeout": No-one picked up in the defined time this call should be ringing on others devices. * The call failed. If desired this can be used as a trigger to exit the call. - * - "success": Someone else joined. The call is in a normal state. Stop audiovisual feedback. + * - "success": Someone else joined. The call is in a normal state. No audiovisual feedback. * - null: EC is configured to never show any waiting for answer state. */ - public readonly callPickupState$: Behavior< - "unknown" | "ringing" | "timeout" | "success" | "decline" | null - > = this.scope.behavior( - combineLatest([ - this.rtcNotificationEventState$, - this.someoneElseJoined$, - fromEvent>( - this.matrixRoom, - RoomEvent.Timeline, - ).pipe( - map(([event]) => { - if (event.getType() === EventType.RTCDecline) return event; - else return null; - }), - startWith(null), - ), - ]).pipe( - map(([notificationEventState, someoneJoined, declineEvent]) => { - // Never enter waiting for answer state if the app is not configured with waitingForAnswer. - if (!this.options.shouldWaitForCallPickup) return null; - // As soon as someone joins, we can consider the call "wait for answer" successful - if (someoneJoined) return "success"; - - switch (notificationEventState?.state) { - case "unknown": - return "unknown"; - case "ringing": - // Check if the decline event corresponds to the current notification event - if (declineEvent?.getId() === notificationEventState.event_id) { - return "decline"; - } - return "ringing"; - case "ringEnded": - return "timeout"; - default: - return "timeout"; - } - }), - // Once we reach a terminal state, keep it - scan((prev, next) => { - if (prev === "decline" || prev === "timeout" || prev === "success") { - return prev; - } - return next; - }), - distinctUntilChanged(), - ), - ); + public readonly callPickupState$ = this.options.waitForCallPickup + ? this.scope.behavior< + "unknown" | "ringing" | "timeout" | "decline" | "success" + >( + this.someoneElseJoined$.pipe( + switchMap((someoneElseJoined) => + someoneElseJoined + ? of("success" as const) + : // Show the ringing state of the most recent ringing attempt. + this.ring$.pipe(switchAll()), + ), + // The state starts as 'unknown' because we don't know if the RTC + // session will actually send a notify event yet. It will only be + // known once we send our own membership and see that we were the + // first one to join. + startWith("unknown" as const), + ), + ) + : constant(null); /** * List of MediaItems that we want to display, that are of type ScreenShare diff --git a/yarn.lock b/yarn.lock index fa71cce0..13e9770b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10046,8 +10046,8 @@ __metadata: linkType: hard "livekit-client@npm:^2.13.0": - version: 2.15.5 - resolution: "livekit-client@npm:2.15.5" + version: 2.15.6 + resolution: "livekit-client@npm:2.15.6" dependencies: "@livekit/mutex": "npm:1.1.1" "@livekit/protocol": "npm:1.39.3" @@ -10060,7 +10060,7 @@ __metadata: webrtc-adapter: "npm:^9.0.1" peerDependencies: "@types/dom-mediacapture-record": ^1 - checksum: 10c0/52a70bdd39d802737ed7c25ae5d06daf9921156c4fc74f918009e86204430b2d200b66c55cefab949be4e5411cbc4d25eac92976f62f96b7226057a5b0706baa + checksum: 10c0/f1ab6cdf2b85647036e9de906734c1394dac497da0bd879a29d0c587c437ada262021478fcef24df99b3489a39d97fe67ab33de0785ed0a63335da2fef577192 languageName: node linkType: hard