diff --git a/src/UrlParams.ts b/src/UrlParams.ts index 94fd3c14..30019d36 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -210,6 +210,12 @@ export interface UrlConfiguration { * Whether and what type of notification EC should send, when the user joins the call. */ sendNotificationType?: RTCNotificationType; + /** + * Whether the app should automatically leave the call when there + * is no one left in the call. + * This is one part to make the call matrixRTC session behave like a telephone call. + */ + autoLeaveWhenOthersLeft: boolean; } // If you need to add a new flag to this interface, prefer a name that describes @@ -277,10 +283,16 @@ class ParamParser { ]; } + /** + * Returns true if the flag exists and is not "false". + */ public getFlagParam(name: string, defaultValue = false): boolean { const param = this.getParam(name); return param === null ? defaultValue : param !== "false"; } + /** + * Returns the value of the flag if it exists, or undefined if it does not. + */ public getFlag(name: string): boolean | undefined { const param = this.getParam(name); return param !== null ? param !== "false" : undefined; @@ -334,6 +346,7 @@ export const getUrlParams = ( skipLobby: true, returnToLobby: false, sendNotificationType: "notification" as RTCNotificationType, + autoLeaveWhenOthersLeft: false, }; switch (intent) { case UserIntent.StartNewCall: @@ -352,14 +365,14 @@ export const getUrlParams = ( intentPreset = { ...inAppDefault, skipLobby: true, - // autoLeaveWhenOthersLeft: true, // TODO: add this once available + autoLeaveWhenOthersLeft: true, }; break; case UserIntent.JoinExistingCallDM: intentPreset = { ...inAppDefault, skipLobby: true, - // autoLeaveWhenOthersLeft: true, // TODO: add this once available + autoLeaveWhenOthersLeft: true, }; break; // Non widget usecase defaults @@ -377,6 +390,7 @@ export const getUrlParams = ( skipLobby: false, returnToLobby: false, sendNotificationType: undefined, + autoLeaveWhenOthersLeft: false, }; } @@ -428,12 +442,13 @@ export const getUrlParams = ( "ring", "notification", ]), + autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"), }; return { ...properties, ...intentPreset, - ...pickBy(configuration, (v) => v !== undefined), + ...pickBy(configuration, (v?: unknown) => v !== undefined), }; }; diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index a0d685ff..a39da82a 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -60,7 +60,7 @@ export function CallEventAudioRenderer({ const audioEngineRef = useLatest(audioEngineCtx); useEffect(() => { - const joinSub = vm.memberChanges$ + const joinSub = vm.participantChanges$ .pipe( filter( ({ joined, ids }) => @@ -72,7 +72,7 @@ export function CallEventAudioRenderer({ void audioEngineRef.current?.playSound("join"); }); - const leftSub = vm.memberChanges$ + const leftSub = vm.participantChanges$ .pipe( filter( ({ ids, left }) => diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 4af599bb..76352523 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -166,7 +166,11 @@ export const GroupCallView: FC = ({ const { displayName, avatarUrl } = useProfile(client); const roomName = useRoomName(room); const roomAvatar = useRoomAvatar(room); - const { perParticipantE2EE, returnToLobby } = useUrlParams(); + const { + perParticipantE2EE, + returnToLobby, + password: passwordFromUrl, + } = useUrlParams(); const e2eeSystem = useRoomEncryptionSystem(room.roomId); const [useNewMembershipManager] = useSetting(useNewMembershipManagerSetting); const [useExperimentalToDeviceTransport] = useSetting( @@ -174,7 +178,6 @@ export const GroupCallView: FC = ({ ); // Save the password once we start the groupCallView - const { password: passwordFromUrl } = useUrlParams(); useEffect(() => { if (passwordFromUrl) saveKeyForRoom(room.roomId, passwordFromUrl); }, [passwordFromUrl, room.roomId]); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 5f8e7c28..5aa270d2 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -25,7 +25,7 @@ import useMeasure from "react-use-measure"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import classNames from "classnames"; import { BehaviorSubject, map } from "rxjs"; -import { useObservable } from "observable-hooks"; +import { useObservable, useSubscription } from "observable-hooks"; import { logger } from "matrix-js-sdk/lib/logger"; import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport"; import { @@ -140,11 +140,11 @@ export const ActiveCall: FC = (props) => { useEffect(() => { logger.info( - `[Lifecycle] InCallView Component mounted, livekitroom state ${livekitRoom?.state}`, + `[Lifecycle] InCallView Component mounted, livekit room state ${livekitRoom?.state}`, ); return (): void => { logger.info( - `[Lifecycle] InCallView Component unmounted, livekitroom state ${livekitRoom?.state}`, + `[Lifecycle] InCallView Component unmounted, livekit room state ${livekitRoom?.state}`, ); livekitRoom ?.disconnect() @@ -159,6 +159,8 @@ export const ActiveCall: FC = (props) => { }; }, [livekitRoom]); + const { autoLeaveWhenOthersLeft } = useUrlParams(); + useEffect(() => { if (livekitRoom !== undefined) { const reactionsReader = new ReactionsReader(props.rtcSession); @@ -166,7 +168,10 @@ export const ActiveCall: FC = (props) => { props.rtcSession, livekitRoom, mediaDevices, - props.e2eeSystem, + { + encryptionSystem: props.e2eeSystem, + autoLeaveWhenOthersLeft, + }, connStateObservable$, reactionsReader.raisedHands$, reactionsReader.reactions$, @@ -183,6 +188,7 @@ export const ActiveCall: FC = (props) => { mediaDevices, props.e2eeSystem, connStateObservable$, + autoLeaveWhenOthersLeft, ]); if (livekitRoom === undefined || vm === null) return null; @@ -313,6 +319,7 @@ export const InCallView: FC = ({ const earpieceMode = useBehavior(vm.earpieceMode$); const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); const switchCamera = useSwitchCamera(vm.localVideo$); + useSubscription(vm.autoLeaveWhenOthersLeft$, onLeave); // Ideally we could detect taps by listening for click events and checking // that the pointerType of the event is "touch", but this isn't yet supported diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index a3068c32..511a9431 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -32,7 +32,11 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { deepCompare } from "matrix-js-sdk/lib/utils"; -import { CallViewModel, type Layout } from "./CallViewModel"; +import { + CallViewModel, + type CallViewModelOptions, + type Layout, +} from "./CallViewModel"; import { mockLivekitRoom, mockLocalParticipant, @@ -71,6 +75,7 @@ import { local, localId, localRtcMember, + localRtcMemberDevice2, } from "../utils/test-fixtures"; import { ObservableScope } from "./ObservableScope"; import { MediaDevices } from "./MediaDevices"; @@ -231,6 +236,10 @@ function withCallViewModel( vm: CallViewModel, subjects: { raisedHands$: BehaviorSubject> }, ) => void, + options: CallViewModelOptions = { + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + autoLeaveWhenOthersLeft: false, + }, ): void { const room = mockMatrixRoom({ client: { @@ -281,9 +290,7 @@ function withCallViewModel( rtcSession as unknown as MatrixRTCSession, liveKitRoom, mediaDevices, - { - kind: E2eeType.PER_PARTICIPANT, - }, + options, connectionState$, raisedHands$, new BehaviorSubject({}), @@ -978,7 +985,7 @@ test("should strip RTL characters from displayname", () => { }); it("should rank raised hands above video feeds and below speakers and presenters", () => { - withTestScheduler(({ schedule, expectObservable }) => { + withTestScheduler(({ schedule, expectObservable, behavior }) => { // There should always be one tile for each MatrixRTCSession const expectedLayoutMarbles = "ab"; @@ -1037,6 +1044,176 @@ it("should rank raised hands above video feeds and below speakers and presenters }); }); +function nooneEverThere$( + hot: (marbles: string, values: Record) => Observable, +): Observable { + return hot("a-b-c-d", { + a: [], // Start empty + b: [], // Alice joins + c: [], // Alice still there + d: [], // Alice leaves + }); +} + +function participantJoinLeave$( + hot: ( + marbles: string, + values: Record, + ) => Observable, +): Observable { + return hot("a-b-c-d", { + a: [], // Start empty + b: [aliceParticipant], // Alice joins + c: [aliceParticipant], // Alice still there + d: [], // Alice leaves + }); +} + +function rtcMemberJoinLeave$( + hot: ( + marbles: string, + values: Record, + ) => Observable, +): Observable { + return hot("a-b-c-d", { + a: [], // Start empty + b: [aliceRtcMember], // Alice joins + c: [aliceRtcMember], // Alice still there + d: [], // Alice leaves + }); +} + +test("allOthersLeft$ emits only when someone joined and then all others left", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + // Test scenario 1: No one ever joins - should only emit initial false and never emit again + withCallViewModel( + scope.behavior(nooneEverThere$(hot), []), + scope.behavior(nooneEverThere$(hot), []), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.allOthersLeft$).toBe("n------", { n: false }); + }, + ); + }); +}); + +test("allOthersLeft$ emits true when someone joined and then all others left", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + withCallViewModel( + scope.behavior(participantJoinLeave$(hot), []), + scope.behavior(rtcMemberJoinLeave$(hot), []), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.allOthersLeft$).toBe( + "n-----u", // false initially, then at frame 6: true then false emissions in same frame + { n: false, u: true }, // map(() => {}) + ); + }, + ); + }); +}); + +test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is enabled", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + withCallViewModel( + scope.behavior(participantJoinLeave$(hot), []), + scope.behavior(rtcMemberJoinLeave$(hot), []), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.autoLeaveWhenOthersLeft$).toBe( + "------e", // false initially, then at frame 6: true then false emissions in same frame + { e: undefined }, + ); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); +}); + +test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + withCallViewModel( + scope.behavior(nooneEverThere$(hot), []), + scope.behavior(nooneEverThere$(hot), []), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------"); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); +}); + +test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + withCallViewModel( + scope.behavior(participantJoinLeave$(hot), []), + scope.behavior(rtcMemberJoinLeave$(hot), []), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------"); + }, + { + autoLeaveWhenOthersLeft: false, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); +}); + +test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + withCallViewModel( + scope.behavior( + hot("a-b-c-d", { + a: [], // Alone + b: [aliceParticipant], // Alice joins + c: [aliceParticipant], + d: [], // Local joins with a second device + }), + [], //Alice leaves + ), + scope.behavior( + hot("a-b-c-d", { + a: [localRtcMember], // Start empty + b: [localRtcMember, aliceRtcMember], // Alice joins + c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there + d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves + }), + [], + ), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("------e", { + e: undefined, + }); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); +}); + test("audio output changes when toggling earpiece mode", () => { withTestScheduler(({ schedule, expectObservable }) => { getUrlParams.mockReturnValue({ controlledAudioDevices: true }); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index bd2a7607..70183a37 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -96,6 +96,10 @@ import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; import { type Behavior } from "./Behavior"; +export interface CallViewModelOptions { + encryptionSystem: EncryptionSystem; + autoLeaveWhenOthersLeft?: boolean; +} // How long we wait after a focus switch before showing the real participant // list again const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; @@ -473,49 +477,47 @@ export class CallViewModel extends ViewModel { ), ); + private readonly memberships$: Observable = merge( + // Handle call membership changes. + fromEvent(this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged), + // Handle room membership changes (and displayname updates) + fromEvent(this.matrixRTCSession.room, RoomStateEvent.Members), + ).pipe( + startWith(this.matrixRTCSession.memberships), + map(() => { + return this.matrixRTCSession.memberships; + }), + ); + /** * Displaynames for each member of the call. This will disambiguate * any displaynames that clashes with another member. Only members * joined to the call are considered here. */ - public readonly memberDisplaynames$ = this.scope.behavior( - merge( - // Handle call membership changes. - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ), - // Handle room membership changes (and displayname updates) - fromEvent(this.matrixRTCSession.room, RoomStateEvent.Members), - ).pipe( - startWith(null), - map(() => { - const displaynameMap = new Map(); - const { room, memberships } = this.matrixRTCSession; + public readonly memberDisplaynames$ = this.memberships$.pipe( + map((memberships) => { + const displaynameMap = new Map(); + const { room } = this.matrixRTCSession; - // We only consider RTC members for disambiguation as they are the only visible members. - for (const rtcMember of memberships) { - const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`; - const { member } = getRoomMemberFromRtcMember(rtcMember, room); - if (!member) { - logger.error( - "Could not find member for media id:", - matrixIdentifier, - ); - continue; - } - const disambiguate = shouldDisambiguate(member, memberships, room); - displaynameMap.set( - matrixIdentifier, - calculateDisplayName(member, disambiguate), - ); + // We only consider RTC members for disambiguation as they are the only visible members. + for (const rtcMember of memberships) { + const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`; + const { member } = getRoomMemberFromRtcMember(rtcMember, room); + if (!member) { + logger.error("Could not find member for media id:", matrixIdentifier); + continue; } - return displaynameMap; - }), - // It turns out that doing the disambiguation above is rather expensive on Safari (10x slower - // than on Chrome/Firefox). This means it is important that we multicast the result so that we - // don't do this work more times than we need to. This is achieved by converting to a behavior: - ), + const disambiguate = shouldDisambiguate(member, memberships, room); + displaynameMap.set( + matrixIdentifier, + calculateDisplayName(member, disambiguate), + ); + } + return displaynameMap; + }), + // It turns out that doing the disambiguation above is rather expensive on Safari (10x slower + // than on Chrome/Firefox). This means it is important that we multicast the result so that we + // don't do this work more times than we need to. This is achieved by converting to a behavior: ); public readonly handsRaised$ = this.scope.behavior(this.handsRaisedSubject$); @@ -612,7 +614,7 @@ export class CallViewModel extends ViewModel { indexedMediaId, member, participant, - this.encryptionSystem, + this.options.encryptionSystem, this.livekitRoom, this.memberDisplaynames$.pipe( map((m) => m.get(matrixIdentifier) ?? "[👻]"), @@ -635,7 +637,7 @@ export class CallViewModel extends ViewModel { screenShareId, member, participant, - this.encryptionSystem, + this.options.encryptionSystem, this.livekitRoom, this.memberDisplaynames$.pipe( map((m) => m.get(matrixIdentifier) ?? "[👻]"), @@ -676,7 +678,7 @@ export class CallViewModel extends ViewModel { nonMemberId, undefined, participant, - this.encryptionSystem, + this.options.encryptionSystem, this.livekitRoom, this.memberDisplaynames$.pipe( map( @@ -726,18 +728,77 @@ export class CallViewModel extends ViewModel { ), ); - public readonly memberChanges$ = this.userMedia$ - .pipe(map((mediaItems) => mediaItems.map((m) => m.id))) - .pipe( - scan( - (prev, ids) => { - const left = prev.ids.filter((id) => !ids.includes(id)); - const joined = ids.filter((id) => !prev.ids.includes(id)); - return { ids, joined, left }; - }, - { ids: [], joined: [], left: [] }, - ), - ); + /** + * This observable tracks the currently connected participants. + * + * - Each participant has one livekit connection + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one matrix user. + */ + public readonly participantChanges$ = this.userMedia$.pipe( + map((mediaItems) => mediaItems.map((m) => m.id)), + scan( + (prev, ids) => { + const left = prev.ids.filter((id) => !ids.includes(id)); + const joined = ids.filter((id) => !prev.ids.includes(id)); + return { ids, joined, left }; + }, + { ids: [], joined: [], left: [] }, + ), + ); + + /** + * This observable tracks the matrix users that are currently in the call. + * There can be just one matrix user with multiple participants (see also participantChanges$) + */ + public readonly matrixUserChanges$ = this.userMedia$.pipe( + map( + (mediaItems) => + new Set( + mediaItems + .map((m) => m.vm.member?.userId) + .filter((id) => id !== undefined), + ), + ), + scan< + Set, + { + userIds: Set; + joinedUserIds: Set; + leftUserIds: Set; + } + >( + (prevState, userIds) => { + const left = new Set( + [...prevState.userIds].filter((id) => !userIds.has(id)), + ); + const joined = new Set( + [...userIds].filter((id) => !prevState.userIds.has(id)), + ); + return { userIds: userIds, joinedUserIds: joined, leftUserIds: left }; + }, + { userIds: new Set(), joinedUserIds: new Set(), leftUserIds: new Set() }, + ), + ); + + public readonly allOthersLeft$ = this.matrixUserChanges$.pipe( + map(({ userIds, leftUserIds }) => { + const userId = this.matrixRTCSession.room.client.getUserId(); + if (!userId) { + logger.warn("Could access client.getUserId to compute allOthersLeft"); + return false; + } + return userIds.size === 1 && userIds.has(userId) && leftUserIds.size > 0; + }), + startWith(false), + distinctUntilChanged(), + ); + + public readonly autoLeaveWhenOthersLeft$ = this.allOthersLeft$.pipe( + distinctUntilChanged(), + filter((leave) => (leave && this.options.autoLeaveWhenOthersLeft) ?? false), + map(() => {}), + ); /** * List of MediaItems that we want to display, that are of type ScreenShare @@ -1426,7 +1487,7 @@ export class CallViewModel extends ViewModel { private readonly matrixRTCSession: MatrixRTCSession, private readonly livekitRoom: LivekitRoom, private readonly mediaDevices: MediaDevices, - private readonly encryptionSystem: EncryptionSystem, + private readonly options: CallViewModelOptions, private readonly connectionState$: Observable, private readonly handsRaisedSubject$: Observable< Record diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index c13c1cf1..6a8b641b 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -12,7 +12,11 @@ import { mockLocalParticipant, } from "./test"; -export const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); +export const localRtcMember = mockRtcMembership("@carol:example.org", "1111"); +export const localRtcMemberDevice2 = mockRtcMembership( + "@carol:example.org", + "2222", +); export const local = mockMatrixRoomMember(localRtcMember); export const localParticipant = mockLocalParticipant({ identity: "" }); export const localId = `${local.userId}:${localRtcMember.deviceId}`; diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 179c38b1..4781bf3d 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -139,7 +139,7 @@ export function getBasicCallViewModelEnvironment( liveKitRoom, mockMediaDevices({}), { - kind: E2eeType.PER_PARTICIPANT, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, }, of(ConnectionState.Connected), handRaisedSubject$, diff --git a/src/utils/test.ts b/src/utils/test.ts index cad2b521..ce964ee8 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -74,6 +74,7 @@ export interface OurRunHelpers extends RunHelpers { values?: { [marble: string]: T }, error?: unknown, ): Behavior; + scope: ObservableScope; } interface TestRunnerGlobal { @@ -96,6 +97,7 @@ export function withTestScheduler( scheduler.run((helpers) => continuation({ ...helpers, + scope, schedule(marbles, actions) { const actionsObservable$ = helpers .cold(marbles)