/* Copyright 2024 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ import { test, vi, onTestFinished, it } from "vitest"; import { combineLatest, debounceTime, distinctUntilChanged, map, type Observable, of, switchMap, } from "rxjs"; import { type MatrixClient } from "matrix-js-sdk/src/matrix"; import { ConnectionState, type LocalParticipant, type Participant, ParticipantEvent, type RemoteParticipant, } from "livekit-client"; import * as ComponentsCore from "@livekit/components-core"; import { isEqual } from "lodash-es"; import { type CallMembership, type MatrixRTCSession, } from "matrix-js-sdk/src/matrixrtc"; import { CallViewModel, type Layout } from "./CallViewModel"; import { mockLivekitRoom, mockLocalParticipant, mockMatrixRoom, mockMatrixRoomMember, mockRemoteParticipant, withTestScheduler, mockRtcMembership, MockRTCSession, } from "../utils/test"; import { ECAddonConnectionState, type ECConnectionState, } from "../livekit/useECConnectionState"; import { E2eeType } from "../e2ee/e2eeType"; vi.mock("@livekit/components-core"); const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); const alice = mockMatrixRoomMember(aliceRtcMember); const bob = mockMatrixRoomMember(bobRtcMember); const carol = mockMatrixRoomMember(localRtcMember); const dave = mockMatrixRoomMember(daveRtcMember); const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`; const bobId = `${bob.userId}:${bobRtcMember.deviceId}`; const daveId = `${dave.userId}:${daveRtcMember.deviceId}`; const localParticipant = mockLocalParticipant({ identity: "" }); const aliceParticipant = mockRemoteParticipant({ identity: aliceId }); const aliceSharingScreen = mockRemoteParticipant({ identity: aliceId, isScreenShareEnabled: true, }); const bobParticipant = mockRemoteParticipant({ identity: bobId }); const bobSharingScreen = mockRemoteParticipant({ identity: bobId, isScreenShareEnabled: true, }); const daveParticipant = mockRemoteParticipant({ identity: daveId }); const roomMembers = new Map( [alice, bob, carol, dave].map((p) => [p.userId, p]), ); export interface GridLayoutSummary { type: "grid"; spotlight?: string[]; grid: string[]; } export interface SpotlightLandscapeLayoutSummary { type: "spotlight-landscape"; spotlight: string[]; grid: string[]; } export interface SpotlightPortraitLayoutSummary { type: "spotlight-portrait"; spotlight: string[]; grid: string[]; } export interface SpotlightExpandedLayoutSummary { type: "spotlight-expanded"; spotlight: string[]; pip?: string; } export interface OneOnOneLayoutSummary { type: "one-on-one"; local: string; remote: string; } export interface PipLayoutSummary { type: "pip"; spotlight: string[]; } export type LayoutSummary = | GridLayoutSummary | SpotlightLandscapeLayoutSummary | SpotlightPortraitLayoutSummary | SpotlightExpandedLayoutSummary | OneOnOneLayoutSummary | PipLayoutSummary; function summarizeLayout(l: Observable): Observable { return l.pipe( switchMap((l) => { switch (l.type) { case "grid": return combineLatest( [ l.spotlight?.media ?? of(undefined), ...l.grid.map((vm) => vm.media), ], (spotlight, ...grid) => ({ type: l.type, spotlight: spotlight?.map((vm) => vm.id), grid: grid.map((vm) => vm.id), }), ); case "spotlight-landscape": case "spotlight-portrait": return combineLatest( [l.spotlight.media, ...l.grid.map((vm) => vm.media)], (spotlight, ...grid) => ({ type: l.type, spotlight: spotlight.map((vm) => vm.id), grid: grid.map((vm) => vm.id), }), ); case "spotlight-expanded": return combineLatest( [l.spotlight.media, l.pip?.media ?? of(undefined)], (spotlight, pip) => ({ type: l.type, spotlight: spotlight.map((vm) => vm.id), pip: pip?.id, }), ); case "one-on-one": return combineLatest( [l.local.media, l.remote.media], (local, remote) => ({ type: l.type, local: local.id, remote: remote.id, }), ); case "pip": return l.spotlight.media.pipe( map((spotlight) => ({ type: l.type, spotlight: spotlight.map((vm) => vm.id), })), ); } }), // Sometimes there can be multiple (synchronous) updates per frame. We only // care about the most recent value for each time step, so discard these // extra values. debounceTime(0), distinctUntilChanged(isEqual), ); } function withCallViewModel( remoteParticipants: Observable, rtcMembers: Observable[]>, connectionState: Observable, speaking: Map>, continuation: (vm: CallViewModel) => void, ): void { const room = mockMatrixRoom({ client: { getUserId: () => localRtcMember.sender, getDeviceId: () => localRtcMember.deviceId, } as Partial as MatrixClient, getMember: (userId) => roomMembers.get(userId) ?? null, }); const rtcSession = new MockRTCSession( room, localRtcMember, [], ).withMemberships(rtcMembers); const participantsSpy = vi .spyOn(ComponentsCore, "connectedParticipantsObserver") .mockReturnValue(remoteParticipants); const mediaSpy = vi .spyOn(ComponentsCore, "observeParticipantMedia") .mockImplementation((p) => of({ participant: p } as Partial< ComponentsCore.ParticipantMedia > as ComponentsCore.ParticipantMedia), ); const eventsSpy = vi .spyOn(ComponentsCore, "observeParticipantEvents") .mockImplementation((p, ...eventTypes) => { if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) { return (speaking.get(p) ?? of(false)).pipe( map((s) => ({ ...p, isSpeaking: s }) as Participant), ); } else { return of(p); } }); const roomEventSelectorSpy = vi .spyOn(ComponentsCore, "roomEventSelector") .mockImplementation((room, eventType) => of()); const liveKitRoom = mockLivekitRoom( { localParticipant }, { remoteParticipants }, ); const vm = new CallViewModel( rtcSession as unknown as MatrixRTCSession, liveKitRoom, { kind: E2eeType.PER_PARTICIPANT, }, connectionState, ); onTestFinished(() => { vm!.destroy(); participantsSpy!.mockRestore(); mediaSpy!.mockRestore(); eventsSpy!.mockRestore(); roomEventSelectorSpy!.mockRestore(); }); continuation(vm); } test("participants are retained during a focus switch", () => { withTestScheduler(({ hot, expectObservable }) => { // Participants disappear on frame 2 and come back on frame 3 const participantInputMarbles = "a-ba"; // Start switching focus on frame 1 and reconnect on frame 3 const connectionInputMarbles = " cs-c"; // The visible participants should remain the same throughout the switch const expectedLayoutMarbles = " a"; withCallViewModel( hot(participantInputMarbles, { a: [aliceParticipant, bobParticipant], b: [], }), of([aliceRtcMember, bobRtcMember]), hot(connectionInputMarbles, { c: ConnectionState.Connected, s: ECAddonConnectionState.ECSwitchingFocus, }), new Map(), (vm) => { expectObservable(summarizeLayout(vm.layout)).toBe( expectedLayoutMarbles, { a: { type: "grid", spotlight: undefined, grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, }, ); }, ); }); }); test("screen sharing activates spotlight layout", () => { withTestScheduler(({ hot, schedule, expectObservable }) => { // Start with no screen shares, then have Alice and Bob share their screens, // then return to no screen shares, then have just Alice share for a bit const participantInputMarbles = " abcda-ba"; // While there are no screen shares, switch to spotlight manually, and then // switch back to grid at the end const modeInputMarbles = " -----s--g"; // We should automatically enter spotlight for the first round of screen // sharing, then return to grid, then manually go into spotlight, and // remain in spotlight until we manually go back to grid const expectedLayoutMarbles = " abcdaefeg"; const expectedShowSpeakingMarbles = "y----nyny"; withCallViewModel( hot(participantInputMarbles, { a: [aliceParticipant, bobParticipant], b: [aliceSharingScreen, bobParticipant], c: [aliceSharingScreen, bobSharingScreen], d: [aliceParticipant, bobSharingScreen], }), of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { schedule(modeInputMarbles, { s: () => vm.setGridMode("spotlight"), g: () => vm.setGridMode("grid"), }); expectObservable(summarizeLayout(vm.layout)).toBe( expectedLayoutMarbles, { a: { type: "grid", spotlight: undefined, grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, b: { type: "spotlight-landscape", spotlight: [`${aliceId}:0:screen-share`], grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, c: { type: "spotlight-landscape", spotlight: [ `${aliceId}:0:screen-share`, `${bobId}:0:screen-share`, ], grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, d: { type: "spotlight-landscape", spotlight: [`${bobId}:0:screen-share`], grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, e: { type: "spotlight-landscape", spotlight: [`${aliceId}:0`], grid: ["local:0", `${bobId}:0`], }, f: { type: "spotlight-landscape", spotlight: [`${aliceId}:0:screen-share`], grid: ["local:0", `${bobId}:0`, `${aliceId}:0`], }, g: { type: "grid", spotlight: undefined, grid: ["local:0", `${bobId}:0`, `${aliceId}:0`], }, }, ); expectObservable(vm.showSpeakingIndicators).toBe( expectedShowSpeakingMarbles, { y: true, n: false, }, ); }, ); }); }); test("participants stay in the same order unless to appear/disappear", () => { withTestScheduler(({ hot, schedule, expectObservable }) => { const modeInputMarbles = " a"; // First Bob speaks, then Dave, then Alice const aSpeakingInputMarbles = "n- 1998ms - 1999ms y"; const bSpeakingInputMarbles = "ny 1998ms n 1999ms -"; const dSpeakingInputMarbles = "n- 1998ms y 1999ms n"; // Nothing should change when Bob speaks, because Bob is already on screen. // When Dave speaks he should switch with Alice because she's the one who // hasn't spoken at all. Then when Alice speaks, she should return to her // place at the top. const expectedLayoutMarbles = "a 1999ms b 1999ms a 57999ms c 1999ms a"; withCallViewModel( of([aliceParticipant, bobParticipant, daveParticipant]), of([aliceRtcMember, bobRtcMember, daveRtcMember]), of(ConnectionState.Connected), new Map([ [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })], [bobParticipant, hot(bSpeakingInputMarbles, { y: true, n: false })], [daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })], ]), (vm) => { schedule(modeInputMarbles, { a: () => { // We imagine that only three tiles (the first three) will be visible // on screen at a time vm.layout.subscribe((layout) => { if (layout.type === "grid") layout.setVisibleTiles(3); }); }, }); expectObservable(summarizeLayout(vm.layout)).toBe( expectedLayoutMarbles, { a: { type: "grid", spotlight: undefined, grid: ["local:0", `${aliceId}:0`, `${bobId}:0`, `${daveId}:0`], }, b: { type: "grid", spotlight: undefined, grid: ["local:0", `${daveId}:0`, `${bobId}:0`, `${aliceId}:0`], }, c: { type: "grid", spotlight: undefined, grid: ["local:0", `${aliceId}:0`, `${daveId}:0`, `${bobId}:0`], }, }, ); }, ); }); }); test("spotlight speakers swap places", () => { withTestScheduler(({ hot, schedule, expectObservable }) => { // Go immediately into spotlight mode for the test const modeInputMarbles = " s"; // First Bob speaks, then Dave, then Alice const aSpeakingInputMarbles = "n--y"; const bSpeakingInputMarbles = "nyn"; const dSpeakingInputMarbles = "n-yn"; // Alice should start in the spotlight, then Bob, then Dave, then Alice // again. However, the positions of Dave and Bob in the grid should be // reversed by the end because they've been swapped in and out of the // spotlight. const expectedLayoutMarbles = "abcd"; withCallViewModel( of([aliceParticipant, bobParticipant, daveParticipant]), of([aliceRtcMember, bobRtcMember, daveRtcMember]), of(ConnectionState.Connected), new Map([ [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })], [bobParticipant, hot(bSpeakingInputMarbles, { y: true, n: false })], [daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })], ]), (vm) => { schedule(modeInputMarbles, { s: () => vm.setGridMode("spotlight") }); expectObservable(summarizeLayout(vm.layout)).toBe( expectedLayoutMarbles, { a: { type: "spotlight-landscape", spotlight: [`${aliceId}:0`], grid: ["local:0", `${bobId}:0`, `${daveId}:0`], }, b: { type: "spotlight-landscape", spotlight: [`${bobId}:0`], grid: ["local:0", `${aliceId}:0`, `${daveId}:0`], }, c: { type: "spotlight-landscape", spotlight: [`${daveId}:0`], grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, d: { type: "spotlight-landscape", spotlight: [`${aliceId}:0`], grid: ["local:0", `${daveId}:0`, `${bobId}:0`], }, }, ); }, ); }); }); test("layout enters picture-in-picture mode when requested", () => { withTestScheduler(({ schedule, expectObservable }) => { // Enable then disable picture-in-picture const pipControlInputMarbles = "-ed"; // Should go into picture-in-picture layout then back to grid const expectedLayoutMarbles = " aba"; withCallViewModel( of([aliceParticipant, bobParticipant]), of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { schedule(pipControlInputMarbles, { e: () => window.controls.enablePip(), d: () => window.controls.disablePip(), }); expectObservable(summarizeLayout(vm.layout)).toBe( expectedLayoutMarbles, { a: { type: "grid", spotlight: undefined, grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, b: { type: "pip", spotlight: [`${aliceId}:0`], }, }, ); }, ); }); }); test("spotlight remembers whether it's expanded", () => { withTestScheduler(({ schedule, expectObservable }) => { // Start in spotlight mode, then switch to grid and back to spotlight a // couple times const modeInputMarbles = " s-gs-gs"; // Expand and collapse the spotlight const expandInputMarbles = " -a--a"; // Spotlight should stay expanded during the first mode switch, and stay // collapsed during the second mode switch const expectedLayoutMarbles = "abcbada"; withCallViewModel( of([aliceParticipant, bobParticipant]), of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { schedule(modeInputMarbles, { s: () => vm.setGridMode("spotlight"), g: () => vm.setGridMode("grid"), }); schedule(expandInputMarbles, { a: () => { let toggle: () => void; vm.toggleSpotlightExpanded.subscribe((val) => (toggle = val!)); toggle!(); }, }); expectObservable(summarizeLayout(vm.layout)).toBe( expectedLayoutMarbles, { a: { type: "spotlight-landscape", spotlight: [`${aliceId}:0`], grid: ["local:0", `${bobId}:0`], }, b: { type: "spotlight-expanded", spotlight: [`${aliceId}:0`], pip: "local:0", }, c: { type: "grid", spotlight: undefined, grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, d: { type: "grid", spotlight: undefined, grid: ["local:0", `${bobId}:0`, `${aliceId}:0`], }, }, ); }, ); }); }); test("participants must have a MatrixRTCSession to be visible", () => { withTestScheduler(({ hot, expectObservable }) => { // iterate through a number of combinations of participants and MatrixRTC memberships // Bob never has an MatrixRTC membership const scenarioInputMarbles = " abcdec"; // Bob should never be visible const expectedLayoutMarbles = "a-bc-b"; withCallViewModel( hot(scenarioInputMarbles, { a: [], b: [bobParticipant], c: [aliceParticipant, bobParticipant], d: [aliceParticipant, daveParticipant, bobParticipant], e: [aliceParticipant, daveParticipant, bobSharingScreen], }), hot(scenarioInputMarbles, { a: [], b: [], c: [aliceRtcMember], d: [aliceRtcMember, daveRtcMember], e: [aliceRtcMember, daveRtcMember], }), of(ConnectionState.Connected), new Map(), (vm) => { vm.setGridMode("grid"); expectObservable(summarizeLayout(vm.layout)).toBe( expectedLayoutMarbles, { a: { type: "grid", spotlight: undefined, grid: ["local:0"], }, b: { type: "one-on-one", local: "local:0", remote: `${aliceId}:0`, }, c: { type: "grid", spotlight: undefined, grid: ["local:0", `${aliceId}:0`, `${daveId}:0`], }, }, ); }, ); }); }); it("should show at least one tile per MatrixRTCSession", () => { withTestScheduler(({ hot, expectObservable }) => { // iterate through some combinations of MatrixRTC memberships const scenarioInputMarbles = " abcd"; // There should always be one tile for each MatrixRTCSession const expectedLayoutMarbles = "abcd"; withCallViewModel( of([]), hot(scenarioInputMarbles, { a: [], b: [aliceRtcMember], c: [aliceRtcMember, daveRtcMember], d: [daveRtcMember], }), of(ConnectionState.Connected), new Map(), (vm) => { vm.setGridMode("grid"); expectObservable(summarizeLayout(vm.layout)).toBe( expectedLayoutMarbles, { a: { type: "grid", spotlight: undefined, grid: ["local:0"], }, b: { type: "one-on-one", local: "local:0", remote: `${aliceId}:0`, }, c: { type: "grid", spotlight: undefined, grid: ["local:0", `${aliceId}:0`, `${daveId}:0`], }, d: { type: "one-on-one", local: "local:0", remote: `${daveId}:0`, }, }, ); }, ); }); });