diff --git a/locales/en/app.json b/locales/en/app.json index bc37dd7b..e500f66c 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -194,6 +194,7 @@ "expand": "Expand", "mute_for_me": "Mute for me", "muted_for_me": "Muted for me", - "volume": "Volume" + "volume": "Volume", + "waiting_for_media": "Waiting for media..." } } diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index ed4d5bce..3947ba66 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -51,7 +51,7 @@ export interface ConfigOptions { // a livekit service url in the client well-known. // The well known needs to be formatted like so: // {"type":"livekit", "livekit_service_url":"https://livekit.example.com"} - // and stored under the key: "livekit_focus" + // and stored under the key: "org.matrix.msc4143.rtc_foci" livekit_service_url: string; }; diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 9014e60b..78d08f60 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -8,10 +8,14 @@ Please see LICENSE in the repository root for full details. import { render } from "@testing-library/react"; import { beforeEach, expect, test } from "vitest"; import { MatrixClient } from "matrix-js-sdk/src/client"; -import { ConnectionState, RemoteParticipant, Room } from "livekit-client"; -import { of } from "rxjs"; +import { ConnectionState, Room } from "livekit-client"; +import { BehaviorSubject, of } from "rxjs"; import { afterEach } from "node:test"; import { act } from "react"; +import { + CallMembership, + type MatrixRTCSession, +} from "matrix-js-sdk/src/matrixrtc"; import { soundEffectVolumeSetting } from "../settings/settings"; import { @@ -22,6 +26,8 @@ import { mockMatrixRoomMember, mockMediaPlay, mockRemoteParticipant, + mockRtcMembership, + MockRTCSession, } from "../utils/test"; import { E2eeType } from "../e2ee/e2eeType"; import { CallViewModel } from "../state/CallViewModel"; @@ -30,11 +36,15 @@ import { MAX_PARTICIPANT_COUNT_FOR_SOUND, } from "./CallEventAudioRenderer"; -const alice = mockMatrixRoomMember({ userId: "@alice:example.org" }); -const bob = mockMatrixRoomMember({ userId: "@bob:example.org" }); -const aliceId = `${alice.userId}:AAAA`; -const bobId = `${bob.userId}:BBBB`; +const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); +const local = mockMatrixRoomMember(localRtcMember); +const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); +const alice = mockMatrixRoomMember(aliceRtcMember); +const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); +const bob = mockMatrixRoomMember(bobRtcMember); const localParticipant = mockLocalParticipant({ identity: "" }); +const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`; +const bobId = `${bob.userId}:${bobRtcMember.deviceId}`; const aliceParticipant = mockRemoteParticipant({ identity: aliceId }); const bobParticipant = mockRemoteParticipant({ identity: bobId }); @@ -53,20 +63,28 @@ afterEach(() => { test("plays a sound when entering a call", () => { const audioIsPlaying: string[] = mockMediaPlay(); - const members = new Map([alice, bob].map((p) => [p.userId, p])); + const matrixRoomMembers = new Map( + [local, alice, bob].map((p) => [p.userId, p]), + ); const remoteParticipants = of([aliceParticipant]); const liveKitRoom = mockLivekitRoom( { localParticipant }, { remoteParticipants }, ); + const matrixRoom = mockMatrixRoom({ + client: { + getUserId: () => localRtcMember.sender, + getDeviceId: () => localRtcMember.deviceId, + } as Partial as MatrixClient, + getMember: (userId) => matrixRoomMembers.get(userId) ?? null, + }); + + const session = new MockRTCSession(matrixRoom, localRtcMember, [ + aliceRtcMember, + ]) as unknown as MatrixRTCSession; const vm = new CallViewModel( - mockMatrixRoom({ - client: { - getUserId: () => "@carol:example.org", - } as Partial as MatrixClient, - getMember: (userId) => members.get(userId) ?? null, - }), + session, liveKitRoom, { kind: E2eeType.PER_PARTICIPANT, @@ -84,20 +102,29 @@ test("plays a sound when entering a call", () => { test("plays no sound when muted", () => { soundEffectVolumeSetting.setValue(0); const audioIsPlaying: string[] = mockMediaPlay(); - const members = new Map([alice, bob].map((p) => [p.userId, p])); + const matrixRoomMembers = new Map( + [local, alice, bob].map((p) => [p.userId, p]), + ); const remoteParticipants = of([aliceParticipant, bobParticipant]); const liveKitRoom = mockLivekitRoom( { localParticipant }, { remoteParticipants }, ); + const matrixRoom = mockMatrixRoom({ + client: { + getUserId: () => localRtcMember.sender, + getDeviceId: () => localRtcMember.deviceId, + } as Partial as MatrixClient, + getMember: (userId) => matrixRoomMembers.get(userId) ?? null, + }); + + const session = new MockRTCSession(matrixRoom, localRtcMember, [ + aliceRtcMember, + ]) as unknown as MatrixRTCSession; + const vm = new CallViewModel( - mockMatrixRoom({ - client: { - getUserId: () => "@carol:example.org", - } as Partial as MatrixClient, - getMember: (userId) => members.get(userId) ?? null, - }), + session, liveKitRoom, { kind: E2eeType.PER_PARTICIPANT, @@ -112,7 +139,7 @@ test("plays no sound when muted", () => { test("plays a sound when a user joins", () => { const audioIsPlaying: string[] = mockMediaPlay(); - const members = new Map([alice].map((p) => [p.userId, p])); + const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p])); const remoteParticipants = new Map( [aliceParticipant].map((p) => [p.identity, p]), ); @@ -121,13 +148,27 @@ test("plays a sound when a user joins", () => { remoteParticipants, }); + const matrixRoom = mockMatrixRoom({ + client: { + getUserId: () => localRtcMember.sender, + getDeviceId: () => localRtcMember.deviceId, + } as Partial as MatrixClient, + getMember: (userId) => matrixRoomMembers.get(userId) ?? null, + }); + + const remoteRtcMemberships = new BehaviorSubject([ + aliceRtcMember, + ]); + // we give Bob an RTC session now, but no participant yet + const session = new MockRTCSession( + matrixRoom, + localRtcMember, + ).withMemberships( + remoteRtcMemberships.asObservable(), + ) as unknown as MatrixRTCSession; + const vm = new CallViewModel( - mockMatrixRoom({ - client: { - getUserId: () => "@carol:example.org", - } as Partial as MatrixClient, - getMember: (userId) => members.get(userId) ?? null, - }), + session, liveKitRoom as unknown as Room, { kind: E2eeType.PER_PARTICIPANT, @@ -137,20 +178,20 @@ test("plays a sound when a user joins", () => { render(); act(() => { - liveKitRoom.addParticipant(bobParticipant); + remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]); }); // Play a sound when joining a call. expect(audioIsPlaying).toEqual([ // Joining the call enterSound, - // Bob leaves + // Bob joins enterSound, ]); }); test("plays a sound when a user leaves", () => { const audioIsPlaying: string[] = mockMediaPlay(); - const members = new Map([alice].map((p) => [p.userId, p])); + const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p])); const remoteParticipants = new Map( [aliceParticipant].map((p) => [p.identity, p]), ); @@ -159,13 +200,25 @@ test("plays a sound when a user leaves", () => { remoteParticipants, }); + const matrixRoom = mockMatrixRoom({ + client: { + getUserId: () => localRtcMember.sender, + getDeviceId: () => localRtcMember.deviceId, + } as Partial as MatrixClient, + getMember: (userId) => matrixRoomMembers.get(userId) ?? null, + }); + + const remoteRtcMemberships = new BehaviorSubject([ + aliceRtcMember, + ]); + + const session = new MockRTCSession( + matrixRoom, + localRtcMember, + ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession; + const vm = new CallViewModel( - mockMatrixRoom({ - client: { - getUserId: () => "@carol:example.org", - } as Partial as MatrixClient, - getMember: (userId) => members.get(userId) ?? null, - }), + session, liveKitRoom as unknown as Room, { kind: E2eeType.PER_PARTICIPANT, @@ -175,7 +228,7 @@ test("plays a sound when a user leaves", () => { render(); act(() => { - liveKitRoom.removeParticipant(aliceParticipant); + remoteRtcMemberships.next([]); }); expect(audioIsPlaying).toEqual([ // Joining the call @@ -185,30 +238,45 @@ test("plays a sound when a user leaves", () => { ]); }); -test("plays no sound when the participant list", () => { +test("plays no sound when the session member count is larger than the max, until decreased", () => { const audioIsPlaying: string[] = mockMediaPlay(); - const members = new Map([alice].map((p) => [p.userId, p])); - const remoteParticipants = new Map([ - [aliceParticipant.identity, aliceParticipant], - ...Array.from({ length: MAX_PARTICIPANT_COUNT_FOR_SOUND - 1 }).map< - [string, RemoteParticipant] - >((_, index) => { - const p = mockRemoteParticipant({ identity: `user${index}` }); - return [p.identity, p]; - }), - ]); + const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p])); + const remoteParticipants = new Map( + [aliceParticipant].map((p) => [p.identity, p]), + ); + + const mockRtcMemberships: CallMembership[] = []; + + for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) { + mockRtcMemberships.push( + mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`), + ); + } + + const remoteRtcMemberships = new BehaviorSubject( + mockRtcMemberships, + ); + const liveKitRoom = new EmittableMockLivekitRoom({ localParticipant, remoteParticipants, }); + const matrixRoom = mockMatrixRoom({ + client: { + getUserId: () => localRtcMember.sender, + getDeviceId: () => localRtcMember.deviceId, + } as Partial as MatrixClient, + getMember: (userId) => matrixRoomMembers.get(userId) ?? null, + }); + + const session = new MockRTCSession( + matrixRoom, + localRtcMember, + ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession; + const vm = new CallViewModel( - mockMatrixRoom({ - client: { - getUserId: () => "@carol:example.org", - } as Partial as MatrixClient, - getMember: (userId) => members.get(userId) ?? null, - }), + session, liveKitRoom as unknown as Room, { kind: E2eeType.PER_PARTICIPANT, @@ -217,9 +285,11 @@ test("plays no sound when the participant list", () => { ); render(); expect(audioIsPlaying).toEqual([]); - // When the count drops + // When the count drops to the max we should play the leave sound act(() => { - liveKitRoom.removeParticipant(aliceParticipant); + remoteRtcMemberships.next( + mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1), + ); }); expect(audioIsPlaying).toEqual([leaveSound]); }); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index f1afa3e4..ab35f164 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -124,7 +124,7 @@ export const ActiveCall: FC = (props) => { useEffect(() => { if (livekitRoom !== undefined) { const vm = new CallViewModel( - props.rtcSession.room, + props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable, @@ -132,12 +132,7 @@ export const ActiveCall: FC = (props) => { setVm(vm); return (): void => vm.destroy(); } - }, [ - props.rtcSession.room, - livekitRoom, - props.e2eeSystem, - connStateObservable, - ]); + }, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable]); if (livekitRoom === undefined || vm === null) return null; diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index 9b2e5ee7..5dbfb1ca 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { test, vi, onTestFinished } from "vitest"; +import { test, vi, onTestFinished, it } from "vitest"; import { combineLatest, debounceTime, @@ -25,6 +25,7 @@ import { } from "livekit-client"; import * as ComponentsCore from "@livekit/components-core"; import { isEqual } from "lodash-es"; +import { CallMembership, MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc"; import { CallViewModel, Layout } from "./CallViewModel"; import { @@ -34,6 +35,8 @@ import { mockMatrixRoomMember, mockRemoteParticipant, withTestScheduler, + mockRtcMembership, + MockRTCSession, } from "../utils/test"; import { ECAddonConnectionState, @@ -43,14 +46,19 @@ import { E2eeType } from "../e2ee/e2eeType"; vi.mock("@livekit/components-core"); -const alice = mockMatrixRoomMember({ userId: "@alice:example.org" }); -const bob = mockMatrixRoomMember({ userId: "@bob:example.org" }); -const carol = mockMatrixRoomMember({ userId: "@carol:example.org" }); -const dave = mockMatrixRoomMember({ userId: "@dave:example.org" }); +const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); +const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); +const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); +const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); -const aliceId = `${alice.userId}:AAAA`; -const bobId = `${bob.userId}:BBBB`; -const daveId = `${dave.userId}:DDDD`; +const alice = mockMatrixRoomMember(aliceRtcMember); +const bob = mockMatrixRoomMember(bobRtcMember); +const carol = mockMatrixRoomMember(localRtcMember); +const dave = mockMatrixRoomMember(daveRtcMember); + +const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`; +const bobId = `${bob.userId}:${bobRtcMember.deviceId}`; +const daveId = `${dave.userId}:${daveRtcMember.deviceId}`; const localParticipant = mockLocalParticipant({ identity: "" }); const aliceParticipant = mockRemoteParticipant({ identity: aliceId }); @@ -65,7 +73,9 @@ const bobSharingScreen = mockRemoteParticipant({ }); const daveParticipant = mockRemoteParticipant({ identity: daveId }); -const members = new Map([alice, bob, carol, dave].map((p) => [p.userId, p])); +const roomMembers = new Map( + [alice, bob, carol, dave].map((p) => [p.userId, p]), +); export interface GridLayoutSummary { type: "grid"; @@ -173,10 +183,23 @@ function summarizeLayout(l: Observable): Observable { function withCallViewModel( remoteParticipants: Observable, + rtcMembers: Observable[]>, connectionState: Observable, speaking: Map>, continuation: (vm: CallViewModel) => void, ): void { + const room = mockMatrixRoom({ + client: { + getUserId: () => localRtcMember.sender, + getDeviceId: () => localRtcMember.deviceId, + } as Partial as MatrixClient, + getMember: (userId) => roomMembers.get(userId) ?? null, + }); + const rtcSession = new MockRTCSession( + room, + localRtcMember, + [], + ).withMemberships(rtcMembers); const participantsSpy = vi .spyOn(ComponentsCore, "connectedParticipantsObserver") .mockReturnValue(remoteParticipants); @@ -209,12 +232,7 @@ function withCallViewModel( ); const vm = new CallViewModel( - mockMatrixRoom({ - client: { - getUserId: () => "@carol:example.org", - } as Partial as MatrixClient, - getMember: (userId) => members.get(userId) ?? null, - }), + rtcSession as unknown as MatrixRTCSession, liveKitRoom, { kind: E2eeType.PER_PARTICIPANT, @@ -247,6 +265,7 @@ test("participants are retained during a focus switch", () => { a: [aliceParticipant, bobParticipant], b: [], }), + of([aliceRtcMember, bobRtcMember]), hot(connectionInputMarbles, { c: ConnectionState.Connected, s: ECAddonConnectionState.ECSwitchingFocus, @@ -288,6 +307,7 @@ test("screen sharing activates spotlight layout", () => { c: [aliceSharingScreen, bobSharingScreen], d: [aliceParticipant, bobSharingScreen], }), + of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { @@ -356,7 +376,7 @@ test("participants stay in the same order unless to appear/disappear", () => { const modeInputMarbles = " a"; // First Bob speaks, then Dave, then Alice const aSpeakingInputMarbles = "n- 1998ms - 1999ms y"; - const bSpeakingInputMarbles = "ny 1998ms n 1999ms "; + const bSpeakingInputMarbles = "ny 1998ms n 1999ms -"; const dSpeakingInputMarbles = "n- 1998ms y 1999ms n"; // Nothing should change when Bob speaks, because Bob is already on screen. // When Dave speaks he should switch with Alice because she's the one who @@ -366,6 +386,7 @@ test("participants stay in the same order unless to appear/disappear", () => { withCallViewModel( of([aliceParticipant, bobParticipant, daveParticipant]), + of([aliceRtcMember, bobRtcMember, daveRtcMember]), of(ConnectionState.Connected), new Map([ [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })], @@ -427,6 +448,7 @@ test("spotlight speakers swap places", () => { withCallViewModel( of([aliceParticipant, bobParticipant, daveParticipant]), + of([aliceRtcMember, bobRtcMember, daveRtcMember]), of(ConnectionState.Connected), new Map([ [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })], @@ -475,6 +497,7 @@ test("layout enters picture-in-picture mode when requested", () => { withCallViewModel( of([aliceParticipant, bobParticipant]), + of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { @@ -515,6 +538,7 @@ test("spotlight remembers whether it's expanded", () => { withCallViewModel( of([aliceParticipant, bobParticipant]), + of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { @@ -559,3 +583,104 @@ test("spotlight remembers whether it's expanded", () => { ); }); }); + +test("participants must have a MatrixRTCSession to be visible", () => { + withTestScheduler(({ hot, expectObservable }) => { + // iterate through a number of combinations of participants and MatrixRTC memberships + // Bob never has an MatrixRTC membership + const scenarioInputMarbles = " abcdec"; + // Bob should never be visible + const expectedLayoutMarbles = "a-bc-b"; + + withCallViewModel( + hot(scenarioInputMarbles, { + a: [], + b: [bobParticipant], + c: [aliceParticipant, bobParticipant], + d: [aliceParticipant, daveParticipant, bobParticipant], + e: [aliceParticipant, daveParticipant, bobSharingScreen], + }), + hot(scenarioInputMarbles, { + a: [], + b: [], + c: [aliceRtcMember], + d: [aliceRtcMember, daveRtcMember], + e: [aliceRtcMember, daveRtcMember], + }), + of(ConnectionState.Connected), + new Map(), + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout(vm.layout)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0"], + }, + b: { + type: "one-on-one", + local: "local:0", + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${daveId}:0`], + }, + }, + ); + }, + ); + }); +}); + +it("should show at least one tile per MatrixRTCSession", () => { + withTestScheduler(({ hot, expectObservable }) => { + // iterate through some combinations of MatrixRTC memberships + const scenarioInputMarbles = " abcd"; + // There should always be one tile for each MatrixRTCSession + const expectedLayoutMarbles = "abcd"; + + withCallViewModel( + of([]), + hot(scenarioInputMarbles, { + a: [], + b: [aliceRtcMember], + c: [aliceRtcMember, daveRtcMember], + d: [daveRtcMember], + }), + of(ConnectionState.Connected), + new Map(), + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout(vm.layout)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0"], + }, + b: { + type: "one-on-one", + local: "local:0", + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${daveId}:0`], + }, + d: { + type: "one-on-one", + local: "local:0", + remote: `${daveId}:0`, + }, + }, + ); + }, + ); + }); +}); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 8d8aaa43..95762c3f 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -18,12 +18,9 @@ import { RemoteParticipant, Track, } from "livekit-client"; +import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix"; import { - Room as MatrixRoom, - RoomMember, - RoomStateEvent, -} from "matrix-js-sdk/src/matrix"; -import { + BehaviorSubject, EMPTY, Observable, Subject, @@ -49,6 +46,10 @@ import { withLatestFrom, } from "rxjs"; import { logger } from "matrix-js-sdk/src/logger"; +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc"; import { ViewModel } from "./ViewModel"; import { @@ -222,41 +223,67 @@ interface LayoutScanState { class UserMedia { private readonly scope = new ObservableScope(); public readonly vm: UserMediaViewModel; + private readonly participant: BehaviorSubject< + LocalParticipant | RemoteParticipant | undefined + >; + public readonly speaker: Observable; public readonly presenter: Observable; - public constructor( public readonly id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: LocalParticipant | RemoteParticipant | undefined, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { - this.vm = participant.isLocal - ? new LocalUserMediaViewModel( - id, - member, - participant as LocalParticipant, - encryptionSystem, - livekitRoom, - ) - : new RemoteUserMediaViewModel( - id, - member, - participant as RemoteParticipant, - encryptionSystem, - livekitRoom, - ); + this.participant = new BehaviorSubject(participant); + + if (participant?.isLocal) { + this.vm = new LocalUserMediaViewModel( + this.id, + member, + this.participant.asObservable() as Observable, + encryptionSystem, + livekitRoom, + ); + } else { + this.vm = new RemoteUserMediaViewModel( + id, + member, + this.participant.asObservable() as Observable< + RemoteParticipant | undefined + >, + encryptionSystem, + livekitRoom, + ); + } this.speaker = observeSpeaker(this.vm.speaking).pipe(this.scope.state()); - this.presenter = observeParticipantEvents( - participant, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled)); + this.presenter = this.participant.pipe( + switchMap( + (p) => + (p && + observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled))) ?? + of(false), + ), + this.scope.state(), + ); + } + + public updateParticipant( + newParticipant: LocalParticipant | RemoteParticipant | undefined, + ): void { + if (this.participant.value !== newParticipant) { + // Update the BehaviourSubject in the UserMedia. + this.participant.next(newParticipant); + } } public destroy(): void { @@ -267,6 +294,9 @@ class UserMedia { class ScreenShare { public readonly vm: ScreenShareViewModel; + private readonly participant: BehaviorSubject< + LocalParticipant | RemoteParticipant + >; public constructor( id: string, @@ -275,12 +305,15 @@ class ScreenShare { encryptionSystem: EncryptionSystem, liveKitRoom: LivekitRoom, ) { + this.participant = new BehaviorSubject(participant); + this.vm = new ScreenShareViewModel( id, member, - participant, + this.participant.asObservable(), encryptionSystem, liveKitRoom, + participant.isLocal, ); } @@ -317,11 +350,11 @@ function findMatrixRoomMember( export class CallViewModel extends ViewModel { public readonly localVideo: Observable = observeTrackReference( - this.livekitRoom.localParticipant, + of(this.livekitRoom.localParticipant), Track.Source.Camera, ).pipe( map((trackRef) => { - const track = trackRef.publication?.track; + const track = trackRef?.publication?.track; return track instanceof LocalVideoTrack ? track : null; }), ); @@ -401,49 +434,87 @@ export class CallViewModel extends ViewModel { this.remoteParticipants, observeParticipantMedia(this.livekitRoom.localParticipant), duplicateTiles.value, - // Also react to changes in the list of members - fromEvent(this.matrixRoom, RoomStateEvent.Update).pipe(startWith(null)), + // Also react to changes in the MatrixRTC session list. + // The session list will also be update if a room membership changes. + // No additional RoomState event listener needs to be set up. + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe(startWith(null)), ]).pipe( scan( ( prevItems, - [remoteParticipants, { participant: localParticipant }, duplicateTiles], + [ + remoteParticipants, + { participant: localParticipant }, + duplicateTiles, + _membershipsChanged, + ], ) => { const newItems = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const p of [localParticipant, ...remoteParticipants]) { - const id = p === localParticipant ? "local" : p.identity; - const member = findMatrixRoomMember(this.matrixRoom, id); - if (member === undefined) - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`, - ); + // m.rtc.members are the basis for calculating what is visible in the call + for (const rtcMember of this.matrixRTCSession.memberships) { + const room = this.matrixRTCSession.room; + // WARN! This is not exactly the sender but the user defined in the state key. + // This will be available once we change to the new "member as object" format in the MatrixRTC object. + let livekitParticipantId = + rtcMember.sender + ":" + rtcMember.deviceId; - // Create as many tiles for this participant as called for by - // the duplicateTiles option + let participant: + | LocalParticipant + | RemoteParticipant + | undefined = undefined; + if ( + rtcMember.sender === room.client.getUserId()! && + rtcMember.deviceId === room.client.getDeviceId() + ) { + livekitParticipantId = "local"; + participant = localParticipant; + } else { + participant = remoteParticipants.find( + (p) => p.identity === livekitParticipantId, + ); + } + + const member = findMatrixRoomMember(room, livekitParticipantId); + if (!member) { + logger.error( + "Could not find member for media id: ", + livekitParticipantId, + ); + } for (let i = 0; i < 1 + duplicateTiles; i++) { - const userMediaId = `${id}:${i}`; + const indexedMediaId = `${livekitParticipantId}:${i}`; + const prevMedia = prevItems.get(indexedMediaId); + if (prevMedia && prevMedia instanceof UserMedia) { + prevMedia.updateParticipant(participant); + } yield [ - userMediaId, - prevItems.get(userMediaId) ?? + indexedMediaId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevMedia ?? new UserMedia( - userMediaId, + indexedMediaId, member, - p, + participant, this.encryptionSystem, this.livekitRoom, ), ]; - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; + if (participant?.isScreenShareEnabled) { + const screenShareId = `${indexedMediaId}:screen-share`; yield [ screenShareId, prevItems.get(screenShareId) ?? new ScreenShare( screenShareId, member, - p, + participant, this.encryptionSystem, this.livekitRoom, ), @@ -454,7 +525,6 @@ export class CallViewModel extends ViewModel { }.bind(this)(), ); - for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); return newItems; }, new Map(), @@ -488,11 +558,6 @@ export class CallViewModel extends ViewModel { ), ); - private readonly localUserMedia: Observable = - this.mediaItems.pipe( - map((ms) => ms.find((m) => m.vm.local)!.vm as LocalUserMediaViewModel), - ); - /** * List of MediaItems that we want to display, that are of type ScreenShare */ @@ -504,7 +569,7 @@ export class CallViewModel extends ViewModel { this.scope.state(), ); - private readonly spotlightSpeaker: Observable = + private readonly spotlightSpeaker: Observable = this.userMedia.pipe( switchMap((mediaItems) => mediaItems.length === 0 @@ -515,7 +580,7 @@ export class CallViewModel extends ViewModel { ), ), ), - scan<(readonly [UserMedia, boolean])[], UserMedia, null>( + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( (prev, mediaItems) => { // Only remote users that are still in the call should be sticky const [stickyMedia, stickySpeaking] = @@ -532,11 +597,11 @@ export class CallViewModel extends ViewModel { // Otherwise, spotlight an arbitrary remote user mediaItems.find(([m]) => !m.vm.local)?.[0] ?? // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)![0]); + mediaItems.find(([m]) => m.vm.local)?.[0]); }, null, ), - map((speaker) => speaker.vm), + map((speaker) => speaker?.vm ?? null), this.scope.state(), ); @@ -576,37 +641,57 @@ export class CallViewModel extends ViewModel { }), ); - private readonly spotlightAndPip: Observable< - [Observable, Observable] - > = this.screenShares.pipe( - map((screenShares) => - screenShares.length > 0 - ? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const) - : ([ - this.spotlightSpeaker.pipe(map((speaker) => [speaker!])), - this.spotlightSpeaker.pipe( - switchMap((speaker) => - speaker.local - ? of(null) - : this.localUserMedia.pipe( - switchMap((vm) => - vm.alwaysShow.pipe( - map((alwaysShow) => (alwaysShow ? vm : null)), - ), - ), - ), - ), - ), - ] as const), - ), - ); - private readonly spotlight: Observable = - this.spotlightAndPip.pipe( - switchMap(([spotlight]) => spotlight), + this.screenShares.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } + + return this.spotlightSpeaker.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), this.scope.state(), ); + private readonly pip: Observable = combineLatest([ + this.screenShares, + this.spotlightSpeaker, + this.mediaItems, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return this.spotlightSpeaker; + } + if (!spotlight || spotlight.local) { + return of(null); + } + + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; + + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; + + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } + + return null; + }), + ); + }), + this.scope.state(), + ); + private readonly hasRemoteScreenShares: Observable = this.spotlight.pipe( map((spotlight) => @@ -615,9 +700,6 @@ export class CallViewModel extends ViewModel { distinctUntilChanged(), ); - private readonly pip: Observable = - this.spotlightAndPip.pipe(switchMap(([, pip]) => pip)); - private readonly pipEnabled: Observable = setPipEnabled.pipe( startWith(false), ); @@ -721,15 +803,16 @@ export class CallViewModel extends ViewModel { this.mediaItems.pipe( map((mediaItems) => { if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)! - .vm as LocalUserMediaViewModel; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as | RemoteUserMediaViewModel | undefined; // There might not be a remote tile if there are screen shares, or if // only the local user is in the call and they're using the duplicate // tiles option - if (remote === undefined) return null; + if (!remote || !local) return null; return { type: "one-on-one", local, remote }; }), @@ -1010,7 +1093,7 @@ export class CallViewModel extends ViewModel { public constructor( // A call is permanently tied to a single Matrix room and LiveKit room - private readonly matrixRoom: MatrixRoom, + private readonly matrixRTCSession: MatrixRTCSession, private readonly livekitRoom: LivekitRoom, private readonly encryptionSystem: EncryptionSystem, private readonly connectionState: Observable, diff --git a/src/state/MediaViewModel.test.ts b/src/state/MediaViewModel.test.ts index 5b5e59a7..c4e0bee6 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/MediaViewModel.test.ts @@ -8,14 +8,17 @@ Please see LICENSE in the repository root for full details. import { expect, test, vi } from "vitest"; import { + mockRtcMembership, withLocalMedia, withRemoteMedia, withTestScheduler, } from "../utils/test"; +const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); + test("control a participant's volume", async () => { const setVolumeSpy = vi.fn(); - await withRemoteMedia({}, { setVolume: setVolumeSpy }, (vm) => + await withRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy }, (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab---c---d|", { a() { @@ -60,7 +63,7 @@ test("control a participant's volume", async () => { }); test("toggle fit/contain for a participant's video", async () => { - await withRemoteMedia({}, {}, (vm) => + await withRemoteMedia(rtcMembership, {}, {}, (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab|", { a: () => vm.toggleFitContain(), @@ -76,17 +79,21 @@ test("toggle fit/contain for a participant's video", async () => { }); test("local media remembers whether it should always be shown", async () => { - await withLocalMedia({}, (vm) => + await withLocalMedia(rtcMembership, {}, (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-a|", { a: () => vm.setAlwaysShow(false) }); expectObservable(vm.alwaysShow).toBe("ab", { a: true, b: false }); }), ); // Next local media should start out *not* always shown - await withLocalMedia({}, (vm) => - withTestScheduler(({ expectObservable, schedule }) => { - schedule("-a|", { a: () => vm.setAlwaysShow(true) }); - expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true }); - }), + await withLocalMedia( + rtcMembership, + + {}, + (vm) => + withTestScheduler(({ expectObservable, schedule }) => { + schedule("-a|", { a: () => vm.setAlwaysShow(true) }); + expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true }); + }), ); }); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 36e76d38..ceaca57c 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -32,7 +32,6 @@ import { Observable, Subject, combineLatest, - distinctUntilChanged, distinctUntilKeyChanged, filter, fromEvent, @@ -40,7 +39,6 @@ import { map, merge, of, - shareReplay, startWith, switchMap, throttleTime, @@ -77,16 +75,24 @@ export function useDisplayName(vm: MediaViewModel): string { } export function observeTrackReference( - participant: Participant, + participant: Observable, source: Track.Source, -): Observable { - return observeParticipantMedia(participant).pipe( - map(() => ({ - participant, - publication: participant.getTrackPublication(source), - source, - })), - distinctUntilKeyChanged("publication"), +): Observable { + return participant.pipe( + switchMap((p) => { + if (p) { + return observeParticipantMedia(p).pipe( + map(() => ({ + participant: p, + publication: p.getTrackPublication(source), + source, + })), + distinctUntilKeyChanged("publication"), + ); + } else { + return of(undefined); + } + }), ); } @@ -105,11 +111,11 @@ function observeRemoteTrackReceivingOkay( }; return combineLatest([ - observeTrackReference(participant, source), + observeTrackReference(of(participant), source), interval(1000).pipe(startWith(0)), ]).pipe( switchMap(async ([trackReference]) => { - const track = trackReference.publication?.track; + const track = trackReference?.publication?.track; if (!track || !(track instanceof RemoteTrack)) { return undefined; } @@ -200,14 +206,10 @@ export enum EncryptionStatus { } abstract class BaseMediaViewModel extends ViewModel { - /** - * Whether the media belongs to the local user. - */ - public readonly local = this.participant.isLocal; /** * The LiveKit video track for this media. */ - public readonly video: Observable; + public readonly video: Observable; /** * Whether there should be a warning that this media is unencrypted. */ @@ -215,6 +217,11 @@ abstract class BaseMediaViewModel extends ViewModel { public readonly encryptionStatus: Observable; + /** + * Whether this media corresponds to the local participant. + */ + public abstract readonly local: boolean; + public constructor( /** * An opaque identifier for this media. @@ -226,7 +233,12 @@ abstract class BaseMediaViewModel extends ViewModel { // TODO: Fully separate the data layer from the UI layer by keeping the // member object internal public readonly member: RoomMember | undefined, - protected readonly participant: LocalParticipant | RemoteParticipant, + // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through + // livekit. + protected readonly participant: Observable< + LocalParticipant | RemoteParticipant | undefined + >, + encryptionSystem: EncryptionSystem, audioSource: AudioSource, videoSource: VideoSource, @@ -243,69 +255,72 @@ abstract class BaseMediaViewModel extends ViewModel { [audio, this.video], (a, v) => encryptionSystem.kind !== E2eeType.NONE && - (a.publication?.isEncrypted === false || - v.publication?.isEncrypted === false), - ).pipe( - distinctUntilChanged(), - shareReplay({ bufferSize: 1, refCount: false }), - ); + (a?.publication?.isEncrypted === false || + v?.publication?.isEncrypted === false), + ).pipe(this.scope.state()); - if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) { - this.encryptionStatus = of(EncryptionStatus.Okay).pipe( - this.scope.state(), - ); - } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { - this.encryptionStatus = combineLatest([ - encryptionErrorObservable( - livekitRoom, - participant, - encryptionSystem, - "MissingKey", - ), - encryptionErrorObservable( - livekitRoom, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay(participant, audioSource), - observeRemoteTrackReceivingOkay(participant, videoSource), - ]).pipe( - map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { - if (keyMissing) return EncryptionStatus.KeyMissing; - if (keyInvalid) return EncryptionStatus.KeyInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - this.scope.state(), - ); - } else { - this.encryptionStatus = combineLatest([ - encryptionErrorObservable( - livekitRoom, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay(participant, audioSource), - observeRemoteTrackReceivingOkay(participant, videoSource), - ]).pipe( - map( - ([keyInvalid, audioOkay, videoOkay]): - | EncryptionStatus - | undefined => { - if (keyInvalid) return EncryptionStatus.PasswordInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }, - ), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - this.scope.state(), - ); - } + this.encryptionStatus = this.participant.pipe( + switchMap((participant): Observable => { + if (!participant) { + return of(EncryptionStatus.Connecting); + } else if ( + participant.isLocal || + encryptionSystem.kind === E2eeType.NONE + ) { + return of(EncryptionStatus.Okay); + } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { + return combineLatest([ + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "MissingKey", + ), + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay(participant, audioSource), + observeRemoteTrackReceivingOkay(participant, videoSource), + ]).pipe( + map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { + if (keyMissing) return EncryptionStatus.KeyMissing; + if (keyInvalid) return EncryptionStatus.KeyInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + ); + } else { + return combineLatest([ + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay(participant, audioSource), + observeRemoteTrackReceivingOkay(participant, videoSource), + ]).pipe( + map( + ([keyInvalid, audioOkay, videoOkay]): + | EncryptionStatus + | undefined => { + if (keyInvalid) return EncryptionStatus.PasswordInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }, + ), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + ); + } + }), + this.scope.state(), + ); } } @@ -324,11 +339,14 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { /** * Whether the participant is speaking. */ - public readonly speaking = observeParticipantEvents( - this.participant, - ParticipantEvent.IsSpeakingChanged, - ).pipe( - map((p) => p.isSpeaking), + public readonly speaking = this.participant.pipe( + switchMap((p) => + p + ? observeParticipantEvents(p, ParticipantEvent.IsSpeakingChanged).pipe( + map((p) => p.isSpeaking), + ) + : of(false), + ), this.scope.state(), ); @@ -350,7 +368,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { @@ -364,18 +382,25 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { livekitRoom, ); - const media = observeParticipantMedia(participant).pipe(this.scope.state()); + const media = participant.pipe( + switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), + this.scope.state(), + ); this.audioEnabled = media.pipe( - map((m) => m.microphoneTrack?.isMuted === false), + map((m) => m?.microphoneTrack?.isMuted === false), ); this.videoEnabled = media.pipe( - map((m) => m.cameraTrack?.isMuted === false), + map((m) => m?.cameraTrack?.isMuted === false), ); } public toggleFitContain(): void { this._cropVideo.next(!this._cropVideo.value); } + + public get local(): boolean { + return this instanceof LocalUserMediaViewModel; + } } /** @@ -387,7 +412,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { */ public readonly mirror = this.video.pipe( switchMap((v) => { - const track = v.publication?.track; + const track = v?.publication?.track; if (!(track instanceof LocalTrack)) return of(false); // Watch for track restarts, because they indicate a camera switch return fromEvent(track, TrackEvent.Restarted).pipe( @@ -409,7 +434,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { @@ -470,18 +495,17 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { super(id, member, participant, encryptionSystem, livekitRoom); // Sync the local volume with LiveKit - this.localVolume - .pipe(this.scope.bind()) - .subscribe((volume) => - (this.participant as RemoteParticipant).setVolume(volume), - ); + combineLatest([ + participant, + this.localVolume.pipe(this.scope.bind()), + ]).subscribe(([p, volume]) => p && p.setVolume(volume)); } public toggleLocallyMuted(): void { @@ -504,9 +528,10 @@ export class ScreenShareViewModel extends BaseMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + public readonly local: boolean, ) { super( id, diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 9b03a5ea..c0cf9c48 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -13,7 +13,7 @@ import { of } from "rxjs"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import { GridTile } from "./GridTile"; -import { withRemoteMedia } from "../utils/test"; +import { mockRtcMembership, withRemoteMedia } from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsProvider } from "../useReactions"; @@ -25,6 +25,7 @@ global.IntersectionObserver = class MockIntersectionObserver { test("GridTile is accessible", async () => { await withRemoteMedia( + mockRtcMembership("@alice:example.org", "AAAA"), { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 27695b65..15f7c295 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -175,6 +175,7 @@ const UserMediaTile = forwardRef( raisedHandTime={handRaised} currentReaction={currentReaction} raisedHandOnClick={raisedHandOnClick} + localParticipant={vm.local} {...props} /> ); diff --git a/src/tile/MediaView.module.css b/src/tile/MediaView.module.css index 47446118..70d6fead 100644 --- a/src/tile/MediaView.module.css +++ b/src/tile/MediaView.module.css @@ -74,9 +74,9 @@ unconditionally select the container so we can use cqmin units */ calc(var(--media-view-border-radius) - var(--cpd-space-3x)) ); display: grid; - grid-template-columns: 1fr auto; + grid-template-columns: 30px 1fr 30px; grid-template-rows: 1fr auto; - grid-template-areas: "status status" "nameTag button"; + grid-template-areas: "reactions status ." "nameTag nameTag button"; gap: var(--cpd-space-1x); place-items: start; } @@ -101,8 +101,8 @@ unconditionally select the container so we can use cqmin units */ grid-area: status; justify-self: center; align-self: start; - padding: var(--cpd-space-1x); - padding-block: var(--cpd-space-1x); + padding: var(--cpd-space-2x); + padding-block: var(--cpd-space-2x); color: var(--cpd-color-text-primary); background-color: var(--cpd-color-bg-canvas-default); display: flex; @@ -116,6 +116,12 @@ unconditionally select the container so we can use cqmin units */ text-align: center; } +.reactions { + grid-area: reactions; + display: flex; + gap: var(--cpd-space-1x); +} + .nameTag > svg, .nameTag > span { flex-shrink: 0; diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 238ffdd1..fea4303f 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { describe, expect, test } from "vitest"; +import { describe, expect, it, test } from "vitest"; import { render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import { TooltipProvider } from "@vector-im/compound-web"; @@ -42,6 +42,7 @@ describe("MediaView", () => { unencryptedWarning: false, video: trackReference, member: undefined, + localParticipant: false, }; test("is accessible", async () => { @@ -59,6 +60,25 @@ describe("MediaView", () => { }); }); + describe("with no participant", () => { + it("shows avatar for local user", () => { + render( + , + ); + expect(screen.getByRole("img", { name: "some name" })).toBeVisible(); + expect(screen.queryAllByText("video_tile.waiting_for_media").length).toBe( + 0, + ); + }); + it("shows avatar and label for remote user", () => { + render( + , + ); + expect(screen.getByRole("img", { name: "some name" })).toBeVisible(); + expect(screen.getByText("video_tile.waiting_for_media")).toBeVisible(); + }); + }); + describe("name tag", () => { test("is shown with name", () => { render(); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 4e69bf41..48871abd 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -28,7 +28,7 @@ interface Props extends ComponentProps { style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; - video: TrackReferenceOrPlaceholder; + video: TrackReferenceOrPlaceholder | undefined; videoFit: "cover" | "contain"; mirror: boolean; member: RoomMember | undefined; @@ -41,6 +41,7 @@ interface Props extends ComponentProps { raisedHandTime?: Date; currentReaction?: ReactionOption; raisedHandOnClick?: () => void; + localParticipant: boolean; } export const MediaView = forwardRef( @@ -63,6 +64,7 @@ export const MediaView = forwardRef( raisedHandTime, currentReaction, raisedHandOnClick, + localParticipant, ...props }, ref, @@ -90,21 +92,21 @@ export const MediaView = forwardRef( size={avatarSize} src={member?.getMxcAvatarUrl()} className={styles.avatar} - style={{ display: videoEnabled ? "none" : "initial" }} + style={{ display: video && videoEnabled ? "none" : "initial" }} /> - {video.publication !== undefined && ( + {video?.publication !== undefined && ( )}
-
+
( /> )}
+ {!video && !localParticipant && ( +
+ {t("video_tile.waiting_for_media")} +
+ )} {/* TODO: Bring this back once encryption status is less broken */} {/*encryptionStatus !== EncryptionStatus.Okay && (
diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index cedeea62..29b574a2 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -12,7 +12,11 @@ import userEvent from "@testing-library/user-event"; import { of } from "rxjs"; import { SpotlightTile } from "./SpotlightTile"; -import { withLocalMedia, withRemoteMedia } from "../utils/test"; +import { + mockRtcMembership, + withLocalMedia, + withRemoteMedia, +} from "../utils/test"; import { SpotlightTileViewModel } from "../state/TileViewModel"; global.IntersectionObserver = class MockIntersectionObserver { @@ -22,6 +26,7 @@ global.IntersectionObserver = class MockIntersectionObserver { test("SpotlightTile is accessible", async () => { await withRemoteMedia( + mockRtcMembership("@alice:example.org", "AAAA"), { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", @@ -29,6 +34,7 @@ test("SpotlightTile is accessible", async () => { {}, async (vm1) => { await withLocalMedia( + mockRtcMembership("@bob:example.org", "BBBB"), { rawDisplayName: "Bob", getMxcAvatarUrl: () => "mxc://dlskf", diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 1c85df92..dce30d5f 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -49,12 +49,13 @@ interface SpotlightItemBaseProps { "data-id": string; targetWidth: number; targetHeight: number; - video: TrackReferenceOrPlaceholder; + video: TrackReferenceOrPlaceholder | undefined; member: RoomMember | undefined; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; displayName: string; "aria-hidden"?: boolean; + localParticipant: boolean; } interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps { @@ -163,6 +164,7 @@ const SpotlightItem = forwardRef( displayName, encryptionStatus, "aria-hidden": ariaHidden, + localParticipant: vm.local, }; return vm instanceof ScreenShareViewModel ? ( @@ -210,7 +212,9 @@ export const SpotlightTile = forwardRef( const ref = useMergedRefs(ourRef, theirRef); const maximised = useObservableEagerState(vm.maximised); const media = useObservableEagerState(vm.media); - const [visibleId, setVisibleId] = useState(media[0].id); + const [visibleId, setVisibleId] = useState( + media[0]?.id, + ); const latestMedia = useLatest(media); const latestVisibleId = useLatest(visibleId); const visibleIndex = media.findIndex((vm) => vm.id === visibleId); diff --git a/src/utils/test.ts b/src/utils/test.ts index 99a9264b..dca98825 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -7,7 +7,20 @@ Please see LICENSE in the repository root for full details. import { map, Observable, of, SchedulerLike } from "rxjs"; import { RunHelpers, TestScheduler } from "rxjs/testing"; import { expect, vi } from "vitest"; -import { RoomMember, Room as MatrixRoom } from "matrix-js-sdk/src/matrix"; +import { + RoomMember, + Room as MatrixRoom, + MatrixEvent, + Room, + TypedEventEmitter, +} from "matrix-js-sdk/src/matrix"; +import { + CallMembership, + Focus, + MatrixRTCSessionEvent, + MatrixRTCSessionEventHandlerMap, + SessionMembershipData, +} from "matrix-js-sdk/src/matrixrtc"; import { LocalParticipant, LocalTrackPublication, @@ -100,11 +113,40 @@ function mockEmitter(): EmitterMock { }; } +export function mockRtcMembership( + user: string | RoomMember, + deviceId: string, + callId = "", + fociPreferred: Focus[] = [], + focusActive: Focus = { type: "oldest_membership" }, + membership: Partial = {}, +): CallMembership { + const data: SessionMembershipData = { + application: "m.call", + call_id: callId, + device_id: deviceId, + foci_preferred: fociPreferred, + focus_active: focusActive, + ...membership, + }; + const event = new MatrixEvent({ + sender: typeof user === "string" ? user : user.userId, + }); + return new CallMembership(event, data); +} + // Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are // rather simple, but if one util to mock a member is good enough for us, maybe // it's useful for matrix-js-sdk consumers in general. -export function mockMatrixRoomMember(member: Partial): RoomMember { - return { ...mockEmitter(), ...member } as RoomMember; +export function mockMatrixRoomMember( + rtcMembership: CallMembership, + member: Partial = {}, +): RoomMember { + return { + ...mockEmitter(), + userId: rtcMembership.sender, + ...member, + } as RoomMember; } export function mockMatrixRoom(room: Partial): MatrixRoom { @@ -174,14 +216,15 @@ export function mockLocalParticipant( } export async function withLocalMedia( - member: Partial, + localRtcMember: CallMembership, + roomMember: Partial, continuation: (vm: LocalUserMediaViewModel) => void | Promise, ): Promise { const localParticipant = mockLocalParticipant({}); const vm = new LocalUserMediaViewModel( "local", - mockMatrixRoomMember(member), - localParticipant, + mockMatrixRoomMember(localRtcMember, roomMember), + of(localParticipant), { kind: E2eeType.PER_PARTICIPANT, }, @@ -208,15 +251,16 @@ export function mockRemoteParticipant( } export async function withRemoteMedia( - member: Partial, + localRtcMember: CallMembership, + roomMember: Partial, participant: Partial, continuation: (vm: RemoteUserMediaViewModel) => void | Promise, ): Promise { const remoteParticipant = mockRemoteParticipant(participant); const vm = new RemoteUserMediaViewModel( "remote", - mockMatrixRoomMember(member), - remoteParticipant, + mockMatrixRoomMember(localRtcMember, roomMember), + of(remoteParticipant), { kind: E2eeType.PER_PARTICIPANT, }, @@ -244,3 +288,30 @@ export function mockMediaPlay(): string[] { }; return audioIsPlaying; } + +export class MockRTCSession extends TypedEventEmitter< + MatrixRTCSessionEvent, + MatrixRTCSessionEventHandlerMap +> { + public constructor( + public readonly room: Room, + private localMembership: CallMembership, + public memberships: CallMembership[] = [], + ) { + super(); + } + + public withMemberships( + rtcMembers: Observable[]>, + ): MockRTCSession { + rtcMembers.subscribe((m) => { + const old = this.memberships; + // always prepend the local participant + const updated = [this.localMembership, ...(m as CallMembership[])]; + this.memberships = updated; + this.emit(MatrixRTCSessionEvent.MembershipsChanged, old, updated); + }); + + return this; + } +}