diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index c85cddf5..8d4233a1 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -42,7 +42,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: ${{ inputs.docker_tags}} @@ -51,7 +51,7 @@ jobs: uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - name: Build and push Docker image - uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 + uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0 with: context: . platforms: linux/amd64,linux/arm64 diff --git a/README.md b/README.md index a0af77fc..ffd73d5e 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,9 @@ work: experimental_features: # MSC3266: Room summary API. Used for knocking over federation msc3266_enabled: true + # MSC4222 needed for syncv2 state_after. This allow clients to + # correctly track the state of the room. + msc4222_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. @@ -109,6 +112,10 @@ summary contains the room join rules. We need that to decide if the user gets prompted with the option to knock ("Request to join call"), a cannot join error or the join view. +MSC4222 allow clients to opt-in to a change of the sync v2 API that allows them +to correctly track the state of the room. This is required by Element Call to +track room state reliably. + Element Call requires a Livekit SFU alongside a [Livekit JWT service](https://github.com/element-hq/lk-jwt-service) to work. The url to the Livekit JWT service can either be configured in the config of Element Call @@ -213,7 +220,7 @@ To add a new translation key you can do these steps: 1. Add the new key entry to the code where the new key is used: `t("some_new_key")` 1. Run `yarn i18n` to extract the new key and update the translation files. This - will add a skeleton entry to the `locales/en-GB/app.json` file: + will add a skeleton entry to the `locales/en/app.json` file: ```jsonc { ... @@ -221,7 +228,7 @@ To add a new translation key you can do these steps: ... } ``` -1. Update the skeleton entry in the `locales/en-GB/app.json` file with +1. Update the skeleton entry in the `locales/en/app.json` file with the English translation: ```jsonc diff --git a/backend/dev_homeserver.yaml b/backend/dev_homeserver.yaml index b41de45b..5697c32e 100644 --- a/backend/dev_homeserver.yaml +++ b/backend/dev_homeserver.yaml @@ -25,6 +25,9 @@ trusted_key_servers: experimental_features: # MSC3266: Room summary API. Used for knocking over federation msc3266_enabled: true + # MSC4222 needed for syncv2 state_after. This allow clients to + # correctly track the state of the room. + msc4222_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. Must be a positive value if set. Defaults to no diff --git a/i18next-parser.config.ts b/i18next-parser.config.ts index 7d71d727..3acf2b5e 100644 --- a/i18next-parser.config.ts +++ b/i18next-parser.config.ts @@ -21,7 +21,7 @@ export default { }, ], }, - locales: ["en-GB"], + locales: ["en"], output: "locales/$LOCALE/$NAMESPACE.json", input: ["src/**/*.{ts,tsx}"], sort: true, diff --git a/localazy.json b/localazy.json index 2b9f713c..823e4a3e 100644 --- a/localazy.json +++ b/localazy.json @@ -7,13 +7,13 @@ "features": ["plural_postfix_us", "filter_untranslated"], "files": [ { - "pattern": "locales/en-GB/*.json", + "pattern": "locales/en/*.json", "lang": "inherited" }, { "group": "existing", "pattern": "locales/*/*.json", - "excludes": ["locales/en-GB/*.json"], + "excludes": ["locales/en/*.json"], "lang": "${autodetectLang}" } ] @@ -25,9 +25,6 @@ "output": "locales/${langLsrDash}/${file}" } ], - "includeSourceLang": "${includeSourceLang|false}", - "langAliases": { - "en": "en_GB" - } + "includeSourceLang": "${includeSourceLang|false}" } } diff --git a/locales/en-GB/app.json b/locales/en/app.json similarity index 99% rename from locales/en-GB/app.json rename to locales/en/app.json index bc37dd7b..e500f66c 100644 --- a/locales/en-GB/app.json +++ b/locales/en/app.json @@ -194,6 +194,7 @@ "expand": "Expand", "mute_for_me": "Mute for me", "muted_for_me": "Muted for me", - "volume": "Volume" + "volume": "Volume", + "waiting_for_media": "Waiting for media..." } } diff --git a/package.json b/package.json index ebaa59ff..71e3f9c9 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@types/grecaptcha": "^3.0.9", "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", - "@types/node": "^20.0.0", + "@types/node": "^22.0.0", "@types/pako": "^2.0.3", "@types/qrcode": "^1.5.5", "@types/react-dom": "^18.3.0", @@ -63,7 +63,7 @@ "@typescript-eslint/eslint-plugin": "^8.0.0", "@typescript-eslint/parser": "^8.0.0", "@use-gesture/react": "^10.2.11", - "@vector-im/compound-design-tokens": "^1.9.1", + "@vector-im/compound-design-tokens": "^2.0.0", "@vector-im/compound-web": "^7.2.0", "@vitejs/plugin-basic-ssl": "^1.0.1", "@vitejs/plugin-react": "^4.0.1", diff --git a/src/@types/i18next.d.ts b/src/@types/i18next.d.ts index 4a8830da..3c65e620 100644 --- a/src/@types/i18next.d.ts +++ b/src/@types/i18next.d.ts @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import "i18next"; // import all namespaces (for the default language, only) -import app from "../../locales/en-GB/app.json"; +import app from "../../locales/en/app.json"; declare module "i18next" { interface CustomTypeOptions { diff --git a/src/TranslatedError.ts b/src/TranslatedError.ts index 0dbe675a..420556be 100644 --- a/src/TranslatedError.ts +++ b/src/TranslatedError.ts @@ -22,7 +22,7 @@ export abstract class TranslatedError extends Error { messageKey: ParseKeys, translationFn: TFunction, ) { - super(translationFn(messageKey, { lng: "en-GB" } as TOptions)); + super(translationFn(messageKey, { lng: "en" } as TOptions)); this.translatedMessage = translationFn(messageKey); } } diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index ed4d5bce..3947ba66 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -51,7 +51,7 @@ export interface ConfigOptions { // a livekit service url in the client well-known. // The well known needs to be formatted like so: // {"type":"livekit", "livekit_service_url":"https://livekit.example.com"} - // and stored under the key: "livekit_focus" + // and stored under the key: "org.matrix.msc4143.rtc_foci" livekit_service_url: string; }; diff --git a/src/index.css b/src/index.css index bf6d1605..aeeccaf4 100644 --- a/src/index.css +++ b/src/index.css @@ -47,6 +47,11 @@ layer(compound); --background-gradient: url("graphics/backgroundGradient.svg"); } +:root, +[class*="cpd-theme-"] { + --video-tile-background: var(--cpd-color-bg-subtle-secondary); +} + .cpd-theme-dark { --cpd-color-border-accent: var(--cpd-color-green-1100); --stopgap-color-on-solid-accent: var(--cpd-color-text-primary); diff --git a/src/initializer.tsx b/src/initializer.tsx index 47634078..e9290504 100644 --- a/src/initializer.tsx +++ b/src/initializer.tsx @@ -24,7 +24,7 @@ import { platform } from "./Platform"; // This generates a map of locale names to their URL (based on import.meta.url), which looks like this: // { -// "../locales/en-GB/app.json": "/whatever/assets/root/locales/en-aabbcc.json", +// "../locales/en/app.json": "/whatever/assets/root/locales/en-aabbcc.json", // ... // } const locales = import.meta.glob("../locales/*/*.json", { @@ -41,7 +41,7 @@ const getLocaleUrl = ( const supportedLngs = [ ...new Set( Object.keys(locales).map((url) => { - // The URLs are of the form ../locales/en-GB/app.json + // The URLs are of the form ../locales/en/app.json // This extracts the language code from the URL const lang = url.match(/\/([^/]+)\/[^/]+\.json$/)?.[1]; if (!lang) { @@ -133,7 +133,7 @@ export class Initializer { .use(languageDetector) .use(initReactI18next) .init({ - fallbackLng: "en-GB", + fallbackLng: "en", defaultNS: "app", keySeparator: ".", nsSeparator: false, diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 46f517a8..4a823a59 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -8,23 +8,30 @@ Please see LICENSE in the repository root for full details. import { render } from "@testing-library/react"; import { afterAll, - afterEach, beforeEach, expect, - Mock, MockedFunction, test, vitest, } from "vitest"; -import { ConnectionState, RemoteParticipant, Room } from "livekit-client"; -import { of } from "rxjs"; +import { MatrixClient } from "matrix-js-sdk/src/client"; +import { ConnectionState } from "livekit-client"; +import { BehaviorSubject, of } from "rxjs"; +import { afterEach } from "node:test"; import { act, ReactNode } from "react"; +import { + CallMembership, + type MatrixRTCSession, +} from "matrix-js-sdk/src/matrixrtc"; import { - EmittableMockLivekitRoom, + mockLivekitRoom, mockLocalParticipant, + mockMatrixRoom, mockMatrixRoomMember, mockRemoteParticipant, + mockRtcMembership, + MockRTCSession, } from "../utils/test"; import { E2eeType } from "../e2ee/e2eeType"; import { CallViewModel } from "../state/CallViewModel"; @@ -33,19 +40,17 @@ import { MAX_PARTICIPANT_COUNT_FOR_SOUND, } from "./CallEventAudioRenderer"; import { prefetchSounds, useAudioContext } from "../useAudioContext"; -import { - MockRoom, - MockRTCSession, - TestReactionsWrapper, -} from "../utils/testReactions"; +import { TestReactionsWrapper } from "../utils/testReactions"; +import { RoomMember } from "matrix-js-sdk/src/matrix"; -const alice = mockMatrixRoomMember({ userId: "@alice:example.org" }); -const bob = mockMatrixRoomMember({ userId: "@bob:example.org" }); -const aliceId = `${alice.userId}:AAAA`; -const bobId = `${bob.userId}:BBBB`; +const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); +const local = mockMatrixRoomMember(localRtcMember); +const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); +const alice = mockMatrixRoomMember(aliceRtcMember); +const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); const localParticipant = mockLocalParticipant({ identity: "" }); +const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`; const aliceParticipant = mockRemoteParticipant({ identity: aliceId }); -const bobParticipant = mockRemoteParticipant({ identity: bobId }); vitest.mock("../useAudioContext"); @@ -57,7 +62,7 @@ afterAll(() => { vitest.restoreAllMocks(); }); -let playSound: Mock< +let playSound: MockedFunction< NonNullable>["playSound"] >; @@ -72,19 +77,63 @@ beforeEach(() => { }); function TestComponent({ - room, + rtcSession, vm, }: { - room: MockRoom; + rtcSession: MockRTCSession; vm: CallViewModel; }): ReactNode { return ( - + ); } +function getMockEnv( + members: RoomMember[], + initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember], +): { + vm: CallViewModel; + session: MockRTCSession; + remoteRtcMemberships: BehaviorSubject; +} { + const matrixRoomMembers = new Map(members.map((p) => [p.userId, p])); + const remoteParticipants = of([aliceParticipant]); + const liveKitRoom = mockLivekitRoom( + { localParticipant }, + { remoteParticipants }, + ); + const matrixRoom = mockMatrixRoom({ + client: { + getUserId: () => localRtcMember.sender, + getDeviceId: () => localRtcMember.deviceId, + on: vitest.fn(), + off: vitest.fn(), + } as Partial as MatrixClient, + getMember: (userId) => matrixRoomMembers.get(userId) ?? null, + }); + + const remoteRtcMemberships = new BehaviorSubject( + initialRemoteRtcMemberships, + ); + + const session = new MockRTCSession( + matrixRoom, + localRtcMember, + ).withMemberships(remoteRtcMemberships); + + const vm = new CallViewModel( + session as unknown as MatrixRTCSession, + liveKitRoom, + { + kind: E2eeType.PER_PARTICIPANT, + }, + of(ConnectionState.Connected), + ); + return { vm, session, remoteRtcMemberships }; +} + /** * We don't want to play a sound when loading the call state * because typically this occurs in two stages. We first join @@ -93,118 +142,56 @@ function TestComponent({ * a noise every time. */ test("plays one sound when entering a call", () => { - const liveKitRoom = new EmittableMockLivekitRoom({ - localParticipant, - remoteParticipants: new Map(), - }); - - const room = new MockRoom(alice.userId); - const vm = new CallViewModel( - room.testGetAsMatrixRoom(), - liveKitRoom.getAsLivekitRoom(), - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), - ); - + const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]); + render(); // Joining a call usually means remote participants are added later. act(() => { - liveKitRoom.addParticipant(bobParticipant); + remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]); }); - - render(); - expect(playSound).toBeCalled(); + expect(playSound).toHaveBeenCalledOnce(); }); +// TODO: Same test? test("plays a sound when a user joins", () => { - const remoteParticipants = new Map( - [aliceParticipant].map((p) => [p.identity, p]), - ); - const liveKitRoom = new EmittableMockLivekitRoom({ - localParticipant, - remoteParticipants, - }); - - const room = new MockRoom(alice.userId); - const vm = new CallViewModel( - room.testGetAsMatrixRoom(), - liveKitRoom as unknown as Room, - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), - ); - render(); + const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]); + render(); act(() => { - liveKitRoom.addParticipant(bobParticipant); + remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]); }); // Play a sound when joining a call. expect(playSound).toBeCalledWith("join"); }); test("plays a sound when a user leaves", () => { - const remoteParticipants = new Map( - [aliceParticipant].map((p) => [p.identity, p]), - ); - const liveKitRoom = new EmittableMockLivekitRoom({ - localParticipant, - remoteParticipants, - }); - const room = new MockRoom(alice.userId); - - const vm = new CallViewModel( - room.testGetAsMatrixRoom(), - liveKitRoom.getAsLivekitRoom(), - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), - ); - render(); + const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]); + render(); act(() => { - liveKitRoom.removeParticipant(aliceParticipant); + remoteRtcMemberships.next([]); }); expect(playSound).toBeCalledWith("left"); }); test("plays no sound when the participant list is more than the maximum size", () => { - expect(playSound).not.toBeCalled(); - const remoteParticipants = new Map([ - [aliceParticipant.identity, aliceParticipant], - // You + other participants to hit the max. - ...Array.from({ length: MAX_PARTICIPANT_COUNT_FOR_SOUND - 1 }).map< - [string, RemoteParticipant] - >((_, index) => { - const p = mockRemoteParticipant({ - identity: `@user${index}:example.com:DEV${index}`, - }); - return [p.identity, p]; - }), - ]); + const mockRtcMemberships: CallMembership[] = []; + for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) { + mockRtcMemberships.push( + mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`), + ); + } - // Preload the call with the maximum members, assume that - // we're already in the call by this point rather than - // joining. - const liveKitRoom = new EmittableMockLivekitRoom({ - localParticipant, - remoteParticipants, - }); - const room = new MockRoom(alice.userId); - const vm = new CallViewModel( - room.testGetAsMatrixRoom(), - liveKitRoom.getAsLivekitRoom(), - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), + const { session, vm, remoteRtcMemberships } = getMockEnv( + [local, alice], + mockRtcMemberships, ); - render(); - // When the count drops, play a leave sound. + + render(); + expect(playSound).not.toBeCalled(); act(() => { - liveKitRoom.removeParticipant(aliceParticipant); + remoteRtcMemberships.next( + mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1), + ); }); expect(playSound).toBeCalledWith("left"); }); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 2586182f..8710d3e8 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -120,7 +120,7 @@ export const ActiveCall: FC = (props) => { useEffect(() => { if (livekitRoom !== undefined) { const vm = new CallViewModel( - props.rtcSession.room, + props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable, @@ -128,12 +128,7 @@ export const ActiveCall: FC = (props) => { setVm(vm); return (): void => vm.destroy(); } - }, [ - props.rtcSession.room, - livekitRoom, - props.e2eeSystem, - connStateObservable, - ]); + }, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable]); if (livekitRoom === undefined || vm === null) return null; diff --git a/src/room/VideoPreview.module.css b/src/room/VideoPreview.module.css index 573807fe..89422af7 100644 --- a/src/room/VideoPreview.module.css +++ b/src/room/VideoPreview.module.css @@ -18,8 +18,7 @@ Please see LICENSE in the repository root for full details. width: 100%; height: 100%; object-fit: cover; - background-color: black; - background-color: var(--cpd-color-bg-subtle-primary); + background-color: var(--video-tile-background); } video.mirror { @@ -35,7 +34,7 @@ video.mirror { display: flex; justify-content: center; align-items: center; - background-color: var(--cpd-color-bg-subtle-secondary); + background-color: var(--video-tile-background); } .buttonBar { diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index 9b2e5ee7..5dbfb1ca 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { test, vi, onTestFinished } from "vitest"; +import { test, vi, onTestFinished, it } from "vitest"; import { combineLatest, debounceTime, @@ -25,6 +25,7 @@ import { } from "livekit-client"; import * as ComponentsCore from "@livekit/components-core"; import { isEqual } from "lodash-es"; +import { CallMembership, MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc"; import { CallViewModel, Layout } from "./CallViewModel"; import { @@ -34,6 +35,8 @@ import { mockMatrixRoomMember, mockRemoteParticipant, withTestScheduler, + mockRtcMembership, + MockRTCSession, } from "../utils/test"; import { ECAddonConnectionState, @@ -43,14 +46,19 @@ import { E2eeType } from "../e2ee/e2eeType"; vi.mock("@livekit/components-core"); -const alice = mockMatrixRoomMember({ userId: "@alice:example.org" }); -const bob = mockMatrixRoomMember({ userId: "@bob:example.org" }); -const carol = mockMatrixRoomMember({ userId: "@carol:example.org" }); -const dave = mockMatrixRoomMember({ userId: "@dave:example.org" }); +const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); +const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); +const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); +const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); -const aliceId = `${alice.userId}:AAAA`; -const bobId = `${bob.userId}:BBBB`; -const daveId = `${dave.userId}:DDDD`; +const alice = mockMatrixRoomMember(aliceRtcMember); +const bob = mockMatrixRoomMember(bobRtcMember); +const carol = mockMatrixRoomMember(localRtcMember); +const dave = mockMatrixRoomMember(daveRtcMember); + +const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`; +const bobId = `${bob.userId}:${bobRtcMember.deviceId}`; +const daveId = `${dave.userId}:${daveRtcMember.deviceId}`; const localParticipant = mockLocalParticipant({ identity: "" }); const aliceParticipant = mockRemoteParticipant({ identity: aliceId }); @@ -65,7 +73,9 @@ const bobSharingScreen = mockRemoteParticipant({ }); const daveParticipant = mockRemoteParticipant({ identity: daveId }); -const members = new Map([alice, bob, carol, dave].map((p) => [p.userId, p])); +const roomMembers = new Map( + [alice, bob, carol, dave].map((p) => [p.userId, p]), +); export interface GridLayoutSummary { type: "grid"; @@ -173,10 +183,23 @@ function summarizeLayout(l: Observable): Observable { function withCallViewModel( remoteParticipants: Observable, + rtcMembers: Observable[]>, connectionState: Observable, speaking: Map>, continuation: (vm: CallViewModel) => void, ): void { + const room = mockMatrixRoom({ + client: { + getUserId: () => localRtcMember.sender, + getDeviceId: () => localRtcMember.deviceId, + } as Partial as MatrixClient, + getMember: (userId) => roomMembers.get(userId) ?? null, + }); + const rtcSession = new MockRTCSession( + room, + localRtcMember, + [], + ).withMemberships(rtcMembers); const participantsSpy = vi .spyOn(ComponentsCore, "connectedParticipantsObserver") .mockReturnValue(remoteParticipants); @@ -209,12 +232,7 @@ function withCallViewModel( ); const vm = new CallViewModel( - mockMatrixRoom({ - client: { - getUserId: () => "@carol:example.org", - } as Partial as MatrixClient, - getMember: (userId) => members.get(userId) ?? null, - }), + rtcSession as unknown as MatrixRTCSession, liveKitRoom, { kind: E2eeType.PER_PARTICIPANT, @@ -247,6 +265,7 @@ test("participants are retained during a focus switch", () => { a: [aliceParticipant, bobParticipant], b: [], }), + of([aliceRtcMember, bobRtcMember]), hot(connectionInputMarbles, { c: ConnectionState.Connected, s: ECAddonConnectionState.ECSwitchingFocus, @@ -288,6 +307,7 @@ test("screen sharing activates spotlight layout", () => { c: [aliceSharingScreen, bobSharingScreen], d: [aliceParticipant, bobSharingScreen], }), + of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { @@ -356,7 +376,7 @@ test("participants stay in the same order unless to appear/disappear", () => { const modeInputMarbles = " a"; // First Bob speaks, then Dave, then Alice const aSpeakingInputMarbles = "n- 1998ms - 1999ms y"; - const bSpeakingInputMarbles = "ny 1998ms n 1999ms "; + const bSpeakingInputMarbles = "ny 1998ms n 1999ms -"; const dSpeakingInputMarbles = "n- 1998ms y 1999ms n"; // Nothing should change when Bob speaks, because Bob is already on screen. // When Dave speaks he should switch with Alice because she's the one who @@ -366,6 +386,7 @@ test("participants stay in the same order unless to appear/disappear", () => { withCallViewModel( of([aliceParticipant, bobParticipant, daveParticipant]), + of([aliceRtcMember, bobRtcMember, daveRtcMember]), of(ConnectionState.Connected), new Map([ [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })], @@ -427,6 +448,7 @@ test("spotlight speakers swap places", () => { withCallViewModel( of([aliceParticipant, bobParticipant, daveParticipant]), + of([aliceRtcMember, bobRtcMember, daveRtcMember]), of(ConnectionState.Connected), new Map([ [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })], @@ -475,6 +497,7 @@ test("layout enters picture-in-picture mode when requested", () => { withCallViewModel( of([aliceParticipant, bobParticipant]), + of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { @@ -515,6 +538,7 @@ test("spotlight remembers whether it's expanded", () => { withCallViewModel( of([aliceParticipant, bobParticipant]), + of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { @@ -559,3 +583,104 @@ test("spotlight remembers whether it's expanded", () => { ); }); }); + +test("participants must have a MatrixRTCSession to be visible", () => { + withTestScheduler(({ hot, expectObservable }) => { + // iterate through a number of combinations of participants and MatrixRTC memberships + // Bob never has an MatrixRTC membership + const scenarioInputMarbles = " abcdec"; + // Bob should never be visible + const expectedLayoutMarbles = "a-bc-b"; + + withCallViewModel( + hot(scenarioInputMarbles, { + a: [], + b: [bobParticipant], + c: [aliceParticipant, bobParticipant], + d: [aliceParticipant, daveParticipant, bobParticipant], + e: [aliceParticipant, daveParticipant, bobSharingScreen], + }), + hot(scenarioInputMarbles, { + a: [], + b: [], + c: [aliceRtcMember], + d: [aliceRtcMember, daveRtcMember], + e: [aliceRtcMember, daveRtcMember], + }), + of(ConnectionState.Connected), + new Map(), + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout(vm.layout)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0"], + }, + b: { + type: "one-on-one", + local: "local:0", + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${daveId}:0`], + }, + }, + ); + }, + ); + }); +}); + +it("should show at least one tile per MatrixRTCSession", () => { + withTestScheduler(({ hot, expectObservable }) => { + // iterate through some combinations of MatrixRTC memberships + const scenarioInputMarbles = " abcd"; + // There should always be one tile for each MatrixRTCSession + const expectedLayoutMarbles = "abcd"; + + withCallViewModel( + of([]), + hot(scenarioInputMarbles, { + a: [], + b: [aliceRtcMember], + c: [aliceRtcMember, daveRtcMember], + d: [daveRtcMember], + }), + of(ConnectionState.Connected), + new Map(), + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout(vm.layout)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0"], + }, + b: { + type: "one-on-one", + local: "local:0", + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${daveId}:0`], + }, + d: { + type: "one-on-one", + local: "local:0", + remote: `${daveId}:0`, + }, + }, + ); + }, + ); + }); +}); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 73320329..af8780b1 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -18,12 +18,9 @@ import { RemoteParticipant, Track, } from "livekit-client"; +import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix"; import { - Room as MatrixRoom, - RoomMember, - RoomStateEvent, -} from "matrix-js-sdk/src/matrix"; -import { + BehaviorSubject, EMPTY, Observable, Subject, @@ -49,6 +46,10 @@ import { withLatestFrom, } from "rxjs"; import { logger } from "matrix-js-sdk/src/logger"; +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc"; import { ViewModel } from "./ViewModel"; import { @@ -222,41 +223,67 @@ interface LayoutScanState { class UserMedia { private readonly scope = new ObservableScope(); public readonly vm: UserMediaViewModel; + private readonly participant: BehaviorSubject< + LocalParticipant | RemoteParticipant | undefined + >; + public readonly speaker: Observable; public readonly presenter: Observable; - public constructor( public readonly id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: LocalParticipant | RemoteParticipant | undefined, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { - this.vm = participant.isLocal - ? new LocalUserMediaViewModel( - id, - member, - participant as LocalParticipant, - encryptionSystem, - livekitRoom, - ) - : new RemoteUserMediaViewModel( - id, - member, - participant as RemoteParticipant, - encryptionSystem, - livekitRoom, - ); + this.participant = new BehaviorSubject(participant); + + if (participant?.isLocal) { + this.vm = new LocalUserMediaViewModel( + this.id, + member, + this.participant.asObservable() as Observable, + encryptionSystem, + livekitRoom, + ); + } else { + this.vm = new RemoteUserMediaViewModel( + id, + member, + this.participant.asObservable() as Observable< + RemoteParticipant | undefined + >, + encryptionSystem, + livekitRoom, + ); + } this.speaker = observeSpeaker(this.vm.speaking).pipe(this.scope.state()); - this.presenter = observeParticipantEvents( - participant, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled)); + this.presenter = this.participant.pipe( + switchMap( + (p) => + (p && + observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled))) ?? + of(false), + ), + this.scope.state(), + ); + } + + public updateParticipant( + newParticipant: LocalParticipant | RemoteParticipant | undefined, + ): void { + if (this.participant.value !== newParticipant) { + // Update the BehaviourSubject in the UserMedia. + this.participant.next(newParticipant); + } } public destroy(): void { @@ -267,6 +294,9 @@ class UserMedia { class ScreenShare { public readonly vm: ScreenShareViewModel; + private readonly participant: BehaviorSubject< + LocalParticipant | RemoteParticipant + >; public constructor( id: string, @@ -275,12 +305,15 @@ class ScreenShare { encryptionSystem: EncryptionSystem, liveKitRoom: LivekitRoom, ) { + this.participant = new BehaviorSubject(participant); + this.vm = new ScreenShareViewModel( id, member, - participant, + this.participant.asObservable(), encryptionSystem, liveKitRoom, + participant.isLocal, ); } @@ -317,11 +350,11 @@ function findMatrixRoomMember( export class CallViewModel extends ViewModel { public readonly localVideo: Observable = observeTrackReference( - this.livekitRoom.localParticipant, + of(this.livekitRoom.localParticipant), Track.Source.Camera, ).pipe( map((trackRef) => { - const track = trackRef.publication?.track; + const track = trackRef?.publication?.track; return track instanceof LocalVideoTrack ? track : null; }), ); @@ -401,49 +434,87 @@ export class CallViewModel extends ViewModel { this.remoteParticipants, observeParticipantMedia(this.livekitRoom.localParticipant), duplicateTiles.value, - // Also react to changes in the list of members - fromEvent(this.matrixRoom, RoomStateEvent.Update).pipe(startWith(null)), + // Also react to changes in the MatrixRTC session list. + // The session list will also be update if a room membership changes. + // No additional RoomState event listener needs to be set up. + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe(startWith(null)), ]).pipe( scan( ( prevItems, - [remoteParticipants, { participant: localParticipant }, duplicateTiles], + [ + remoteParticipants, + { participant: localParticipant }, + duplicateTiles, + _membershipsChanged, + ], ) => { const newItems = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const p of [localParticipant, ...remoteParticipants]) { - const id = p === localParticipant ? "local" : p.identity; - const member = findMatrixRoomMember(this.matrixRoom, id); - if (member === undefined) - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`, - ); + // m.rtc.members are the basis for calculating what is visible in the call + for (const rtcMember of this.matrixRTCSession.memberships) { + const room = this.matrixRTCSession.room; + // WARN! This is not exactly the sender but the user defined in the state key. + // This will be available once we change to the new "member as object" format in the MatrixRTC object. + let livekitParticipantId = + rtcMember.sender + ":" + rtcMember.deviceId; - // Create as many tiles for this participant as called for by - // the duplicateTiles option + let participant: + | LocalParticipant + | RemoteParticipant + | undefined = undefined; + if ( + rtcMember.sender === room.client.getUserId()! && + rtcMember.deviceId === room.client.getDeviceId() + ) { + livekitParticipantId = "local"; + participant = localParticipant; + } else { + participant = remoteParticipants.find( + (p) => p.identity === livekitParticipantId, + ); + } + + const member = findMatrixRoomMember(room, livekitParticipantId); + if (!member) { + logger.error( + "Could not find member for media id: ", + livekitParticipantId, + ); + } for (let i = 0; i < 1 + duplicateTiles; i++) { - const userMediaId = `${id}:${i}`; + const indexedMediaId = `${livekitParticipantId}:${i}`; + const prevMedia = prevItems.get(indexedMediaId); + if (prevMedia && prevMedia instanceof UserMedia) { + prevMedia.updateParticipant(participant); + } yield [ - userMediaId, - prevItems.get(userMediaId) ?? + indexedMediaId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevMedia ?? new UserMedia( - userMediaId, + indexedMediaId, member, - p, + participant, this.encryptionSystem, this.livekitRoom, ), ]; - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; + if (participant?.isScreenShareEnabled) { + const screenShareId = `${indexedMediaId}:screen-share`; yield [ screenShareId, prevItems.get(screenShareId) ?? new ScreenShare( screenShareId, member, - p, + participant, this.encryptionSystem, this.livekitRoom, ), @@ -454,7 +525,6 @@ export class CallViewModel extends ViewModel { }.bind(this)(), ); - for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); return newItems; }, new Map(), @@ -488,11 +558,6 @@ export class CallViewModel extends ViewModel { ), ); - private readonly localUserMedia: Observable = - this.mediaItems.pipe( - map((ms) => ms.find((m) => m.vm.local)!.vm as LocalUserMediaViewModel), - ); - /** * List of MediaItems that we want to display, that are of type ScreenShare */ @@ -504,7 +569,7 @@ export class CallViewModel extends ViewModel { this.scope.state(), ); - private readonly spotlightSpeaker: Observable = + private readonly spotlightSpeaker: Observable = this.userMedia.pipe( switchMap((mediaItems) => mediaItems.length === 0 @@ -515,7 +580,7 @@ export class CallViewModel extends ViewModel { ), ), ), - scan<(readonly [UserMedia, boolean])[], UserMedia, null>( + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( (prev, mediaItems) => { // Only remote users that are still in the call should be sticky const [stickyMedia, stickySpeaking] = @@ -532,11 +597,11 @@ export class CallViewModel extends ViewModel { // Otherwise, spotlight an arbitrary remote user mediaItems.find(([m]) => !m.vm.local)?.[0] ?? // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)![0]); + mediaItems.find(([m]) => m.vm.local)?.[0]); }, null, ), - map((speaker) => speaker.vm), + map((speaker) => speaker?.vm ?? null), this.scope.state(), ); @@ -576,37 +641,57 @@ export class CallViewModel extends ViewModel { }), ); - private readonly spotlightAndPip: Observable< - [Observable, Observable] - > = this.screenShares.pipe( - map((screenShares) => - screenShares.length > 0 - ? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const) - : ([ - this.spotlightSpeaker.pipe(map((speaker) => [speaker!])), - this.spotlightSpeaker.pipe( - switchMap((speaker) => - speaker.local - ? of(null) - : this.localUserMedia.pipe( - switchMap((vm) => - vm.alwaysShow.pipe( - map((alwaysShow) => (alwaysShow ? vm : null)), - ), - ), - ), - ), - ), - ] as const), - ), - ); - private readonly spotlight: Observable = - this.spotlightAndPip.pipe( - switchMap(([spotlight]) => spotlight), + this.screenShares.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } + + return this.spotlightSpeaker.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), this.scope.state(), ); + private readonly pip: Observable = combineLatest([ + this.screenShares, + this.spotlightSpeaker, + this.mediaItems, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return this.spotlightSpeaker; + } + if (!spotlight || spotlight.local) { + return of(null); + } + + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; + + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; + + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } + + return null; + }), + ); + }), + this.scope.state(), + ); + private readonly hasRemoteScreenShares: Observable = this.spotlight.pipe( map((spotlight) => @@ -615,9 +700,6 @@ export class CallViewModel extends ViewModel { distinctUntilChanged(), ); - private readonly pip: Observable = - this.spotlightAndPip.pipe(switchMap(([, pip]) => pip)); - private readonly pipEnabled: Observable = setPipEnabled.pipe( startWith(false), ); @@ -721,15 +803,16 @@ export class CallViewModel extends ViewModel { this.mediaItems.pipe( map((mediaItems) => { if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)! - .vm as LocalUserMediaViewModel; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as | RemoteUserMediaViewModel | undefined; // There might not be a remote tile if there are screen shares, or if // only the local user is in the call and they're using the duplicate // tiles option - if (remote === undefined) return null; + if (!remote || !local) return null; return { type: "one-on-one", local, remote }; }), @@ -1010,7 +1093,7 @@ export class CallViewModel extends ViewModel { public constructor( // A call is permanently tied to a single Matrix room and LiveKit room - private readonly matrixRoom: MatrixRoom, + private readonly matrixRTCSession: MatrixRTCSession, private readonly livekitRoom: LivekitRoom, private readonly encryptionSystem: EncryptionSystem, private readonly connectionState: Observable, diff --git a/src/state/MediaViewModel.test.ts b/src/state/MediaViewModel.test.ts index 5b5e59a7..c4e0bee6 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/MediaViewModel.test.ts @@ -8,14 +8,17 @@ Please see LICENSE in the repository root for full details. import { expect, test, vi } from "vitest"; import { + mockRtcMembership, withLocalMedia, withRemoteMedia, withTestScheduler, } from "../utils/test"; +const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); + test("control a participant's volume", async () => { const setVolumeSpy = vi.fn(); - await withRemoteMedia({}, { setVolume: setVolumeSpy }, (vm) => + await withRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy }, (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab---c---d|", { a() { @@ -60,7 +63,7 @@ test("control a participant's volume", async () => { }); test("toggle fit/contain for a participant's video", async () => { - await withRemoteMedia({}, {}, (vm) => + await withRemoteMedia(rtcMembership, {}, {}, (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab|", { a: () => vm.toggleFitContain(), @@ -76,17 +79,21 @@ test("toggle fit/contain for a participant's video", async () => { }); test("local media remembers whether it should always be shown", async () => { - await withLocalMedia({}, (vm) => + await withLocalMedia(rtcMembership, {}, (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-a|", { a: () => vm.setAlwaysShow(false) }); expectObservable(vm.alwaysShow).toBe("ab", { a: true, b: false }); }), ); // Next local media should start out *not* always shown - await withLocalMedia({}, (vm) => - withTestScheduler(({ expectObservable, schedule }) => { - schedule("-a|", { a: () => vm.setAlwaysShow(true) }); - expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true }); - }), + await withLocalMedia( + rtcMembership, + + {}, + (vm) => + withTestScheduler(({ expectObservable, schedule }) => { + schedule("-a|", { a: () => vm.setAlwaysShow(true) }); + expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true }); + }), ); }); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 36e76d38..ceaca57c 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -32,7 +32,6 @@ import { Observable, Subject, combineLatest, - distinctUntilChanged, distinctUntilKeyChanged, filter, fromEvent, @@ -40,7 +39,6 @@ import { map, merge, of, - shareReplay, startWith, switchMap, throttleTime, @@ -77,16 +75,24 @@ export function useDisplayName(vm: MediaViewModel): string { } export function observeTrackReference( - participant: Participant, + participant: Observable, source: Track.Source, -): Observable { - return observeParticipantMedia(participant).pipe( - map(() => ({ - participant, - publication: participant.getTrackPublication(source), - source, - })), - distinctUntilKeyChanged("publication"), +): Observable { + return participant.pipe( + switchMap((p) => { + if (p) { + return observeParticipantMedia(p).pipe( + map(() => ({ + participant: p, + publication: p.getTrackPublication(source), + source, + })), + distinctUntilKeyChanged("publication"), + ); + } else { + return of(undefined); + } + }), ); } @@ -105,11 +111,11 @@ function observeRemoteTrackReceivingOkay( }; return combineLatest([ - observeTrackReference(participant, source), + observeTrackReference(of(participant), source), interval(1000).pipe(startWith(0)), ]).pipe( switchMap(async ([trackReference]) => { - const track = trackReference.publication?.track; + const track = trackReference?.publication?.track; if (!track || !(track instanceof RemoteTrack)) { return undefined; } @@ -200,14 +206,10 @@ export enum EncryptionStatus { } abstract class BaseMediaViewModel extends ViewModel { - /** - * Whether the media belongs to the local user. - */ - public readonly local = this.participant.isLocal; /** * The LiveKit video track for this media. */ - public readonly video: Observable; + public readonly video: Observable; /** * Whether there should be a warning that this media is unencrypted. */ @@ -215,6 +217,11 @@ abstract class BaseMediaViewModel extends ViewModel { public readonly encryptionStatus: Observable; + /** + * Whether this media corresponds to the local participant. + */ + public abstract readonly local: boolean; + public constructor( /** * An opaque identifier for this media. @@ -226,7 +233,12 @@ abstract class BaseMediaViewModel extends ViewModel { // TODO: Fully separate the data layer from the UI layer by keeping the // member object internal public readonly member: RoomMember | undefined, - protected readonly participant: LocalParticipant | RemoteParticipant, + // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through + // livekit. + protected readonly participant: Observable< + LocalParticipant | RemoteParticipant | undefined + >, + encryptionSystem: EncryptionSystem, audioSource: AudioSource, videoSource: VideoSource, @@ -243,69 +255,72 @@ abstract class BaseMediaViewModel extends ViewModel { [audio, this.video], (a, v) => encryptionSystem.kind !== E2eeType.NONE && - (a.publication?.isEncrypted === false || - v.publication?.isEncrypted === false), - ).pipe( - distinctUntilChanged(), - shareReplay({ bufferSize: 1, refCount: false }), - ); + (a?.publication?.isEncrypted === false || + v?.publication?.isEncrypted === false), + ).pipe(this.scope.state()); - if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) { - this.encryptionStatus = of(EncryptionStatus.Okay).pipe( - this.scope.state(), - ); - } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { - this.encryptionStatus = combineLatest([ - encryptionErrorObservable( - livekitRoom, - participant, - encryptionSystem, - "MissingKey", - ), - encryptionErrorObservable( - livekitRoom, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay(participant, audioSource), - observeRemoteTrackReceivingOkay(participant, videoSource), - ]).pipe( - map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { - if (keyMissing) return EncryptionStatus.KeyMissing; - if (keyInvalid) return EncryptionStatus.KeyInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - this.scope.state(), - ); - } else { - this.encryptionStatus = combineLatest([ - encryptionErrorObservable( - livekitRoom, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay(participant, audioSource), - observeRemoteTrackReceivingOkay(participant, videoSource), - ]).pipe( - map( - ([keyInvalid, audioOkay, videoOkay]): - | EncryptionStatus - | undefined => { - if (keyInvalid) return EncryptionStatus.PasswordInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }, - ), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - this.scope.state(), - ); - } + this.encryptionStatus = this.participant.pipe( + switchMap((participant): Observable => { + if (!participant) { + return of(EncryptionStatus.Connecting); + } else if ( + participant.isLocal || + encryptionSystem.kind === E2eeType.NONE + ) { + return of(EncryptionStatus.Okay); + } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { + return combineLatest([ + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "MissingKey", + ), + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay(participant, audioSource), + observeRemoteTrackReceivingOkay(participant, videoSource), + ]).pipe( + map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { + if (keyMissing) return EncryptionStatus.KeyMissing; + if (keyInvalid) return EncryptionStatus.KeyInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + ); + } else { + return combineLatest([ + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay(participant, audioSource), + observeRemoteTrackReceivingOkay(participant, videoSource), + ]).pipe( + map( + ([keyInvalid, audioOkay, videoOkay]): + | EncryptionStatus + | undefined => { + if (keyInvalid) return EncryptionStatus.PasswordInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }, + ), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + ); + } + }), + this.scope.state(), + ); } } @@ -324,11 +339,14 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { /** * Whether the participant is speaking. */ - public readonly speaking = observeParticipantEvents( - this.participant, - ParticipantEvent.IsSpeakingChanged, - ).pipe( - map((p) => p.isSpeaking), + public readonly speaking = this.participant.pipe( + switchMap((p) => + p + ? observeParticipantEvents(p, ParticipantEvent.IsSpeakingChanged).pipe( + map((p) => p.isSpeaking), + ) + : of(false), + ), this.scope.state(), ); @@ -350,7 +368,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { @@ -364,18 +382,25 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { livekitRoom, ); - const media = observeParticipantMedia(participant).pipe(this.scope.state()); + const media = participant.pipe( + switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), + this.scope.state(), + ); this.audioEnabled = media.pipe( - map((m) => m.microphoneTrack?.isMuted === false), + map((m) => m?.microphoneTrack?.isMuted === false), ); this.videoEnabled = media.pipe( - map((m) => m.cameraTrack?.isMuted === false), + map((m) => m?.cameraTrack?.isMuted === false), ); } public toggleFitContain(): void { this._cropVideo.next(!this._cropVideo.value); } + + public get local(): boolean { + return this instanceof LocalUserMediaViewModel; + } } /** @@ -387,7 +412,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { */ public readonly mirror = this.video.pipe( switchMap((v) => { - const track = v.publication?.track; + const track = v?.publication?.track; if (!(track instanceof LocalTrack)) return of(false); // Watch for track restarts, because they indicate a camera switch return fromEvent(track, TrackEvent.Restarted).pipe( @@ -409,7 +434,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { @@ -470,18 +495,17 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { super(id, member, participant, encryptionSystem, livekitRoom); // Sync the local volume with LiveKit - this.localVolume - .pipe(this.scope.bind()) - .subscribe((volume) => - (this.participant as RemoteParticipant).setVolume(volume), - ); + combineLatest([ + participant, + this.localVolume.pipe(this.scope.bind()), + ]).subscribe(([p, volume]) => p && p.setVolume(volume)); } public toggleLocallyMuted(): void { @@ -504,9 +528,10 @@ export class ScreenShareViewModel extends BaseMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + public readonly local: boolean, ) { super( id, diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 9b03a5ea..c0cf9c48 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -13,7 +13,7 @@ import { of } from "rxjs"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import { GridTile } from "./GridTile"; -import { withRemoteMedia } from "../utils/test"; +import { mockRtcMembership, withRemoteMedia } from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsProvider } from "../useReactions"; @@ -25,6 +25,7 @@ global.IntersectionObserver = class MockIntersectionObserver { test("GridTile is accessible", async () => { await withRemoteMedia( + mockRtcMembership("@alice:example.org", "AAAA"), { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 27695b65..15f7c295 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -175,6 +175,7 @@ const UserMediaTile = forwardRef( raisedHandTime={handRaised} currentReaction={currentReaction} raisedHandOnClick={raisedHandOnClick} + localParticipant={vm.local} {...props} /> ); diff --git a/src/tile/MediaView.module.css b/src/tile/MediaView.module.css index 4594c284..70d6fead 100644 --- a/src/tile/MediaView.module.css +++ b/src/tile/MediaView.module.css @@ -15,7 +15,7 @@ Please see LICENSE in the repository root for full details. inline-size: 100%; block-size: 100%; object-fit: contain; - background-color: var(--cpd-color-bg-subtle-primary); + background-color: var(--video-tile-background); /* This transform is a no-op, but it forces Firefox to use a different rendering path, one that actually clips the corners of