diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index c85cddf5..8d4233a1 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -42,7 +42,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: ${{ inputs.docker_tags}} @@ -51,7 +51,7 @@ jobs: uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - name: Build and push Docker image - uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 + uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0 with: context: . platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/element-call.yaml b/.github/workflows/element-call.yaml index 7924140d..a424fb74 100644 --- a/.github/workflows/element-call.yaml +++ b/.github/workflows/element-call.yaml @@ -28,7 +28,7 @@ jobs: uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 with: cache: "yarn" - node-version: "lts/*" + node-version-file: ".node-version" - name: Install dependencies run: "yarn install" - name: Build diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 69493ff6..d9367626 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -12,7 +12,7 @@ jobs: uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 with: cache: "yarn" - node-version: "lts/*" + node-version-file: ".node-version" - name: Install dependencies run: "yarn install" - name: Prettier diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b63eb283..a1c7f232 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,7 +14,7 @@ jobs: uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 with: cache: "yarn" - node-version: "lts/*" + node-version-file: ".node-version" - name: Install dependencies run: "yarn install" - name: Vitest diff --git a/.github/workflows/translations-download.yaml b/.github/workflows/translations-download.yaml index 7359f781..30ce6ff9 100644 --- a/.github/workflows/translations-download.yaml +++ b/.github/workflows/translations-download.yaml @@ -18,7 +18,7 @@ jobs: - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 with: cache: "yarn" - node-version: "lts/*" + node-version-file: ".node-version" - name: Install Deps run: "yarn install --frozen-lockfile" diff --git a/.node-version b/.node-version new file mode 100644 index 00000000..2bd5a0a9 --- /dev/null +++ b/.node-version @@ -0,0 +1 @@ +22 diff --git a/README.md b/README.md index a0af77fc..ffd73d5e 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,9 @@ work: experimental_features: # MSC3266: Room summary API. Used for knocking over federation msc3266_enabled: true + # MSC4222 needed for syncv2 state_after. This allow clients to + # correctly track the state of the room. + msc4222_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. @@ -109,6 +112,10 @@ summary contains the room join rules. We need that to decide if the user gets prompted with the option to knock ("Request to join call"), a cannot join error or the join view. +MSC4222 allow clients to opt-in to a change of the sync v2 API that allows them +to correctly track the state of the room. This is required by Element Call to +track room state reliably. + Element Call requires a Livekit SFU alongside a [Livekit JWT service](https://github.com/element-hq/lk-jwt-service) to work. The url to the Livekit JWT service can either be configured in the config of Element Call @@ -213,7 +220,7 @@ To add a new translation key you can do these steps: 1. Add the new key entry to the code where the new key is used: `t("some_new_key")` 1. Run `yarn i18n` to extract the new key and update the translation files. This - will add a skeleton entry to the `locales/en-GB/app.json` file: + will add a skeleton entry to the `locales/en/app.json` file: ```jsonc { ... @@ -221,7 +228,7 @@ To add a new translation key you can do these steps: ... } ``` -1. Update the skeleton entry in the `locales/en-GB/app.json` file with +1. Update the skeleton entry in the `locales/en/app.json` file with the English translation: ```jsonc diff --git a/backend/dev_homeserver.yaml b/backend/dev_homeserver.yaml index b41de45b..5697c32e 100644 --- a/backend/dev_homeserver.yaml +++ b/backend/dev_homeserver.yaml @@ -25,6 +25,9 @@ trusted_key_servers: experimental_features: # MSC3266: Room summary API. Used for knocking over federation msc3266_enabled: true + # MSC4222 needed for syncv2 state_after. This allow clients to + # correctly track the state of the room. + msc4222_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. Must be a positive value if set. Defaults to no diff --git a/i18next-parser.config.ts b/i18next-parser.config.ts index 7d71d727..3acf2b5e 100644 --- a/i18next-parser.config.ts +++ b/i18next-parser.config.ts @@ -21,7 +21,7 @@ export default { }, ], }, - locales: ["en-GB"], + locales: ["en"], output: "locales/$LOCALE/$NAMESPACE.json", input: ["src/**/*.{ts,tsx}"], sort: true, diff --git a/localazy.json b/localazy.json index 2b9f713c..823e4a3e 100644 --- a/localazy.json +++ b/localazy.json @@ -7,13 +7,13 @@ "features": ["plural_postfix_us", "filter_untranslated"], "files": [ { - "pattern": "locales/en-GB/*.json", + "pattern": "locales/en/*.json", "lang": "inherited" }, { "group": "existing", "pattern": "locales/*/*.json", - "excludes": ["locales/en-GB/*.json"], + "excludes": ["locales/en/*.json"], "lang": "${autodetectLang}" } ] @@ -25,9 +25,6 @@ "output": "locales/${langLsrDash}/${file}" } ], - "includeSourceLang": "${includeSourceLang|false}", - "langAliases": { - "en": "en_GB" - } + "includeSourceLang": "${includeSourceLang|false}" } } diff --git a/locales/de/app.json b/locales/de/app.json index aa57f824..67138ca5 100644 --- a/locales/de/app.json +++ b/locales/de/app.json @@ -69,9 +69,10 @@ "video": "Video" }, "developer_mode": { - "crypto_version": "Krypto-Version:{{version}}", + "crypto_version": "Krypto-Version: {{version}}", "device_id": "Geräte-ID: {{id}}", "duplicate_tiles_label": "Anzahl zusätzlicher Kachelkopien pro Teilnehmer", + "hostname": "Hostname: {{hostname}}", "matrix_id": "Matrix-ID: {{id}}" }, "disconnected_banner": "Die Verbindung zum Server wurde getrennt.", @@ -146,7 +147,6 @@ "screenshare_button_label": "Bildschirm teilen", "settings": { "audio_tab": { - "effect_volume_description": "Lautstärke anpassen, mit der Reaktionen und Handmeldungen abgespielt werden", "effect_volume_label": "Lautstärke der Soundeffekte" }, "developer_settings_label": "Entwicklereinstellungen", @@ -193,6 +193,7 @@ "expand": "Erweitern", "mute_for_me": "Für mich stumm schalten", "muted_for_me": "Für mich stumm geschaltet", - "volume": "Lautstärke" + "volume": "Lautstärke", + "waiting_for_media": "Warten auf Medien..." } } diff --git a/locales/en-GB/app.json b/locales/en/app.json similarity index 97% rename from locales/en-GB/app.json rename to locales/en/app.json index bc37dd7b..07dd53e3 100644 --- a/locales/en-GB/app.json +++ b/locales/en/app.json @@ -147,7 +147,7 @@ "screenshare_button_label": "Share screen", "settings": { "audio_tab": { - "effect_volume_description": "Adjust the volume at which reactions and hand raised effects play", + "effect_volume_description": "Volume for sound effects such as: joining or leaving a call, and reactions.", "effect_volume_label": "Sound effect volume" }, "developer_settings_label": "Developer Settings", @@ -194,6 +194,7 @@ "expand": "Expand", "mute_for_me": "Mute for me", "muted_for_me": "Muted for me", - "volume": "Volume" + "volume": "Volume", + "waiting_for_media": "Waiting for media..." } } diff --git a/locales/ro/app.json b/locales/ro/app.json new file mode 100644 index 00000000..ab55c511 --- /dev/null +++ b/locales/ro/app.json @@ -0,0 +1,195 @@ +{ + "a11y": { + "user_menu": "Meniul utilizatorului" + }, + "action": { + "close": "Închide", + "copy_link": "Copiază linkul", + "edit": "Editare", + "go": "Du-te", + "invite": "Invită", + "lower_hand": "Mâna inferioară", + "no": "No", + "pick_reaction": "Alegeți reacția", + "raise_hand": "Ridicați mâna", + "register": "Inregistrare", + "remove": "elimina", + "show_less": "Arată mai puțin", + "show_more": "Arată mai mult", + "sign_in": "Autentificare", + "sign_out": "Sign out", + "submit": "Trimiteți", + "upload_file": "Încărcați fișierul" + }, + "analytics_notice": "Prin participarea la această versiune beta, sunteți de acord cu colectarea de date anonime, pe care le folosim pentru a îmbunătăți produsul. Puteți găsi mai multe informații despre datele pe care le urmărim în Politica noastră de <2> confidențialitate și Politica noastră <6> privind cookie-urile.", + "app_selection_modal": { + "continue_in_browser": "Continuați în browser", + "open_in_app": "Deschideți în aplicație", + "text": "Sunteți gata să vă alăturați?", + "title": "Selectați aplicația" + }, + "application_opened_another_tab": "Această aplicație a fost deschisă într-o altă filă.", + "browser_media_e2ee_unsupported": "Browserul dvs. web nu acceptă criptarea media end-to-end. Browserele acceptate sunt Chrome, Safari, Firefox > = 117", + "browser_media_e2ee_unsupported_heading": "Browser incompatibil", + "call_ended_view": { + "body": "Ai fost deconectat de la apel", + "create_account_button": "Creează cont", + "create_account_prompt": "<0>De ce să nu terminați prin configurarea unei parole pentru a vă păstra contul? <1>Veți putea să vă păstrați numele și să setați un avatar pentru a fi utilizat la apelurile viitoare ", + "feedback_done": "<0>Vă mulțumim pentru feedback! ", + "feedback_prompt": "<0>Ne-ar plăcea să auzim feedback-ul dvs., astfel încât să vă putem îmbunătăți experiența. ", + "headline": "{{displayName}}, apelul tău s-a încheiat.", + "not_now_button": "Nu acum, reveniți la ecranul de pornire", + "reconnect_button": "Reconecta", + "survey_prompt": "Cum a mers?" + }, + "call_name": "Numele apelului", + "common": { + "analytics": "Analiză", + "audio": "Audio", + "avatar": "avatar", + "back": "Înapoi", + "camera": "Aparat foto", + "display_name": "Nume afișat", + "encrypted": "Criptat", + "error": "Eroare", + "home": "Acasa", + "loading": "Se încarcă...", + "microphone": "Microfon", + "next": "Urmator\n", + "options": "Opțiuni", + "password": "Parolă", + "preferences": "preferinte", + "profile": "Profil", + "reaction": "Reacție", + "reactions": "Reacții", + "settings": "Settings", + "something_went_wrong": "Ceva nu a mers bine", + "unencrypted": "Nu este criptat", + "username": "Nume utilizator", + "video": "Videoclip" + }, + "developer_mode": { + "crypto_version": "Versiunea Crypto: {{version}}", + "device_id": "ID-ul dispozitivului: {{id}}", + "duplicate_tiles_label": "Numărul de exemplare suplimentare de cartonașe per participant", + "hostname": "Numele gazdei: {{hostname}}", + "matrix_id": "ID-ul matricei: {{id}}" + }, + "disconnected_banner": "Conectivitatea la server a fost pierdută.", + "full_screen_view_description": "<0>Trimiterea jurnalelor de depanare ne va ajuta să urmărim problema. ", + "full_screen_view_h1": "<0>Hopa, ceva nu a mers bine. ", + "group_call_loader": { + "banned_body": "Ai fost interzis să ieși din cameră.", + "banned_heading": "Interzis", + "call_ended_body": "Ați fost eliminat din apel.", + "call_ended_heading": "Apel încheiat", + "failed_heading": "Nu s-a putut alătura", + "failed_text": "Apelul nu a fost găsit sau nu este accesibil.", + "knock_reject_body": "Cererea dvs. de a vă alătura a fost respinsă.", + "knock_reject_heading": "Acces refuzat", + "reason": "Motivul" + }, + "hangup_button_label": "Încheiați apelul", + "header_label": "Element Call Home", + "header_participants_label": "Participanți", + "invite_modal": { + "link_copied_toast": "Link copiat în clipboard", + "title": "Invitați la acest apel" + }, + "join_existing_call_modal": { + "join_button": "Da, alăturați-vă apelului", + "text": "Acest apel există deja, doriți să vă alăturați?", + "title": "Alăturați-vă apelului existent?" + }, + "layout_grid_label": "GRILĂ", + "layout_spotlight_label": "Spotlight", + "lobby": { + "ask_to_join": "Solicitare de participare la apel", + "join_as_guest": "Alăturați-vă ca invitat", + "join_button": "Alăturați-vă apelului", + "leave_button": "Înapoi la cele mai recente", + "waiting_for_invite": "Solicitare trimisă! În așteptarea permisiunii de a participa..." + }, + "log_in": "Autentificare", + "logging_in": "Autentificare...", + "login_auth_links": "<0>Creați un cont sau <2> accesați ca invitat ", + "login_auth_links_prompt": "Nu sunteți încă înregistrat?", + "login_subheading": "Pentru a continua la Element", + "login_title": "Logare", + "microphone_off": "Microfon oprit", + "microphone_on": "Microfon pornit", + "mute_microphone_button_label": "Dezactivați microfonul", + "qr_code": "COD QR", + "rageshake_button_error_caption": "Încearcă din nou trimiterea jurnalelor", + "rageshake_request_modal": { + "body": "Un alt utilizator al acestui apel are o problemă. Pentru a diagnostica mai bine aceste probleme, am dori să colectăm un jurnal de depanare.", + "title": "Solicitare jurnal de depanare" + }, + "rageshake_send_logs": "Trimiteți jurnale de depanare", + "rageshake_sending": "Trimiterea...", + "rageshake_sending_logs": "Trimiterea jurnalelor de depanare...", + "rageshake_sent": "Multumesc!", + "recaptcha_caption": "Acest site este protejat de reCAPTCHA și se aplică Politica de <2> confidențialitate Google și <6> Termenii și condițiile. <9>Făcând clic pe „Înregistrare”, sunteți de acord cu Acordul nostru de licențiere pentru utilizatorul <12> final (EULA) ", + "recaptcha_dismissed": "Recaptcha a fost respins", + "recaptcha_not_loaded": "Recaptcha nu a fost încărcat", + "register": { + "passwords_must_match": "Parolele trebuie să se potrivească", + "registering": "Înregistrare..." + }, + "register_auth_links": "<0>Ai deja un cont? <1><0>Conectați-vă sau <2> accesați ca invitat ", + "register_confirm_password_label": "Confirmă Parola", + "register_heading": "Creează-ți contul", + "return_home_button": "Reveniți la ecranul de pornire", + "room_auth_view_continue_button": "Continuă", + "room_auth_view_eula_caption": "Făcând clic pe „Continuați”, sunteți de acord cu Acordul nostru de licențiere pentru utilizatorul <2> final (EULA) ", + "screenshare_button_label": "Partajare ecran", + "settings": { + "audio_tab": { + "effect_volume_description": "Reglați volumul la care reacționează reacțiile și efectele ridicate de mână", + "effect_volume_label": "Volumul efectului sonor" + }, + "developer_settings_label": "Setări pentru dezvoltatori", + "developer_settings_label_description": "Expuneți setările dezvoltatorului în fereastra de setări.", + "developer_tab_title": "dezvoltator", + "feedback_tab_body": "Dacă întâmpinați probleme sau pur și simplu doriți să oferiți feedback, vă rugăm să ne trimiteți o scurtă descriere mai jos.", + "feedback_tab_description_label": "Feedback-ul tău", + "feedback_tab_h4": "Trimiteți Feedback", + "feedback_tab_send_logs_label": "Includeți jurnale de depanare", + "feedback_tab_thank_you": "Vă mulțumim, am primit feedback-ul dvs.!", + "feedback_tab_title": "Feedback", + "more_tab_title": "Mai mult", + "opt_in_description": "<0><1>Puteți retrage consimțământul debifând această casetă. Dacă sunteți în prezent la un apel, această setare va intra în vigoare la sfârșitul apelului.", + "preferences_tab": { + "reactions_play_sound_description": "Redați un efect sonor atunci când cineva trimite o reacție la un apel.", + "reactions_play_sound_label": "Redați sunete de reacție", + "reactions_show_description": "Afișați o animație atunci când cineva trimite o reacție.", + "reactions_show_label": "Afișați reacțiile", + "reactions_title": "Reacții" + }, + "preferences_tab_body": "Aici puteți configura opțiuni suplimentare pentru o experiență îmbunătățită", + "preferences_tab_h4": "preferinte", + "preferences_tab_show_hand_raised_timer_description": "Afișați un cronometru atunci când un participant ridică mâna", + "preferences_tab_show_hand_raised_timer_label": "Afișați durata ridicării mâinii", + "speaker_device_selection_label": "vorbitor" + }, + "start_new_call": "Începe un nou apel", + "start_video_button_label": "Începeți videoclipul", + "stop_screenshare_button_label": "Partajarea ecranului", + "stop_video_button_label": "Opriți videoclipul", + "submitting": "Trimiterea...", + "switch_camera": "Comutați camera", + "unauthenticated_view_body": "Nu sunteți încă înregistrat? <2>Creați un cont ", + "unauthenticated_view_eula_caption": "Făcând clic pe „Go”, sunteți de acord cu Acordul nostru de licențiere pentru utilizatorul <2> final (EULA) ", + "unauthenticated_view_login_button": "Conectați-vă la contul dvs.", + "unmute_microphone_button_label": "Anulează microfonul", + "version": "{{productName}}Versiune: {{version}}", + "video_tile": { + "always_show": "Arată întotdeauna", + "change_fit_contain": "Se potrivește cadrului", + "collapse": "colaps", + "expand": "Extindeți", + "mute_for_me": "Mute pentru mine", + "muted_for_me": "Dezactivat pentru mine", + "volume": "VOLUM" + } +} diff --git a/package.json b/package.json index ebaa59ff..71e3f9c9 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@types/grecaptcha": "^3.0.9", "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", - "@types/node": "^20.0.0", + "@types/node": "^22.0.0", "@types/pako": "^2.0.3", "@types/qrcode": "^1.5.5", "@types/react-dom": "^18.3.0", @@ -63,7 +63,7 @@ "@typescript-eslint/eslint-plugin": "^8.0.0", "@typescript-eslint/parser": "^8.0.0", "@use-gesture/react": "^10.2.11", - "@vector-im/compound-design-tokens": "^1.9.1", + "@vector-im/compound-design-tokens": "^2.0.0", "@vector-im/compound-web": "^7.2.0", "@vitejs/plugin-basic-ssl": "^1.0.1", "@vitejs/plugin-react": "^4.0.1", diff --git a/src/@types/i18next.d.ts b/src/@types/i18next.d.ts index 4a8830da..3c65e620 100644 --- a/src/@types/i18next.d.ts +++ b/src/@types/i18next.d.ts @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import "i18next"; // import all namespaces (for the default language, only) -import app from "../../locales/en-GB/app.json"; +import app from "../../locales/en/app.json"; declare module "i18next" { interface CustomTypeOptions { diff --git a/src/Avatar.test.tsx b/src/Avatar.test.tsx new file mode 100644 index 00000000..7eee2e90 --- /dev/null +++ b/src/Avatar.test.tsx @@ -0,0 +1,156 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, expect, test, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { MatrixClient } from "matrix-js-sdk/src/client"; +import { FC, PropsWithChildren } from "react"; + +import { ClientContextProvider } from "./ClientContext"; +import { Avatar } from "./Avatar"; +import { mockMatrixRoomMember, mockRtcMembership } from "./utils/test"; + +const TestComponent: FC< + PropsWithChildren<{ client: MatrixClient; supportsThumbnails?: boolean }> +> = ({ client, children, supportsThumbnails }) => { + return ( + + {children} + + ); +}; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +test("should just render a placeholder when the user has no avatar", () => { + const client = vi.mocked({ + getAccessToken: () => "my-access-token", + mxcUrlToHttp: () => vi.fn(), + } as unknown as MatrixClient); + + vi.spyOn(client, "mxcUrlToHttp"); + const member = mockMatrixRoomMember( + mockRtcMembership("@alice:example.org", "AAAA"), + { + getMxcAvatarUrl: () => undefined, + }, + ); + const displayName = "Alice"; + render( + + + , + ); + const element = screen.getByRole("img", { name: "@alice:example.org" }); + expect(element.tagName).toEqual("SPAN"); + expect(client.mxcUrlToHttp).toBeCalledTimes(0); +}); + +test("should just render a placeholder when thumbnails are not supported", () => { + const client = vi.mocked({ + getAccessToken: () => "my-access-token", + mxcUrlToHttp: () => vi.fn(), + } as unknown as MatrixClient); + + vi.spyOn(client, "mxcUrlToHttp"); + const member = mockMatrixRoomMember( + mockRtcMembership("@alice:example.org", "AAAA"), + { + getMxcAvatarUrl: () => "mxc://example.org/alice-avatar", + }, + ); + const displayName = "Alice"; + render( + + + , + ); + const element = screen.getByRole("img", { name: "@alice:example.org" }); + expect(element.tagName).toEqual("SPAN"); + expect(client.mxcUrlToHttp).toBeCalledTimes(0); +}); + +test("should attempt to fetch authenticated media", async () => { + const expectedAuthUrl = "http://example.org/media/alice-avatar"; + const expectedObjectURL = "my-object-url"; + const accessToken = "my-access-token"; + const theBlob = new Blob([]); + + // vitest doesn't have a implementation of create/revokeObjectURL, so we need + // to delete the property. It's a bit odd, but it works. + Reflect.deleteProperty(global.window.URL, "createObjectURL"); + globalThis.URL.createObjectURL = vi.fn().mockReturnValue(expectedObjectURL); + Reflect.deleteProperty(global.window.URL, "revokeObjectURL"); + globalThis.URL.revokeObjectURL = vi.fn(); + + const fetchFn = vi.fn().mockResolvedValue({ + blob: async () => Promise.resolve(theBlob), + }); + vi.stubGlobal("fetch", fetchFn); + + const client = vi.mocked({ + getAccessToken: () => accessToken, + mxcUrlToHttp: () => vi.fn(), + } as unknown as MatrixClient); + + vi.spyOn(client, "mxcUrlToHttp").mockReturnValue(expectedAuthUrl); + const member = mockMatrixRoomMember( + mockRtcMembership("@alice:example.org", "AAAA"), + { + getMxcAvatarUrl: () => "mxc://example.org/alice-avatar", + }, + ); + const displayName = "Alice"; + render( + + + , + ); + + // Fetch is asynchronous, so wait for this to resolve. + await vi.waitUntil(() => + document.querySelector(`img[src='${expectedObjectURL}']`), + ); + + expect(client.mxcUrlToHttp).toBeCalledTimes(1); + expect(globalThis.fetch).toBeCalledWith(expectedAuthUrl, { + headers: { Authorization: `Bearer ${accessToken}` }, + }); +}); diff --git a/src/Avatar.tsx b/src/Avatar.tsx index 29ab5236..f3fe6cd8 100644 --- a/src/Avatar.tsx +++ b/src/Avatar.tsx @@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { useMemo, FC, CSSProperties } from "react"; +import { useMemo, FC, CSSProperties, useState, useEffect } from "react"; import { Avatar as CompoundAvatar } from "@vector-im/compound-web"; +import { MatrixClient } from "matrix-js-sdk/src/client"; -import { getAvatarUrl } from "./utils/matrix"; -import { useClient } from "./ClientContext"; +import { useClientState } from "./ClientContext"; export enum Size { XS = "xs", @@ -36,6 +36,28 @@ interface Props { style?: CSSProperties; } +export function getAvatarUrl( + client: MatrixClient, + mxcUrl: string | null, + avatarSize = 96, +): string | null { + const width = Math.floor(avatarSize * window.devicePixelRatio); + const height = Math.floor(avatarSize * window.devicePixelRatio); + // scale is more suitable for larger sizes + const resizeMethod = avatarSize <= 96 ? "crop" : "scale"; + return mxcUrl + ? client.mxcUrlToHttp( + mxcUrl, + width, + height, + resizeMethod, + false, + true, + true, + ) + : null; +} + export const Avatar: FC = ({ className, id, @@ -45,7 +67,7 @@ export const Avatar: FC = ({ style, ...props }) => { - const { client } = useClient(); + const clientState = useClientState(); const sizePx = useMemo( () => @@ -55,10 +77,50 @@ export const Avatar: FC = ({ [size], ); - const resolvedSrc = useMemo(() => { - if (!client || !src || !sizePx) return undefined; - return src.startsWith("mxc://") ? getAvatarUrl(client, src, sizePx) : src; - }, [client, src, sizePx]); + const [avatarUrl, setAvatarUrl] = useState(undefined); + + useEffect(() => { + if (clientState?.state !== "valid") { + return; + } + const { authenticated, supportedFeatures } = clientState; + const client = authenticated?.client; + + if (!client || !src || !sizePx || !supportedFeatures.thumbnails) { + return; + } + + const token = client.getAccessToken(); + if (!token) { + return; + } + const resolveSrc = getAvatarUrl(client, src, sizePx); + if (!resolveSrc) { + setAvatarUrl(undefined); + return; + } + + let objectUrl: string | undefined; + fetch(resolveSrc, { + headers: { + Authorization: `Bearer ${token}`, + }, + }) + .then(async (req) => req.blob()) + .then((blob) => { + objectUrl = URL.createObjectURL(blob); + setAvatarUrl(objectUrl); + }) + .catch((ex) => { + setAvatarUrl(undefined); + }); + + return (): void => { + if (objectUrl) { + URL.revokeObjectURL(objectUrl); + } + }; + }, [clientState, src, sizePx]); return ( = ({ id={id} name={name} size={`${sizePx}px`} - src={resolvedSrc} + src={avatarUrl} style={style} {...props} /> diff --git a/src/ClientContext.tsx b/src/ClientContext.tsx index 8b5589d5..7a37e750 100644 --- a/src/ClientContext.tsx +++ b/src/ClientContext.tsx @@ -48,6 +48,7 @@ export type ValidClientState = { disconnected: boolean; supportedFeatures: { reactions: boolean; + thumbnails: boolean; }; setClient: (params?: SetClientParams) => void; }; @@ -71,6 +72,8 @@ export type SetClientParams = { const ClientContext = createContext(undefined); +export const ClientContextProvider = ClientContext.Provider; + export const useClientState = (): ClientState | undefined => useContext(ClientContext); @@ -253,6 +256,7 @@ export const ClientProvider: FC = ({ children }) => { const [isDisconnected, setIsDisconnected] = useState(false); const [supportsReactions, setSupportsReactions] = useState(false); + const [supportsThumbnails, setSupportsThumbnails] = useState(false); const state: ClientState | undefined = useMemo(() => { if (alreadyOpenedErr) { @@ -278,6 +282,7 @@ export const ClientProvider: FC = ({ children }) => { disconnected: isDisconnected, supportedFeatures: { reactions: supportsReactions, + thumbnails: supportsThumbnails, }, }; }, [ @@ -288,6 +293,7 @@ export const ClientProvider: FC = ({ children }) => { setClient, isDisconnected, supportsReactions, + supportsThumbnails, ]); const onSync = useCallback( @@ -313,6 +319,8 @@ export const ClientProvider: FC = ({ children }) => { } if (initClientState.widgetApi) { + // There is currently no widget API for authenticated media thumbnails. + setSupportsThumbnails(false); const reactSend = initClientState.widgetApi.hasCapability( "org.matrix.msc2762.send.event:m.reaction", ); @@ -334,6 +342,7 @@ export const ClientProvider: FC = ({ children }) => { } } else { setSupportsReactions(true); + setSupportsThumbnails(true); } return (): void => { diff --git a/src/TranslatedError.ts b/src/TranslatedError.ts index 0dbe675a..420556be 100644 --- a/src/TranslatedError.ts +++ b/src/TranslatedError.ts @@ -22,7 +22,7 @@ export abstract class TranslatedError extends Error { messageKey: ParseKeys, translationFn: TFunction, ) { - super(translationFn(messageKey, { lng: "en-GB" } as TOptions)); + super(translationFn(messageKey, { lng: "en" } as TOptions)); this.translatedMessage = translationFn(messageKey); } } diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index ed4d5bce..3947ba66 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -51,7 +51,7 @@ export interface ConfigOptions { // a livekit service url in the client well-known. // The well known needs to be formatted like so: // {"type":"livekit", "livekit_service_url":"https://livekit.example.com"} - // and stored under the key: "livekit_focus" + // and stored under the key: "org.matrix.msc4143.rtc_foci" livekit_service_url: string; }; diff --git a/src/index.css b/src/index.css index bf6d1605..aeeccaf4 100644 --- a/src/index.css +++ b/src/index.css @@ -47,6 +47,11 @@ layer(compound); --background-gradient: url("graphics/backgroundGradient.svg"); } +:root, +[class*="cpd-theme-"] { + --video-tile-background: var(--cpd-color-bg-subtle-secondary); +} + .cpd-theme-dark { --cpd-color-border-accent: var(--cpd-color-green-1100); --stopgap-color-on-solid-accent: var(--cpd-color-text-primary); diff --git a/src/initializer.tsx b/src/initializer.tsx index 47634078..e9290504 100644 --- a/src/initializer.tsx +++ b/src/initializer.tsx @@ -24,7 +24,7 @@ import { platform } from "./Platform"; // This generates a map of locale names to their URL (based on import.meta.url), which looks like this: // { -// "../locales/en-GB/app.json": "/whatever/assets/root/locales/en-aabbcc.json", +// "../locales/en/app.json": "/whatever/assets/root/locales/en-aabbcc.json", // ... // } const locales = import.meta.glob("../locales/*/*.json", { @@ -41,7 +41,7 @@ const getLocaleUrl = ( const supportedLngs = [ ...new Set( Object.keys(locales).map((url) => { - // The URLs are of the form ../locales/en-GB/app.json + // The URLs are of the form ../locales/en/app.json // This extracts the language code from the URL const lang = url.match(/\/([^/]+)\/[^/]+\.json$/)?.[1]; if (!lang) { @@ -133,7 +133,7 @@ export class Initializer { .use(languageDetector) .use(initReactI18next) .init({ - fallbackLng: "en-GB", + fallbackLng: "en", defaultNS: "app", keySeparator: ".", nsSeparator: false, diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 46f517a8..5bb1ba19 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -8,23 +8,31 @@ Please see LICENSE in the repository root for full details. import { render } from "@testing-library/react"; import { afterAll, - afterEach, beforeEach, expect, - Mock, MockedFunction, test, vitest, } from "vitest"; -import { ConnectionState, RemoteParticipant, Room } from "livekit-client"; -import { of } from "rxjs"; +import { MatrixClient } from "matrix-js-sdk/src/client"; +import { ConnectionState } from "livekit-client"; +import { BehaviorSubject, of } from "rxjs"; +import { afterEach } from "node:test"; import { act, ReactNode } from "react"; +import { + CallMembership, + type MatrixRTCSession, +} from "matrix-js-sdk/src/matrixrtc"; +import { RoomMember } from "matrix-js-sdk/src/matrix"; import { - EmittableMockLivekitRoom, + mockLivekitRoom, mockLocalParticipant, + mockMatrixRoom, mockMatrixRoomMember, mockRemoteParticipant, + mockRtcMembership, + MockRTCSession, } from "../utils/test"; import { E2eeType } from "../e2ee/e2eeType"; import { CallViewModel } from "../state/CallViewModel"; @@ -32,22 +40,21 @@ import { CallEventAudioRenderer, MAX_PARTICIPANT_COUNT_FOR_SOUND, } from "./CallEventAudioRenderer"; -import { prefetchSounds, useAudioContext } from "../useAudioContext"; -import { - MockRoom, - MockRTCSession, - TestReactionsWrapper, -} from "../utils/testReactions"; +import { useAudioContext } from "../useAudioContext"; +import { TestReactionsWrapper } from "../utils/testReactions"; +import { prefetchSounds } from "../soundUtils"; -const alice = mockMatrixRoomMember({ userId: "@alice:example.org" }); -const bob = mockMatrixRoomMember({ userId: "@bob:example.org" }); -const aliceId = `${alice.userId}:AAAA`; -const bobId = `${bob.userId}:BBBB`; +const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); +const local = mockMatrixRoomMember(localRtcMember); +const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); +const alice = mockMatrixRoomMember(aliceRtcMember); +const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); const localParticipant = mockLocalParticipant({ identity: "" }); +const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`; const aliceParticipant = mockRemoteParticipant({ identity: aliceId }); -const bobParticipant = mockRemoteParticipant({ identity: bobId }); vitest.mock("../useAudioContext"); +vitest.mock("../soundUtils"); afterEach(() => { vitest.resetAllMocks(); @@ -57,7 +64,7 @@ afterAll(() => { vitest.restoreAllMocks(); }); -let playSound: Mock< +let playSound: MockedFunction< NonNullable>["playSound"] >; @@ -72,19 +79,65 @@ beforeEach(() => { }); function TestComponent({ - room, + rtcSession, vm, }: { - room: MockRoom; + rtcSession: MockRTCSession; vm: CallViewModel; }): ReactNode { return ( - + ); } +function getMockEnv( + members: RoomMember[], + initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember], +): { + vm: CallViewModel; + session: MockRTCSession; + remoteRtcMemberships: BehaviorSubject; +} { + const matrixRoomMembers = new Map(members.map((p) => [p.userId, p])); + const remoteParticipants = of([aliceParticipant]); + const liveKitRoom = mockLivekitRoom( + { localParticipant }, + { remoteParticipants }, + ); + const matrixRoom = mockMatrixRoom({ + client: { + getUserId: () => localRtcMember.sender, + getDeviceId: () => localRtcMember.deviceId, + on: vitest.fn(), + off: vitest.fn(), + } as Partial as MatrixClient, + getMember: (userId) => matrixRoomMembers.get(userId) ?? null, + }); + + const remoteRtcMemberships = new BehaviorSubject( + initialRemoteRtcMemberships, + ); + + const session = new MockRTCSession( + matrixRoom, + localRtcMember, + ).withMemberships(remoteRtcMemberships); + + const vm = new CallViewModel( + session as unknown as MatrixRTCSession, + liveKitRoom, + { + kind: E2eeType.PER_PARTICIPANT, + }, + of(ConnectionState.Connected), + ); + return { vm, session, remoteRtcMemberships }; +} + /** * We don't want to play a sound when loading the call state * because typically this occurs in two stages. We first join @@ -93,118 +146,56 @@ function TestComponent({ * a noise every time. */ test("plays one sound when entering a call", () => { - const liveKitRoom = new EmittableMockLivekitRoom({ - localParticipant, - remoteParticipants: new Map(), - }); - - const room = new MockRoom(alice.userId); - const vm = new CallViewModel( - room.testGetAsMatrixRoom(), - liveKitRoom.getAsLivekitRoom(), - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), - ); - + const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]); + render(); // Joining a call usually means remote participants are added later. act(() => { - liveKitRoom.addParticipant(bobParticipant); + remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]); }); - - render(); - expect(playSound).toBeCalled(); + expect(playSound).toHaveBeenCalledOnce(); }); +// TODO: Same test? test("plays a sound when a user joins", () => { - const remoteParticipants = new Map( - [aliceParticipant].map((p) => [p.identity, p]), - ); - const liveKitRoom = new EmittableMockLivekitRoom({ - localParticipant, - remoteParticipants, - }); - - const room = new MockRoom(alice.userId); - const vm = new CallViewModel( - room.testGetAsMatrixRoom(), - liveKitRoom as unknown as Room, - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), - ); - render(); + const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]); + render(); act(() => { - liveKitRoom.addParticipant(bobParticipant); + remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]); }); // Play a sound when joining a call. expect(playSound).toBeCalledWith("join"); }); test("plays a sound when a user leaves", () => { - const remoteParticipants = new Map( - [aliceParticipant].map((p) => [p.identity, p]), - ); - const liveKitRoom = new EmittableMockLivekitRoom({ - localParticipant, - remoteParticipants, - }); - const room = new MockRoom(alice.userId); - - const vm = new CallViewModel( - room.testGetAsMatrixRoom(), - liveKitRoom.getAsLivekitRoom(), - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), - ); - render(); + const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]); + render(); act(() => { - liveKitRoom.removeParticipant(aliceParticipant); + remoteRtcMemberships.next([]); }); expect(playSound).toBeCalledWith("left"); }); test("plays no sound when the participant list is more than the maximum size", () => { - expect(playSound).not.toBeCalled(); - const remoteParticipants = new Map([ - [aliceParticipant.identity, aliceParticipant], - // You + other participants to hit the max. - ...Array.from({ length: MAX_PARTICIPANT_COUNT_FOR_SOUND - 1 }).map< - [string, RemoteParticipant] - >((_, index) => { - const p = mockRemoteParticipant({ - identity: `@user${index}:example.com:DEV${index}`, - }); - return [p.identity, p]; - }), - ]); + const mockRtcMemberships: CallMembership[] = []; + for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) { + mockRtcMemberships.push( + mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`), + ); + } - // Preload the call with the maximum members, assume that - // we're already in the call by this point rather than - // joining. - const liveKitRoom = new EmittableMockLivekitRoom({ - localParticipant, - remoteParticipants, - }); - const room = new MockRoom(alice.userId); - const vm = new CallViewModel( - room.testGetAsMatrixRoom(), - liveKitRoom.getAsLivekitRoom(), - { - kind: E2eeType.PER_PARTICIPANT, - }, - of(ConnectionState.Connected), + const { session, vm, remoteRtcMemberships } = getMockEnv( + [local, alice], + mockRtcMemberships, ); - render(); - // When the count drops, play a leave sound. + + render(); + expect(playSound).not.toBeCalled(); act(() => { - liveKitRoom.removeParticipant(aliceParticipant); + remoteRtcMemberships.next( + mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1), + ); }); expect(playSound).toBeCalledWith("left"); }); diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index 6648536a..51c0de7a 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -15,7 +15,8 @@ import leftCallSoundMp3 from "../sound/left_call.mp3"; import leftCallSoundOgg from "../sound/left_call.ogg"; import handSoundOgg from "../sound/raise_hand.ogg?url"; import handSoundMp3 from "../sound/raise_hand.mp3?url"; -import { prefetchSounds, useAudioContext } from "../useAudioContext"; +import { useAudioContext } from "../useAudioContext"; +import { prefetchSounds } from "../soundUtils"; import { useReactions } from "../useReactions"; import { useLatest } from "../useLatest"; @@ -24,7 +25,7 @@ import { useLatest } from "../useLatest"; export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; export const THROTTLE_SOUND_EFFECT_MS = 500; -export const CallEventAudioSounds = prefetchSounds({ +const sounds = prefetchSounds({ join: { mp3: joinCallSoundMp3, ogg: joinCallSoundOgg, @@ -45,7 +46,7 @@ export function CallEventAudioRenderer({ vm: CallViewModel; }): ReactNode { const audioEngineCtx = useAudioContext({ - sounds: CallEventAudioSounds, + sounds, latencyHint: "interactive", }); const audioEngineRef = useLatest(audioEngineCtx); diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 0b185b07..39b8a2c5 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -343,7 +343,7 @@ export const GroupCallView: FC = ({ = (props) => { useEffect(() => { if (livekitRoom !== undefined) { const vm = new CallViewModel( - props.rtcSession.room, + props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable, @@ -128,12 +128,7 @@ export const ActiveCall: FC = (props) => { setVm(vm); return (): void => vm.destroy(); } - }, [ - props.rtcSession.room, - livekitRoom, - props.e2eeSystem, - connStateObservable, - ]); + }, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable]); if (livekitRoom === undefined || vm === null) return null; diff --git a/src/room/ReactionAudioRenderer.test.tsx b/src/room/ReactionAudioRenderer.test.tsx index a6471114..2fec8a9a 100644 --- a/src/room/ReactionAudioRenderer.test.tsx +++ b/src/room/ReactionAudioRenderer.test.tsx @@ -29,8 +29,9 @@ import { playReactionsSound, soundEffectVolumeSetting, } from "../settings/settings"; -import { prefetchSounds, useAudioContext } from "../useAudioContext"; +import { useAudioContext } from "../useAudioContext"; import { GenericReaction, ReactionSet } from "../reactions"; +import { prefetchSounds } from "../soundUtils"; const memberUserIdAlice = "@alice:example.org"; const memberUserIdBob = "@bob:example.org"; @@ -60,6 +61,7 @@ function TestComponent({ } vitest.mock("../useAudioContext"); +vitest.mock("../soundUtils"); afterEach(() => { vitest.resetAllMocks(); diff --git a/src/room/ReactionAudioRenderer.tsx b/src/room/ReactionAudioRenderer.tsx index 1e29beb0..7f6936a2 100644 --- a/src/room/ReactionAudioRenderer.tsx +++ b/src/room/ReactionAudioRenderer.tsx @@ -5,15 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { ReactNode, useDeferredValue, useEffect } from "react"; +import { ReactNode, useDeferredValue, useEffect, useState } from "react"; import { useReactions } from "../useReactions"; import { playReactionsSound, useSetting } from "../settings/settings"; import { GenericReaction, ReactionSet } from "../reactions"; -import { prefetchSounds, useAudioContext } from "../useAudioContext"; +import { useAudioContext } from "../useAudioContext"; +import { prefetchSounds } from "../soundUtils"; import { useLatest } from "../useLatest"; -const SoundMap = Object.fromEntries([ +const soundMap = Object.fromEntries([ ...ReactionSet.filter((v) => v.sound !== undefined).map((v) => [ v.name, v.sound!, @@ -21,18 +22,29 @@ const SoundMap = Object.fromEntries([ [GenericReaction.name, GenericReaction.sound], ]); -const Sounds = prefetchSounds(SoundMap); - export function ReactionsAudioRenderer(): ReactNode { const { reactions } = useReactions(); const [shouldPlay] = useSetting(playReactionsSound); + const [soundCache, setSoundCache] = useState | null>(null); const audioEngineCtx = useAudioContext({ - sounds: Sounds, + sounds: soundCache, latencyHint: "interactive", }); const audioEngineRef = useLatest(audioEngineCtx); const oldReactions = useDeferredValue(reactions); + useEffect(() => { + if (!shouldPlay || soundCache) { + return; + } + // This is fine even if we load the component multiple times, + // as the browser's cache should ensure once the media is loaded + // once that future fetches come via the cache. + setSoundCache(prefetchSounds(soundMap)); + }, [soundCache, shouldPlay]); + useEffect(() => { if (!shouldPlay || !audioEngineRef.current) { return; @@ -47,7 +59,7 @@ export function ReactionsAudioRenderer(): ReactNode { // Don't replay old reactions return; } - if (SoundMap[reactionName]) { + if (soundMap[reactionName]) { void audioEngineRef.current.playSound(reactionName); } else { // Fallback sounds. @@ -55,5 +67,5 @@ export function ReactionsAudioRenderer(): ReactNode { } } }, [audioEngineRef, shouldPlay, oldReactions, reactions]); - return <>; + return null; } diff --git a/src/room/VideoPreview.module.css b/src/room/VideoPreview.module.css index 573807fe..89422af7 100644 --- a/src/room/VideoPreview.module.css +++ b/src/room/VideoPreview.module.css @@ -18,8 +18,7 @@ Please see LICENSE in the repository root for full details. width: 100%; height: 100%; object-fit: cover; - background-color: black; - background-color: var(--cpd-color-bg-subtle-primary); + background-color: var(--video-tile-background); } video.mirror { @@ -35,7 +34,7 @@ video.mirror { display: flex; justify-content: center; align-items: center; - background-color: var(--cpd-color-bg-subtle-secondary); + background-color: var(--video-tile-background); } .buttonBar { diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 65642ece..9d7eb5a1 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { ChangeEvent, FC, useCallback, useEffect, useState } from "react"; +import { ChangeEvent, FC, useCallback, useState } from "react"; import { Trans, useTranslation } from "react-i18next"; import { MatrixClient } from "matrix-js-sdk/src/matrix"; import { Root as Form, Text } from "@vector-im/compound-web"; @@ -86,9 +86,6 @@ export const SettingsModal: FC = ({ const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting); const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume); - // Debounce saving the sound volume as it triggers certain components to reload. - useEffect(() => {}); - const audioTab: Tab = { key: "audio", name: t("common.audio"), diff --git a/src/soundUtils.ts b/src/soundUtils.ts new file mode 100644 index 00000000..162091b7 --- /dev/null +++ b/src/soundUtils.ts @@ -0,0 +1,63 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { logger } from "matrix-js-sdk/src/logger"; + +type SoundDefinition = { mp3?: string; ogg: string }; + +export type PrefetchedSounds = Promise< + Record +>; + +/** + * Determine the best format we can use to play our sounds + * through. We prefer ogg support if possible, but will fall + * back to MP3. + * @returns "ogg" if the browser is likely to support it, or "mp3" otherwise. + */ +function getPreferredAudioFormat(): "ogg" | "mp3" { + const a = document.createElement("audio"); + if (a.canPlayType("audio/ogg") === "maybe") { + return "ogg"; + } + // Otherwise just assume MP3, as that has a chance of being more widely supported. + return "mp3"; +} + +const preferredFormat = getPreferredAudioFormat(); + +/** + * Prefetch sounds to be used by the AudioContext. This can + * be called outside the scope of a component to ensure the + * sounds load ahead of time. + * @param sounds A set of sound files that may be played. + * @returns A map of sound files to buffers. + */ +export async function prefetchSounds( + sounds: Record, +): PrefetchedSounds { + const buffers: Record = {}; + await Promise.all( + Object.entries(sounds).map(async ([name, file]) => { + const { mp3, ogg } = file as SoundDefinition; + // Use preferred format, fallback to ogg if no mp3 is provided. + // Load an audio file + const response = await fetch( + preferredFormat === "ogg" ? ogg : (mp3 ?? ogg), + ); + if (!response.ok) { + // If the sound doesn't load, it's not the end of the world. We won't play + // the sound when requested, but it's better than failing the whole application. + logger.warn(`Could not load sound ${name}, response was not okay`); + return; + } + // Decode it + buffers[name] = await response.arrayBuffer(); + }), + ); + return buffers as Record; +} diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index 9b2e5ee7..5dbfb1ca 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { test, vi, onTestFinished } from "vitest"; +import { test, vi, onTestFinished, it } from "vitest"; import { combineLatest, debounceTime, @@ -25,6 +25,7 @@ import { } from "livekit-client"; import * as ComponentsCore from "@livekit/components-core"; import { isEqual } from "lodash-es"; +import { CallMembership, MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc"; import { CallViewModel, Layout } from "./CallViewModel"; import { @@ -34,6 +35,8 @@ import { mockMatrixRoomMember, mockRemoteParticipant, withTestScheduler, + mockRtcMembership, + MockRTCSession, } from "../utils/test"; import { ECAddonConnectionState, @@ -43,14 +46,19 @@ import { E2eeType } from "../e2ee/e2eeType"; vi.mock("@livekit/components-core"); -const alice = mockMatrixRoomMember({ userId: "@alice:example.org" }); -const bob = mockMatrixRoomMember({ userId: "@bob:example.org" }); -const carol = mockMatrixRoomMember({ userId: "@carol:example.org" }); -const dave = mockMatrixRoomMember({ userId: "@dave:example.org" }); +const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); +const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); +const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); +const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); -const aliceId = `${alice.userId}:AAAA`; -const bobId = `${bob.userId}:BBBB`; -const daveId = `${dave.userId}:DDDD`; +const alice = mockMatrixRoomMember(aliceRtcMember); +const bob = mockMatrixRoomMember(bobRtcMember); +const carol = mockMatrixRoomMember(localRtcMember); +const dave = mockMatrixRoomMember(daveRtcMember); + +const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`; +const bobId = `${bob.userId}:${bobRtcMember.deviceId}`; +const daveId = `${dave.userId}:${daveRtcMember.deviceId}`; const localParticipant = mockLocalParticipant({ identity: "" }); const aliceParticipant = mockRemoteParticipant({ identity: aliceId }); @@ -65,7 +73,9 @@ const bobSharingScreen = mockRemoteParticipant({ }); const daveParticipant = mockRemoteParticipant({ identity: daveId }); -const members = new Map([alice, bob, carol, dave].map((p) => [p.userId, p])); +const roomMembers = new Map( + [alice, bob, carol, dave].map((p) => [p.userId, p]), +); export interface GridLayoutSummary { type: "grid"; @@ -173,10 +183,23 @@ function summarizeLayout(l: Observable): Observable { function withCallViewModel( remoteParticipants: Observable, + rtcMembers: Observable[]>, connectionState: Observable, speaking: Map>, continuation: (vm: CallViewModel) => void, ): void { + const room = mockMatrixRoom({ + client: { + getUserId: () => localRtcMember.sender, + getDeviceId: () => localRtcMember.deviceId, + } as Partial as MatrixClient, + getMember: (userId) => roomMembers.get(userId) ?? null, + }); + const rtcSession = new MockRTCSession( + room, + localRtcMember, + [], + ).withMemberships(rtcMembers); const participantsSpy = vi .spyOn(ComponentsCore, "connectedParticipantsObserver") .mockReturnValue(remoteParticipants); @@ -209,12 +232,7 @@ function withCallViewModel( ); const vm = new CallViewModel( - mockMatrixRoom({ - client: { - getUserId: () => "@carol:example.org", - } as Partial as MatrixClient, - getMember: (userId) => members.get(userId) ?? null, - }), + rtcSession as unknown as MatrixRTCSession, liveKitRoom, { kind: E2eeType.PER_PARTICIPANT, @@ -247,6 +265,7 @@ test("participants are retained during a focus switch", () => { a: [aliceParticipant, bobParticipant], b: [], }), + of([aliceRtcMember, bobRtcMember]), hot(connectionInputMarbles, { c: ConnectionState.Connected, s: ECAddonConnectionState.ECSwitchingFocus, @@ -288,6 +307,7 @@ test("screen sharing activates spotlight layout", () => { c: [aliceSharingScreen, bobSharingScreen], d: [aliceParticipant, bobSharingScreen], }), + of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { @@ -356,7 +376,7 @@ test("participants stay in the same order unless to appear/disappear", () => { const modeInputMarbles = " a"; // First Bob speaks, then Dave, then Alice const aSpeakingInputMarbles = "n- 1998ms - 1999ms y"; - const bSpeakingInputMarbles = "ny 1998ms n 1999ms "; + const bSpeakingInputMarbles = "ny 1998ms n 1999ms -"; const dSpeakingInputMarbles = "n- 1998ms y 1999ms n"; // Nothing should change when Bob speaks, because Bob is already on screen. // When Dave speaks he should switch with Alice because she's the one who @@ -366,6 +386,7 @@ test("participants stay in the same order unless to appear/disappear", () => { withCallViewModel( of([aliceParticipant, bobParticipant, daveParticipant]), + of([aliceRtcMember, bobRtcMember, daveRtcMember]), of(ConnectionState.Connected), new Map([ [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })], @@ -427,6 +448,7 @@ test("spotlight speakers swap places", () => { withCallViewModel( of([aliceParticipant, bobParticipant, daveParticipant]), + of([aliceRtcMember, bobRtcMember, daveRtcMember]), of(ConnectionState.Connected), new Map([ [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })], @@ -475,6 +497,7 @@ test("layout enters picture-in-picture mode when requested", () => { withCallViewModel( of([aliceParticipant, bobParticipant]), + of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { @@ -515,6 +538,7 @@ test("spotlight remembers whether it's expanded", () => { withCallViewModel( of([aliceParticipant, bobParticipant]), + of([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), (vm) => { @@ -559,3 +583,104 @@ test("spotlight remembers whether it's expanded", () => { ); }); }); + +test("participants must have a MatrixRTCSession to be visible", () => { + withTestScheduler(({ hot, expectObservable }) => { + // iterate through a number of combinations of participants and MatrixRTC memberships + // Bob never has an MatrixRTC membership + const scenarioInputMarbles = " abcdec"; + // Bob should never be visible + const expectedLayoutMarbles = "a-bc-b"; + + withCallViewModel( + hot(scenarioInputMarbles, { + a: [], + b: [bobParticipant], + c: [aliceParticipant, bobParticipant], + d: [aliceParticipant, daveParticipant, bobParticipant], + e: [aliceParticipant, daveParticipant, bobSharingScreen], + }), + hot(scenarioInputMarbles, { + a: [], + b: [], + c: [aliceRtcMember], + d: [aliceRtcMember, daveRtcMember], + e: [aliceRtcMember, daveRtcMember], + }), + of(ConnectionState.Connected), + new Map(), + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout(vm.layout)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0"], + }, + b: { + type: "one-on-one", + local: "local:0", + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${daveId}:0`], + }, + }, + ); + }, + ); + }); +}); + +it("should show at least one tile per MatrixRTCSession", () => { + withTestScheduler(({ hot, expectObservable }) => { + // iterate through some combinations of MatrixRTC memberships + const scenarioInputMarbles = " abcd"; + // There should always be one tile for each MatrixRTCSession + const expectedLayoutMarbles = "abcd"; + + withCallViewModel( + of([]), + hot(scenarioInputMarbles, { + a: [], + b: [aliceRtcMember], + c: [aliceRtcMember, daveRtcMember], + d: [daveRtcMember], + }), + of(ConnectionState.Connected), + new Map(), + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout(vm.layout)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0"], + }, + b: { + type: "one-on-one", + local: "local:0", + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${daveId}:0`], + }, + d: { + type: "one-on-one", + local: "local:0", + remote: `${daveId}:0`, + }, + }, + ); + }, + ); + }); +}); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 73320329..af8780b1 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -18,12 +18,9 @@ import { RemoteParticipant, Track, } from "livekit-client"; +import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix"; import { - Room as MatrixRoom, - RoomMember, - RoomStateEvent, -} from "matrix-js-sdk/src/matrix"; -import { + BehaviorSubject, EMPTY, Observable, Subject, @@ -49,6 +46,10 @@ import { withLatestFrom, } from "rxjs"; import { logger } from "matrix-js-sdk/src/logger"; +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc"; import { ViewModel } from "./ViewModel"; import { @@ -222,41 +223,67 @@ interface LayoutScanState { class UserMedia { private readonly scope = new ObservableScope(); public readonly vm: UserMediaViewModel; + private readonly participant: BehaviorSubject< + LocalParticipant | RemoteParticipant | undefined + >; + public readonly speaker: Observable; public readonly presenter: Observable; - public constructor( public readonly id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: LocalParticipant | RemoteParticipant | undefined, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { - this.vm = participant.isLocal - ? new LocalUserMediaViewModel( - id, - member, - participant as LocalParticipant, - encryptionSystem, - livekitRoom, - ) - : new RemoteUserMediaViewModel( - id, - member, - participant as RemoteParticipant, - encryptionSystem, - livekitRoom, - ); + this.participant = new BehaviorSubject(participant); + + if (participant?.isLocal) { + this.vm = new LocalUserMediaViewModel( + this.id, + member, + this.participant.asObservable() as Observable, + encryptionSystem, + livekitRoom, + ); + } else { + this.vm = new RemoteUserMediaViewModel( + id, + member, + this.participant.asObservable() as Observable< + RemoteParticipant | undefined + >, + encryptionSystem, + livekitRoom, + ); + } this.speaker = observeSpeaker(this.vm.speaking).pipe(this.scope.state()); - this.presenter = observeParticipantEvents( - participant, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled)); + this.presenter = this.participant.pipe( + switchMap( + (p) => + (p && + observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled))) ?? + of(false), + ), + this.scope.state(), + ); + } + + public updateParticipant( + newParticipant: LocalParticipant | RemoteParticipant | undefined, + ): void { + if (this.participant.value !== newParticipant) { + // Update the BehaviourSubject in the UserMedia. + this.participant.next(newParticipant); + } } public destroy(): void { @@ -267,6 +294,9 @@ class UserMedia { class ScreenShare { public readonly vm: ScreenShareViewModel; + private readonly participant: BehaviorSubject< + LocalParticipant | RemoteParticipant + >; public constructor( id: string, @@ -275,12 +305,15 @@ class ScreenShare { encryptionSystem: EncryptionSystem, liveKitRoom: LivekitRoom, ) { + this.participant = new BehaviorSubject(participant); + this.vm = new ScreenShareViewModel( id, member, - participant, + this.participant.asObservable(), encryptionSystem, liveKitRoom, + participant.isLocal, ); } @@ -317,11 +350,11 @@ function findMatrixRoomMember( export class CallViewModel extends ViewModel { public readonly localVideo: Observable = observeTrackReference( - this.livekitRoom.localParticipant, + of(this.livekitRoom.localParticipant), Track.Source.Camera, ).pipe( map((trackRef) => { - const track = trackRef.publication?.track; + const track = trackRef?.publication?.track; return track instanceof LocalVideoTrack ? track : null; }), ); @@ -401,49 +434,87 @@ export class CallViewModel extends ViewModel { this.remoteParticipants, observeParticipantMedia(this.livekitRoom.localParticipant), duplicateTiles.value, - // Also react to changes in the list of members - fromEvent(this.matrixRoom, RoomStateEvent.Update).pipe(startWith(null)), + // Also react to changes in the MatrixRTC session list. + // The session list will also be update if a room membership changes. + // No additional RoomState event listener needs to be set up. + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe(startWith(null)), ]).pipe( scan( ( prevItems, - [remoteParticipants, { participant: localParticipant }, duplicateTiles], + [ + remoteParticipants, + { participant: localParticipant }, + duplicateTiles, + _membershipsChanged, + ], ) => { const newItems = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const p of [localParticipant, ...remoteParticipants]) { - const id = p === localParticipant ? "local" : p.identity; - const member = findMatrixRoomMember(this.matrixRoom, id); - if (member === undefined) - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`, - ); + // m.rtc.members are the basis for calculating what is visible in the call + for (const rtcMember of this.matrixRTCSession.memberships) { + const room = this.matrixRTCSession.room; + // WARN! This is not exactly the sender but the user defined in the state key. + // This will be available once we change to the new "member as object" format in the MatrixRTC object. + let livekitParticipantId = + rtcMember.sender + ":" + rtcMember.deviceId; - // Create as many tiles for this participant as called for by - // the duplicateTiles option + let participant: + | LocalParticipant + | RemoteParticipant + | undefined = undefined; + if ( + rtcMember.sender === room.client.getUserId()! && + rtcMember.deviceId === room.client.getDeviceId() + ) { + livekitParticipantId = "local"; + participant = localParticipant; + } else { + participant = remoteParticipants.find( + (p) => p.identity === livekitParticipantId, + ); + } + + const member = findMatrixRoomMember(room, livekitParticipantId); + if (!member) { + logger.error( + "Could not find member for media id: ", + livekitParticipantId, + ); + } for (let i = 0; i < 1 + duplicateTiles; i++) { - const userMediaId = `${id}:${i}`; + const indexedMediaId = `${livekitParticipantId}:${i}`; + const prevMedia = prevItems.get(indexedMediaId); + if (prevMedia && prevMedia instanceof UserMedia) { + prevMedia.updateParticipant(participant); + } yield [ - userMediaId, - prevItems.get(userMediaId) ?? + indexedMediaId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevMedia ?? new UserMedia( - userMediaId, + indexedMediaId, member, - p, + participant, this.encryptionSystem, this.livekitRoom, ), ]; - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; + if (participant?.isScreenShareEnabled) { + const screenShareId = `${indexedMediaId}:screen-share`; yield [ screenShareId, prevItems.get(screenShareId) ?? new ScreenShare( screenShareId, member, - p, + participant, this.encryptionSystem, this.livekitRoom, ), @@ -454,7 +525,6 @@ export class CallViewModel extends ViewModel { }.bind(this)(), ); - for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); return newItems; }, new Map(), @@ -488,11 +558,6 @@ export class CallViewModel extends ViewModel { ), ); - private readonly localUserMedia: Observable = - this.mediaItems.pipe( - map((ms) => ms.find((m) => m.vm.local)!.vm as LocalUserMediaViewModel), - ); - /** * List of MediaItems that we want to display, that are of type ScreenShare */ @@ -504,7 +569,7 @@ export class CallViewModel extends ViewModel { this.scope.state(), ); - private readonly spotlightSpeaker: Observable = + private readonly spotlightSpeaker: Observable = this.userMedia.pipe( switchMap((mediaItems) => mediaItems.length === 0 @@ -515,7 +580,7 @@ export class CallViewModel extends ViewModel { ), ), ), - scan<(readonly [UserMedia, boolean])[], UserMedia, null>( + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( (prev, mediaItems) => { // Only remote users that are still in the call should be sticky const [stickyMedia, stickySpeaking] = @@ -532,11 +597,11 @@ export class CallViewModel extends ViewModel { // Otherwise, spotlight an arbitrary remote user mediaItems.find(([m]) => !m.vm.local)?.[0] ?? // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)![0]); + mediaItems.find(([m]) => m.vm.local)?.[0]); }, null, ), - map((speaker) => speaker.vm), + map((speaker) => speaker?.vm ?? null), this.scope.state(), ); @@ -576,37 +641,57 @@ export class CallViewModel extends ViewModel { }), ); - private readonly spotlightAndPip: Observable< - [Observable, Observable] - > = this.screenShares.pipe( - map((screenShares) => - screenShares.length > 0 - ? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const) - : ([ - this.spotlightSpeaker.pipe(map((speaker) => [speaker!])), - this.spotlightSpeaker.pipe( - switchMap((speaker) => - speaker.local - ? of(null) - : this.localUserMedia.pipe( - switchMap((vm) => - vm.alwaysShow.pipe( - map((alwaysShow) => (alwaysShow ? vm : null)), - ), - ), - ), - ), - ), - ] as const), - ), - ); - private readonly spotlight: Observable = - this.spotlightAndPip.pipe( - switchMap(([spotlight]) => spotlight), + this.screenShares.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } + + return this.spotlightSpeaker.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), this.scope.state(), ); + private readonly pip: Observable = combineLatest([ + this.screenShares, + this.spotlightSpeaker, + this.mediaItems, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return this.spotlightSpeaker; + } + if (!spotlight || spotlight.local) { + return of(null); + } + + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; + + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; + + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } + + return null; + }), + ); + }), + this.scope.state(), + ); + private readonly hasRemoteScreenShares: Observable = this.spotlight.pipe( map((spotlight) => @@ -615,9 +700,6 @@ export class CallViewModel extends ViewModel { distinctUntilChanged(), ); - private readonly pip: Observable = - this.spotlightAndPip.pipe(switchMap(([, pip]) => pip)); - private readonly pipEnabled: Observable = setPipEnabled.pipe( startWith(false), ); @@ -721,15 +803,16 @@ export class CallViewModel extends ViewModel { this.mediaItems.pipe( map((mediaItems) => { if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)! - .vm as LocalUserMediaViewModel; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as | RemoteUserMediaViewModel | undefined; // There might not be a remote tile if there are screen shares, or if // only the local user is in the call and they're using the duplicate // tiles option - if (remote === undefined) return null; + if (!remote || !local) return null; return { type: "one-on-one", local, remote }; }), @@ -1010,7 +1093,7 @@ export class CallViewModel extends ViewModel { public constructor( // A call is permanently tied to a single Matrix room and LiveKit room - private readonly matrixRoom: MatrixRoom, + private readonly matrixRTCSession: MatrixRTCSession, private readonly livekitRoom: LivekitRoom, private readonly encryptionSystem: EncryptionSystem, private readonly connectionState: Observable, diff --git a/src/state/MediaViewModel.test.ts b/src/state/MediaViewModel.test.ts index 5b5e59a7..c4e0bee6 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/MediaViewModel.test.ts @@ -8,14 +8,17 @@ Please see LICENSE in the repository root for full details. import { expect, test, vi } from "vitest"; import { + mockRtcMembership, withLocalMedia, withRemoteMedia, withTestScheduler, } from "../utils/test"; +const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); + test("control a participant's volume", async () => { const setVolumeSpy = vi.fn(); - await withRemoteMedia({}, { setVolume: setVolumeSpy }, (vm) => + await withRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy }, (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab---c---d|", { a() { @@ -60,7 +63,7 @@ test("control a participant's volume", async () => { }); test("toggle fit/contain for a participant's video", async () => { - await withRemoteMedia({}, {}, (vm) => + await withRemoteMedia(rtcMembership, {}, {}, (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab|", { a: () => vm.toggleFitContain(), @@ -76,17 +79,21 @@ test("toggle fit/contain for a participant's video", async () => { }); test("local media remembers whether it should always be shown", async () => { - await withLocalMedia({}, (vm) => + await withLocalMedia(rtcMembership, {}, (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-a|", { a: () => vm.setAlwaysShow(false) }); expectObservable(vm.alwaysShow).toBe("ab", { a: true, b: false }); }), ); // Next local media should start out *not* always shown - await withLocalMedia({}, (vm) => - withTestScheduler(({ expectObservable, schedule }) => { - schedule("-a|", { a: () => vm.setAlwaysShow(true) }); - expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true }); - }), + await withLocalMedia( + rtcMembership, + + {}, + (vm) => + withTestScheduler(({ expectObservable, schedule }) => { + schedule("-a|", { a: () => vm.setAlwaysShow(true) }); + expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true }); + }), ); }); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 36e76d38..ceaca57c 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -32,7 +32,6 @@ import { Observable, Subject, combineLatest, - distinctUntilChanged, distinctUntilKeyChanged, filter, fromEvent, @@ -40,7 +39,6 @@ import { map, merge, of, - shareReplay, startWith, switchMap, throttleTime, @@ -77,16 +75,24 @@ export function useDisplayName(vm: MediaViewModel): string { } export function observeTrackReference( - participant: Participant, + participant: Observable, source: Track.Source, -): Observable { - return observeParticipantMedia(participant).pipe( - map(() => ({ - participant, - publication: participant.getTrackPublication(source), - source, - })), - distinctUntilKeyChanged("publication"), +): Observable { + return participant.pipe( + switchMap((p) => { + if (p) { + return observeParticipantMedia(p).pipe( + map(() => ({ + participant: p, + publication: p.getTrackPublication(source), + source, + })), + distinctUntilKeyChanged("publication"), + ); + } else { + return of(undefined); + } + }), ); } @@ -105,11 +111,11 @@ function observeRemoteTrackReceivingOkay( }; return combineLatest([ - observeTrackReference(participant, source), + observeTrackReference(of(participant), source), interval(1000).pipe(startWith(0)), ]).pipe( switchMap(async ([trackReference]) => { - const track = trackReference.publication?.track; + const track = trackReference?.publication?.track; if (!track || !(track instanceof RemoteTrack)) { return undefined; } @@ -200,14 +206,10 @@ export enum EncryptionStatus { } abstract class BaseMediaViewModel extends ViewModel { - /** - * Whether the media belongs to the local user. - */ - public readonly local = this.participant.isLocal; /** * The LiveKit video track for this media. */ - public readonly video: Observable; + public readonly video: Observable; /** * Whether there should be a warning that this media is unencrypted. */ @@ -215,6 +217,11 @@ abstract class BaseMediaViewModel extends ViewModel { public readonly encryptionStatus: Observable; + /** + * Whether this media corresponds to the local participant. + */ + public abstract readonly local: boolean; + public constructor( /** * An opaque identifier for this media. @@ -226,7 +233,12 @@ abstract class BaseMediaViewModel extends ViewModel { // TODO: Fully separate the data layer from the UI layer by keeping the // member object internal public readonly member: RoomMember | undefined, - protected readonly participant: LocalParticipant | RemoteParticipant, + // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through + // livekit. + protected readonly participant: Observable< + LocalParticipant | RemoteParticipant | undefined + >, + encryptionSystem: EncryptionSystem, audioSource: AudioSource, videoSource: VideoSource, @@ -243,69 +255,72 @@ abstract class BaseMediaViewModel extends ViewModel { [audio, this.video], (a, v) => encryptionSystem.kind !== E2eeType.NONE && - (a.publication?.isEncrypted === false || - v.publication?.isEncrypted === false), - ).pipe( - distinctUntilChanged(), - shareReplay({ bufferSize: 1, refCount: false }), - ); + (a?.publication?.isEncrypted === false || + v?.publication?.isEncrypted === false), + ).pipe(this.scope.state()); - if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) { - this.encryptionStatus = of(EncryptionStatus.Okay).pipe( - this.scope.state(), - ); - } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { - this.encryptionStatus = combineLatest([ - encryptionErrorObservable( - livekitRoom, - participant, - encryptionSystem, - "MissingKey", - ), - encryptionErrorObservable( - livekitRoom, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay(participant, audioSource), - observeRemoteTrackReceivingOkay(participant, videoSource), - ]).pipe( - map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { - if (keyMissing) return EncryptionStatus.KeyMissing; - if (keyInvalid) return EncryptionStatus.KeyInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - this.scope.state(), - ); - } else { - this.encryptionStatus = combineLatest([ - encryptionErrorObservable( - livekitRoom, - participant, - encryptionSystem, - "InvalidKey", - ), - observeRemoteTrackReceivingOkay(participant, audioSource), - observeRemoteTrackReceivingOkay(participant, videoSource), - ]).pipe( - map( - ([keyInvalid, audioOkay, videoOkay]): - | EncryptionStatus - | undefined => { - if (keyInvalid) return EncryptionStatus.PasswordInvalid; - if (audioOkay || videoOkay) return EncryptionStatus.Okay; - return undefined; // no change - }, - ), - filter((x) => !!x), - startWith(EncryptionStatus.Connecting), - this.scope.state(), - ); - } + this.encryptionStatus = this.participant.pipe( + switchMap((participant): Observable => { + if (!participant) { + return of(EncryptionStatus.Connecting); + } else if ( + participant.isLocal || + encryptionSystem.kind === E2eeType.NONE + ) { + return of(EncryptionStatus.Okay); + } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { + return combineLatest([ + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "MissingKey", + ), + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay(participant, audioSource), + observeRemoteTrackReceivingOkay(participant, videoSource), + ]).pipe( + map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { + if (keyMissing) return EncryptionStatus.KeyMissing; + if (keyInvalid) return EncryptionStatus.KeyInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + ); + } else { + return combineLatest([ + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay(participant, audioSource), + observeRemoteTrackReceivingOkay(participant, videoSource), + ]).pipe( + map( + ([keyInvalid, audioOkay, videoOkay]): + | EncryptionStatus + | undefined => { + if (keyInvalid) return EncryptionStatus.PasswordInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }, + ), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + ); + } + }), + this.scope.state(), + ); } } @@ -324,11 +339,14 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { /** * Whether the participant is speaking. */ - public readonly speaking = observeParticipantEvents( - this.participant, - ParticipantEvent.IsSpeakingChanged, - ).pipe( - map((p) => p.isSpeaking), + public readonly speaking = this.participant.pipe( + switchMap((p) => + p + ? observeParticipantEvents(p, ParticipantEvent.IsSpeakingChanged).pipe( + map((p) => p.isSpeaking), + ) + : of(false), + ), this.scope.state(), ); @@ -350,7 +368,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { @@ -364,18 +382,25 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { livekitRoom, ); - const media = observeParticipantMedia(participant).pipe(this.scope.state()); + const media = participant.pipe( + switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), + this.scope.state(), + ); this.audioEnabled = media.pipe( - map((m) => m.microphoneTrack?.isMuted === false), + map((m) => m?.microphoneTrack?.isMuted === false), ); this.videoEnabled = media.pipe( - map((m) => m.cameraTrack?.isMuted === false), + map((m) => m?.cameraTrack?.isMuted === false), ); } public toggleFitContain(): void { this._cropVideo.next(!this._cropVideo.value); } + + public get local(): boolean { + return this instanceof LocalUserMediaViewModel; + } } /** @@ -387,7 +412,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { */ public readonly mirror = this.video.pipe( switchMap((v) => { - const track = v.publication?.track; + const track = v?.publication?.track; if (!(track instanceof LocalTrack)) return of(false); // Watch for track restarts, because they indicate a camera switch return fromEvent(track, TrackEvent.Restarted).pipe( @@ -409,7 +434,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { @@ -470,18 +495,17 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, ) { super(id, member, participant, encryptionSystem, livekitRoom); // Sync the local volume with LiveKit - this.localVolume - .pipe(this.scope.bind()) - .subscribe((volume) => - (this.participant as RemoteParticipant).setVolume(volume), - ); + combineLatest([ + participant, + this.localVolume.pipe(this.scope.bind()), + ]).subscribe(([p, volume]) => p && p.setVolume(volume)); } public toggleLocallyMuted(): void { @@ -504,9 +528,10 @@ export class ScreenShareViewModel extends BaseMediaViewModel { public constructor( id: string, member: RoomMember | undefined, - participant: LocalParticipant | RemoteParticipant, + participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + public readonly local: boolean, ) { super( id, diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 9b03a5ea..c0cf9c48 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -13,7 +13,7 @@ import { of } from "rxjs"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import { GridTile } from "./GridTile"; -import { withRemoteMedia } from "../utils/test"; +import { mockRtcMembership, withRemoteMedia } from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsProvider } from "../useReactions"; @@ -25,6 +25,7 @@ global.IntersectionObserver = class MockIntersectionObserver { test("GridTile is accessible", async () => { await withRemoteMedia( + mockRtcMembership("@alice:example.org", "AAAA"), { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 27695b65..15f7c295 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -175,6 +175,7 @@ const UserMediaTile = forwardRef( raisedHandTime={handRaised} currentReaction={currentReaction} raisedHandOnClick={raisedHandOnClick} + localParticipant={vm.local} {...props} /> ); diff --git a/src/tile/MediaView.module.css b/src/tile/MediaView.module.css index 4594c284..70d6fead 100644 --- a/src/tile/MediaView.module.css +++ b/src/tile/MediaView.module.css @@ -15,7 +15,7 @@ Please see LICENSE in the repository root for full details. inline-size: 100%; block-size: 100%; object-fit: contain; - background-color: var(--cpd-color-bg-subtle-primary); + background-color: var(--video-tile-background); /* This transform is a no-op, but it forces Firefox to use a different rendering path, one that actually clips the corners of