Merge branch 'livekit' into hs/fix-avatar-media

This commit is contained in:
Hugh Nimmo-Smith
2024-12-06 18:22:48 +00:00
45 changed files with 1363 additions and 326 deletions

View File

@@ -42,7 +42,7 @@ jobs:
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: ${{ inputs.docker_tags}}
@@ -51,7 +51,7 @@ jobs:
uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
- name: Build and push Docker image
uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0
uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0
with:
context: .
platforms: linux/amd64,linux/arm64

View File

@@ -28,7 +28,7 @@ jobs:
uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"
node-version-file: ".node-version"
- name: Install dependencies
run: "yarn install"
- name: Build

View File

@@ -12,7 +12,7 @@ jobs:
uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"
node-version-file: ".node-version"
- name: Install dependencies
run: "yarn install"
- name: Prettier

View File

@@ -14,7 +14,7 @@ jobs:
uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"
node-version-file: ".node-version"
- name: Install dependencies
run: "yarn install"
- name: Vitest

View File

@@ -18,7 +18,7 @@ jobs:
- uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"
node-version-file: ".node-version"
- name: Install Deps
run: "yarn install --frozen-lockfile"

1
.node-version Normal file
View File

@@ -0,0 +1 @@
22

View File

@@ -92,6 +92,9 @@ work:
experimental_features:
# MSC3266: Room summary API. Used for knocking over federation
msc3266_enabled: true
# MSC4222 needed for syncv2 state_after. This allow clients to
# correctly track the state of the room.
msc4222_enabled: true
# The maximum allowed duration by which sent events can be delayed, as
# per MSC4140.
@@ -109,6 +112,10 @@ summary contains the room join rules. We need that to decide if the user gets
prompted with the option to knock ("Request to join call"), a cannot join error or the
join view.
MSC4222 allow clients to opt-in to a change of the sync v2 API that allows them
to correctly track the state of the room. This is required by Element Call to
track room state reliably.
Element Call requires a Livekit SFU alongside a [Livekit JWT
service](https://github.com/element-hq/lk-jwt-service) to work. The url to the
Livekit JWT service can either be configured in the config of Element Call
@@ -213,7 +220,7 @@ To add a new translation key you can do these steps:
1. Add the new key entry to the code where the new key is used: `t("some_new_key")`
1. Run `yarn i18n` to extract the new key and update the translation files. This
will add a skeleton entry to the `locales/en-GB/app.json` file:
will add a skeleton entry to the `locales/en/app.json` file:
```jsonc
{
...
@@ -221,7 +228,7 @@ To add a new translation key you can do these steps:
...
}
```
1. Update the skeleton entry in the `locales/en-GB/app.json` file with
1. Update the skeleton entry in the `locales/en/app.json` file with
the English translation:
```jsonc

View File

@@ -25,6 +25,9 @@ trusted_key_servers:
experimental_features:
# MSC3266: Room summary API. Used for knocking over federation
msc3266_enabled: true
# MSC4222 needed for syncv2 state_after. This allow clients to
# correctly track the state of the room.
msc4222_enabled: true
# The maximum allowed duration by which sent events can be delayed, as
# per MSC4140. Must be a positive value if set. Defaults to no

View File

@@ -21,7 +21,7 @@ export default {
},
],
},
locales: ["en-GB"],
locales: ["en"],
output: "locales/$LOCALE/$NAMESPACE.json",
input: ["src/**/*.{ts,tsx}"],
sort: true,

View File

@@ -7,13 +7,13 @@
"features": ["plural_postfix_us", "filter_untranslated"],
"files": [
{
"pattern": "locales/en-GB/*.json",
"pattern": "locales/en/*.json",
"lang": "inherited"
},
{
"group": "existing",
"pattern": "locales/*/*.json",
"excludes": ["locales/en-GB/*.json"],
"excludes": ["locales/en/*.json"],
"lang": "${autodetectLang}"
}
]
@@ -25,9 +25,6 @@
"output": "locales/${langLsrDash}/${file}"
}
],
"includeSourceLang": "${includeSourceLang|false}",
"langAliases": {
"en": "en_GB"
}
"includeSourceLang": "${includeSourceLang|false}"
}
}

View File

@@ -68,8 +68,13 @@
"username": "Benutzername",
"video": "Video"
},
"crypto_version": "Krypto-Version:{{version}}",
"device_id": "Geräte-ID: {{id}}",
"developer_mode": {
"crypto_version": "Krypto-Version: {{version}}",
"device_id": "Geräte-ID: {{id}}",
"duplicate_tiles_label": "Anzahl zusätzlicher Kachelkopien pro Teilnehmer",
"hostname": "Hostname: {{hostname}}",
"matrix_id": "Matrix-ID: {{id}}"
},
"disconnected_banner": "Die Verbindung zum Server wurde getrennt.",
"full_screen_view_description": "<0>Übermittelte Problemberichte helfen uns, Fehler zu beheben.</0>",
"full_screen_view_h1": "<0>Hoppla, etwas ist schiefgelaufen.</0>",
@@ -111,7 +116,6 @@
"login_auth_links_prompt": "Noch nicht registriert?",
"login_subheading": "Weiter zu Element",
"login_title": "Anmelden",
"matrix_id": "Matrix-ID: {{id}}",
"microphone_off": "Mikrofon aus",
"microphone_on": "Mikrofon an",
"mute_microphone_button_label": "Mikrofon stumm schalten",
@@ -149,7 +153,6 @@
"developer_settings_label": "Entwicklereinstellungen",
"developer_settings_label_description": "Zeige die Entwicklereinstellungen im Einstellungsfenster.",
"developer_tab_title": "Entwickler",
"duplicate_tiles_label": "Anzahl zusätzlicher Kachelkopien pro Teilnehmer",
"feedback_tab_body": "Falls du auf Probleme stößt oder einfach nur eine Rückmeldung geben möchtest, sende uns bitte eine kurze Beschreibung.",
"feedback_tab_description_label": "Deine Rückmeldung",
"feedback_tab_h4": "Rückmeldung geben",
@@ -191,6 +194,7 @@
"expand": "Erweitern",
"mute_for_me": "Für mich stumm schalten",
"muted_for_me": "Für mich stumm geschaltet",
"volume": "Lautstärke"
"volume": "Lautstärke",
"waiting_for_media": "Warten auf Medien..."
}
}

View File

@@ -68,8 +68,13 @@
"username": "Username",
"video": "Video"
},
"crypto_version": "Crypto version: {{version}}",
"device_id": "Device ID: {{id}}",
"developer_mode": {
"crypto_version": "Crypto version: {{version}}",
"device_id": "Device ID: {{id}}",
"duplicate_tiles_label": "Number of additional tile copies per participant",
"hostname": "Hostname: {{hostname}}",
"matrix_id": "Matrix ID: {{id}}"
},
"disconnected_banner": "Connectivity to the server has been lost.",
"full_screen_view_description": "<0>Submitting debug logs will help us track down the problem.</0>",
"full_screen_view_h1": "<0>Oops, something's gone wrong.</0>",
@@ -111,7 +116,6 @@
"login_auth_links_prompt": "Not registered yet?",
"login_subheading": "To continue to Element",
"login_title": "Login",
"matrix_id": "Matrix ID: {{id}}",
"microphone_off": "Microphone off",
"microphone_on": "Microphone on",
"mute_microphone_button_label": "Mute microphone",
@@ -149,7 +153,6 @@
"developer_settings_label": "Developer Settings",
"developer_settings_label_description": "Expose developer settings in the settings window.",
"developer_tab_title": "Developer",
"duplicate_tiles_label": "Number of additional tile copies per participant",
"feedback_tab_body": "If you are experiencing issues or simply would like to provide some feedback, please send us a short description below.",
"feedback_tab_description_label": "Your feedback",
"feedback_tab_h4": "Submit feedback",
@@ -191,6 +194,7 @@
"expand": "Expand",
"mute_for_me": "Mute for me",
"muted_for_me": "Muted for me",
"volume": "Volume"
"volume": "Volume",
"waiting_for_media": "Waiting for media..."
}
}

195
locales/ro/app.json Normal file
View File

@@ -0,0 +1,195 @@
{
"a11y": {
"user_menu": "Meniul utilizatorului"
},
"action": {
"close": "Închide",
"copy_link": "Copiază linkul",
"edit": "Editare",
"go": "Du-te",
"invite": "Invită",
"lower_hand": "Mâna inferioară",
"no": "No",
"pick_reaction": "Alegeți reacția",
"raise_hand": "Ridicați mâna",
"register": "Inregistrare",
"remove": "elimina",
"show_less": "Arată mai puțin",
"show_more": "Arată mai mult",
"sign_in": "Autentificare",
"sign_out": "Sign out",
"submit": "Trimiteți",
"upload_file": "Încărcați fișierul"
},
"analytics_notice": "Prin participarea la această versiune beta, sunteți de acord cu colectarea de date anonime, pe care le folosim pentru a îmbunătăți produsul. Puteți găsi mai multe informații despre datele pe care le urmărim în Politica noastră de <2> confidențialitate </2> și Politica noastră <6> privind cookie-urile</6>.",
"app_selection_modal": {
"continue_in_browser": "Continuați în browser",
"open_in_app": "Deschideți în aplicație",
"text": "Sunteți gata să vă alăturați?",
"title": "Selectați aplicația"
},
"application_opened_another_tab": "Această aplicație a fost deschisă într-o altă filă.",
"browser_media_e2ee_unsupported": "Browserul dvs. web nu acceptă criptarea media end-to-end. Browserele acceptate sunt Chrome, Safari, Firefox > = 117",
"browser_media_e2ee_unsupported_heading": "Browser incompatibil",
"call_ended_view": {
"body": "Ai fost deconectat de la apel",
"create_account_button": "Creează cont",
"create_account_prompt": "<0>De ce să nu terminați prin configurarea unei parole pentru a vă păstra contul? </0><1>Veți putea să vă păstrați numele și să setați un avatar pentru a fi utilizat la apelurile viitoare </1>",
"feedback_done": "<0>Vă mulțumim pentru feedback! </0>",
"feedback_prompt": "<0>Ne-ar plăcea să auzim feedback-ul dvs., astfel încât să vă putem îmbunătăți experiența. </0>",
"headline": "{{displayName}}, apelul tău s-a încheiat.",
"not_now_button": "Nu acum, reveniți la ecranul de pornire",
"reconnect_button": "Reconecta",
"survey_prompt": "Cum a mers?"
},
"call_name": "Numele apelului",
"common": {
"analytics": "Analiză",
"audio": "Audio",
"avatar": "avatar",
"back": "Înapoi",
"camera": "Aparat foto",
"display_name": "Nume afișat",
"encrypted": "Criptat",
"error": "Eroare",
"home": "Acasa",
"loading": "Se încarcă...",
"microphone": "Microfon",
"next": "Urmator\n",
"options": "Opțiuni",
"password": "Parolă",
"preferences": "preferinte",
"profile": "Profil",
"reaction": "Reacție",
"reactions": "Reacții",
"settings": "Settings",
"something_went_wrong": "Ceva nu a mers bine",
"unencrypted": "Nu este criptat",
"username": "Nume utilizator",
"video": "Videoclip"
},
"developer_mode": {
"crypto_version": "Versiunea Crypto: {{version}}",
"device_id": "ID-ul dispozitivului: {{id}}",
"duplicate_tiles_label": "Numărul de exemplare suplimentare de cartonașe per participant",
"hostname": "Numele gazdei: {{hostname}}",
"matrix_id": "ID-ul matricei: {{id}}"
},
"disconnected_banner": "Conectivitatea la server a fost pierdută.",
"full_screen_view_description": "<0>Trimiterea jurnalelor de depanare ne va ajuta să urmărim problema. </0>",
"full_screen_view_h1": "<0>Hopa, ceva nu a mers bine. </0>",
"group_call_loader": {
"banned_body": "Ai fost interzis să ieși din cameră.",
"banned_heading": "Interzis",
"call_ended_body": "Ați fost eliminat din apel.",
"call_ended_heading": "Apel încheiat",
"failed_heading": "Nu s-a putut alătura",
"failed_text": "Apelul nu a fost găsit sau nu este accesibil.",
"knock_reject_body": "Cererea dvs. de a vă alătura a fost respinsă.",
"knock_reject_heading": "Acces refuzat",
"reason": "Motivul"
},
"hangup_button_label": "Încheiați apelul",
"header_label": "Element Call Home",
"header_participants_label": "Participanți",
"invite_modal": {
"link_copied_toast": "Link copiat în clipboard",
"title": "Invitați la acest apel"
},
"join_existing_call_modal": {
"join_button": "Da, alăturați-vă apelului",
"text": "Acest apel există deja, doriți să vă alăturați?",
"title": "Alăturați-vă apelului existent?"
},
"layout_grid_label": "GRILĂ",
"layout_spotlight_label": "Spotlight",
"lobby": {
"ask_to_join": "Solicitare de participare la apel",
"join_as_guest": "Alăturați-vă ca invitat",
"join_button": "Alăturați-vă apelului",
"leave_button": "Înapoi la cele mai recente",
"waiting_for_invite": "Solicitare trimisă! În așteptarea permisiunii de a participa..."
},
"log_in": "Autentificare",
"logging_in": "Autentificare...",
"login_auth_links": "<0>Creați un cont </0> sau <2> accesați ca invitat </2>",
"login_auth_links_prompt": "Nu sunteți încă înregistrat?",
"login_subheading": "Pentru a continua la Element",
"login_title": "Logare",
"microphone_off": "Microfon oprit",
"microphone_on": "Microfon pornit",
"mute_microphone_button_label": "Dezactivați microfonul",
"qr_code": "COD QR",
"rageshake_button_error_caption": "Încearcă din nou trimiterea jurnalelor",
"rageshake_request_modal": {
"body": "Un alt utilizator al acestui apel are o problemă. Pentru a diagnostica mai bine aceste probleme, am dori să colectăm un jurnal de depanare.",
"title": "Solicitare jurnal de depanare"
},
"rageshake_send_logs": "Trimiteți jurnale de depanare",
"rageshake_sending": "Trimiterea...",
"rageshake_sending_logs": "Trimiterea jurnalelor de depanare...",
"rageshake_sent": "Multumesc!",
"recaptcha_caption": "Acest site este protejat de reCAPTCHA și se aplică Politica de <2> confidențialitate Google </2> și <6> Termenii și condițiile. </6> <9></9>Făcând clic pe „Înregistrare”, sunteți de acord cu Acordul nostru de licențiere pentru utilizatorul <12> final (EULA) </12>",
"recaptcha_dismissed": "Recaptcha a fost respins",
"recaptcha_not_loaded": "Recaptcha nu a fost încărcat",
"register": {
"passwords_must_match": "Parolele trebuie să se potrivească",
"registering": "Înregistrare..."
},
"register_auth_links": "<0>Ai deja un cont? </0><1><0>Conectați-vă </0> sau <2> accesați ca invitat </2> </1>",
"register_confirm_password_label": "Confirmă Parola",
"register_heading": "Creează-ți contul",
"return_home_button": "Reveniți la ecranul de pornire",
"room_auth_view_continue_button": "Continuă",
"room_auth_view_eula_caption": "Făcând clic pe „Continuați”, sunteți de acord cu Acordul nostru de licențiere pentru utilizatorul <2> final (EULA) </2>",
"screenshare_button_label": "Partajare ecran",
"settings": {
"audio_tab": {
"effect_volume_description": "Reglați volumul la care reacționează reacțiile și efectele ridicate de mână",
"effect_volume_label": "Volumul efectului sonor"
},
"developer_settings_label": "Setări pentru dezvoltatori",
"developer_settings_label_description": "Expuneți setările dezvoltatorului în fereastra de setări.",
"developer_tab_title": "dezvoltator",
"feedback_tab_body": "Dacă întâmpinați probleme sau pur și simplu doriți să oferiți feedback, vă rugăm să ne trimiteți o scurtă descriere mai jos.",
"feedback_tab_description_label": "Feedback-ul tău",
"feedback_tab_h4": "Trimiteți Feedback",
"feedback_tab_send_logs_label": "Includeți jurnale de depanare",
"feedback_tab_thank_you": "Vă mulțumim, am primit feedback-ul dvs.!",
"feedback_tab_title": "Feedback",
"more_tab_title": "Mai mult",
"opt_in_description": "<0></0><1></1>Puteți retrage consimțământul debifând această casetă. Dacă sunteți în prezent la un apel, această setare va intra în vigoare la sfârșitul apelului.",
"preferences_tab": {
"reactions_play_sound_description": "Redați un efect sonor atunci când cineva trimite o reacție la un apel.",
"reactions_play_sound_label": "Redați sunete de reacție",
"reactions_show_description": "Afișați o animație atunci când cineva trimite o reacție.",
"reactions_show_label": "Afișați reacțiile",
"reactions_title": "Reacții"
},
"preferences_tab_body": "Aici puteți configura opțiuni suplimentare pentru o experiență îmbunătățită",
"preferences_tab_h4": "preferinte",
"preferences_tab_show_hand_raised_timer_description": "Afișați un cronometru atunci când un participant ridică mâna",
"preferences_tab_show_hand_raised_timer_label": "Afișați durata ridicării mâinii",
"speaker_device_selection_label": "vorbitor"
},
"start_new_call": "Începe un nou apel",
"start_video_button_label": "Începeți videoclipul",
"stop_screenshare_button_label": "Partajarea ecranului",
"stop_video_button_label": "Opriți videoclipul",
"submitting": "Trimiterea...",
"switch_camera": "Comutați camera",
"unauthenticated_view_body": "Nu sunteți încă înregistrat? <2>Creați un cont </2>",
"unauthenticated_view_eula_caption": "Făcând clic pe „Go”, sunteți de acord cu Acordul nostru de licențiere pentru utilizatorul <2> final (EULA) </2>",
"unauthenticated_view_login_button": "Conectați-vă la contul dvs.",
"unmute_microphone_button_label": "Anulează microfonul",
"version": "{{productName}}Versiune: {{version}}",
"video_tile": {
"always_show": "Arată întotdeauna",
"change_fit_contain": "Se potrivește cadrului",
"collapse": "colaps",
"expand": "Extindeți",
"mute_for_me": "Mute pentru mine",
"muted_for_me": "Dezactivat pentru mine",
"volume": "VOLUM"
}
}

View File

@@ -53,7 +53,7 @@
"@types/grecaptcha": "^3.0.9",
"@types/jsdom": "^21.1.7",
"@types/lodash-es": "^4.17.12",
"@types/node": "^20.0.0",
"@types/node": "^22.0.0",
"@types/pako": "^2.0.3",
"@types/qrcode": "^1.5.5",
"@types/react-dom": "^18.3.0",
@@ -63,7 +63,7 @@
"@typescript-eslint/eslint-plugin": "^8.0.0",
"@typescript-eslint/parser": "^8.0.0",
"@use-gesture/react": "^10.2.11",
"@vector-im/compound-design-tokens": "^1.9.1",
"@vector-im/compound-design-tokens": "^2.0.0",
"@vector-im/compound-web": "^7.2.0",
"@vitejs/plugin-basic-ssl": "^1.0.1",
"@vitejs/plugin-react": "^4.0.1",

View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import "i18next";
// import all namespaces (for the default language, only)
import app from "../../locales/en-GB/app.json";
import app from "../../locales/en/app.json";
declare module "i18next" {
interface CustomTypeOptions {

View File

@@ -22,7 +22,7 @@ export abstract class TranslatedError extends Error {
messageKey: ParseKeys<DefaultNamespace, TOptions>,
translationFn: TFunction<DefaultNamespace>,
) {
super(translationFn(messageKey, { lng: "en-GB" } as TOptions));
super(translationFn(messageKey, { lng: "en" } as TOptions));
this.translatedMessage = translationFn(messageKey);
}
}

View File

@@ -51,7 +51,7 @@ export interface ConfigOptions {
// a livekit service url in the client well-known.
// The well known needs to be formatted like so:
// {"type":"livekit", "livekit_service_url":"https://livekit.example.com"}
// and stored under the key: "livekit_focus"
// and stored under the key: "org.matrix.msc4143.rtc_foci"
livekit_service_url: string;
};

View File

@@ -47,6 +47,11 @@ layer(compound);
--background-gradient: url("graphics/backgroundGradient.svg");
}
:root,
[class*="cpd-theme-"] {
--video-tile-background: var(--cpd-color-bg-subtle-secondary);
}
.cpd-theme-dark {
--cpd-color-border-accent: var(--cpd-color-green-1100);
--stopgap-color-on-solid-accent: var(--cpd-color-text-primary);

View File

@@ -24,7 +24,7 @@ import { platform } from "./Platform";
// This generates a map of locale names to their URL (based on import.meta.url), which looks like this:
// {
// "../locales/en-GB/app.json": "/whatever/assets/root/locales/en-aabbcc.json",
// "../locales/en/app.json": "/whatever/assets/root/locales/en-aabbcc.json",
// ...
// }
const locales = import.meta.glob<string>("../locales/*/*.json", {
@@ -41,7 +41,7 @@ const getLocaleUrl = (
const supportedLngs = [
...new Set(
Object.keys(locales).map((url) => {
// The URLs are of the form ../locales/en-GB/app.json
// The URLs are of the form ../locales/en/app.json
// This extracts the language code from the URL
const lang = url.match(/\/([^/]+)\/[^/]+\.json$/)?.[1];
if (!lang) {
@@ -133,7 +133,7 @@ export class Initializer {
.use(languageDetector)
.use(initReactI18next)
.init({
fallbackLng: "en-GB",
fallbackLng: "en",
defaultNS: "app",
keySeparator: ".",
nsSeparator: false,

View File

@@ -73,6 +73,7 @@ interface InputFieldProps {
defaultValue?: string;
placeholder?: string;
defaultChecked?: boolean;
min?: number;
onChange?: (event: ChangeEvent<HTMLInputElement>) => void;
}
@@ -91,6 +92,7 @@ export const InputField = forwardRef<
suffix,
description,
disabled,
min,
...rest
},
ref,
@@ -127,6 +129,7 @@ export const InputField = forwardRef<
checked={checked}
disabled={disabled}
aria-describedby={descriptionId}
min={min}
{...rest}
/>
)}

View File

@@ -0,0 +1,295 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { beforeEach, expect, test } from "vitest";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { ConnectionState, Room } from "livekit-client";
import { BehaviorSubject, of } from "rxjs";
import { afterEach } from "node:test";
import { act } from "react";
import {
CallMembership,
type MatrixRTCSession,
} from "matrix-js-sdk/src/matrixrtc";
import { soundEffectVolumeSetting } from "../settings/settings";
import {
EmittableMockLivekitRoom,
mockLivekitRoom,
mockLocalParticipant,
mockMatrixRoom,
mockMatrixRoomMember,
mockMediaPlay,
mockRemoteParticipant,
mockRtcMembership,
MockRTCSession,
} from "../utils/test";
import { E2eeType } from "../e2ee/e2eeType";
import { CallViewModel } from "../state/CallViewModel";
import {
CallEventAudioRenderer,
MAX_PARTICIPANT_COUNT_FOR_SOUND,
} from "./CallEventAudioRenderer";
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const local = mockMatrixRoomMember(localRtcMember);
const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
const alice = mockMatrixRoomMember(aliceRtcMember);
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const bob = mockMatrixRoomMember(bobRtcMember);
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
const bobParticipant = mockRemoteParticipant({ identity: bobId });
const originalPlayFn = window.HTMLMediaElement.prototype.play;
const enterSound = "http://localhost:3000/src/sound/join_call.ogg";
const leaveSound = "http://localhost:3000/src/sound/left_call.ogg";
beforeEach(() => {
soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
});
afterEach(() => {
window.HTMLMediaElement.prototype.play = originalPlayFn;
});
test("plays a sound when entering a call", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map(
[local, alice, bob].map((p) => [p.userId, p]),
);
const remoteParticipants = of([aliceParticipant]);
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
{ remoteParticipants },
);
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const session = new MockRTCSession(matrixRoom, localRtcMember, [
aliceRtcMember,
]) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
expect(audioIsPlaying).toEqual([
// Joining the call
enterSound,
]);
});
test("plays no sound when muted", () => {
soundEffectVolumeSetting.setValue(0);
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map(
[local, alice, bob].map((p) => [p.userId, p]),
);
const remoteParticipants = of([aliceParticipant, bobParticipant]);
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
{ remoteParticipants },
);
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const session = new MockRTCSession(matrixRoom, localRtcMember, [
aliceRtcMember,
]) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
// Play a sound when joining a call.
expect(audioIsPlaying).toHaveLength(0);
});
test("plays a sound when a user joins", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
const remoteParticipants = new Map(
[aliceParticipant].map((p) => [p.identity, p]),
);
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
aliceRtcMember,
]);
// we give Bob an RTC session now, but no participant yet
const session = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(
remoteRtcMemberships.asObservable(),
) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom as unknown as Room,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
act(() => {
remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
});
// Play a sound when joining a call.
expect(audioIsPlaying).toEqual([
// Joining the call
enterSound,
// Bob joins
enterSound,
]);
});
test("plays a sound when a user leaves", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
const remoteParticipants = new Map(
[aliceParticipant].map((p) => [p.identity, p]),
);
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
aliceRtcMember,
]);
const session = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom as unknown as Room,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
act(() => {
remoteRtcMemberships.next([]);
});
expect(audioIsPlaying).toEqual([
// Joining the call
enterSound,
// Alice leaves
leaveSound,
]);
});
test("plays no sound when the session member count is larger than the max, until decreased", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
const remoteParticipants = new Map(
[aliceParticipant].map((p) => [p.identity, p]),
);
const mockRtcMemberships: CallMembership[] = [];
for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) {
mockRtcMemberships.push(
mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`),
);
}
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
mockRtcMemberships,
);
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const session = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom as unknown as Room,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
expect(audioIsPlaying).toEqual([]);
// When the count drops to the max we should play the leave sound
act(() => {
remoteRtcMemberships.next(
mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1),
);
});
expect(audioIsPlaying).toEqual([leaveSound]);
});

View File

@@ -0,0 +1,117 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ReactNode, useEffect, useRef } from "react";
import { filter } from "rxjs";
import {
soundEffectVolumeSetting as effectSoundVolumeSetting,
useSetting,
} from "../settings/settings";
import { CallViewModel } from "../state/CallViewModel";
import enterCallSoundMp3 from "../sound/join_call.mp3";
import enterCallSoundOgg from "../sound/join_call.ogg";
import leftCallSoundMp3 from "../sound/left_call.mp3";
import leftCallSoundOgg from "../sound/left_call.ogg";
// Do not play any sounds if the participant count has exceeded this
// number.
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const CONCURRENT_AUDIO_CHANNELS = 2;
export function CallEventAudioRenderer({
vm,
}: {
vm: CallViewModel;
}): ReactNode {
const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
const callEntered = useRef<(HTMLAudioElement | null)[]>([]);
const callLeft = useRef<(HTMLAudioElement | null)[]>([]);
useEffect(() => {
if (effectSoundVolume === 0) {
return;
}
const joinSub = vm.memberChanges
.pipe(
filter(
({ joined, ids }) =>
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0,
),
)
.subscribe(({ joined }) => {
const availablePlayer = callEntered.current.find((v) => v?.paused);
void availablePlayer?.play();
});
const leftSub = vm.memberChanges
.pipe(
filter(
({ ids, left }) =>
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
),
)
.subscribe(() => {
const availablePlayer = callLeft.current.find((v) => v?.paused);
void availablePlayer?.play();
});
return (): void => {
joinSub.unsubscribe();
leftSub.unsubscribe();
};
}, [effectSoundVolume, callEntered, callLeft, vm]);
// Set volume.
useEffect(() => {
callEntered.current.forEach((a) => {
if (a) {
a.volume = effectSoundVolume;
}
});
callLeft.current.forEach((a) => {
if (a) {
a.volume = effectSoundVolume;
}
});
}, [callEntered, callLeft, effectSoundVolume]);
// Do not render any audio elements if playback is disabled. Will save
// audio file fetches.
if (effectSoundVolume === 0) {
return null;
}
return (
// Will play as soon as it's mounted, which is what we want as this will
// play when the call is entered.
<>
{Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
<audio
key={index}
ref={(r) => (callEntered.current[index] = r)}
preload="auto"
hidden
>
<source src={enterCallSoundOgg} type="audio/ogg; codecs=vorbis" />
<source src={enterCallSoundMp3} type="audio/mpeg" />
</audio>
))}
{Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
<audio
key={index}
ref={(r) => (callLeft.current[index] = r)}
preload="auto"
hidden
>
<source src={leftCallSoundOgg} type="audio/ogg; codecs=vorbis" />
<source src={leftCallSoundMp3} type="audio/mpeg" />
</audio>
))}
</>
);
}

View File

@@ -87,6 +87,7 @@ import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import { useSwitchCamera } from "./useSwitchCamera";
import { soundEffectVolumeSetting, useSetting } from "../settings/settings";
import { ReactionsOverlay } from "./ReactionsOverlay";
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@@ -123,7 +124,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
useEffect(() => {
if (livekitRoom !== undefined) {
const vm = new CallViewModel(
props.rtcSession.room,
props.rtcSession,
livekitRoom,
props.e2eeSystem,
connStateObservable,
@@ -131,12 +132,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
setVm(vm);
return (): void => vm.destroy();
}
}, [
props.rtcSession.room,
livekitRoom,
props.e2eeSystem,
connStateObservable,
]);
}, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable]);
if (livekitRoom === undefined || vm === null) return null;
@@ -670,6 +666,7 @@ export const InCallView: FC<InCallViewProps> = ({
))}
<RoomAudioRenderer />
{renderContent()}
<CallEventAudioRenderer vm={vm} />
<audio ref={handRaisePlayer} preload="auto" hidden>
<source src={handSoundOgg} type="audio/ogg; codecs=vorbis" />
<source src={handSoundMp3} type="audio/mpeg" />

View File

@@ -21,6 +21,7 @@ import {
playReactionsSound,
soundEffectVolumeSetting,
} from "../settings/settings";
import { mockMediaPlay } from "../utils/test";
const memberUserIdAlice = "@alice:example.org";
const memberUserIdBob = "@bob:example.org";
@@ -80,11 +81,7 @@ test("loads no audio elements when disabled in settings", () => {
});
test("will play an audio sound when there is a reaction", () => {
const audioIsPlaying: string[] = [];
window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
return Promise.resolve();
};
const audioIsPlaying: string[] = mockMediaPlay();
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
@@ -105,11 +102,7 @@ test("will play an audio sound when there is a reaction", () => {
});
test("will play the generic audio sound when there is soundless reaction", () => {
const audioIsPlaying: string[] = [];
window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
return Promise.resolve();
};
const audioIsPlaying: string[] = mockMediaPlay();
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
@@ -152,11 +145,7 @@ test("will play an audio sound with the correct volume", () => {
});
test("will play multiple audio sounds when there are multiple different reactions", () => {
const audioIsPlaying: string[] = [];
window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
return Promise.resolve();
};
const audioIsPlaying: string[] = mockMediaPlay();
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);

View File

@@ -18,8 +18,7 @@ Please see LICENSE in the repository root for full details.
width: 100%;
height: 100%;
object-fit: cover;
background-color: black;
background-color: var(--cpd-color-bg-subtle-primary);
background-color: var(--video-tile-background);
}
video.mirror {
@@ -35,7 +34,7 @@ video.mirror {
display: flex;
justify-content: center;
align-items: center;
background-color: var(--cpd-color-bg-subtle-secondary);
background-color: var(--video-tile-background);
}
.buttonBar {

View File

@@ -195,6 +195,11 @@ export const SettingsModal: FC<Props> = ({
name: t("settings.developer_tab_title"),
content: (
<>
<p>
{t("developer_mode.hostname", {
hostname: window.location.hostname || "unknown",
})}
</p>
<p>
{t("version", {
productName: import.meta.env.VITE_PRODUCT_NAME || "Element Call",
@@ -202,17 +207,17 @@ export const SettingsModal: FC<Props> = ({
})}
</p>
<p>
{t("crypto_version", {
{t("developer_mode.crypto_version", {
version: client.getCrypto()?.getVersion() || "unknown",
})}
</p>
<p>
{t("matrix_id", {
{t("developer_mode.matrix_id", {
id: client.getUserId() || "unknown",
})}
</p>
<p>
{t("device_id", {
{t("developer_mode.device_id", {
id: client.getDeviceId() || "unknown",
})}
</p>
@@ -220,11 +225,15 @@ export const SettingsModal: FC<Props> = ({
<InputField
id="duplicateTiles"
type="number"
label={t("settings.duplicate_tiles_label")}
label={t("developer_mode.duplicate_tiles_label")}
value={duplicateTiles.toString()}
min={0}
onChange={useCallback(
(event: ChangeEvent<HTMLInputElement>): void => {
const value = event.target.valueAsNumber;
if (value < 0) {
return;
}
setDuplicateTiles(Number.isNaN(value) ? 0 : value);
},
[setDuplicateTiles],

View File

@@ -19,4 +19,6 @@ The following sound effects have been originally created by Element.
- `end_talk`
- `start_talk_local`
- `start_talk_remote`
- `join_call`
- `end_call`
- `reactions/rock`

BIN
src/sound/join_call.mp3 Normal file

Binary file not shown.

BIN
src/sound/join_call.ogg Normal file

Binary file not shown.

BIN
src/sound/left_call.mp3 Normal file

Binary file not shown.

BIN
src/sound/left_call.ogg Normal file

Binary file not shown.

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { test, vi, onTestFinished } from "vitest";
import { test, vi, onTestFinished, it } from "vitest";
import {
combineLatest,
debounceTime,
@@ -25,6 +25,7 @@ import {
} from "livekit-client";
import * as ComponentsCore from "@livekit/components-core";
import { isEqual } from "lodash-es";
import { CallMembership, MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";
import { CallViewModel, Layout } from "./CallViewModel";
import {
@@ -34,6 +35,8 @@ import {
mockMatrixRoomMember,
mockRemoteParticipant,
withTestScheduler,
mockRtcMembership,
MockRTCSession,
} from "../utils/test";
import {
ECAddonConnectionState,
@@ -43,14 +46,19 @@ import { E2eeType } from "../e2ee/e2eeType";
vi.mock("@livekit/components-core");
const alice = mockMatrixRoomMember({ userId: "@alice:example.org" });
const bob = mockMatrixRoomMember({ userId: "@bob:example.org" });
const carol = mockMatrixRoomMember({ userId: "@carol:example.org" });
const dave = mockMatrixRoomMember({ userId: "@dave:example.org" });
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD");
const aliceId = `${alice.userId}:AAAA`;
const bobId = `${bob.userId}:BBBB`;
const daveId = `${dave.userId}:DDDD`;
const alice = mockMatrixRoomMember(aliceRtcMember);
const bob = mockMatrixRoomMember(bobRtcMember);
const carol = mockMatrixRoomMember(localRtcMember);
const dave = mockMatrixRoomMember(daveRtcMember);
const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
const daveId = `${dave.userId}:${daveRtcMember.deviceId}`;
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
@@ -65,7 +73,9 @@ const bobSharingScreen = mockRemoteParticipant({
});
const daveParticipant = mockRemoteParticipant({ identity: daveId });
const members = new Map([alice, bob, carol, dave].map((p) => [p.userId, p]));
const roomMembers = new Map(
[alice, bob, carol, dave].map((p) => [p.userId, p]),
);
export interface GridLayoutSummary {
type: "grid";
@@ -173,10 +183,23 @@ function summarizeLayout(l: Observable<Layout>): Observable<LayoutSummary> {
function withCallViewModel(
remoteParticipants: Observable<RemoteParticipant[]>,
rtcMembers: Observable<Partial<CallMembership>[]>,
connectionState: Observable<ECConnectionState>,
speaking: Map<Participant, Observable<boolean>>,
continuation: (vm: CallViewModel) => void,
): void {
const room = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => roomMembers.get(userId) ?? null,
});
const rtcSession = new MockRTCSession(
room,
localRtcMember,
[],
).withMemberships(rtcMembers);
const participantsSpy = vi
.spyOn(ComponentsCore, "connectedParticipantsObserver")
.mockReturnValue(remoteParticipants);
@@ -209,12 +232,7 @@ function withCallViewModel(
);
const vm = new CallViewModel(
mockMatrixRoom({
client: {
getUserId: () => "@carol:example.org",
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => members.get(userId) ?? null,
}),
rtcSession as unknown as MatrixRTCSession,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
@@ -247,6 +265,7 @@ test("participants are retained during a focus switch", () => {
a: [aliceParticipant, bobParticipant],
b: [],
}),
of([aliceRtcMember, bobRtcMember]),
hot(connectionInputMarbles, {
c: ConnectionState.Connected,
s: ECAddonConnectionState.ECSwitchingFocus,
@@ -288,6 +307,7 @@ test("screen sharing activates spotlight layout", () => {
c: [aliceSharingScreen, bobSharingScreen],
d: [aliceParticipant, bobSharingScreen],
}),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
@@ -356,7 +376,7 @@ test("participants stay in the same order unless to appear/disappear", () => {
const modeInputMarbles = " a";
// First Bob speaks, then Dave, then Alice
const aSpeakingInputMarbles = "n- 1998ms - 1999ms y";
const bSpeakingInputMarbles = "ny 1998ms n 1999ms ";
const bSpeakingInputMarbles = "ny 1998ms n 1999ms -";
const dSpeakingInputMarbles = "n- 1998ms y 1999ms n";
// Nothing should change when Bob speaks, because Bob is already on screen.
// When Dave speaks he should switch with Alice because she's the one who
@@ -366,6 +386,7 @@ test("participants stay in the same order unless to appear/disappear", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })],
@@ -427,6 +448,7 @@ test("spotlight speakers swap places", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })],
@@ -475,6 +497,7 @@ test("layout enters picture-in-picture mode when requested", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
@@ -515,6 +538,7 @@ test("spotlight remembers whether it's expanded", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
@@ -559,3 +583,104 @@ test("spotlight remembers whether it's expanded", () => {
);
});
});
test("participants must have a MatrixRTCSession to be visible", () => {
withTestScheduler(({ hot, expectObservable }) => {
// iterate through a number of combinations of participants and MatrixRTC memberships
// Bob never has an MatrixRTC membership
const scenarioInputMarbles = " abcdec";
// Bob should never be visible
const expectedLayoutMarbles = "a-bc-b";
withCallViewModel(
hot(scenarioInputMarbles, {
a: [],
b: [bobParticipant],
c: [aliceParticipant, bobParticipant],
d: [aliceParticipant, daveParticipant, bobParticipant],
e: [aliceParticipant, daveParticipant, bobSharingScreen],
}),
hot(scenarioInputMarbles, {
a: [],
b: [],
c: [aliceRtcMember],
d: [aliceRtcMember, daveRtcMember],
e: [aliceRtcMember, daveRtcMember],
}),
of(ConnectionState.Connected),
new Map(),
(vm) => {
vm.setGridMode("grid");
expectObservable(summarizeLayout(vm.layout)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0"],
},
b: {
type: "one-on-one",
local: "local:0",
remote: `${aliceId}:0`,
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
},
},
);
},
);
});
});
it("should show at least one tile per MatrixRTCSession", () => {
withTestScheduler(({ hot, expectObservable }) => {
// iterate through some combinations of MatrixRTC memberships
const scenarioInputMarbles = " abcd";
// There should always be one tile for each MatrixRTCSession
const expectedLayoutMarbles = "abcd";
withCallViewModel(
of([]),
hot(scenarioInputMarbles, {
a: [],
b: [aliceRtcMember],
c: [aliceRtcMember, daveRtcMember],
d: [daveRtcMember],
}),
of(ConnectionState.Connected),
new Map(),
(vm) => {
vm.setGridMode("grid");
expectObservable(summarizeLayout(vm.layout)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0"],
},
b: {
type: "one-on-one",
local: "local:0",
remote: `${aliceId}:0`,
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
},
d: {
type: "one-on-one",
local: "local:0",
remote: `${daveId}:0`,
},
},
);
},
);
});
});

View File

@@ -18,12 +18,9 @@ import {
RemoteParticipant,
Track,
} from "livekit-client";
import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix";
import {
Room as MatrixRoom,
RoomMember,
RoomStateEvent,
} from "matrix-js-sdk/src/matrix";
import {
BehaviorSubject,
EMPTY,
Observable,
Subject,
@@ -49,6 +46,10 @@ import {
withLatestFrom,
} from "rxjs";
import { logger } from "matrix-js-sdk/src/logger";
import {
MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/src/matrixrtc";
import { ViewModel } from "./ViewModel";
import {
@@ -222,41 +223,67 @@ interface LayoutScanState {
class UserMedia {
private readonly scope = new ObservableScope();
public readonly vm: UserMediaViewModel;
private readonly participant: BehaviorSubject<
LocalParticipant | RemoteParticipant | undefined
>;
public readonly speaker: Observable<boolean>;
public readonly presenter: Observable<boolean>;
public constructor(
public readonly id: string,
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
this.vm = participant.isLocal
? new LocalUserMediaViewModel(
id,
member,
participant as LocalParticipant,
encryptionSystem,
livekitRoom,
)
: new RemoteUserMediaViewModel(
id,
member,
participant as RemoteParticipant,
encryptionSystem,
livekitRoom,
);
this.participant = new BehaviorSubject(participant);
if (participant?.isLocal) {
this.vm = new LocalUserMediaViewModel(
this.id,
member,
this.participant.asObservable() as Observable<LocalParticipant>,
encryptionSystem,
livekitRoom,
);
} else {
this.vm = new RemoteUserMediaViewModel(
id,
member,
this.participant.asObservable() as Observable<
RemoteParticipant | undefined
>,
encryptionSystem,
livekitRoom,
);
}
this.speaker = observeSpeaker(this.vm.speaking).pipe(this.scope.state());
this.presenter = observeParticipantEvents(
participant,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
this.presenter = this.participant.pipe(
switchMap(
(p) =>
(p &&
observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled))) ??
of(false),
),
this.scope.state(),
);
}
public updateParticipant(
newParticipant: LocalParticipant | RemoteParticipant | undefined,
): void {
if (this.participant.value !== newParticipant) {
// Update the BehaviourSubject in the UserMedia.
this.participant.next(newParticipant);
}
}
public destroy(): void {
@@ -267,6 +294,9 @@ class UserMedia {
class ScreenShare {
public readonly vm: ScreenShareViewModel;
private readonly participant: BehaviorSubject<
LocalParticipant | RemoteParticipant
>;
public constructor(
id: string,
@@ -275,12 +305,15 @@ class ScreenShare {
encryptionSystem: EncryptionSystem,
liveKitRoom: LivekitRoom,
) {
this.participant = new BehaviorSubject(participant);
this.vm = new ScreenShareViewModel(
id,
member,
participant,
this.participant.asObservable(),
encryptionSystem,
liveKitRoom,
participant.isLocal,
);
}
@@ -317,11 +350,11 @@ function findMatrixRoomMember(
export class CallViewModel extends ViewModel {
public readonly localVideo: Observable<LocalVideoTrack | null> =
observeTrackReference(
this.livekitRoom.localParticipant,
of(this.livekitRoom.localParticipant),
Track.Source.Camera,
).pipe(
map((trackRef) => {
const track = trackRef.publication?.track;
const track = trackRef?.publication?.track;
return track instanceof LocalVideoTrack ? track : null;
}),
);
@@ -401,49 +434,87 @@ export class CallViewModel extends ViewModel {
this.remoteParticipants,
observeParticipantMedia(this.livekitRoom.localParticipant),
duplicateTiles.value,
// Also react to changes in the list of members
fromEvent(this.matrixRoom, RoomStateEvent.Update).pipe(startWith(null)),
// Also react to changes in the MatrixRTC session list.
// The session list will also be update if a room membership changes.
// No additional RoomState event listener needs to be set up.
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
).pipe(startWith(null)),
]).pipe(
scan(
(
prevItems,
[remoteParticipants, { participant: localParticipant }, duplicateTiles],
[
remoteParticipants,
{ participant: localParticipant },
duplicateTiles,
_membershipsChanged,
],
) => {
const newItems = new Map(
function* (this: CallViewModel): Iterable<[string, MediaItem]> {
for (const p of [localParticipant, ...remoteParticipants]) {
const id = p === localParticipant ? "local" : p.identity;
const member = findMatrixRoomMember(this.matrixRoom, id);
if (member === undefined)
logger.warn(
`Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`,
);
// m.rtc.members are the basis for calculating what is visible in the call
for (const rtcMember of this.matrixRTCSession.memberships) {
const room = this.matrixRTCSession.room;
// WARN! This is not exactly the sender but the user defined in the state key.
// This will be available once we change to the new "member as object" format in the MatrixRTC object.
let livekitParticipantId =
rtcMember.sender + ":" + rtcMember.deviceId;
// Create as many tiles for this participant as called for by
// the duplicateTiles option
let participant:
| LocalParticipant
| RemoteParticipant
| undefined = undefined;
if (
rtcMember.sender === room.client.getUserId()! &&
rtcMember.deviceId === room.client.getDeviceId()
) {
livekitParticipantId = "local";
participant = localParticipant;
} else {
participant = remoteParticipants.find(
(p) => p.identity === livekitParticipantId,
);
}
const member = findMatrixRoomMember(room, livekitParticipantId);
if (!member) {
logger.error(
"Could not find member for media id: ",
livekitParticipantId,
);
}
for (let i = 0; i < 1 + duplicateTiles; i++) {
const userMediaId = `${id}:${i}`;
const indexedMediaId = `${livekitParticipantId}:${i}`;
const prevMedia = prevItems.get(indexedMediaId);
if (prevMedia && prevMedia instanceof UserMedia) {
prevMedia.updateParticipant(participant);
}
yield [
userMediaId,
prevItems.get(userMediaId) ??
indexedMediaId,
// We create UserMedia with or without a participant.
// This will be the initial value of a BehaviourSubject.
// Once a participant appears we will update the BehaviourSubject. (see above)
prevMedia ??
new UserMedia(
userMediaId,
indexedMediaId,
member,
p,
participant,
this.encryptionSystem,
this.livekitRoom,
),
];
if (p.isScreenShareEnabled) {
const screenShareId = `${userMediaId}:screen-share`;
if (participant?.isScreenShareEnabled) {
const screenShareId = `${indexedMediaId}:screen-share`;
yield [
screenShareId,
prevItems.get(screenShareId) ??
new ScreenShare(
screenShareId,
member,
p,
participant,
this.encryptionSystem,
this.livekitRoom,
),
@@ -454,7 +525,6 @@ export class CallViewModel extends ViewModel {
}.bind(this)(),
);
for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy();
return newItems;
},
new Map<string, MediaItem>(),
@@ -475,9 +545,17 @@ export class CallViewModel extends ViewModel {
),
);
private readonly localUserMedia: Observable<LocalUserMediaViewModel> =
this.mediaItems.pipe(
map((ms) => ms.find((m) => m.vm.local)!.vm as LocalUserMediaViewModel),
public readonly memberChanges = this.userMedia
.pipe(map((mediaItems) => mediaItems.map((m) => m.id)))
.pipe(
scan<string[], { ids: string[]; joined: string[]; left: string[] }>(
(prev, ids) => {
const left = prev.ids.filter((id) => !ids.includes(id));
const joined = ids.filter((id) => !prev.ids.includes(id));
return { ids, joined, left };
},
{ ids: [], joined: [], left: [] },
),
);
/**
@@ -491,7 +569,7 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
private readonly spotlightSpeaker: Observable<UserMediaViewModel> =
private readonly spotlightSpeaker: Observable<UserMediaViewModel | null> =
this.userMedia.pipe(
switchMap((mediaItems) =>
mediaItems.length === 0
@@ -502,7 +580,7 @@ export class CallViewModel extends ViewModel {
),
),
),
scan<(readonly [UserMedia, boolean])[], UserMedia, null>(
scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>(
(prev, mediaItems) => {
// Only remote users that are still in the call should be sticky
const [stickyMedia, stickySpeaking] =
@@ -519,11 +597,11 @@ export class CallViewModel extends ViewModel {
// Otherwise, spotlight an arbitrary remote user
mediaItems.find(([m]) => !m.vm.local)?.[0] ??
// Otherwise, spotlight the local user
mediaItems.find(([m]) => m.vm.local)![0]);
mediaItems.find(([m]) => m.vm.local)?.[0]);
},
null,
),
map((speaker) => speaker.vm),
map((speaker) => speaker?.vm ?? null),
this.scope.state(),
);
@@ -563,37 +641,57 @@ export class CallViewModel extends ViewModel {
}),
);
private readonly spotlightAndPip: Observable<
[Observable<MediaViewModel[]>, Observable<UserMediaViewModel | null>]
> = this.screenShares.pipe(
map((screenShares) =>
screenShares.length > 0
? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const)
: ([
this.spotlightSpeaker.pipe(map((speaker) => [speaker!])),
this.spotlightSpeaker.pipe(
switchMap((speaker) =>
speaker.local
? of(null)
: this.localUserMedia.pipe(
switchMap((vm) =>
vm.alwaysShow.pipe(
map((alwaysShow) => (alwaysShow ? vm : null)),
),
),
),
),
),
] as const),
),
);
private readonly spotlight: Observable<MediaViewModel[]> =
this.spotlightAndPip.pipe(
switchMap(([spotlight]) => spotlight),
this.screenShares.pipe(
switchMap((screenShares) => {
if (screenShares.length > 0) {
return of(screenShares.map((m) => m.vm));
}
return this.spotlightSpeaker.pipe(
map((speaker) => (speaker ? [speaker] : [])),
);
}),
this.scope.state(),
);
private readonly pip: Observable<UserMediaViewModel | null> = combineLatest([
this.screenShares,
this.spotlightSpeaker,
this.mediaItems,
]).pipe(
switchMap(([screenShares, spotlight, mediaItems]) => {
if (screenShares.length > 0) {
return this.spotlightSpeaker;
}
if (!spotlight || spotlight.local) {
return of(null);
}
const localUserMedia = mediaItems.find(
(m) => m.vm instanceof LocalUserMediaViewModel,
) as UserMedia | undefined;
const localUserMediaViewModel = localUserMedia?.vm as
| LocalUserMediaViewModel
| undefined;
if (!localUserMediaViewModel) {
return of(null);
}
return localUserMediaViewModel.alwaysShow.pipe(
map((alwaysShow) => {
if (alwaysShow) {
return localUserMediaViewModel;
}
return null;
}),
);
}),
this.scope.state(),
);
private readonly hasRemoteScreenShares: Observable<boolean> =
this.spotlight.pipe(
map((spotlight) =>
@@ -602,9 +700,6 @@ export class CallViewModel extends ViewModel {
distinctUntilChanged(),
);
private readonly pip: Observable<UserMediaViewModel | null> =
this.spotlightAndPip.pipe(switchMap(([, pip]) => pip));
private readonly pipEnabled: Observable<boolean> = setPipEnabled.pipe(
startWith(false),
);
@@ -708,15 +803,16 @@ export class CallViewModel extends ViewModel {
this.mediaItems.pipe(
map((mediaItems) => {
if (mediaItems.length !== 2) return null;
const local = mediaItems.find((vm) => vm.vm.local)!
.vm as LocalUserMediaViewModel;
const local = mediaItems.find((vm) => vm.vm.local)?.vm as
| LocalUserMediaViewModel
| undefined;
const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as
| RemoteUserMediaViewModel
| undefined;
// There might not be a remote tile if there are screen shares, or if
// only the local user is in the call and they're using the duplicate
// tiles option
if (remote === undefined) return null;
if (!remote || !local) return null;
return { type: "one-on-one", local, remote };
}),
@@ -997,7 +1093,7 @@ export class CallViewModel extends ViewModel {
public constructor(
// A call is permanently tied to a single Matrix room and LiveKit room
private readonly matrixRoom: MatrixRoom,
private readonly matrixRTCSession: MatrixRTCSession,
private readonly livekitRoom: LivekitRoom,
private readonly encryptionSystem: EncryptionSystem,
private readonly connectionState: Observable<ECConnectionState>,

View File

@@ -8,14 +8,17 @@ Please see LICENSE in the repository root for full details.
import { expect, test, vi } from "vitest";
import {
mockRtcMembership,
withLocalMedia,
withRemoteMedia,
withTestScheduler,
} from "../utils/test";
const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
test("control a participant's volume", async () => {
const setVolumeSpy = vi.fn();
await withRemoteMedia({}, { setVolume: setVolumeSpy }, (vm) =>
await withRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy }, (vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab---c---d|", {
a() {
@@ -60,7 +63,7 @@ test("control a participant's volume", async () => {
});
test("toggle fit/contain for a participant's video", async () => {
await withRemoteMedia({}, {}, (vm) =>
await withRemoteMedia(rtcMembership, {}, {}, (vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab|", {
a: () => vm.toggleFitContain(),
@@ -76,17 +79,21 @@ test("toggle fit/contain for a participant's video", async () => {
});
test("local media remembers whether it should always be shown", async () => {
await withLocalMedia({}, (vm) =>
await withLocalMedia(rtcMembership, {}, (vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm.setAlwaysShow(false) });
expectObservable(vm.alwaysShow).toBe("ab", { a: true, b: false });
}),
);
// Next local media should start out *not* always shown
await withLocalMedia({}, (vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm.setAlwaysShow(true) });
expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true });
}),
await withLocalMedia(
rtcMembership,
{},
(vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm.setAlwaysShow(true) });
expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true });
}),
);
});

View File

@@ -32,7 +32,6 @@ import {
Observable,
Subject,
combineLatest,
distinctUntilChanged,
distinctUntilKeyChanged,
filter,
fromEvent,
@@ -40,7 +39,6 @@ import {
map,
merge,
of,
shareReplay,
startWith,
switchMap,
throttleTime,
@@ -77,16 +75,24 @@ export function useDisplayName(vm: MediaViewModel): string {
}
export function observeTrackReference(
participant: Participant,
participant: Observable<Participant | undefined>,
source: Track.Source,
): Observable<TrackReferenceOrPlaceholder> {
return observeParticipantMedia(participant).pipe(
map(() => ({
participant,
publication: participant.getTrackPublication(source),
source,
})),
distinctUntilKeyChanged("publication"),
): Observable<TrackReferenceOrPlaceholder | undefined> {
return participant.pipe(
switchMap((p) => {
if (p) {
return observeParticipantMedia(p).pipe(
map(() => ({
participant: p,
publication: p.getTrackPublication(source),
source,
})),
distinctUntilKeyChanged("publication"),
);
} else {
return of(undefined);
}
}),
);
}
@@ -105,11 +111,11 @@ function observeRemoteTrackReceivingOkay(
};
return combineLatest([
observeTrackReference(participant, source),
observeTrackReference(of(participant), source),
interval(1000).pipe(startWith(0)),
]).pipe(
switchMap(async ([trackReference]) => {
const track = trackReference.publication?.track;
const track = trackReference?.publication?.track;
if (!track || !(track instanceof RemoteTrack)) {
return undefined;
}
@@ -200,14 +206,10 @@ export enum EncryptionStatus {
}
abstract class BaseMediaViewModel extends ViewModel {
/**
* Whether the media belongs to the local user.
*/
public readonly local = this.participant.isLocal;
/**
* The LiveKit video track for this media.
*/
public readonly video: Observable<TrackReferenceOrPlaceholder>;
public readonly video: Observable<TrackReferenceOrPlaceholder | undefined>;
/**
* Whether there should be a warning that this media is unencrypted.
*/
@@ -215,6 +217,11 @@ abstract class BaseMediaViewModel extends ViewModel {
public readonly encryptionStatus: Observable<EncryptionStatus>;
/**
* Whether this media corresponds to the local participant.
*/
public abstract readonly local: boolean;
public constructor(
/**
* An opaque identifier for this media.
@@ -226,7 +233,12 @@ abstract class BaseMediaViewModel extends ViewModel {
// TODO: Fully separate the data layer from the UI layer by keeping the
// member object internal
public readonly member: RoomMember | undefined,
protected readonly participant: LocalParticipant | RemoteParticipant,
// We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through
// livekit.
protected readonly participant: Observable<
LocalParticipant | RemoteParticipant | undefined
>,
encryptionSystem: EncryptionSystem,
audioSource: AudioSource,
videoSource: VideoSource,
@@ -243,69 +255,72 @@ abstract class BaseMediaViewModel extends ViewModel {
[audio, this.video],
(a, v) =>
encryptionSystem.kind !== E2eeType.NONE &&
(a.publication?.isEncrypted === false ||
v.publication?.isEncrypted === false),
).pipe(
distinctUntilChanged(),
shareReplay({ bufferSize: 1, refCount: false }),
);
(a?.publication?.isEncrypted === false ||
v?.publication?.isEncrypted === false),
).pipe(this.scope.state());
if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) {
this.encryptionStatus = of(EncryptionStatus.Okay).pipe(
this.scope.state(),
);
} else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
this.encryptionStatus = combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"MissingKey",
),
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
if (keyMissing) return EncryptionStatus.KeyMissing;
if (keyInvalid) return EncryptionStatus.KeyInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
}),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
this.scope.state(),
);
} else {
this.encryptionStatus = combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(
([keyInvalid, audioOkay, videoOkay]):
| EncryptionStatus
| undefined => {
if (keyInvalid) return EncryptionStatus.PasswordInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
},
),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
this.scope.state(),
);
}
this.encryptionStatus = this.participant.pipe(
switchMap((participant): Observable<EncryptionStatus> => {
if (!participant) {
return of(EncryptionStatus.Connecting);
} else if (
participant.isLocal ||
encryptionSystem.kind === E2eeType.NONE
) {
return of(EncryptionStatus.Okay);
} else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"MissingKey",
),
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
if (keyMissing) return EncryptionStatus.KeyMissing;
if (keyInvalid) return EncryptionStatus.KeyInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
}),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
);
} else {
return combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(
([keyInvalid, audioOkay, videoOkay]):
| EncryptionStatus
| undefined => {
if (keyInvalid) return EncryptionStatus.PasswordInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
},
),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
);
}
}),
this.scope.state(),
);
}
}
@@ -324,11 +339,14 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
/**
* Whether the participant is speaking.
*/
public readonly speaking = observeParticipantEvents(
this.participant,
ParticipantEvent.IsSpeakingChanged,
).pipe(
map((p) => p.isSpeaking),
public readonly speaking = this.participant.pipe(
switchMap((p) =>
p
? observeParticipantEvents(p, ParticipantEvent.IsSpeakingChanged).pipe(
map((p) => p.isSpeaking),
)
: of(false),
),
this.scope.state(),
);
@@ -350,7 +368,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
public constructor(
id: string,
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
participant: Observable<LocalParticipant | RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
@@ -364,18 +382,25 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
livekitRoom,
);
const media = observeParticipantMedia(participant).pipe(this.scope.state());
const media = participant.pipe(
switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)),
this.scope.state(),
);
this.audioEnabled = media.pipe(
map((m) => m.microphoneTrack?.isMuted === false),
map((m) => m?.microphoneTrack?.isMuted === false),
);
this.videoEnabled = media.pipe(
map((m) => m.cameraTrack?.isMuted === false),
map((m) => m?.cameraTrack?.isMuted === false),
);
}
public toggleFitContain(): void {
this._cropVideo.next(!this._cropVideo.value);
}
public get local(): boolean {
return this instanceof LocalUserMediaViewModel;
}
}
/**
@@ -387,7 +412,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
*/
public readonly mirror = this.video.pipe(
switchMap((v) => {
const track = v.publication?.track;
const track = v?.publication?.track;
if (!(track instanceof LocalTrack)) return of(false);
// Watch for track restarts, because they indicate a camera switch
return fromEvent(track, TrackEvent.Restarted).pipe(
@@ -409,7 +434,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
public constructor(
id: string,
member: RoomMember | undefined,
participant: LocalParticipant,
participant: Observable<LocalParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
@@ -470,18 +495,17 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
public constructor(
id: string,
member: RoomMember | undefined,
participant: RemoteParticipant,
participant: Observable<RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(id, member, participant, encryptionSystem, livekitRoom);
// Sync the local volume with LiveKit
this.localVolume
.pipe(this.scope.bind())
.subscribe((volume) =>
(this.participant as RemoteParticipant).setVolume(volume),
);
combineLatest([
participant,
this.localVolume.pipe(this.scope.bind()),
]).subscribe(([p, volume]) => p && p.setVolume(volume));
}
public toggleLocallyMuted(): void {
@@ -504,9 +528,10 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
public constructor(
id: string,
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
participant: Observable<LocalParticipant | RemoteParticipant>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
public readonly local: boolean,
) {
super(
id,

View File

@@ -13,7 +13,7 @@ import { of } from "rxjs";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { GridTile } from "./GridTile";
import { withRemoteMedia } from "../utils/test";
import { mockRtcMembership, withRemoteMedia } from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsProvider } from "../useReactions";
@@ -25,6 +25,7 @@ global.IntersectionObserver = class MockIntersectionObserver {
test("GridTile is accessible", async () => {
await withRemoteMedia(
mockRtcMembership("@alice:example.org", "AAAA"),
{
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",

View File

@@ -175,6 +175,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
raisedHandTime={handRaised}
currentReaction={currentReaction}
raisedHandOnClick={raisedHandOnClick}
localParticipant={vm.local}
{...props}
/>
);

View File

@@ -15,7 +15,7 @@ Please see LICENSE in the repository root for full details.
inline-size: 100%;
block-size: 100%;
object-fit: contain;
background-color: var(--cpd-color-bg-subtle-primary);
background-color: var(--video-tile-background);
/* This transform is a no-op, but it forces Firefox to use a different
rendering path, one that actually clips the corners of <video> elements into
the intended rounded shape. We can remove this if Firefox stops being broken. */
@@ -35,7 +35,7 @@ Please see LICENSE in the repository root for full details.
}
.bg {
background-color: var(--cpd-color-bg-subtle-secondary);
background-color: var(--video-tile-background);
inline-size: 100%;
block-size: 100%;
border-radius: inherit;
@@ -74,9 +74,9 @@ unconditionally select the container so we can use cqmin units */
calc(var(--media-view-border-radius) - var(--cpd-space-3x))
);
display: grid;
grid-template-columns: 1fr auto;
grid-template-columns: 30px 1fr 30px;
grid-template-rows: 1fr auto;
grid-template-areas: "status status" "nameTag button";
grid-template-areas: "reactions status ." "nameTag nameTag button";
gap: var(--cpd-space-1x);
place-items: start;
}
@@ -101,8 +101,8 @@ unconditionally select the container so we can use cqmin units */
grid-area: status;
justify-self: center;
align-self: start;
padding: var(--cpd-space-1x);
padding-block: var(--cpd-space-1x);
padding: var(--cpd-space-2x);
padding-block: var(--cpd-space-2x);
color: var(--cpd-color-text-primary);
background-color: var(--cpd-color-bg-canvas-default);
display: flex;
@@ -116,6 +116,12 @@ unconditionally select the container so we can use cqmin units */
text-align: center;
}
.reactions {
grid-area: reactions;
display: flex;
gap: var(--cpd-space-1x);
}
.nameTag > svg,
.nameTag > span {
flex-shrink: 0;

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, test } from "vitest";
import { describe, expect, it, test } from "vitest";
import { render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import { TooltipProvider } from "@vector-im/compound-web";
@@ -42,6 +42,7 @@ describe("MediaView", () => {
unencryptedWarning: false,
video: trackReference,
member: undefined,
localParticipant: false,
};
test("is accessible", async () => {
@@ -59,6 +60,25 @@ describe("MediaView", () => {
});
});
describe("with no participant", () => {
it("shows avatar for local user", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={true} />,
);
expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
expect(screen.queryAllByText("video_tile.waiting_for_media").length).toBe(
0,
);
});
it("shows avatar and label for remote user", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={false} />,
);
expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
expect(screen.getByText("video_tile.waiting_for_media")).toBeVisible();
});
});
describe("name tag", () => {
test("is shown with name", () => {
render(<MediaView {...baseProps} displayName="Bob" />);

View File

@@ -28,7 +28,7 @@ interface Props extends ComponentProps<typeof animated.div> {
style?: ComponentProps<typeof animated.div>["style"];
targetWidth: number;
targetHeight: number;
video: TrackReferenceOrPlaceholder;
video: TrackReferenceOrPlaceholder | undefined;
videoFit: "cover" | "contain";
mirror: boolean;
member: RoomMember | undefined;
@@ -41,6 +41,7 @@ interface Props extends ComponentProps<typeof animated.div> {
raisedHandTime?: Date;
currentReaction?: ReactionOption;
raisedHandOnClick?: () => void;
localParticipant: boolean;
}
export const MediaView = forwardRef<HTMLDivElement, Props>(
@@ -63,6 +64,7 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
raisedHandTime,
currentReaction,
raisedHandOnClick,
localParticipant,
...props
},
ref,
@@ -90,21 +92,21 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
size={avatarSize}
src={member?.getMxcAvatarUrl()}
className={styles.avatar}
style={{ display: videoEnabled ? "none" : "initial" }}
style={{ display: video && videoEnabled ? "none" : "initial" }}
/>
{video.publication !== undefined && (
{video?.publication !== undefined && (
<VideoTrack
trackRef={video}
// There's no reason for this to be focusable
tabIndex={-1}
disablePictureInPicture
style={{ display: videoEnabled ? "block" : "none" }}
style={{ display: video && videoEnabled ? "block" : "none" }}
data-testid="video"
/>
)}
</div>
<div className={styles.fg}>
<div style={{ display: "flex", gap: "var(--cpd-space-1x)" }}>
<div className={styles.reactions}>
<RaisedHandIndicator
raisedHandTime={raisedHandTime}
miniature={avatarSize < 96}
@@ -118,6 +120,11 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
/>
)}
</div>
{!video && !localParticipant && (
<div className={styles.status}>
{t("video_tile.waiting_for_media")}
</div>
)}
{/* TODO: Bring this back once encryption status is less broken */}
{/*encryptionStatus !== EncryptionStatus.Okay && (
<div className={styles.status}>

View File

@@ -12,7 +12,11 @@ import userEvent from "@testing-library/user-event";
import { of } from "rxjs";
import { SpotlightTile } from "./SpotlightTile";
import { withLocalMedia, withRemoteMedia } from "../utils/test";
import {
mockRtcMembership,
withLocalMedia,
withRemoteMedia,
} from "../utils/test";
import { SpotlightTileViewModel } from "../state/TileViewModel";
global.IntersectionObserver = class MockIntersectionObserver {
@@ -22,6 +26,7 @@ global.IntersectionObserver = class MockIntersectionObserver {
test("SpotlightTile is accessible", async () => {
await withRemoteMedia(
mockRtcMembership("@alice:example.org", "AAAA"),
{
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
@@ -29,6 +34,7 @@ test("SpotlightTile is accessible", async () => {
{},
async (vm1) => {
await withLocalMedia(
mockRtcMembership("@bob:example.org", "BBBB"),
{
rawDisplayName: "Bob",
getMxcAvatarUrl: () => "mxc://dlskf",

View File

@@ -49,12 +49,13 @@ interface SpotlightItemBaseProps {
"data-id": string;
targetWidth: number;
targetHeight: number;
video: TrackReferenceOrPlaceholder;
video: TrackReferenceOrPlaceholder | undefined;
member: RoomMember | undefined;
unencryptedWarning: boolean;
encryptionStatus: EncryptionStatus;
displayName: string;
"aria-hidden"?: boolean;
localParticipant: boolean;
}
interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps {
@@ -163,6 +164,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
displayName,
encryptionStatus,
"aria-hidden": ariaHidden,
localParticipant: vm.local,
};
return vm instanceof ScreenShareViewModel ? (
@@ -210,7 +212,9 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
const ref = useMergedRefs(ourRef, theirRef);
const maximised = useObservableEagerState(vm.maximised);
const media = useObservableEagerState(vm.media);
const [visibleId, setVisibleId] = useState(media[0].id);
const [visibleId, setVisibleId] = useState<string | undefined>(
media[0]?.id,
);
const latestMedia = useLatest(media);
const latestVisibleId = useLatest(visibleId);
const visibleIndex = media.findIndex((vm) => vm.id === visibleId);

View File

@@ -7,14 +7,29 @@ Please see LICENSE in the repository root for full details.
import { map, Observable, of, SchedulerLike } from "rxjs";
import { RunHelpers, TestScheduler } from "rxjs/testing";
import { expect, vi } from "vitest";
import { RoomMember, Room as MatrixRoom } from "matrix-js-sdk/src/matrix";
import {
RoomMember,
Room as MatrixRoom,
MatrixEvent,
Room,
TypedEventEmitter,
} from "matrix-js-sdk/src/matrix";
import {
CallMembership,
Focus,
MatrixRTCSessionEvent,
MatrixRTCSessionEventHandlerMap,
SessionMembershipData,
} from "matrix-js-sdk/src/matrixrtc";
import {
LocalParticipant,
LocalTrackPublication,
RemoteParticipant,
RemoteTrackPublication,
Room as LivekitRoom,
RoomEvent,
} from "livekit-client";
import { EventEmitter } from "stream";
import {
LocalUserMediaViewModel,
@@ -98,17 +113,73 @@ function mockEmitter<T>(): EmitterMock<T> {
};
}
export function mockRtcMembership(
user: string | RoomMember,
deviceId: string,
callId = "",
fociPreferred: Focus[] = [],
focusActive: Focus = { type: "oldest_membership" },
membership: Partial<SessionMembershipData> = {},
): CallMembership {
const data: SessionMembershipData = {
application: "m.call",
call_id: callId,
device_id: deviceId,
foci_preferred: fociPreferred,
focus_active: focusActive,
...membership,
};
const event = new MatrixEvent({
sender: typeof user === "string" ? user : user.userId,
});
return new CallMembership(event, data);
}
// Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are
// rather simple, but if one util to mock a member is good enough for us, maybe
// it's useful for matrix-js-sdk consumers in general.
export function mockMatrixRoomMember(member: Partial<RoomMember>): RoomMember {
return { ...mockEmitter(), ...member } as RoomMember;
export function mockMatrixRoomMember(
rtcMembership: CallMembership,
member: Partial<RoomMember> = {},
): RoomMember {
return {
...mockEmitter(),
userId: rtcMembership.sender,
...member,
} as RoomMember;
}
export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
}
/**
* A mock of a Livekit Room that can emit events.
*/
export class EmittableMockLivekitRoom extends EventEmitter {
public localParticipant?: LocalParticipant;
public remoteParticipants: Map<string, RemoteParticipant>;
public constructor(room: {
localParticipant?: LocalParticipant;
remoteParticipants: Map<string, RemoteParticipant>;
}) {
super();
this.localParticipant = room.localParticipant;
this.remoteParticipants = room.remoteParticipants ?? new Map();
}
public addParticipant(remoteParticipant: RemoteParticipant): void {
this.remoteParticipants.set(remoteParticipant.identity, remoteParticipant);
this.emit(RoomEvent.ParticipantConnected, remoteParticipant);
}
public removeParticipant(remoteParticipant: RemoteParticipant): void {
this.remoteParticipants.delete(remoteParticipant.identity);
this.emit(RoomEvent.ParticipantDisconnected, remoteParticipant);
}
}
export function mockLivekitRoom(
room: Partial<LivekitRoom>,
{
@@ -145,14 +216,15 @@ export function mockLocalParticipant(
}
export async function withLocalMedia(
member: Partial<RoomMember>,
localRtcMember: CallMembership,
roomMember: Partial<RoomMember>,
continuation: (vm: LocalUserMediaViewModel) => void | Promise<void>,
): Promise<void> {
const localParticipant = mockLocalParticipant({});
const vm = new LocalUserMediaViewModel(
"local",
mockMatrixRoomMember(member),
localParticipant,
mockMatrixRoomMember(localRtcMember, roomMember),
of(localParticipant),
{
kind: E2eeType.PER_PARTICIPANT,
},
@@ -179,15 +251,16 @@ export function mockRemoteParticipant(
}
export async function withRemoteMedia(
member: Partial<RoomMember>,
localRtcMember: CallMembership,
roomMember: Partial<RoomMember>,
participant: Partial<RemoteParticipant>,
continuation: (vm: RemoteUserMediaViewModel) => void | Promise<void>,
): Promise<void> {
const remoteParticipant = mockRemoteParticipant(participant);
const vm = new RemoteUserMediaViewModel(
"remote",
mockMatrixRoomMember(member),
remoteParticipant,
mockMatrixRoomMember(localRtcMember, roomMember),
of(remoteParticipant),
{
kind: E2eeType.PER_PARTICIPANT,
},
@@ -206,3 +279,39 @@ export function mockConfig(config: Partial<ResolvedConfigOptions> = {}): void {
...config,
});
}
export function mockMediaPlay(): string[] {
const audioIsPlaying: string[] = [];
window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
return Promise.resolve();
};
return audioIsPlaying;
}
export class MockRTCSession extends TypedEventEmitter<
MatrixRTCSessionEvent,
MatrixRTCSessionEventHandlerMap
> {
public constructor(
public readonly room: Room,
private localMembership: CallMembership,
public memberships: CallMembership[] = [],
) {
super();
}
public withMemberships(
rtcMembers: Observable<Partial<CallMembership>[]>,
): MockRTCSession {
rtcMembers.subscribe((m) => {
const old = this.memberships;
// always prepend the local participant
const updated = [this.localMembership, ...(m as CallMembership[])];
this.memberships = updated;
this.emit(MatrixRTCSessionEvent.MembershipsChanged, old, updated);
});
return this;
}
}

View File

@@ -17,20 +17,20 @@ import "vitest-axe/extend-expect";
import { logger } from "matrix-js-sdk/src/logger";
import "@testing-library/jest-dom/vitest";
import EN_GB from "../locales/en-GB/app.json";
import EN from "../locales/en/app.json";
import { Config } from "./config/Config";
// Bare-minimum i18n config
i18n
.use(initReactI18next)
.init({
lng: "en-GB",
fallbackLng: "en-GB",
supportedLngs: ["en-GB"],
lng: "en",
fallbackLng: "en",
supportedLngs: ["en"],
// We embed the translations, so that it never needs to fetch
resources: {
"en-GB": {
app: EN_GB,
en: {
app: EN,
},
},
interpolation: {

View File

@@ -3051,12 +3051,12 @@
dependencies:
undici-types "~6.19.8"
"@types/node@^20.0.0":
version "20.17.9"
resolved "https://registry.yarnpkg.com/@types/node/-/node-20.17.9.tgz#5f141d4b7ee125cdee5faefe28de095398865bab"
integrity sha512-0JOXkRyLanfGPE2QRCwgxhzlBAvaRdCNMcvbd7jFfpmD4eEXll7LRwy5ymJmyeZqk7Nh7eD2LeUyQ68BbndmXw==
"@types/node@^22.0.0":
version "22.10.1"
resolved "https://registry.yarnpkg.com/@types/node/-/node-22.10.1.tgz#41ffeee127b8975a05f8c4f83fb89bcb2987d766"
integrity sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==
dependencies:
undici-types "~6.19.2"
undici-types "~6.20.0"
"@types/normalize-package-data@^2.4.0":
version "2.4.4"
@@ -3307,12 +3307,10 @@
dependencies:
"@use-gesture/core" "10.3.1"
"@vector-im/compound-design-tokens@^1.9.1":
version "1.9.2"
resolved "https://registry.yarnpkg.com/@vector-im/compound-design-tokens/-/compound-design-tokens-1.9.2.tgz#0b76e5475da3bc36443f7dc87951b937b5013d6f"
integrity sha512-gQmK4dHR2iws3ZskDv8Il6A4/rvQV7TPSmEOXLsahDhBTInWqexXeQnNRSt9Z5DsLPrkxL3/KoCt9lfYu/yiag==
dependencies:
prettier "^3.3.3"
"@vector-im/compound-design-tokens@^2.0.0":
version "2.1.1"
resolved "https://registry.yarnpkg.com/@vector-im/compound-design-tokens/-/compound-design-tokens-2.1.1.tgz#d6175a99fe4b97688464126f255386990f3048d6"
integrity sha512-QnUi2K14D9KTXxcLQKUU3V75cforZLMwhaaJDNftT8F5mG86950hAM+qhgDNEpEU+pkTffQj0/g/5859YmqWzQ==
"@vector-im/compound-web@^7.2.0":
version "7.4.0"
@@ -7007,7 +7005,7 @@ prelude-ls@^1.2.1:
resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396"
integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==
prettier@^3.0.0, prettier@^3.3.3:
prettier@^3.0.0:
version "3.4.1"
resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.4.1.tgz#e211d451d6452db0a291672ca9154bc8c2579f7b"
integrity sha512-G+YdqtITVZmOJje6QkXQWzl3fSfMxFwm1tjTyo9exhkmWSqC4Yhd1+lug++IlR2mvRVAxEDDWYkQdeSztajqgg==
@@ -8262,11 +8260,16 @@ underscore.string@~3.3.4:
sprintf-js "^1.1.1"
util-deprecate "^1.0.2"
undici-types@~6.19.2, undici-types@~6.19.8:
undici-types@~6.19.8:
version "6.19.8"
resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02"
integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==
undici-types@~6.20.0:
version "6.20.0"
resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.20.0.tgz#8171bf22c1f588d1554d55bf204bc624af388433"
integrity sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==
undici@^5.25.4:
version "5.28.4"
resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.4.tgz#6b280408edb6a1a604a9b20340f45b422e373068"
@@ -8411,9 +8414,9 @@ value-or-function@^4.0.0:
integrity sha512-aeVK81SIuT6aMJfNo9Vte8Dw0/FZINGBV8BfCraGtqVxIeLAEhJyoWs8SmvRVmXfGss2PmmOwZCuBPbZR+IYWg==
vaul@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/vaul/-/vaul-1.1.0.tgz#7da4bc965e0b184ada632f1208096b0f5575d920"
integrity sha512-YhO/bikcauk48hzhMhvIvT+U87cuCbNbKk9fF4Ou5UkI9t2KkBMernmdP37pCzF15hrv55fcny1YhexK8h6GVQ==
version "1.1.1"
resolved "https://registry.yarnpkg.com/vaul/-/vaul-1.1.1.tgz#93aceaad16f7c53aacf28a2609b2dd43b5a91fa0"
integrity sha512-+ejzF6ffQKPcfgS7uOrGn017g39F8SO4yLPXbBhpC7a0H+oPqPna8f1BUfXaz8eU4+pxbQcmjxW+jWBSbxjaFg==
dependencies:
"@radix-ui/react-dialog" "^1.1.1"