diff --git a/.eslintrc.cjs b/.eslintrc.cjs index cada6b46..b734c520 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -44,7 +44,7 @@ module.exports = { ], // To encourage good usage of RxJS: "rxjs/no-exposed-subjects": "error", - "rxjs/finnish": "error", + "rxjs/finnish": ["error", { names: { "^this$": false } }], }, settings: { react: { diff --git a/.github/workflows/changelog-label.yml b/.github/workflows/changelog-label.yml index d8a82832..8d9acbc2 100644 --- a/.github/workflows/changelog-label.yml +++ b/.github/workflows/changelog-label.yml @@ -9,6 +9,6 @@ jobs: steps: - uses: yogevbd/enforce-label-action@a3c219da6b8fa73f6ba62b68ff09c469b3a1c024 # 2.2.2 with: - REQUIRED_LABELS_ANY: "PR-Bug-Fix,PR-Documentation,PR-Task,PR-Feature,PR-Improvement,PR-Developer-Experience,dependencies" + REQUIRED_LABELS_ANY: "PR-Bug-Fix,PR-Documentation,PR-Task,PR-Feature,PR-Improvement,PR-Developer-Experience,dependencies,PR-Breaking-Change" REQUIRED_LABELS_ANY_DESCRIPTION: "Select at least one 'PR-' label" BANNED_LABELS: "banned" diff --git a/.gitignore b/.gitignore index d15cee0b..3e9016a6 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ dist-ssr public/config.json backend/synapse_tmp/* /coverage +config.json # Yarn yarn-error.log diff --git a/backend/dev_livekit.yaml b/backend/dev_livekit.yaml index 0e0c5c7b..f0c5b3a4 100644 --- a/backend/dev_livekit.yaml +++ b/backend/dev_livekit.yaml @@ -21,3 +21,5 @@ turn: external_tls: true keys: devkey: secret +room: + auto_create: false diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index d927bc9c..e6180710 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -14,6 +14,7 @@ services: # If the configured homeserver runs on localhost, it'll probably be using # a self-signed certificate - LIVEKIT_INSECURE_SKIP_VERIFY_TLS=YES_I_KNOW_WHAT_I_AM_DOING + - LIVEKIT_FULL_ACCESS_HOMESERVERS=* deploy: restart_policy: condition: on-failure @@ -101,4 +102,6 @@ services: depends_on: - synapse networks: - - ecbackend + ecbackend: + aliases: + - matrix-rtc.m.localhost diff --git a/docs/MSC4195_setup.drawio.png b/docs/MSC4195_setup.drawio.png index 18566448..e9c7c8a7 100644 Binary files a/docs/MSC4195_setup.drawio.png and b/docs/MSC4195_setup.drawio.png differ diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 50d76b44..d7a1fbdf 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -145,10 +145,6 @@ server { { "type": "livekit", "livekit_service_url": "https://matrix-rtc-2.example.com/livekit/jwt" - }, - { - "type": "nextgen_new_foci_type", - "props_for_nextgen_foci": "val" } ] ``` diff --git a/docs/url-params.md b/docs/url-params.md index bc3846cd..3fac185a 100644 --- a/docs/url-params.md +++ b/docs/url-params.md @@ -48,6 +48,7 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st | Name | Values | Required for widget | Required for SPA | Description | | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `intent` | `start_call`, `join_existing`, `start_call_dm`, `join_existing_dm. | No, defaults to `start_call` | No, defaults to `start_call` | The intent is a special url parameter that defines the defaults for all the other parameters. In most cases it should be enough to only set the intent to setup element-call. | | `allowIceFallback` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Allows use of fallback STUN servers for ICE if the user's homeserver doesn’t provide any. | | `analyticsID` (deprecated: use `posthogUserId` instead) | Posthog analytics ID | No | No | Available only with user's consent for sharing telemetry in Element Web. | | `appPrompt` | `true` or `false` | No, defaults to `true` | No, defaults to `true` | Prompts the user to launch the native mobile app upon entering a room, applicable only on Android and iOS, and must be enabled in config. | @@ -59,7 +60,6 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st | `header` | `none`, `standard` or `app_bar` | No, defaults to `standard` | No, defaults to `standard` | The style of headers to show. `standard` is the default arrangement, `none` hides the header entirely, and `app_bar` produces a header with a back button like you might see in mobile apps. The callback for the back button is `window.controls.onBackButtonPressed`. | | `hideScreensharing` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Hides the screen-sharing button. | | `homeserver` | | Not applicable | No | Homeserver for registering a new (guest) user, configures non-default guest user server when creating a spa link. | -| `intent` | `start_call` or `join_existing` | No, defaults to `start_call` | No, defaults to `start_call` | The intent of the user with respect to the call. e.g. if they clicked a Start Call button, this would be `start_call`. If it was a Join Call button, it would be `join_existing`. | | `lang` | [BCP 47](https://www.rfc-editor.org/info/bcp47) code | No | No | The language the app should use. | | `password` | | No | No | E2EE password when using a shared secret. (For individual sender keys in embedded mode this is not required.) | | `perParticipantE2EE` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Enables per participant encryption with Keys exchanged over encrypted matrix room messages. | @@ -69,6 +69,7 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st | `skipLobby` (deprecated: use `intent` instead) | `true` or `false` | No. If `intent` is explicitly `start_call` then defaults to `true`. Otherwise defaults to `false` | No, defaults to `false` | Skips the lobby to join a call directly, can be combined with preload in widget. When `true` the audio and video inputs will be muted by default. (This means there currently is no way to start without muted video if one wants to skip the lobby. Also not in widget mode.) | | `theme` | One of: `light`, `dark`, `light-high-contrast`, `dark-high-contrast` | No, defaults to `dark` | No, defaults to `dark` | UI theme to use. | | `viaServers` | Comma separated list of [Matrix Server Names](https://spec.matrix.org/v1.12/appendices/#server-name) | Not applicable | No | Homeserver for joining a room, non-empty value required for rooms not on the user’s default homeserver. | +| `sendNotificationType` | `ring` or `notification` | No | No | Will send a "ring" or "notification" `m.rtc.notification` event if the user is the first one in the call. | ### Widget-only parameters diff --git a/embedded/android/gradle/libs.versions.toml b/embedded/android/gradle/libs.versions.toml index 095521b5..9982f14d 100644 --- a/embedded/android/gradle/libs.versions.toml +++ b/embedded/android/gradle/libs.versions.toml @@ -2,11 +2,11 @@ # https://docs.gradle.org/current/userguide/platforms.html#sub::toml-dependencies-format [versions] -android_gradle_plugin = "8.11.0" +android_gradle_plugin = "8.11.1" [libraries] android_gradle_plugin = { module = "com.android.tools.build:gradle", version.ref = "android_gradle_plugin" } [plugins] android_library = { id = "com.android.library", version.ref = "android_gradle_plugin" } -maven_publish = { id = "com.vanniktech.maven.publish", version = "0.33.0" } \ No newline at end of file +maven_publish = { id = "com.vanniktech.maven.publish", version = "0.34.0" } \ No newline at end of file diff --git a/embedded/android/gradle/wrapper/gradle-wrapper.properties b/embedded/android/gradle/wrapper/gradle-wrapper.properties index be2dc79a..7705927e 100644 --- a/embedded/android/gradle/wrapper/gradle-wrapper.properties +++ b/embedded/android/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/embedded/android/lib/build.gradle.kts b/embedded/android/lib/build.gradle.kts index 4684f21b..0a1863fd 100644 --- a/embedded/android/lib/build.gradle.kts +++ b/embedded/android/lib/build.gradle.kts @@ -5,8 +5,6 @@ * Please see LICENSE files in the repository root for full details. */ -import com.vanniktech.maven.publish.SonatypeHost - plugins { alias(libs.plugins.android.library) alias(libs.plugins.maven.publish) @@ -27,7 +25,7 @@ android { } mavenPublishing { - publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL, automaticRelease = true) + publishToMavenCentral(automaticRelease = true) signAllPublications() diff --git a/locales/cs/app.json b/locales/cs/app.json index 81f1ef9a..f988c336 100644 --- a/locales/cs/app.json +++ b/locales/cs/app.json @@ -61,6 +61,7 @@ "video": "Video" }, "developer_mode": { + "always_show_iphone_earpiece": "Zobrazit možnost sluchátek pro iPhone na všech platformách", "crypto_version": "Kryptografická verze: {{version}}", "debug_tile_layout_label": "Ladění rozložení dlaždic", "device_id": "ID zařízení: {{id}}", @@ -173,6 +174,7 @@ "devices": { "camera": "Fotoaparát", "camera_numbered": "Fotoaparát {{n}}", + "change_device_button": "Změnit zvukové zařízení", "default": "Výchozí", "default_named": "Výchozí <2> ({{name}}) ", "microphone": "Mikrofon", diff --git a/locales/da/app.json b/locales/da/app.json index 4461e467..5142ba08 100644 --- a/locales/da/app.json +++ b/locales/da/app.json @@ -61,6 +61,7 @@ "video": "Video" }, "developer_mode": { + "always_show_iphone_earpiece": "Vis mulighed for iPhone-høretelefon på alle platforme", "crypto_version": "Krypto-version: {{version}}", "debug_tile_layout_label": "Fejlfinding af fliselayout", "device_id": "Enheds-id: {{id}}", @@ -70,6 +71,7 @@ "livekit_server_info": "LiveKit Serverinfo", "livekit_sfu": "LiveKit SFU: {{url}}", "matrix_id": "Matrix ID: {{id}}", + "mute_all_audio": "Slå al lyd fra (deltagere, reaktioner, deltagelseslyde)", "show_connection_stats": "Vis forbindelsesstatistik", "show_non_member_tiles": "Vis fliser for medier fra ikke-medlemmer", "url_params": "URL-parametre", @@ -80,7 +82,7 @@ "error": { "call_is_not_supported": "Opkald er ikke understøttet", "call_not_found": "Opkald ikke fundet", - "call_not_found_description": "<0>Det link ser ikke ud til at høre til et eksisterende opkald. Tjek at du har det rigtige link, eller<1> opret et nyt.", + "call_not_found_description": "<0>Det link ser ikke ud til at høre til et eksisterende opkald. Tjek at du har det rigtige link, eller <2> opret et nyt.", "connection_lost": "Forbindelsen gik tabt", "connection_lost_description": "Du blev afbrudt fra opkaldet.", "e2ee_unsupported": "Inkompatibel browser", @@ -164,12 +166,17 @@ "effect_volume_description": "Juster den lydstyrke som reaktioner og håndsoprækninger afspilles med.", "effect_volume_label": "Lydstyrke for lydeffekter" }, + "background_blur_header": "Baggrund", + "background_blur_label": "Gør videoens baggrund sløret", + "blur_not_supported_by_browser": "(Baggrundssløring understøttes ikke af denne enhed.)", "developer_tab_title": "Udvikler", "devices": { "camera": "Kamera", "camera_numbered": "Kamera {{n}}", + "change_device_button": "Skift lydenhed", "default": "Standard", "default_named": "Standard <2>({{name}})", + "loudspeaker": "Højttaler", "microphone": "Mikrofon", "microphone_numbered": "Mikrofon {{n}}", "speaker": "Højttaler", diff --git a/locales/de/app.json b/locales/de/app.json index 67aab44c..fd6d2017 100644 --- a/locales/de/app.json +++ b/locales/de/app.json @@ -82,7 +82,7 @@ "error": { "call_is_not_supported": "Anrufe werden nicht unterstützt", "call_not_found": "Anruf nicht gefunden", - "call_not_found_description": "<0>Dieser Link scheint zu keinem bestehenden Anruf zu gehören. Vergewissern Sie sich, dass Sie den richtigen Link haben, oder <1> erstellen Sie einen neuen. ", + "call_not_found_description": "<0>Dieser Link scheint zu keinem bestehenden Anruf zu gehören. Es sollte geprüft werden, ob der Link korrekt ist, oder <2>ein neuer erstellt werden.", "connection_lost": "Verbindung verloren", "connection_lost_description": "Ihre Verbindung zum Anruf wurde unterbrochen.", "e2ee_unsupported": "Inkompatibler Browser", @@ -94,6 +94,8 @@ "matrix_rtc_focus_missing": "Der Server ist nicht für die Verwendung mit {{brand}} konfiguriert. Bitte den Serveradministrator kontaktieren (Domain: {{domain}}, Fehlercode: {{ errorCode }}).", "open_elsewhere": "In einem anderen Tab geöffnet", "open_elsewhere_description": "{{brand}} wurde in einem anderen Tab geöffnet. Wenn das nicht richtig klingt, versuchen Sie, die Seite neu zu laden.", + "room_creation_restricted": "Anruf konnte nicht erstellt werden", + "room_creation_restricted_description": "Das Erstellen von Anrufen ist nur für autorisierte Nutzer möglich. Versuche es später erneut oder kontaktiere deinen Serveradministrator, falls das Problem weiterhin besteht.", "unexpected_ec_error": "Ein unerwarteter Fehler ist aufgetreten (<0>Fehlercode: <1>{{ errorCode }}). Bitte den Serveradministrator kontaktieren." }, "group_call_loader": { @@ -105,6 +107,11 @@ "knock_reject_heading": "Zugriff verweigert", "reason": "Grund: {{reason}}" }, + "handset": { + "overlay_back_button": "Zurück zum Lautsprechermodus", + "overlay_description": "Nur wenn App im Vordergrund nutzbar", + "overlay_title": "Ohrhörer Modus" + }, "hangup_button_label": "Anruf beenden", "header_label": "Element Call-Startseite", "header_participants_label": "Teilnehmende", @@ -173,9 +180,11 @@ "devices": { "camera": "Kamera", "camera_numbered": "Kamera {{n}}", + "change_device_button": "Audiogerät wechseln", "default": "Standard", "default_named": "Standard<2> ({{name}} )", - "earpiece": "Ohrhörer", + "handset": "Ohrhörer", + "loudspeaker": "Lautsprecher", "microphone": "Mikrofon", "microphone_numbered": "Mikrofon{{n}}", "speaker": "Lautsprecher", diff --git a/locales/en/app.json b/locales/en/app.json index f7d7bf0b..d375b629 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -79,11 +79,6 @@ "use_to_device_key_transport": "Use to device key transport. This will fallback to room key transport when another call member sent a room key" }, "disconnected_banner": "Connectivity to the server has been lost.", - "earpiece": { - "overlay_back_button": "Back to Speaker Mode", - "overlay_description": "Only works while using app", - "overlay_title": "Earpiece Mode" - }, "error": { "call_is_not_supported": "Call is not supported", "call_not_found": "Call not found", @@ -99,6 +94,8 @@ "matrix_rtc_focus_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).", "open_elsewhere": "Opened in another tab", "open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.", + "room_creation_restricted": "Failed to create call", + "room_creation_restricted_description": "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists.", "unexpected_ec_error": "An unexpected error occurred (<0>Error Code: <1>{{ errorCode }}). Please contact your server admin." }, "group_call_loader": { @@ -110,6 +107,11 @@ "knock_reject_heading": "Access denied", "reason": "Reason: {{reason}}" }, + "handset": { + "overlay_back_button": "Back to Speaker Mode", + "overlay_description": "Only works while using app", + "overlay_title": "Handset Mode" + }, "hangup_button_label": "End call", "header_label": "Element Call Home", "header_participants_label": "Participants", @@ -181,7 +183,7 @@ "change_device_button": "Change audio device", "default": "Default", "default_named": "Default <2>({{name}})", - "earpiece": "Earpiece", + "handset": "Handset", "loudspeaker": "Loudspeaker", "microphone": "Microphone", "microphone_numbered": "Microphone {{n}}", diff --git a/locales/et/app.json b/locales/et/app.json index ed32a6fc..e269e53f 100644 --- a/locales/et/app.json +++ b/locales/et/app.json @@ -61,6 +61,7 @@ "video": "Video" }, "developer_mode": { + "always_show_iphone_earpiece": "Näita iPhone'i kuulari valikut kõikidel platvormidel", "crypto_version": "Krüptoteekide versioon: {{version}}", "debug_tile_layout_label": "Meediapaanide paigutus", "device_id": "Seadme tunnus: {{id}}", @@ -81,7 +82,7 @@ "error": { "call_is_not_supported": "Kõne pole toetatud", "call_not_found": "Kõnet ei leidu", - "call_not_found_description": "<0>See link ei tundu olema seotud ühegi olemasoleva kõnega. Kontrolli, et sul on õige link või <1>loo uus.", + "call_not_found_description": "<0>See link ei tundu olema seotud ühegi olemasoleva kõnega. Kontrolli, et sul on õige link või <2>loo uus.", "connection_lost": "Ühendus on katkenud", "connection_lost_description": "Sinu ühendus selle kõnega on katkenud.", "e2ee_unsupported": "Mitteühilduv brauser", @@ -93,6 +94,8 @@ "matrix_rtc_focus_missing": "See server pole seadistatud töötama rakendusega {{brand}}. Palun võta ühendust serveri halduriga (domeen: {{domain}}, veakood: {{ errorCode }}).", "open_elsewhere": "Avatud teisel vahekaardil", "open_elsewhere_description": "{{brand}} on avatud teisel vahekaardil. Kui see ei tundu olema õige, proovi selle lehe uuesti laadimist.", + "room_creation_restricted": "Kõne loomine ei õnnestunud", + "room_creation_restricted_description": "Kõne loomine võib olla lubatud ainult volitatud kasutajatele. Proovi hiljem uuesti või probleemi püsimisel võta ühendust oma serveri haldajaga.", "unexpected_ec_error": "Tekkis ootamatu viga (<0>Veakood: <1>{{ errorCode }}). Palun võta ühendust serveri haldajaga." }, "group_call_loader": { @@ -104,6 +107,10 @@ "knock_reject_heading": "Liitumine pole lubatud", "reason": "Põhjus" }, + "handset": { + "overlay_back_button": "Tagasi esineja vaatesse", + "overlay_description": "See toimib vaid rakenduse kasutamise ajal" + }, "hangup_button_label": "Lõpeta kõne", "header_label": "Avaleht: Element Call", "header_participants_label": "Osalejad", @@ -172,8 +179,10 @@ "devices": { "camera": "Kaamera", "camera_numbered": "Kaamera {{n}}", + "change_device_button": "Muuda heliseadet", "default": "Vaikimisi", "default_named": "Vaikimisi <2>({{name}})", + "loudspeaker": "Valjuhääldi", "microphone": "Mikrofon", "microphone_numbered": "Mikrofon {{n}}", "speaker": "Kõlar", diff --git a/locales/ru/app.json b/locales/ru/app.json index b6ec8a6a..99b8775a 100644 --- a/locales/ru/app.json +++ b/locales/ru/app.json @@ -61,6 +61,7 @@ "video": "Видео" }, "developer_mode": { + "always_show_iphone_earpiece": "Показать опцию наушников для iPhone на всех платформах", "crypto_version": "Версия криптографии: {{version}}", "debug_tile_layout_label": "Отладка расположения плиток", "device_id": "Идентификатор устройства: {{id}}", @@ -70,6 +71,7 @@ "livekit_server_info": "Информация о сервере LiveKit", "livekit_sfu": "LiveKit SFU: {{url}}", "matrix_id": "Matrix ID: {{id}}", + "mute_all_audio": "Отключить все звуки (участников, реакции, звуки присоединения)", "show_connection_stats": "Показать статистику подключений", "show_non_member_tiles": "Показать плитки для медиафайлов, не являющихся участниками", "url_params": "Параметры URL-адреса", @@ -172,6 +174,7 @@ "devices": { "camera": "Камера", "camera_numbered": "Камера {{n}}", + "change_device_button": "Изменить аудиоустройство", "default": "По умолчанию", "default_named": "По умолчанию <2>({{name}})", "microphone": "Микрофон", diff --git a/locales/sk/app.json b/locales/sk/app.json index 20e41408..d017220b 100644 --- a/locales/sk/app.json +++ b/locales/sk/app.json @@ -61,6 +61,7 @@ "video": "Video" }, "developer_mode": { + "always_show_iphone_earpiece": "Zobraziť možnosť slúchadla iPhone na všetkých platformách", "crypto_version": "Krypto verzia: {{version}}", "debug_tile_layout_label": "Ladenie rozloženia dlaždíc", "device_id": "ID zariadenia: {{id}}", @@ -81,7 +82,7 @@ "error": { "call_is_not_supported": "Hovor nie je podporovaný", "call_not_found": "Hovor nebol nájdený", - "call_not_found_description": "<0>Zdá sa, že tento odkaz nepatrí k žiadnemu existujúcemu hovoru. Skontrolujte, či máte správny odkaz, alebo <1> vytvorte nový. ", + "call_not_found_description": "<0>Zdá sa, že tento odkaz nepatrí k žiadnemu existujúcemu hovoru. Skontrolujte, či máte správny odkaz, alebo <2>vytvorte nový.", "connection_lost": "Strata spojenia", "connection_lost_description": "Boli ste odpojení od hovoru.", "e2ee_unsupported": "Nekompatibilný prehliadač", @@ -173,8 +174,10 @@ "devices": { "camera": "Kamera", "camera_numbered": "Kamera {{n}}", + "change_device_button": "Zmeniť zvukové zariadenie", "default": "Predvolené", "default_named": "Predvolené <2>({{name}})", + "loudspeaker": "Reproduktor", "microphone": "Mikrofón", "microphone_numbered": "Mikrofón {{n}}", "speaker": "Reproduktor", diff --git a/locales/sv/app.json b/locales/sv/app.json index 500c4ee7..f8424225 100644 --- a/locales/sv/app.json +++ b/locales/sv/app.json @@ -61,6 +61,7 @@ "video": "Video" }, "developer_mode": { + "always_show_iphone_earpiece": "Visa iPhone-hörsnäckealternativ på alla plattformar", "crypto_version": "Kryptoversion: {{version}}", "debug_tile_layout_label": "Felsök panelarrangemang", "device_id": "Enhets-ID: {{id}}", @@ -70,6 +71,7 @@ "livekit_server_info": "LiveKit-serverinfo", "livekit_sfu": "LiveKit SFU: {{url}}", "matrix_id": "Matrix-ID: {{id}}", + "mute_all_audio": "Tysta allt ljud (deltagare, reaktioner, anslutningsljud)", "show_connection_stats": "Visa anslutningsstatistik", "show_non_member_tiles": "Visa paneler för media som inte är medlemmar", "url_params": "URL-parametrar", @@ -164,10 +166,14 @@ "effect_volume_description": "Justera volymen vid vilken reaktioner och handuppräckningseffekter spelas", "effect_volume_label": "Ljudeffektsvolym" }, + "background_blur_header": "Bakgrund", + "background_blur_label": "Gör bakgrunden i videon suddig", + "blur_not_supported_by_browser": "(Bakgrundssuddighet stöds inte av den här enheten.)", "developer_tab_title": "Utvecklare", "devices": { "camera": "Kamera", "camera_numbered": "Kamera {{n}}", + "change_device_button": "Byt ljudenhet", "default": "Förval", "default_named": "Förval <2>({{name}})", "microphone": "Mikrofon", diff --git a/package.json b/package.json index 689e3eee..1df79c26 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "@mediapipe/tasks-vision": "^0.10.18", "@opentelemetry/api": "^1.4.0", "@opentelemetry/core": "^2.0.0", - "@opentelemetry/exporter-trace-otlp-http": "^0.202.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.203.0", "@opentelemetry/resources": "^2.0.0", "@opentelemetry/sdk-trace-base": "^2.0.0", "@opentelemetry/sdk-trace-web": "^2.0.0", @@ -81,7 +81,7 @@ "@typescript-eslint/eslint-plugin": "^8.31.0", "@typescript-eslint/parser": "^8.31.0", "@use-gesture/react": "^10.2.11", - "@vector-im/compound-design-tokens": "^4.0.0", + "@vector-im/compound-design-tokens": "^6.0.0", "@vector-im/compound-web": "^8.0.0", "@vitejs/plugin-react": "^4.0.1", "@vitest/coverage-v8": "^3.0.0", diff --git a/playwright/errors.spec.ts b/playwright/errors.spec.ts index 7671c103..851e448d 100644 --- a/playwright/errors.spec.ts +++ b/playwright/errors.spec.ts @@ -72,3 +72,56 @@ test("Should automatically retry non fatal JWT errors", async ({ await hasRetriedPromise; await expect(page.getByTestId("video").first()).toBeVisible(); }); + +test("Should show error screen if call creation is restricted", async ({ + page, +}) => { + await page.goto("/"); + + // We need the socket connection to fail, but this cannot be done by using the websocket route. + // Instead, we will trick the app by returning a bad URL for the SFU that will not be reachable an error out. + await page.route( + "**/matrix-rtc.m.localhost/livekit/jwt/sfu/get", + async (route) => + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + url: "wss://badurltotricktest/livekit/sfu", + jwt: "FAKE", + }), + }), + ); + + // Then if the socket connection fails, livekit will try to validate the token! + // Livekit will not auto_create anymore and will return a 404 error. + await page.route( + "**/badurltotricktest/livekit/sfu/rtc/validate?**", + async (route) => + await route.fulfill({ + status: 404, + contentType: "text/plain", + body: "requested room does not exist", + }), + ); + + await page.pause(); + + await page.getByTestId("home_callName").click(); + await page.getByTestId("home_callName").fill("HelloCall"); + await page.getByTestId("home_displayName").click(); + await page.getByTestId("home_displayName").fill("John Doe"); + await page.getByTestId("home_go").click(); + + // Join the call + await page.getByTestId("lobby_joinCall").click(); + + await page.pause(); + // Should fail + await expect(page.getByText("Failed to create call")).toBeVisible(); + await expect( + page.getByText( + /Call creation might be restricted to authorized users only/, + ), + ).toBeVisible(); +}); diff --git a/playwright/restricted-sfu.spec.ts b/playwright/restricted-sfu.spec.ts new file mode 100644 index 00000000..a9e07d38 --- /dev/null +++ b/playwright/restricted-sfu.spec.ts @@ -0,0 +1,75 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "@playwright/test"; +import { sleep } from "matrix-js-sdk/lib/utils.js"; + +test("Should request JWT token before starting the call", async ({ page }) => { + await page.goto("/"); + + let sfGetTimestamp = 0; + let sendStateEventTimestamp = 0; + await page.route( + "**/matrix-rtc.m.localhost/livekit/jwt/sfu/get", + async (route) => { + await sleep(2000); // Simulate very slow request + await route.continue(); + sfGetTimestamp = Date.now(); + }, + ); + + await page.route( + "**/state/org.matrix.msc3401.call.member/**", + async (route) => { + await route.continue(); + sendStateEventTimestamp = Date.now(); + }, + ); + + await page.getByTestId("home_callName").click(); + await page.getByTestId("home_callName").fill("HelloCall"); + await page.getByTestId("home_displayName").click(); + await page.getByTestId("home_displayName").fill("John Doe"); + await page.getByTestId("home_go").click(); + + // Join the call + await page.getByTestId("lobby_joinCall").click(); + await page.waitForTimeout(4000); + // Ensure that the call is connected + await page + .locator("div") + .filter({ hasText: /^HelloCall$/ }) + .click(); + + expect(sfGetTimestamp).toBeGreaterThan(0); + expect(sendStateEventTimestamp).toBeGreaterThan(0); + expect(sfGetTimestamp).toBeLessThan(sendStateEventTimestamp); +}); + +test("Error when pre-warming the focus are caught by the ErrorBoundary", async ({ + page, +}) => { + await page.goto("/"); + + await page.route("**/openid/request_token", async (route) => { + await route.fulfill({ + status: 418, // Simulate an error not retryable + }); + }); + + await page.getByTestId("home_callName").click(); + await page.getByTestId("home_callName").fill("HelloCall"); + await page.getByTestId("home_displayName").click(); + await page.getByTestId("home_displayName").fill("John Doe"); + await page.getByTestId("home_go").click(); + + // Join the call + await page.getByTestId("lobby_joinCall").click(); + + // Should fail + await expect(page.getByText("Something went wrong")).toBeVisible(); +}); diff --git a/playwright/sfu-reconnect-bug.spec.ts b/playwright/sfu-reconnect-bug.spec.ts index c756570a..6138eb78 100644 --- a/playwright/sfu-reconnect-bug.spec.ts +++ b/playwright/sfu-reconnect-bug.spec.ts @@ -100,5 +100,5 @@ test("When creator left, avoid reconnect to the same SFU", async ({ // https://github.com/element-hq/element-call/issues/3344 // The app used to request a new jwt token then to reconnect to the SFU expect(wsConnectionCount).toBe(1); - expect(sfuGetCallCount).toBe(1); + expect(sfuGetCallCount).toBe(2 /* the first one is for the warmup */); }); diff --git a/src/AppBar.tsx b/src/AppBar.tsx index 337c31c5..e70bb50d 100644 --- a/src/AppBar.tsx +++ b/src/AppBar.tsx @@ -17,10 +17,7 @@ import { useState, } from "react"; import { Heading, IconButton, Tooltip } from "@vector-im/compound-web"; -import { - ArrowLeftIcon, - CollapseIcon, -} from "@vector-im/compound-design-tokens/assets/web/icons"; +import { CollapseIcon } from "@vector-im/compound-design-tokens/assets/web/icons"; import { useTranslation } from "react-i18next"; import { Header, LeftNav, RightNav } from "./Header"; @@ -45,7 +42,6 @@ interface Props { */ export const AppBar: FC = ({ children }) => { const { t } = useTranslation(); - const BackIcon = platform === "ios" ? CollapseIcon : ArrowLeftIcon; const onBackClick = useCallback((e: MouseEvent) => { e.preventDefault(); window.controls.onBackButtonPressed?.(); @@ -69,7 +65,7 @@ export const AppBar: FC = ({ children }) => { - + diff --git a/src/UrlParams.test.ts b/src/UrlParams.test.ts index b65638e0..fbf0c095 100644 --- a/src/UrlParams.test.ts +++ b/src/UrlParams.test.ts @@ -10,7 +10,7 @@ import { describe, expect, it } from "vitest"; import { getRoomIdentifierFromUrl, getUrlParams, - UserIntent, + HeaderStyle, } from "../src/UrlParams"; const ROOM_NAME = "roomNameHere"; @@ -82,6 +82,16 @@ describe("UrlParams", () => { getRoomIdentifierFromUrl("", `?roomId=${ROOM_ID}`, "").roomId, ).toBe(ROOM_ID); }); + it("(roomId with unprintable characters)", () => { + const invisibleChar = "\u2066"; + expect( + getRoomIdentifierFromUrl( + "", + `?roomId=${invisibleChar}${ROOM_ID}${invisibleChar}`, + "", + ).roomId, + ).toBe(ROOM_ID); + }); }); it("ignores room alias", () => { @@ -201,24 +211,68 @@ describe("UrlParams", () => { }); describe("intent", () => { - it("defaults to unknown", () => { - expect(getUrlParams().intent).toBe(UserIntent.Unknown); + const noIntentDefaults = { + confineToRoom: false, + appPrompt: true, + preload: false, + header: HeaderStyle.Standard, + showControls: true, + hideScreensharing: false, + allowIceFallback: false, + perParticipantE2EE: false, + controlledAudioDevices: false, + skipLobby: false, + returnToLobby: false, + sendNotificationType: undefined, + }; + const startNewCallDefaults = (platform: string): object => ({ + confineToRoom: true, + appPrompt: false, + preload: true, + header: platform === "desktop" ? HeaderStyle.None : HeaderStyle.AppBar, + showControls: true, + hideScreensharing: false, + allowIceFallback: true, + perParticipantE2EE: true, + controlledAudioDevices: platform === "desktop" ? false : true, + skipLobby: true, + returnToLobby: false, + sendNotificationType: "notification", + }); + const joinExistingCallDefaults = (platform: string): object => ({ + confineToRoom: true, + appPrompt: false, + preload: true, + header: platform === "desktop" ? HeaderStyle.None : HeaderStyle.AppBar, + showControls: true, + hideScreensharing: false, + allowIceFallback: true, + perParticipantE2EE: true, + controlledAudioDevices: platform === "desktop" ? false : true, + skipLobby: false, + returnToLobby: false, + sendNotificationType: "notification", + }); + it("use no-intent-defaults with unknown intent", () => { + expect(getUrlParams()).toMatchObject(noIntentDefaults); }); it("ignores intent if it is not a valid value", () => { - expect(getUrlParams("?intent=foo").intent).toBe(UserIntent.Unknown); + expect(getUrlParams("?intent=foo")).toMatchObject(noIntentDefaults); }); it("accepts start_call", () => { - expect(getUrlParams("?intent=start_call").intent).toBe( - UserIntent.StartNewCall, - ); + expect( + getUrlParams("?intent=start_call&widgetId=1234&parentUrl=parent.org"), + ).toMatchObject(startNewCallDefaults("desktop")); }); it("accepts join_existing", () => { - expect(getUrlParams("?intent=join_existing").intent).toBe( - UserIntent.JoinExistingCall, - ); + expect( + getUrlParams( + "?intent=join_existing&widgetId=1234&parentUrl=parent.org", + ), + ).toMatchObject(joinExistingCallDefaults("desktop")); }); }); @@ -250,9 +304,5 @@ describe("UrlParams", () => { ); expect(getUrlParams("?header=none&hideHeader=false").header).toBe("none"); }); - it("converts hideHeader to the correct header value", () => { - expect(getUrlParams("?hideHeader=true").header).toBe("none"); - expect(getUrlParams("?hideHeader=false").header).toBe("standard"); - }); }); }); diff --git a/src/UrlParams.ts b/src/UrlParams.ts index 9f89fd47..30019d36 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -8,10 +8,13 @@ Please see LICENSE in the repository root for full details. import { useMemo } from "react"; import { useLocation } from "react-router-dom"; import { logger } from "matrix-js-sdk/lib/logger"; +import { type RTCNotificationType } from "matrix-js-sdk/lib/matrixrtc"; +import { pickBy } from "lodash-es"; import { Config } from "./config/Config"; import { type EncryptionSystem } from "./e2ee/sharedKeyManagement"; import { E2eeType } from "./e2ee/e2eeType"; +import { platform } from "./Platform"; interface RoomIdentifier { roomAlias: string | null; @@ -22,6 +25,8 @@ interface RoomIdentifier { export enum UserIntent { StartNewCall = "start_call", JoinExistingCall = "join_existing", + StartNewCallDM = "start_call_dm", + JoinExistingCallDM = "join_existing_dm", Unknown = "unknown", } @@ -31,12 +36,12 @@ export enum HeaderStyle { AppBar = "app_bar", } -// If you need to add a new flag to this interface, prefer a name that describes -// a specific behavior (such as 'confineToRoom'), rather than one that describes -// the situations that call for this behavior ('isEmbedded'). This makes it -// clearer what each flag means, and helps us avoid coupling Element Call's -// behavior to the needs of specific consumers. -export interface UrlParams { +/** + * The UrlProperties are used to pass required data to the widget. + * Those are different in different rooms, users, devices. They do not configure the behavior of the + * widget but provide the required data to the widget. + */ +export interface UrlProperties { // Widget api related params widgetId: string | null; parentUrl: string | null; @@ -48,45 +53,11 @@ export interface UrlParams { * is also not validated, where it is in useRoomIdentifier(). */ roomId: string | null; - /** - * Whether the app should keep the user confined to the current call/room. - */ - confineToRoom: boolean; - /** - * Whether upon entering a room, the user should be prompted to launch the - * native mobile app. (Affects only Android and iOS.) - * - * The app prompt must also be enabled in the config for this to take effect. - */ - appPrompt: boolean; - /** - * Whether the app should pause before joining the call until it sees an - * io.element.join widget action, allowing it to be preloaded. - */ - preload: boolean; - /** - * The style of headers to show. "standard" is the default arrangement, "none" - * hides the header entirely, and "app_bar" produces a header with a back - * button like you might see in mobile apps. The callback for the back button - * is window.controls.onBackButtonPressed. - */ - header: HeaderStyle; - /** - * Whether the controls should be shown. For screen recording no controls can be desired. - */ - showControls: boolean; - /** - * Whether to hide the screen-sharing button. - */ - hideScreensharing: boolean; - /** - * Whether to use end-to-end encryption. - */ - e2eEnabled: boolean; /** * The user's ID (only used in matryoshka mode). */ userId: string | null; + /** * The display name to use for auto-registration. */ @@ -124,14 +95,96 @@ export interface UrlParams { */ posthogApiKey: string | null; /** - * Whether the app is allowed to use fallback STUN servers for ICE in case the - * user's homeserver doesn't provide any. + * Whether to use end-to-end encryption. */ - allowIceFallback: boolean; + e2eEnabled: boolean; /** * E2EE password */ password: string | null; + /** This defines the homeserver that is going to be used when joining a room. + * It has to be set to a non default value for links to rooms + * that are not on the default homeserver, + * that is in use for the current user. + */ + viaServers: string | null; + + /** + * This defines the homeserver that is going to be used when registering + * a new (guest) user. + * This can be user to configure a non default guest user server when + * creating a spa link. + */ + homeserver: string | null; + + /** + * The rageshake submit URL. This is only used in the embedded package of Element Call. + */ + rageshakeSubmitUrl: string | null; + + /** + * The Sentry DSN. This is only used in the embedded package of Element Call. + */ + sentryDsn: string | null; + + /** + * The Sentry environment. This is only used in the embedded package of Element Call. + */ + sentryEnvironment: string | null; + /** + * The theme to use for element call. + * can be "light", "dark", "light-high-contrast" or "dark-high-contrast". + */ + theme: string | null; +} + +/** + * The configuration for the app, which can be set via URL parameters. + * Those property are different to the UrlProperties, since they are all optional + * and configure the behavior of the app. Their value is the same if EC is used in + * the same context but with different accounts/users. + * + * Their defaults can be controlled by the `intent` property. + */ +export interface UrlConfiguration { + /** + * Whether the app should keep the user confined to the current call/room. + */ + confineToRoom: boolean; + /** + * Whether upon entering a room, the user should be prompted to launch the + * native mobile app. (Affects only Android and iOS.) + * + * The app prompt must also be enabled in the config for this to take effect. + */ + appPrompt: boolean; + /** + * Whether the app should pause before joining the call until it sees an + * io.element.join widget action, allowing it to be preloaded. + */ + preload: boolean; + /** + * The style of headers to show. "standard" is the default arrangement, "none" + * hides the header entirely, and "app_bar" produces a header with a back + * button like you might see in mobile apps. The callback for the back button + * is window.controls.onBackButtonPressed. + */ + header: HeaderStyle; + /** + * Whether the controls should be shown. For screen recording no controls can be desired. + */ + showControls: boolean; + /** + * Whether to hide the screen-sharing button. + */ + hideScreensharing: boolean; + + /** + * Whether the app is allowed to use fallback STUN servers for ICE in case the + * user's homeserver doesn't provide any. + */ + allowIceFallback: boolean; + /** * Whether the app should use per participant keys for E2EE. */ @@ -154,47 +207,24 @@ export interface UrlParams { */ returnToLobby: boolean; /** - * The theme to use for element call. - * can be "light", "dark", "light-high-contrast" or "dark-high-contrast". + * Whether and what type of notification EC should send, when the user joins the call. */ - theme: string | null; - /** This defines the homeserver that is going to be used when joining a room. - * It has to be set to a non default value for links to rooms - * that are not on the default homeserver, - * that is in use for the current user. - */ - viaServers: string | null; + sendNotificationType?: RTCNotificationType; /** - * This defines the homeserver that is going to be used when registering - * a new (guest) user. - * This can be user to configure a non default guest user server when - * creating a spa link. + * Whether the app should automatically leave the call when there + * is no one left in the call. + * This is one part to make the call matrixRTC session behave like a telephone call. */ - homeserver: string | null; - - /** - * The user's intent with respect to the call. - * e.g. if they clicked a Start Call button, this would be `start_call`. - * If it was a Join Call button, it would be `join_existing`. - */ - intent: string | null; - - /** - * The rageshake submit URL. This is only used in the embedded package of Element Call. - */ - rageshakeSubmitUrl: string | null; - - /** - * The Sentry DSN. This is only used in the embedded package of Element Call. - */ - sentryDsn: string | null; - - /** - * The Sentry environment. This is only used in the embedded package of Element Call. - */ - sentryEnvironment: string | null; + autoLeaveWhenOthersLeft: boolean; } +// If you need to add a new flag to this interface, prefer a name that describes +// a specific behavior (such as 'confineToRoom'), rather than one that describes +// the situations that call for this behavior ('isEmbedded'). This makes it +// clearer what each flag means, and helps us avoid coupling Element Call's +// behavior to the needs of specific consumers. +export interface UrlParams extends UrlProperties, UrlConfiguration {} + // This is here as a stopgap, but what would be far nicer is a function that // takes a UrlParams and returns a query string. That would enable us to // consolidate all the data about URL parameters and their meanings to this one @@ -235,6 +265,17 @@ class ParamParser { return this.fragmentParams.get(name) ?? this.queryParams.get(name); } + public getEnumParam( + name: string, + type: { [s: string]: T } | ArrayLike, + ): T | undefined { + const value = this.getParam(name); + if (value !== null && Object.values(type).includes(value as T)) { + return value as T; + } + return undefined; + } + public getAllParams(name: string): string[] { return [ ...this.fragmentParams.getAll(name), @@ -242,10 +283,20 @@ class ParamParser { ]; } + /** + * Returns true if the flag exists and is not "false". + */ public getFlagParam(name: string, defaultValue = false): boolean { const param = this.getParam(name); return param === null ? defaultValue : param !== "false"; } + /** + * Returns the value of the flag if it exists, or undefined if it does not. + */ + public getFlag(name: string): boolean | undefined { + const param = this.getParam(name); + return param !== null ? param !== "false" : undefined; + } } /** @@ -262,41 +313,95 @@ export const getUrlParams = ( const fontScale = parseFloat(parser.getParam("fontScale") ?? ""); - let intent = parser.getParam("intent"); - if (!intent || !Object.values(UserIntent).includes(intent as UserIntent)) { - intent = UserIntent.Unknown; - } - - // Check hideHeader for backwards compatibility. If header is set, hideHeader - // is ignored. - const header = - parser.getParam("header") ?? - (parser.getFlagParam("hideHeader") - ? HeaderStyle.None - : HeaderStyle.Standard); - const widgetId = parser.getParam("widgetId"); const parentUrl = parser.getParam("parentUrl"); const isWidget = !!widgetId && !!parentUrl; - return { + /** + * The user's intent with respect to the call. + * e.g. if they clicked a Start Call button, this would be `start_call`. + * If it was a Join Call button, it would be `join_existing`. + * This is a platform specific default set of parameters, that allows to minize the configuration + * needed to start a call. And empowers the EC codebase to control the platform/intent behavior in + * a central place. + * + * In short: either provide url query parameters of UrlConfiguration or set the intent + * (or the global defaults will be used). + */ + const intent = !isWidget + ? UserIntent.Unknown + : (parser.getEnumParam("intent", UserIntent) ?? UserIntent.Unknown); + // Here we only use constants and `platform` to determine the intent preset. + let intentPreset: UrlConfiguration; + const inAppDefault = { + confineToRoom: true, + appPrompt: false, + preload: true, + header: platform === "desktop" ? HeaderStyle.None : HeaderStyle.AppBar, + showControls: true, + hideScreensharing: false, + allowIceFallback: true, + perParticipantE2EE: true, + controlledAudioDevices: platform === "desktop" ? false : true, + skipLobby: true, + returnToLobby: false, + sendNotificationType: "notification" as RTCNotificationType, + autoLeaveWhenOthersLeft: false, + }; + switch (intent) { + case UserIntent.StartNewCall: + intentPreset = { + ...inAppDefault, + skipLobby: true, + }; + break; + case UserIntent.JoinExistingCall: + intentPreset = { + ...inAppDefault, + skipLobby: false, + }; + break; + case UserIntent.StartNewCallDM: + intentPreset = { + ...inAppDefault, + skipLobby: true, + autoLeaveWhenOthersLeft: true, + }; + break; + case UserIntent.JoinExistingCallDM: + intentPreset = { + ...inAppDefault, + skipLobby: true, + autoLeaveWhenOthersLeft: true, + }; + break; + // Non widget usecase defaults + default: + intentPreset = { + confineToRoom: false, + appPrompt: true, + preload: false, + header: HeaderStyle.Standard, + showControls: true, + hideScreensharing: false, + allowIceFallback: false, + perParticipantE2EE: false, + controlledAudioDevices: false, + skipLobby: false, + returnToLobby: false, + sendNotificationType: undefined, + autoLeaveWhenOthersLeft: false, + }; + } + + const properties: UrlProperties = { widgetId, parentUrl, - // NB. we don't validate roomId here as we do in getRoomIdentifierFromUrl: // what would we do if it were invalid? If the widget API says that's what // the room ID is, then that's what it is. roomId: parser.getParam("roomId"), password: parser.getParam("password"), - // This flag has 'embed' as an alias for historical reasons - confineToRoom: - parser.getFlagParam("confineToRoom") || parser.getFlagParam("embed"), - appPrompt: parser.getFlagParam("appPrompt", true), - preload: isWidget ? parser.getFlagParam("preload") : false, - header: header as HeaderStyle, - showControls: parser.getFlagParam("showControls", true), - hideScreensharing: parser.getFlagParam("hideScreensharing"), - e2eEnabled: parser.getFlagParam("enableE2EE", true), userId: isWidget ? parser.getParam("userId") : null, displayName: parser.getParam("displayName"), deviceId: isWidget ? parser.getParam("deviceId") : null, @@ -304,24 +409,9 @@ export const getUrlParams = ( lang: parser.getParam("lang"), fonts: parser.getAllParams("font"), fontScale: Number.isNaN(fontScale) ? null : fontScale, - allowIceFallback: parser.getFlagParam("allowIceFallback"), - perParticipantE2EE: parser.getFlagParam("perParticipantE2EE"), - controlledAudioDevices: parser.getFlagParam( - "controlledAudioDevices", - // the deprecated property name - parser.getFlagParam("controlledMediaDevices"), - ), - skipLobby: parser.getFlagParam( - "skipLobby", - isWidget && intent === UserIntent.StartNewCall, - ), - // In SPA mode the user should always exit to the home screen when hanging - // up, rather than being sent back to the lobby - returnToLobby: isWidget ? parser.getFlagParam("returnToLobby") : false, theme: parser.getParam("theme"), viaServers: !isWidget ? parser.getParam("viaServers") : null, homeserver: !isWidget ? parser.getParam("homeserver") : null, - intent, posthogApiHost: parser.getParam("posthogApiHost"), posthogApiKey: parser.getParam("posthogApiKey"), posthogUserId: @@ -329,6 +419,36 @@ export const getUrlParams = ( rageshakeSubmitUrl: parser.getParam("rageshakeSubmitUrl"), sentryDsn: parser.getParam("sentryDsn"), sentryEnvironment: parser.getParam("sentryEnvironment"), + e2eEnabled: parser.getFlagParam("enableE2EE", true), + }; + + const configuration: Partial = { + confineToRoom: parser.getFlag("confineToRoom"), + appPrompt: parser.getFlag("appPrompt"), + preload: isWidget ? parser.getFlag("preload") : undefined, + // Check hideHeader for backwards compatibility. If header is set, hideHeader + // is ignored. + header: parser.getEnumParam("header", HeaderStyle), + showControls: parser.getFlag("showControls"), + hideScreensharing: parser.getFlag("hideScreensharing"), + allowIceFallback: parser.getFlag("allowIceFallback"), + perParticipantE2EE: parser.getFlag("perParticipantE2EE"), + controlledAudioDevices: parser.getFlag("controlledAudioDevices"), + skipLobby: isWidget ? parser.getFlag("skipLobby") : false, + // In SPA mode the user should always exit to the home screen when hanging + // up, rather than being sent back to the lobby + returnToLobby: isWidget ? parser.getFlag("returnToLobby") : false, + sendNotificationType: parser.getEnumParam("sendNotificationType", [ + "ring", + "notification", + ]), + autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"), + }; + + return { + ...properties, + ...intentPreset, + ...pickBy(configuration, (v?: unknown) => v !== undefined), }; }; @@ -387,10 +507,16 @@ export function getRoomIdentifierFromUrl( // Make sure roomId is valid let roomId: string | null = parser.getParam("roomId"); - if (!roomId?.startsWith("!")) { - roomId = null; - } else if (!roomId.includes("")) { - roomId = null; + if (roomId !== null) { + // Replace any non-printable characters that another client may have inserted. + // For instance on iOS, some copied links end up with zero width characters on the end which get encoded into the URL. + // This isn't valid for a roomId, so we can freely strip the content. + roomId = roomId.replaceAll(/^[^ -~]+|[^ -~]+$/g, ""); + if (!roomId.startsWith("!")) { + roomId = null; + } else if (!roomId.includes("")) { + roomId = null; + } } return { diff --git a/src/__snapshots__/AppBar.test.tsx.snap b/src/__snapshots__/AppBar.test.tsx.snap index e7dc1c46..fe61d09b 100644 --- a/src/__snapshots__/AppBar.test.tsx.snap +++ b/src/__snapshots__/AppBar.test.tsx.snap @@ -32,7 +32,7 @@ exports[`AppBar > renders 1`] = ` xmlns="http://www.w3.org/2000/svg" > diff --git a/src/button/ReactionToggleButton.tsx b/src/button/ReactionToggleButton.tsx index f1a6f0aa..69673293 100644 --- a/src/button/ReactionToggleButton.tsx +++ b/src/button/ReactionToggleButton.tsx @@ -24,8 +24,6 @@ import { import { useTranslation } from "react-i18next"; import { logger } from "matrix-js-sdk/lib/logger"; import classNames from "classnames"; -import { useObservableState } from "observable-hooks"; -import { map } from "rxjs"; import { useReactionsSender } from "../reactions/useReactionsSender"; import styles from "./ReactionToggleButton.module.css"; @@ -36,6 +34,7 @@ import { } from "../reactions"; import { Modal } from "../Modal"; import { type CallViewModel } from "../state/CallViewModel"; +import { useBehavior } from "../useBehavior"; interface InnerButtonProps extends ComponentPropsWithoutRef<"button"> { raised: boolean; @@ -180,12 +179,8 @@ export function ReactionToggleButton({ const [showReactionsMenu, setShowReactionsMenu] = useState(false); const [errorText, setErrorText] = useState(); - const isHandRaised = useObservableState( - vm.handsRaised$.pipe(map((v) => !!v[identifier])), - ); - const canReact = useObservableState( - vm.reactions$.pipe(map((v) => !v[identifier])), - ); + const isHandRaised = !!useBehavior(vm.handsRaised$)[identifier]; + const canReact = !useBehavior(vm.reactions$)[identifier]; useEffect(() => { // Clear whenever the reactions menu state changes. diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index c4dc9144..40b2342b 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -114,24 +114,29 @@ export interface ConfigOptions { * when someone leaves a call. */ wait_for_key_rotation_ms?: number; - /** @deprecated use wait_for_key_rotation_ms instead */ - key_rotation_on_leave_delay?: number; /** * The duration (in milliseconds) after the most recent keep-alive (delayed leave event restart) * that the server waits before sending the leave MatrixRTC membership event. */ delayed_leave_event_delay_ms?: number; - /** @deprecated use delayed_leave_event_delay_ms instead */ - membership_server_side_expiry_timeout?: number; + + /** + * The time (in milliseconds) after which a we consider a delayed event restart http request to have failed. + * Setting this to a lower value will result in more frequent retries but also a higher chance of failiour. + * + * In the presence of network packet loss (hurting TCP connections), the custom delayedEventRestartLocalTimeoutMs + * helps by keeping more delayed event reset candidates in flight, + * improving the chances of a successful reset. (its is equivalent to the js-sdk `localTimeout` configuration, + * but only applies to calls to the `_unstable_updateDelayedEvent` endpoint with a body of `{action:"restart"}`.) + */ + delayed_leave_event_restart_local_timeout_ms?: number; /** * The time interval (in milliseconds) at which the client sends membership keep-alive * messages to the server by restarting the timer for the delayed leave event. */ delayed_leave_event_restart_ms?: number; - /** @deprecated use delayed_leave_event_restart_ms instead */ - membership_keep_alive_period?: number; /** * How long we wait before retrying after a network error on any of the requests. diff --git a/src/controls.ts b/src/controls.ts index 03a12e5a..6a050cb0 100644 --- a/src/controls.ts +++ b/src/controls.ts @@ -6,9 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { Subject } from "rxjs"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; - -const logger = rootLogger.getChild("[controlled-output]"); +import { logger } from "matrix-js-sdk/lib/logger"; export interface Controls { canEnterPip(): boolean; diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx index d0588fb6..6c85b8af 100644 --- a/src/grid/Grid.tsx +++ b/src/grid/Grid.tsx @@ -24,16 +24,16 @@ import { createContext, memo, use, + useCallback, useEffect, useMemo, useRef, useState, + useSyncExternalStore, } from "react"; import useMeasure from "react-use-measure"; import classNames from "classnames"; import { logger } from "matrix-js-sdk/lib/logger"; -import { useObservableEagerState } from "observable-hooks"; -import { fromEvent, map, startWith } from "rxjs"; import styles from "./Grid.module.css"; import { useMergedRefs } from "../useMergedRefs"; @@ -155,11 +155,6 @@ export function useVisibleTiles(callback: VisibleTilesCallback): void { ); } -const windowHeightObservable$ = fromEvent(window, "resize").pipe( - startWith(null), - map(() => window.innerHeight), -); - export interface LayoutProps { ref?: Ref; model: LayoutModel; @@ -261,7 +256,13 @@ export function Grid< const [gridRoot, gridRef2] = useState(null); const gridRef = useMergedRefs(gridRef1, gridRef2); - const windowHeight = useObservableEagerState(windowHeightObservable$); + const windowHeight = useSyncExternalStore( + useCallback((onChange) => { + window.addEventListener("resize", onChange); + return (): void => window.removeEventListener("resize", onChange); + }, []), + useCallback(() => window.innerHeight, []), + ); const [layoutRoot, setLayoutRoot] = useState(null); const [generation, setGeneration] = useState(null); const [visibleTilesCallback, setVisibleTilesCallback] = diff --git a/src/grid/OneOnOneLayout.tsx b/src/grid/OneOnOneLayout.tsx index 8e1bffbe..675e4d0a 100644 --- a/src/grid/OneOnOneLayout.tsx +++ b/src/grid/OneOnOneLayout.tsx @@ -13,6 +13,7 @@ import { type OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewMod import { type CallLayout, arrangeTiles } from "./CallLayout"; import styles from "./OneOnOneLayout.module.css"; import { type DragCallback, useUpdateLayout } from "./Grid"; +import { useBehavior } from "../useBehavior"; /** * An implementation of the "one-on-one" layout, in which the remote participant @@ -32,7 +33,7 @@ export const makeOneOnOneLayout: CallLayout = ({ scrolling: function OneOnOneLayoutScrolling({ ref, model, Slot }): ReactNode { useUpdateLayout(); const { width, height } = useObservableEagerState(minBounds$); - const pipAlignmentValue = useObservableEagerState(pipAlignment$); + const pipAlignmentValue = useBehavior(pipAlignment$); const { tileWidth, tileHeight } = useMemo( () => arrangeTiles(width, height, 1), [width, height], diff --git a/src/grid/SpotlightExpandedLayout.tsx b/src/grid/SpotlightExpandedLayout.tsx index 88271752..9dd2a109 100644 --- a/src/grid/SpotlightExpandedLayout.tsx +++ b/src/grid/SpotlightExpandedLayout.tsx @@ -6,12 +6,12 @@ Please see LICENSE in the repository root for full details. */ import { type ReactNode, useCallback } from "react"; -import { useObservableEagerState } from "observable-hooks"; import { type SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel"; import { type CallLayout } from "./CallLayout"; import { type DragCallback, useUpdateLayout } from "./Grid"; import styles from "./SpotlightExpandedLayout.module.css"; +import { useBehavior } from "../useBehavior"; /** * An implementation of the "expanded spotlight" layout, in which the spotlight @@ -46,7 +46,7 @@ export const makeSpotlightExpandedLayout: CallLayout< Slot, }): ReactNode { useUpdateLayout(); - const pipAlignmentValue = useObservableEagerState(pipAlignment$); + const pipAlignmentValue = useBehavior(pipAlignment$); const onDragPip: DragCallback = useCallback( ({ xRatio, yRatio }) => diff --git a/src/grid/SpotlightPortraitLayout.tsx b/src/grid/SpotlightPortraitLayout.tsx index 3b4de6a1..ad11ed11 100644 --- a/src/grid/SpotlightPortraitLayout.tsx +++ b/src/grid/SpotlightPortraitLayout.tsx @@ -13,6 +13,7 @@ import { type CallLayout, arrangeTiles } from "./CallLayout"; import { type SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel"; import styles from "./SpotlightPortraitLayout.module.css"; import { useUpdateLayout, useVisibleTiles } from "./Grid"; +import { useBehavior } from "../useBehavior"; interface GridCSSProperties extends CSSProperties { "--grid-gap": string; @@ -65,8 +66,7 @@ export const makeSpotlightPortraitLayout: CallLayout< width, model.grid.length, ); - const withIndicators = - useObservableEagerState(model.spotlight.media$).length > 1; + const withIndicators = useBehavior(model.spotlight.media$).length > 1; return (
+ + + + + diff --git a/src/icons/FullScreenMinimise.svg b/src/icons/FullScreenMinimise.svg new file mode 100644 index 00000000..204259e2 --- /dev/null +++ b/src/icons/FullScreenMinimise.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 1afa53e5..24975509 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -14,13 +14,12 @@ import { type AudioTrackProps, } from "@livekit/components-react"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { logger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; -const logger = rootLogger.getChild("[MatrixAudioRenderer]"); export interface MatrixAudioRendererProps { /** * The list of participants to render audio for. @@ -72,7 +71,7 @@ export function MatrixAudioRenderer({ const logInvalid = (identity: string, validIdentities: Set): void => { if (loggedInvalidIdentities.current.has(identity)) return; logger.warn( - `Audio track ${identity} has no matching matrix call member`, + `[MatrixAudioRenderer] Audio track ${identity} has no matching matrix call member`, `current members: ${Array.from(validIdentities.values())}`, `track will not get rendered`, ); @@ -102,7 +101,7 @@ export function MatrixAudioRenderer({ useEffect(() => { if (!tracks.some((t) => !validIdentities.has(t.participant.identity))) { logger.debug( - `All audio tracks have a matching matrix call member identity.`, + `[MatrixAudioRenderer] All audio tracks have a matching matrix call member identity.`, ); loggedInvalidIdentities.current.clear(); } @@ -182,7 +181,7 @@ interface StereoPanAudioTrackProps { /** * This wraps `livekit.AudioTrack` to allow adding audio nodes to a track. * It main purpose is to remount the AudioTrack component when switching from - * audiooContext to normal audio playback. + * audioContext to normal audio playback. * As of now the AudioTrack component does not support adding audio nodes while being mounted. * @param param0 * @returns @@ -202,7 +201,7 @@ function AudioTrackWithAudioNodes({ const [trackReady, setTrackReady] = useReactiveState( () => false, // We only want the track to reset once both (audioNodes and audioContext) are set. - // for unsetting the audioContext its enough if one of the the is undefined. + // for unsetting the audioContext its enough if one of the two is undefined. [audioContext && audioNodes], ); diff --git a/src/livekit/useECConnectionState.ts b/src/livekit/useECConnectionState.ts index 3c7b91f8..83b247e9 100644 --- a/src/livekit/useECConnectionState.ts +++ b/src/livekit/useECConnectionState.ts @@ -22,17 +22,11 @@ import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { ElementCallError, InsufficientCapacityError, + SFURoomCreationRestrictedError, UnknownCallError, } from "../utils/errors.ts"; import { AbortHandle } from "../utils/abortHandle.ts"; -declare global { - interface Window { - peerConnectionTimeout?: number; - websocketTimeout?: number; - } -} - /* * Additional values for states that a call can be in, beyond what livekit * provides in ConnectionState. Also reconnects the call if the SFU Config @@ -169,12 +163,7 @@ async function connectAndPublish( try { logger.info(`[Lifecycle] Connecting to livekit room ${sfuConfig!.url} ...`); - await livekitRoom!.connect(sfuConfig!.url, sfuConfig!.jwt, { - // Due to stability issues on Firefox we are testing the effect of different - // timeouts, and allow these values to be set through the console - peerConnectionTimeout: window.peerConnectionTimeout ?? 45000, - websocketTimeout: window.websocketTimeout ?? 45000, - }); + await livekitRoom!.connect(sfuConfig!.url, sfuConfig!.jwt); logger.info(`[Lifecycle] ... connected to livekit room`); } catch (e) { logger.error("[Lifecycle] Failed to connect", e); @@ -184,11 +173,19 @@ async function connectAndPublish( // participant limits. // LiveKit Cloud uses 429 for connection limits. // Either way, all these errors can be explained as "insufficient capacity". - if ( - e instanceof ConnectionError && - (e.status === 503 || e.status === 200 || e.status === 429) - ) - throw new InsufficientCapacityError(); + if (e instanceof ConnectionError) { + if (e.status === 503 || e.status === 200 || e.status === 429) { + throw new InsufficientCapacityError(); + } + if (e.status === 404) { + // error msg is "Could not establish signal connection: requested room does not exist" + // The room does not exist. There are two different modes of operation for the SFU: + // - the room is created on the fly when connecting (livekit `auto_create` option) + // - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service) + // In the first case there will not be a 404, so we are in the second case. + throw new SFURoomCreationRestrictedError(); + } + } throw e; } diff --git a/src/livekit/useLivekit.ts b/src/livekit/useLivekit.ts index 58f088f6..24e0ca29 100644 --- a/src/livekit/useLivekit.ts +++ b/src/livekit/useLivekit.ts @@ -157,10 +157,13 @@ export function useLivekit( useObservableEagerState( useObservable( (room$) => - observeTrackReference$( - room$.pipe(map(([room]) => room.localParticipant)), - Track.Source.Camera, - ).pipe( + room$.pipe( + switchMap(([room]) => + observeTrackReference$( + room.localParticipant, + Track.Source.Camera, + ), + ), map((trackRef) => { const track = trackRef?.publication?.track; return track instanceof LocalVideoTrack ? track : null; @@ -320,16 +323,18 @@ export function useLivekit( useEffect(() => { // Sync the requested devices with LiveKit's devices - if ( - room !== undefined && - connectionState === ConnectionState.Connected && - !controlledAudioDevices - ) { + if (room !== undefined && connectionState === ConnectionState.Connected) { const syncDevice = ( kind: MediaDeviceKind, selected$: Observable, ): Subscription => selected$.subscribe((device) => { + logger.info( + "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", + room.getActiveDevice(kind), + " !== ", + device?.id, + ); if ( device !== undefined && room.getActiveDevice(kind) !== device.id @@ -344,7 +349,9 @@ export function useLivekit( const subscriptions = [ syncDevice("audioinput", devices.audioInput.selected$), - syncDevice("audiooutput", devices.audioOutput.selected$), + !controlledAudioDevices + ? syncDevice("audiooutput", devices.audioOutput.selected$) + : undefined, syncDevice("videoinput", devices.videoInput.selected$), // Restart the audio input track whenever we detect that the active media // device has changed to refer to a different hardware device. We do this @@ -384,7 +391,7 @@ export function useLivekit( ]; return (): void => { - for (const s of subscriptions) s.unsubscribe(); + for (const s of subscriptions) s?.unsubscribe(); }; } }, [room, devices, connectionState, controlledAudioDevices]); diff --git a/src/reactions/useReactionsSender.tsx b/src/reactions/useReactionsSender.tsx index 30804d4b..5f509a0c 100644 --- a/src/reactions/useReactionsSender.tsx +++ b/src/reactions/useReactionsSender.tsx @@ -16,12 +16,12 @@ import { } from "react"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; -import { useObservableEagerState } from "observable-hooks"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; import { useClientState } from "../ClientContext"; import { ElementCallReactionEventType, type ReactionOption } from "."; import { type CallViewModel } from "../state/CallViewModel"; +import { useBehavior } from "../useBehavior"; interface ReactionsSenderContextType { supportsReactions: boolean; @@ -70,7 +70,7 @@ export const ReactionsSenderProvider = ({ [memberships, myUserId, myDeviceId], ); - const reactions = useObservableEagerState(vm.reactions$); + const reactions = useBehavior(vm.reactions$); const myReaction = useMemo( () => myMembershipIdentifier !== undefined @@ -79,7 +79,7 @@ export const ReactionsSenderProvider = ({ [myMembershipIdentifier, reactions], ); - const handsRaised = useObservableEagerState(vm.handsRaised$); + const handsRaised = useBehavior(vm.handsRaised$); const myRaisedHand = useMemo( () => myMembershipIdentifier !== undefined diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index a0d685ff..a39da82a 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -60,7 +60,7 @@ export function CallEventAudioRenderer({ const audioEngineRef = useLatest(audioEngineCtx); useEffect(() => { - const joinSub = vm.memberChanges$ + const joinSub = vm.participantChanges$ .pipe( filter( ({ joined, ids }) => @@ -72,7 +72,7 @@ export function CallEventAudioRenderer({ void audioEngineRef.current?.playSound("join"); }); - const leftSub = vm.memberChanges$ + const leftSub = vm.participantChanges$ .pipe( filter( ({ ids, left }) => diff --git a/src/room/EarpieceOverlay.module.css b/src/room/EarpieceOverlay.module.css index 20760406..fd46d0e2 100644 --- a/src/room/EarpieceOverlay.module.css +++ b/src/room/EarpieceOverlay.module.css @@ -61,3 +61,7 @@ .overlay > p { text-align: center; } + +.spacer { + min-height: var(--cpd-space-32x); +} diff --git a/src/room/EarpieceOverlay.tsx b/src/room/EarpieceOverlay.tsx index 054e3083..14ce33cc 100644 --- a/src/room/EarpieceOverlay.tsx +++ b/src/room/EarpieceOverlay.tsx @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { type FC } from "react"; import { BigIcon, Button, Heading, Text } from "@vector-im/compound-web"; -import { EarpieceIcon } from "@vector-im/compound-design-tokens/assets/web/icons"; +import { VoiceCallIcon } from "@vector-im/compound-design-tokens/assets/web/icons"; import { useTranslation } from "react-i18next"; import styles from "./EarpieceOverlay.module.css"; @@ -22,12 +22,12 @@ export const EarpieceOverlay: FC = ({ show, onBackToVideoPressed }) => { return (
- + - {t("earpiece.overlay_title")} + {t("handset.overlay_title")} - {t("earpiece.overlay_description")} + {t("handset.overlay_description")} + {/* This spacer is used to give the overlay an offset to the top. */} +
); }; diff --git a/src/room/GroupCallErrorBoundary.tsx b/src/room/GroupCallErrorBoundary.tsx index b5f195c1..3d55d005 100644 --- a/src/room/GroupCallErrorBoundary.tsx +++ b/src/room/GroupCallErrorBoundary.tsx @@ -21,6 +21,7 @@ import { OfflineIcon, WebBrowserIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; +import { Button } from "@vector-im/compound-web"; import { ConnectionLostError, @@ -93,9 +94,13 @@ const ErrorPage: FC = ({

{actions && actions.map((action, index) => ( - + ))} diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index 3a290cc7..4eb32af0 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -16,7 +16,6 @@ import { import { render, waitFor, screen } from "@testing-library/react"; import { type MatrixClient, JoinRule, type RoomState } from "matrix-js-sdk"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { of } from "rxjs"; import { BrowserRouter } from "react-router-dom"; import userEvent from "@testing-library/user-event"; import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container"; @@ -43,6 +42,7 @@ import { MatrixRTCFocusMissingError } from "../utils/errors"; import { ProcessorProvider } from "../livekit/TrackProcessorContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { HeaderStyle } from "../UrlParams"; +import { constant } from "../state/Behavior"; vi.mock("../soundUtils"); vi.mock("../useAudioContext"); @@ -141,7 +141,7 @@ function createGroupCallView( room, localRtcMember, [], - ).withMemberships(of([])); + ).withMemberships(constant([])); rtcSession.joined = joined; const muteState = { audio: { enabled: false }, diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 5b9b1f02..76352523 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -24,7 +24,6 @@ import { type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { useNavigate } from "react-router-dom"; -import { useObservableEagerState } from "observable-hooks"; import type { IWidgetApiRequest } from "matrix-widget-api"; import { @@ -72,6 +71,7 @@ import { import { useTypedEventEmitter } from "../useEvents"; import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; import { useAppBarTitle } from "../AppBar.tsx"; +import { useBehavior } from "../useBehavior.ts"; declare global { interface Window { @@ -110,7 +110,7 @@ export const GroupCallView: FC = ({ ); const memberships = useMatrixRTCSessionMemberships(rtcSession); - const muteAllAudio = useObservableEagerState(muteAllAudio$); + const muteAllAudio = useBehavior(muteAllAudio$); const leaveSoundContext = useLatest( useAudioContext({ sounds: callEventAudioSounds, @@ -166,7 +166,11 @@ export const GroupCallView: FC = ({ const { displayName, avatarUrl } = useProfile(client); const roomName = useRoomName(room); const roomAvatar = useRoomAvatar(room); - const { perParticipantE2EE, returnToLobby } = useUrlParams(); + const { + perParticipantE2EE, + returnToLobby, + password: passwordFromUrl, + } = useUrlParams(); const e2eeSystem = useRoomEncryptionSystem(room.roomId); const [useNewMembershipManager] = useSetting(useNewMembershipManagerSetting); const [useExperimentalToDeviceTransport] = useSetting( @@ -174,7 +178,6 @@ export const GroupCallView: FC = ({ ); // Save the password once we start the groupCallView - const { password: passwordFromUrl } = useUrlParams(); useEffect(() => { if (passwordFromUrl) saveKeyForRoom(room.roomId, passwordFromUrl); }, [passwordFromUrl, room.roomId]); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 53fc0667..9cd5ffb2 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -25,11 +25,11 @@ import useMeasure from "react-use-measure"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import classNames from "classnames"; import { BehaviorSubject, map } from "rxjs"; -import { useObservable, useObservableEagerState } from "observable-hooks"; +import { useObservable, useSubscription } from "observable-hooks"; import { logger } from "matrix-js-sdk/lib/logger"; import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport"; import { - EarpieceIcon, + VoiceCallSolidIcon, VolumeOnSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import { useTranslation } from "react-i18next"; @@ -110,6 +110,7 @@ import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMembership import { useMediaDevices } from "../MediaDevicesContext.ts"; import { EarpieceOverlay } from "./EarpieceOverlay.tsx"; import { useAppBarHidden, useAppBarSecondaryButton } from "../AppBar.tsx"; +import { useBehavior } from "../useBehavior.ts"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -137,17 +138,17 @@ export const ActiveCall: FC = (props) => { useEffect(() => { logger.info( - `[Lifecycle] InCallView Component mounted, livekitroom state ${livekitRoom?.state}`, + `[Lifecycle] InCallView Component mounted, livekit room state ${livekitRoom?.state}`, ); return (): void => { logger.info( - `[Lifecycle] InCallView Component unmounted, livekitroom state ${livekitRoom?.state}`, + `[Lifecycle] InCallView Component unmounted, livekit room state ${livekitRoom?.state}`, ); livekitRoom ?.disconnect() .then(() => { logger.info( - `[Lifecycle] Disconnected from livekite room, state:${livekitRoom?.state}`, + `[Lifecycle] Disconnected from livekit room, state:${livekitRoom?.state}`, ); }) .catch((e) => { @@ -156,6 +157,8 @@ export const ActiveCall: FC = (props) => { }; }, [livekitRoom]); + const { autoLeaveWhenOthersLeft } = useUrlParams(); + useEffect(() => { if (livekitRoom !== undefined) { const reactionsReader = new ReactionsReader(props.rtcSession); @@ -163,7 +166,10 @@ export const ActiveCall: FC = (props) => { props.rtcSession, livekitRoom, mediaDevices, - props.e2eeSystem, + { + encryptionSystem: props.e2eeSystem, + autoLeaveWhenOthersLeft, + }, connStateObservable$, reactionsReader.raisedHands$, reactionsReader.reactions$, @@ -180,6 +186,7 @@ export const ActiveCall: FC = (props) => { mediaDevices, props.e2eeSystem, connStateObservable$, + autoLeaveWhenOthersLeft, ]); if (livekitRoom === undefined || vm === null) return null; @@ -249,7 +256,7 @@ export const InCallView: FC = ({ room: livekitRoom, }); - const muteAllAudio = useObservableEagerState(muteAllAudio$); + const muteAllAudio = useBehavior(muteAllAudio$); // This seems like it might be enough logic to use move it into the call view model? const [didFallbackToRoomKey, setDidFallbackToRoomKey] = useState(false); @@ -300,15 +307,16 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); - const windowMode = useObservableEagerState(vm.windowMode$); - const layout = useObservableEagerState(vm.layout$); - const tileStoreGeneration = useObservableEagerState(vm.tileStoreGeneration$); + const windowMode = useBehavior(vm.windowMode$); + const layout = useBehavior(vm.layout$); + const tileStoreGeneration = useBehavior(vm.tileStoreGeneration$); const [debugTileLayout] = useSetting(debugTileLayoutSetting); - const gridMode = useObservableEagerState(vm.gridMode$); - const showHeader = useObservableEagerState(vm.showHeader$); - const showFooter = useObservableEagerState(vm.showFooter$); - const earpieceMode = useObservableEagerState(vm.earpieceMode$); - const audioOutputSwitcher = useObservableEagerState(vm.audioOutputSwitcher$); + const gridMode = useBehavior(vm.gridMode$); + const showHeader = useBehavior(vm.showHeader$); + const showFooter = useBehavior(vm.showFooter$); + const earpieceMode = useBehavior(vm.earpieceMode$); + const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); + useSubscription(vm.autoLeaveWhenOthersLeft$, onLeave); // Ideally we could detect taps by listening for click events and checking // that the pointerType of the event is "touch", but this isn't yet supported @@ -454,9 +462,9 @@ export const InCallView: FC = ({ useMemo(() => { if (audioOutputSwitcher === null) return null; const isEarpieceTarget = audioOutputSwitcher.targetOutput === "earpiece"; - const Icon = isEarpieceTarget ? EarpieceIcon : VolumeOnSolidIcon; + const Icon = isEarpieceTarget ? VoiceCallSolidIcon : VolumeOnSolidIcon; const label = isEarpieceTarget - ? t("settings.devices.earpiece") + ? t("settings.devices.handset") : t("settings.devices.loudspeaker"); return ( @@ -524,16 +532,12 @@ export const InCallView: FC = ({ targetHeight, model, }: TileProps): ReactNode { - const spotlightExpanded = useObservableEagerState( - vm.spotlightExpanded$, - ); - const onToggleExpanded = useObservableEagerState( - vm.toggleSpotlightExpanded$, - ); - const showSpeakingIndicatorsValue = useObservableEagerState( + const spotlightExpanded = useBehavior(vm.spotlightExpanded$); + const onToggleExpanded = useBehavior(vm.toggleSpotlightExpanded$); + const showSpeakingIndicatorsValue = useBehavior( vm.showSpeakingIndicators$, ); - const showSpotlightIndicatorsValue = useObservableEagerState( + const showSpotlightIndicatorsValue = useBehavior( vm.showSpotlightIndicators$, ); diff --git a/src/room/MuteStates.test.tsx b/src/room/MuteStates.test.tsx index 13dc8ee0..d349a5c6 100644 --- a/src/room/MuteStates.test.tsx +++ b/src/room/MuteStates.test.tsx @@ -191,7 +191,11 @@ describe("useMuteStates", () => { mockConfig(); render( - + diff --git a/src/room/MuteStates.ts b/src/room/MuteStates.ts index 16ba171a..1f4b8e82 100644 --- a/src/room/MuteStates.ts +++ b/src/room/MuteStates.ts @@ -86,6 +86,14 @@ export function useMuteStates(isJoined: boolean): MuteStates { const audio = useMuteState(devices.audioInput, () => { return Config.get().media_devices.enable_audio && !skipLobby && !isJoined; }); + useEffect(() => { + // If audio is enabled, we need to request the device names again, + // because iOS will not be able to switch to the correct device after un-muting. + // This is one of the main changes that makes iOS work with bluetooth audio devices. + if (audio.enabled) { + devices.requestDeviceNames(); + } + }, [audio.enabled, devices]); const isEarpiece = useIsEarpiece(); const video = useMuteState( devices.videoInput, diff --git a/src/room/ReactionsOverlay.tsx b/src/room/ReactionsOverlay.tsx index 9bf7ab66..f3dff848 100644 --- a/src/room/ReactionsOverlay.tsx +++ b/src/room/ReactionsOverlay.tsx @@ -6,16 +6,16 @@ Please see LICENSE in the repository root for full details. */ import { type ReactNode } from "react"; -import { useObservableState } from "observable-hooks"; import styles from "./ReactionsOverlay.module.css"; import { type CallViewModel } from "../state/CallViewModel"; +import { useBehavior } from "../useBehavior"; export function ReactionsOverlay({ vm }: { vm: CallViewModel }): ReactNode { - const reactionsIcons = useObservableState(vm.visibleReactions$); + const reactionsIcons = useBehavior(vm.visibleReactions$); return (
- {reactionsIcons?.map(({ sender, emoji, startX }) => ( + {reactionsIcons.map(({ sender, emoji, startX }) => ( You were disconnected from the call.

-

- Earpiece Mode + Handset Mode

rendering > renders 1`] = ` > Back to Speaker Mode +

{ roomId: "roomId", client: { getDomain: vi.fn().mockReturnValue("example.org"), + getOpenIdToken: vi.fn().mockResolvedValue({ + access_token: "ACCCESS_TOKEN", + token_type: "Bearer", + matrix_server_name: "localhost", + expires_in: 10000, + }), }, }, memberships: [], @@ -195,6 +201,12 @@ test("It should not fail with configuration error if homeserver config has livek roomId: "roomId", client: { getDomain: vi.fn().mockReturnValue("example.org"), + getOpenIdToken: vi.fn().mockResolvedValue({ + access_token: "ACCCESS_TOKEN", + token_type: "Bearer", + matrix_server_name: "localhost", + expires_in: 10000, + }), }, }, memberships: [], diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index e4176dc0..73f58cea 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -5,21 +5,22 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { logger } from "matrix-js-sdk/lib/logger"; import { isLivekitFocus, isLivekitFocusConfig, type LivekitFocus, type LivekitFocusActive, + type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; +import { logger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { PosthogAnalytics } from "./analytics/PosthogAnalytics"; import { Config } from "./config/Config"; import { ElementWidgetActions, widget, type WidgetHelpers } from "./widget"; -import { MatrixRTCFocusMissingError } from "./utils/errors.ts"; -import { getUrlParams } from "./UrlParams.ts"; +import { MatrixRTCFocusMissingError } from "./utils/errors"; +import { getUrlParams } from "./UrlParams"; +import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; @@ -46,6 +47,9 @@ async function makePreferredLivekitFoci( preferredFoci.push(focusInUse); } + // Warm up the first focus we owned, to ensure livekit room is created before any state event sent. + let toWarmUp: LivekitFocus | undefined; + // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); if (domain) { @@ -55,18 +59,17 @@ async function makePreferredLivekitFoci( FOCI_WK_KEY ]; if (Array.isArray(wellKnownFoci)) { - preferredFoci.push( - ...wellKnownFoci - .filter((f) => !!f) - .filter(isLivekitFocusConfig) - .map((wellKnownFocus) => { - logger.log( - "Adding livekit focus from well known: ", - wellKnownFocus, - ); - return { ...wellKnownFocus, livekit_alias: livekitAlias }; - }), - ); + const validWellKnownFoci = wellKnownFoci + .filter((f) => !!f) + .filter(isLivekitFocusConfig) + .map((wellKnownFocus) => { + logger.log("Adding livekit focus from well known: ", wellKnownFocus); + return { ...wellKnownFocus, livekit_alias: livekitAlias }; + }); + if (validWellKnownFoci.length > 0) { + toWarmUp = validWellKnownFoci[0]; + } + preferredFoci.push(...validWellKnownFoci); } } @@ -77,10 +80,15 @@ async function makePreferredLivekitFoci( livekit_service_url: urlFromConf, livekit_alias: livekitAlias, }; + toWarmUp = toWarmUp ?? focusFormConf; logger.log("Adding livekit focus from config: ", focusFormConf); preferredFoci.push(focusFormConf); } + if (toWarmUp) { + // this will call the jwt/sfu/get endpoint to pre create the livekit room. + await getSFUConfigWithOpenID(rtcSession.room.client, toWarmUp); + } if (preferredFoci.length === 0) throw new MatrixRTCFocusMissingError(domain ?? ""); return Promise.resolve(preferredFoci); @@ -116,21 +124,20 @@ export async function enterRTCSession( await makePreferredLivekitFoci(rtcSession, livekitAlias), makeActiveFocus(), { + notificationType: getUrlParams().sendNotificationType, useNewMembershipManager, manageMediaKeys: encryptMedia, ...(useDeviceSessionMemberEvents !== undefined && { useLegacyMemberEvents: !useDeviceSessionMemberEvents, }), delayedLeaveEventRestartMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_ms ?? - matrixRtcSessionConfig?.membership_keep_alive_period, + matrixRtcSessionConfig?.delayed_leave_event_restart_ms, delayedLeaveEventDelayMs: - matrixRtcSessionConfig?.delayed_leave_event_delay_ms ?? - matrixRtcSessionConfig?.membership_server_side_expiry_timeout, + matrixRtcSessionConfig?.delayed_leave_event_delay_ms, + delayedLeaveEventRestartLocalTimeoutMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, - makeKeyDelay: - matrixRtcSessionConfig?.wait_for_key_rotation_ms ?? - matrixRtcSessionConfig?.key_rotation_on_leave_delay, + makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms, useExperimentalToDeviceTransport, diff --git a/src/settings/DeviceSelection.tsx b/src/settings/DeviceSelection.tsx index 314a2e63..197046c3 100644 --- a/src/settings/DeviceSelection.tsx +++ b/src/settings/DeviceSelection.tsx @@ -98,7 +98,7 @@ export const DeviceSelection: FC = ({ labelText = t("settings.devices.loudspeaker"); break; case "earpiece": - labelText = t("settings.devices.earpiece"); + labelText = t("settings.devices.handset"); break; } diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 376fdd64..3272200d 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -10,7 +10,6 @@ import { useTranslation } from "react-i18next"; import { type MatrixClient } from "matrix-js-sdk"; import { Button, Root as Form, Separator } from "@vector-im/compound-web"; import { type Room as LivekitRoom } from "livekit-client"; -import { useObservableEagerState } from "observable-hooks"; import { Modal } from "../Modal"; import styles from "./SettingsModal.module.css"; @@ -34,6 +33,7 @@ import { DeveloperSettingsTab } from "./DeveloperSettingsTab"; import { FieldRow, InputField } from "../input/Input"; import { useSubmitRageshake } from "./submit-rageshake"; import { useUrlParams } from "../UrlParams"; +import { useBehavior } from "../useBehavior"; type SettingsTab = | "audio" @@ -112,7 +112,7 @@ export const SettingsModal: FC = ({ // rather than the input section. const { controlledAudioDevices } = useUrlParams(); // If we are on iOS we will show a button to open the native audio device picker. - const iosDeviceMenu = useObservableEagerState(iosDeviceMenu$); + const iosDeviceMenu = useBehavior(iosDeviceMenu$); const audioTab: Tab = { key: "audio", diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 50e70671..7c7f1250 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -6,10 +6,11 @@ Please see LICENSE in the repository root for full details. */ import { logger } from "matrix-js-sdk/lib/logger"; -import { BehaviorSubject, type Observable } from "rxjs"; -import { useObservableEagerState } from "observable-hooks"; +import { BehaviorSubject } from "rxjs"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; +import { type Behavior } from "../state/Behavior"; +import { useBehavior } from "../useBehavior"; export class Setting { public constructor( @@ -38,7 +39,7 @@ export class Setting { private readonly key: string; private readonly _value$: BehaviorSubject; - public readonly value$: Observable; + public readonly value$: Behavior; public readonly setValue = (value: T): void => { this._value$.next(value); @@ -53,7 +54,7 @@ export class Setting { * React hook that returns a settings's current value and a setter. */ export function useSetting(setting: Setting): [T, (value: T) => void] { - return [useObservableEagerState(setting.value$), setting.setValue]; + return [useBehavior(setting.value$), setting.setValue]; } // null = undecided diff --git a/src/state/Behavior.ts b/src/state/Behavior.ts new file mode 100644 index 00000000..3c88dc00 --- /dev/null +++ b/src/state/Behavior.ts @@ -0,0 +1,26 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { BehaviorSubject } from "rxjs"; + +/** + * A stateful, read-only reactive value. As an Observable, it is "hot" and + * always replays the current value upon subscription. + * + * A Behavior is to BehaviorSubject what Observable is to Subject; it does not + * provide a way to imperatively set new values. For more info on the + * distinction between Behaviors and Observables, see + * https://monoid.dk/post/behaviors-and-streams-why-both/. + */ +export type Behavior = Omit, "next" | "observers">; + +/** + * Creates a Behavior which never changes in value. + */ +export function constant(value: T): Behavior { + return new BehaviorSubject(value); +} diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index 42b04079..511a9431 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -12,9 +12,9 @@ import { debounceTime, distinctUntilChanged, map, + NEVER, type Observable, of, - skip, switchMap, } from "rxjs"; import { type MatrixClient } from "matrix-js-sdk"; @@ -32,7 +32,11 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { deepCompare } from "matrix-js-sdk/lib/utils"; -import { CallViewModel, type Layout } from "./CallViewModel"; +import { + CallViewModel, + type CallViewModelOptions, + type Layout, +} from "./CallViewModel"; import { mockLivekitRoom, mockLocalParticipant, @@ -71,14 +75,23 @@ import { local, localId, localRtcMember, + localRtcMemberDevice2, } from "../utils/test-fixtures"; import { ObservableScope } from "./ObservableScope"; import { MediaDevices } from "./MediaDevices"; import { getValue } from "../utils/observable"; +import { type Behavior, constant } from "./Behavior"; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); vi.mock("../UrlParams", () => ({ getUrlParams })); +vi.mock("rxjs", async (importOriginal) => ({ + ...(await importOriginal()), + // Disable interval Observables for the following tests since the test + // scheduler will loop on them forever and never call the test 'done' + interval: (): Observable => NEVER, +})); + vi.mock("@livekit/components-core"); const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); @@ -157,9 +170,10 @@ function summarizeLayout$(l$: Observable): Observable { case "grid": return combineLatest( [ - l.spotlight?.media$ ?? of(undefined), + l.spotlight?.media$ ?? constant(undefined), ...l.grid.map((vm) => vm.media$), ], + // eslint-disable-next-line rxjs/finnish -- false positive (spotlight, ...grid) => ({ type: l.type, spotlight: spotlight?.map((vm) => vm.id), @@ -178,7 +192,8 @@ function summarizeLayout$(l$: Observable): Observable { ); case "spotlight-expanded": return combineLatest( - [l.spotlight.media$, l.pip?.media$ ?? of(undefined)], + [l.spotlight.media$, l.pip?.media$ ?? constant(undefined)], + // eslint-disable-next-line rxjs/finnish -- false positive (spotlight, pip) => ({ type: l.type, spotlight: spotlight.map((vm) => vm.id), @@ -212,8 +227,8 @@ function summarizeLayout$(l$: Observable): Observable { } function withCallViewModel( - remoteParticipants$: Observable, - rtcMembers$: Observable[]>, + remoteParticipants$: Behavior, + rtcMembers$: Behavior[]>, connectionState$: Observable, speaking: Map>, mediaDevices: MediaDevices, @@ -221,6 +236,10 @@ function withCallViewModel( vm: CallViewModel, subjects: { raisedHands$: BehaviorSubject> }, ) => void, + options: CallViewModelOptions = { + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + autoLeaveWhenOthersLeft: false, + }, ): void { const room = mockMatrixRoom({ client: { @@ -271,9 +290,7 @@ function withCallViewModel( rtcSession as unknown as MatrixRTCSession, liveKitRoom, mediaDevices, - { - kind: E2eeType.PER_PARTICIPANT, - }, + options, connectionState$, raisedHands$, new BehaviorSubject({}), @@ -291,7 +308,7 @@ function withCallViewModel( } test("participants are retained during a focus switch", () => { - withTestScheduler(({ hot, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { // Participants disappear on frame 2 and come back on frame 3 const participantInputMarbles = "a-ba"; // Start switching focus on frame 1 and reconnect on frame 3 @@ -300,12 +317,12 @@ test("participants are retained during a focus switch", () => { const expectedLayoutMarbles = " a"; withCallViewModel( - hot(participantInputMarbles, { + behavior(participantInputMarbles, { a: [aliceParticipant, bobParticipant], b: [], }), - of([aliceRtcMember, bobRtcMember]), - hot(connectionInputMarbles, { + constant([aliceRtcMember, bobRtcMember]), + behavior(connectionInputMarbles, { c: ConnectionState.Connected, s: ECAddonConnectionState.ECSwitchingFocus, }), @@ -328,7 +345,7 @@ test("participants are retained during a focus switch", () => { }); test("screen sharing activates spotlight layout", () => { - withTestScheduler(({ hot, schedule, expectObservable }) => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { // Start with no screen shares, then have Alice and Bob share their screens, // then return to no screen shares, then have just Alice share for a bit const participantInputMarbles = " abcda-ba"; @@ -341,13 +358,13 @@ test("screen sharing activates spotlight layout", () => { const expectedLayoutMarbles = " abcdaefeg"; const expectedShowSpeakingMarbles = "y----nyny"; withCallViewModel( - hot(participantInputMarbles, { + behavior(participantInputMarbles, { a: [aliceParticipant, bobParticipant], b: [aliceSharingScreen, bobParticipant], c: [aliceSharingScreen, bobSharingScreen], d: [aliceParticipant, bobSharingScreen], }), - of([aliceRtcMember, bobRtcMember]), + constant([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), mockMediaDevices({}), @@ -413,7 +430,7 @@ test("screen sharing activates spotlight layout", () => { }); test("participants stay in the same order unless to appear/disappear", () => { - withTestScheduler(({ hot, schedule, expectObservable }) => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { const visibilityInputMarbles = "a"; // First Bob speaks, then Dave, then Alice const aSpeakingInputMarbles = " n- 1998ms - 1999ms y"; @@ -426,13 +443,22 @@ test("participants stay in the same order unless to appear/disappear", () => { const expectedLayoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a"; withCallViewModel( - of([aliceParticipant, bobParticipant, daveParticipant]), - of([aliceRtcMember, bobRtcMember, daveRtcMember]), + constant([aliceParticipant, bobParticipant, daveParticipant]), + constant([aliceRtcMember, bobRtcMember, daveRtcMember]), of(ConnectionState.Connected), new Map([ - [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })], - [bobParticipant, hot(bSpeakingInputMarbles, { y: true, n: false })], - [daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })], + [ + aliceParticipant, + behavior(aSpeakingInputMarbles, { y: true, n: false }), + ], + [ + bobParticipant, + behavior(bSpeakingInputMarbles, { y: true, n: false }), + ], + [ + daveParticipant, + behavior(dSpeakingInputMarbles, { y: true, n: false }), + ], ]), mockMediaDevices({}), (vm) => { @@ -472,7 +498,7 @@ test("participants stay in the same order unless to appear/disappear", () => { }); test("participants adjust order when space becomes constrained", () => { - withTestScheduler(({ hot, schedule, expectObservable }) => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { // Start with all tiles on screen then shrink to 3 const visibilityInputMarbles = "a-b"; // Bob and Dave speak @@ -484,12 +510,18 @@ test("participants adjust order when space becomes constrained", () => { const expectedLayoutMarbles = " a-b"; withCallViewModel( - of([aliceParticipant, bobParticipant, daveParticipant]), - of([aliceRtcMember, bobRtcMember, daveRtcMember]), + constant([aliceParticipant, bobParticipant, daveParticipant]), + constant([aliceRtcMember, bobRtcMember, daveRtcMember]), of(ConnectionState.Connected), new Map([ - [bobParticipant, hot(bSpeakingInputMarbles, { y: true, n: false })], - [daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })], + [ + bobParticipant, + behavior(bSpeakingInputMarbles, { y: true, n: false }), + ], + [ + daveParticipant, + behavior(dSpeakingInputMarbles, { y: true, n: false }), + ], ]), mockMediaDevices({}), (vm) => { @@ -523,7 +555,7 @@ test("participants adjust order when space becomes constrained", () => { }); test("spotlight speakers swap places", () => { - withTestScheduler(({ hot, schedule, expectObservable }) => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { // Go immediately into spotlight mode for the test const modeInputMarbles = " s"; // First Bob speaks, then Dave, then Alice @@ -537,13 +569,22 @@ test("spotlight speakers swap places", () => { const expectedLayoutMarbles = "abcd"; withCallViewModel( - of([aliceParticipant, bobParticipant, daveParticipant]), - of([aliceRtcMember, bobRtcMember, daveRtcMember]), + constant([aliceParticipant, bobParticipant, daveParticipant]), + constant([aliceRtcMember, bobRtcMember, daveRtcMember]), of(ConnectionState.Connected), new Map([ - [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })], - [bobParticipant, hot(bSpeakingInputMarbles, { y: true, n: false })], - [daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })], + [ + aliceParticipant, + behavior(aSpeakingInputMarbles, { y: true, n: false }), + ], + [ + bobParticipant, + behavior(bSpeakingInputMarbles, { y: true, n: false }), + ], + [ + daveParticipant, + behavior(dSpeakingInputMarbles, { y: true, n: false }), + ], ]), mockMediaDevices({}), (vm) => { @@ -587,8 +628,8 @@ test("layout enters picture-in-picture mode when requested", () => { const expectedLayoutMarbles = " aba"; withCallViewModel( - of([aliceParticipant, bobParticipant]), - of([aliceRtcMember, bobRtcMember]), + constant([aliceParticipant, bobParticipant]), + constant([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), mockMediaDevices({}), @@ -629,8 +670,8 @@ test("spotlight remembers whether it's expanded", () => { const expectedLayoutMarbles = "abcbada"; withCallViewModel( - of([aliceParticipant, bobParticipant]), - of([aliceRtcMember, bobRtcMember]), + constant([aliceParticipant, bobParticipant]), + constant([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), mockMediaDevices({}), @@ -678,7 +719,7 @@ test("spotlight remembers whether it's expanded", () => { }); test("participants must have a MatrixRTCSession to be visible", () => { - withTestScheduler(({ hot, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { // iterate through a number of combinations of participants and MatrixRTC memberships // Bob never has an MatrixRTC membership const scenarioInputMarbles = " abcdec"; @@ -686,14 +727,14 @@ test("participants must have a MatrixRTCSession to be visible", () => { const expectedLayoutMarbles = "a-bc-b"; withCallViewModel( - hot(scenarioInputMarbles, { + behavior(scenarioInputMarbles, { a: [], b: [bobParticipant], c: [aliceParticipant, bobParticipant], d: [aliceParticipant, daveParticipant, bobParticipant], e: [aliceParticipant, daveParticipant, bobSharingScreen], }), - hot(scenarioInputMarbles, { + behavior(scenarioInputMarbles, { a: [], b: [], c: [aliceRtcMember], @@ -734,17 +775,17 @@ test("shows participants without MatrixRTCSession when enabled in settings", () try { // enable the setting: showNonMemberTiles.setValue(true); - withTestScheduler(({ hot, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { const scenarioInputMarbles = " abc"; const expectedLayoutMarbles = "abc"; withCallViewModel( - hot(scenarioInputMarbles, { + behavior(scenarioInputMarbles, { a: [], b: [aliceParticipant], c: [aliceParticipant, bobParticipant], }), - of([]), // No one joins the MatrixRTC session + constant([]), // No one joins the MatrixRTC session of(ConnectionState.Connected), new Map(), mockMediaDevices({}), @@ -779,15 +820,15 @@ test("shows participants without MatrixRTCSession when enabled in settings", () }); it("should show at least one tile per MatrixRTCSession", () => { - withTestScheduler(({ hot, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { // iterate through some combinations of MatrixRTC memberships const scenarioInputMarbles = " abcd"; // There should always be one tile for each MatrixRTCSession const expectedLayoutMarbles = "abcd"; withCallViewModel( - of([]), - hot(scenarioInputMarbles, { + constant([]), + behavior(scenarioInputMarbles, { a: [], b: [aliceRtcMember], c: [aliceRtcMember, daveRtcMember], @@ -829,13 +870,13 @@ it("should show at least one tile per MatrixRTCSession", () => { }); test("should disambiguate users with the same displayname", () => { - withTestScheduler(({ hot, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { const scenarioInputMarbles = "abcde"; const expectedLayoutMarbles = "abcde"; withCallViewModel( - of([]), - hot(scenarioInputMarbles, { + constant([]), + behavior(scenarioInputMarbles, { a: [], b: [aliceRtcMember], c: [aliceRtcMember, aliceDoppelgangerRtcMember], @@ -846,50 +887,46 @@ test("should disambiguate users with the same displayname", () => { new Map(), mockMediaDevices({}), (vm) => { - // Skip the null state. - expectObservable(vm.memberDisplaynames$.pipe(skip(1))).toBe( - expectedLayoutMarbles, - { - // Carol has no displayname - So userId is used. - a: new Map([[carolId, carol.userId]]), - b: new Map([ - [carolId, carol.userId], - [aliceId, alice.rawDisplayName], - ]), - // The second alice joins. - c: new Map([ - [carolId, carol.userId], - [aliceId, "Alice (@alice:example.org)"], - [aliceDoppelgangerId, "Alice (@alice2:example.org)"], - ]), - // Bob also joins - d: new Map([ - [carolId, carol.userId], - [aliceId, "Alice (@alice:example.org)"], - [aliceDoppelgangerId, "Alice (@alice2:example.org)"], - [bobId, bob.rawDisplayName], - ]), - // Alice leaves, and the displayname should reset. - e: new Map([ - [carolId, carol.userId], - [aliceDoppelgangerId, "Alice"], - [bobId, bob.rawDisplayName], - ]), - }, - ); + expectObservable(vm.memberDisplaynames$).toBe(expectedLayoutMarbles, { + // Carol has no displayname - So userId is used. + a: new Map([[carolId, carol.userId]]), + b: new Map([ + [carolId, carol.userId], + [aliceId, alice.rawDisplayName], + ]), + // The second alice joins. + c: new Map([ + [carolId, carol.userId], + [aliceId, "Alice (@alice:example.org)"], + [aliceDoppelgangerId, "Alice (@alice2:example.org)"], + ]), + // Bob also joins + d: new Map([ + [carolId, carol.userId], + [aliceId, "Alice (@alice:example.org)"], + [aliceDoppelgangerId, "Alice (@alice2:example.org)"], + [bobId, bob.rawDisplayName], + ]), + // Alice leaves, and the displayname should reset. + e: new Map([ + [carolId, carol.userId], + [aliceDoppelgangerId, "Alice"], + [bobId, bob.rawDisplayName], + ]), + }); }, ); }); }); test("should disambiguate users with invisible characters", () => { - withTestScheduler(({ hot, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { const scenarioInputMarbles = "ab"; const expectedLayoutMarbles = "ab"; withCallViewModel( - of([]), - hot(scenarioInputMarbles, { + constant([]), + behavior(scenarioInputMarbles, { a: [], b: [bobRtcMember, bobZeroWidthSpaceRtcMember], }), @@ -897,36 +934,32 @@ test("should disambiguate users with invisible characters", () => { new Map(), mockMediaDevices({}), (vm) => { - // Skip the null state. - expectObservable(vm.memberDisplaynames$.pipe(skip(1))).toBe( - expectedLayoutMarbles, - { - // Carol has no displayname - So userId is used. - a: new Map([[carolId, carol.userId]]), - // Both Bobs join, and should handle zero width hacks. - b: new Map([ - [carolId, carol.userId], - [bobId, `Bob (${bob.userId})`], - [ - bobZeroWidthSpaceId, - `${bobZeroWidthSpace.rawDisplayName} (${bobZeroWidthSpace.userId})`, - ], - ]), - }, - ); + expectObservable(vm.memberDisplaynames$).toBe(expectedLayoutMarbles, { + // Carol has no displayname - So userId is used. + a: new Map([[carolId, carol.userId]]), + // Both Bobs join, and should handle zero width hacks. + b: new Map([ + [carolId, carol.userId], + [bobId, `Bob (${bob.userId})`], + [ + bobZeroWidthSpaceId, + `${bobZeroWidthSpace.rawDisplayName} (${bobZeroWidthSpace.userId})`, + ], + ]), + }); }, ); }); }); test("should strip RTL characters from displayname", () => { - withTestScheduler(({ hot, expectObservable }) => { + withTestScheduler(({ behavior, expectObservable }) => { const scenarioInputMarbles = "ab"; const expectedLayoutMarbles = "ab"; withCallViewModel( - of([]), - hot(scenarioInputMarbles, { + constant([]), + behavior(scenarioInputMarbles, { a: [], b: [daveRtcMember, daveRTLRtcMember], }), @@ -934,35 +967,31 @@ test("should strip RTL characters from displayname", () => { new Map(), mockMediaDevices({}), (vm) => { - // Skip the null state. - expectObservable(vm.memberDisplaynames$.pipe(skip(1))).toBe( - expectedLayoutMarbles, - { - // Carol has no displayname - So userId is used. - a: new Map([[carolId, carol.userId]]), - // Both Dave's join. Since after stripping - b: new Map([ - [carolId, carol.userId], - // Not disambiguated - [daveId, "Dave"], - // This one is, since it's using RTL. - [daveRTLId, `evaD (${daveRTL.userId})`], - ]), - }, - ); + expectObservable(vm.memberDisplaynames$).toBe(expectedLayoutMarbles, { + // Carol has no displayname - So userId is used. + a: new Map([[carolId, carol.userId]]), + // Both Dave's join. Since after stripping + b: new Map([ + [carolId, carol.userId], + // Not disambiguated + [daveId, "Dave"], + // This one is, since it's using RTL. + [daveRTLId, `evaD (${daveRTL.userId})`], + ]), + }); }, ); }); }); it("should rank raised hands above video feeds and below speakers and presenters", () => { - withTestScheduler(({ schedule, expectObservable }) => { + withTestScheduler(({ schedule, expectObservable, behavior }) => { // There should always be one tile for each MatrixRTCSession const expectedLayoutMarbles = "ab"; withCallViewModel( - of([aliceParticipant, bobParticipant]), - of([aliceRtcMember, bobRtcMember]), + constant([aliceParticipant, bobParticipant]), + constant([aliceRtcMember, bobRtcMember]), of(ConnectionState.Connected), new Map(), mockMediaDevices({}), @@ -1015,6 +1044,176 @@ it("should rank raised hands above video feeds and below speakers and presenters }); }); +function nooneEverThere$( + hot: (marbles: string, values: Record) => Observable, +): Observable { + return hot("a-b-c-d", { + a: [], // Start empty + b: [], // Alice joins + c: [], // Alice still there + d: [], // Alice leaves + }); +} + +function participantJoinLeave$( + hot: ( + marbles: string, + values: Record, + ) => Observable, +): Observable { + return hot("a-b-c-d", { + a: [], // Start empty + b: [aliceParticipant], // Alice joins + c: [aliceParticipant], // Alice still there + d: [], // Alice leaves + }); +} + +function rtcMemberJoinLeave$( + hot: ( + marbles: string, + values: Record, + ) => Observable, +): Observable { + return hot("a-b-c-d", { + a: [], // Start empty + b: [aliceRtcMember], // Alice joins + c: [aliceRtcMember], // Alice still there + d: [], // Alice leaves + }); +} + +test("allOthersLeft$ emits only when someone joined and then all others left", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + // Test scenario 1: No one ever joins - should only emit initial false and never emit again + withCallViewModel( + scope.behavior(nooneEverThere$(hot), []), + scope.behavior(nooneEverThere$(hot), []), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.allOthersLeft$).toBe("n------", { n: false }); + }, + ); + }); +}); + +test("allOthersLeft$ emits true when someone joined and then all others left", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + withCallViewModel( + scope.behavior(participantJoinLeave$(hot), []), + scope.behavior(rtcMemberJoinLeave$(hot), []), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.allOthersLeft$).toBe( + "n-----u", // false initially, then at frame 6: true then false emissions in same frame + { n: false, u: true }, // map(() => {}) + ); + }, + ); + }); +}); + +test("autoLeaveWhenOthersLeft$ emits only when autoLeaveWhenOthersLeft option is enabled", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + withCallViewModel( + scope.behavior(participantJoinLeave$(hot), []), + scope.behavior(rtcMemberJoinLeave$(hot), []), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.autoLeaveWhenOthersLeft$).toBe( + "------e", // false initially, then at frame 6: true then false emissions in same frame + { e: undefined }, + ); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); +}); + +test("autoLeaveWhenOthersLeft$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + withCallViewModel( + scope.behavior(nooneEverThere$(hot), []), + scope.behavior(nooneEverThere$(hot), []), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------"); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); +}); + +test("autoLeaveWhenOthersLeft$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + withCallViewModel( + scope.behavior(participantJoinLeave$(hot), []), + scope.behavior(rtcMemberJoinLeave$(hot), []), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("-------"); + }, + { + autoLeaveWhenOthersLeft: false, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); +}); + +test("autoLeaveWhenOthersLeft$ doesn't emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => { + withTestScheduler(({ hot, expectObservable, scope }) => { + withCallViewModel( + scope.behavior( + hot("a-b-c-d", { + a: [], // Alone + b: [aliceParticipant], // Alice joins + c: [aliceParticipant], + d: [], // Local joins with a second device + }), + [], //Alice leaves + ), + scope.behavior( + hot("a-b-c-d", { + a: [localRtcMember], // Start empty + b: [localRtcMember, aliceRtcMember], // Alice joins + c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there + d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves + }), + [], + ), + of(ConnectionState.Connected), + new Map(), + mockMediaDevices({}), + (vm) => { + expectObservable(vm.autoLeaveWhenOthersLeft$).toBe("------e", { + e: undefined, + }); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); +}); + test("audio output changes when toggling earpiece mode", () => { withTestScheduler(({ schedule, expectObservable }) => { getUrlParams.mockReturnValue({ controlledAudioDevices: true }); @@ -1026,7 +1225,7 @@ test("audio output changes when toggling earpiece mode", () => { window.controls.setAvailableAudioDevices([ { id: "speaker", name: "Speaker", isSpeaker: true }, - { id: "earpiece", name: "Earpiece", isEarpiece: true }, + { id: "earpiece", name: "Handset", isEarpiece: true }, { id: "headphones", name: "Headphones" }, ]); window.controls.setAudioDevice("headphones"); @@ -1036,8 +1235,8 @@ test("audio output changes when toggling earpiece mode", () => { const expectedTargetStateMarbles = " sese"; withCallViewModel( - of([]), - of([]), + constant([]), + constant([]), of(ConnectionState.Connected), new Map(), devices, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 9dd2c3f0..d54d9ae5 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -91,7 +91,12 @@ import { observeSpeaker$ } from "./observeSpeaker"; import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; +import { type Behavior } from "./Behavior"; +export interface CallViewModelOptions { + encryptionSystem: EncryptionSystem; + autoLeaveWhenOthersLeft?: boolean; +} // How long we wait after a focus switch before showing the real participant // list again const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; @@ -247,8 +252,8 @@ class UserMedia { LocalParticipant | RemoteParticipant | undefined >; - public readonly speaker$: Observable; - public readonly presenter$: Observable; + public readonly speaker$: Behavior; + public readonly presenter$: Behavior; public constructor( public readonly id: string, member: RoomMember | undefined, @@ -266,13 +271,13 @@ class UserMedia { this.vm = new LocalUserMediaViewModel( this.id, member, - this.participant$.asObservable() as Observable, + this.participant$ as Behavior, encryptionSystem, livekitRoom, mediaDevices, - displayname$, - handRaised$, - reaction$, + this.scope.behavior(displayname$), + this.scope.behavior(handRaised$), + this.scope.behavior(reaction$), ); } else { this.vm = new RemoteUserMediaViewModel( @@ -283,28 +288,29 @@ class UserMedia { >, encryptionSystem, livekitRoom, - displayname$, - handRaised$, - reaction$, + this.scope.behavior(displayname$), + this.scope.behavior(handRaised$), + this.scope.behavior(reaction$), ); } - this.speaker$ = observeSpeaker$(this.vm.speaking$).pipe(this.scope.state()); + this.speaker$ = this.scope.behavior(observeSpeaker$(this.vm.speaking$)); - this.presenter$ = this.participant$.pipe( - switchMap( - (p) => - (p && - observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled))) ?? - of(false), + this.presenter$ = this.scope.behavior( + this.participant$.pipe( + switchMap( + (p) => + (p && + observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled))) ?? + of(false), + ), ), - this.scope.state(), ); } @@ -324,6 +330,7 @@ class UserMedia { } class ScreenShare { + private readonly scope = new ObservableScope(); public readonly vm: ScreenShareViewModel; private readonly participant$: BehaviorSubject< LocalParticipant | RemoteParticipant @@ -335,7 +342,7 @@ class ScreenShare { participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, liveKitRoom: LivekitRoom, - displayname$: Observable, + displayName$: Observable, ) { this.participant$ = new BehaviorSubject(participant); @@ -345,12 +352,13 @@ class ScreenShare { this.participant$.asObservable(), encryptionSystem, liveKitRoom, - displayname$, + this.scope.behavior(displayName$), participant.isLocal, ); } public destroy(): void { + this.scope.end(); this.vm.destroy(); } } @@ -384,14 +392,17 @@ export class CallViewModel extends ViewModel { /** * The raw list of RemoteParticipants as reported by LiveKit */ - private readonly rawRemoteParticipants$: Observable = - connectedParticipantsObserver(this.livekitRoom).pipe(this.scope.state()); + private readonly rawRemoteParticipants$ = this.scope.behavior< + RemoteParticipant[] + >(connectedParticipantsObserver(this.livekitRoom), []); /** * Lists of RemoteParticipants to "hold" on display, even if LiveKit claims that * they've left */ - private readonly remoteParticipantHolds$: Observable = + private readonly remoteParticipantHolds$ = this.scope.behavior< + RemoteParticipant[][] + >( this.connectionState$.pipe( withLatestFrom(this.rawRemoteParticipants$), mergeMap(([s, ps]) => { @@ -423,12 +434,15 @@ export class CallViewModel extends ViewModel { ? [instruction.hold, ...holds] : holds.filter((h) => h !== instruction.unhold), ), - ); + ), + ); /** * The RemoteParticipants including those that are being "held" on the screen */ - private readonly remoteParticipants$: Observable = + private readonly remoteParticipants$ = this.scope.behavior< + RemoteParticipant[] + >( combineLatest( [this.rawRemoteParticipants$, this.remoteParticipantHolds$], (raw, holds) => { @@ -447,23 +461,30 @@ export class CallViewModel extends ViewModel { return result; }, - ); + ), + ); + + private readonly memberships$: Observable = merge( + // Handle call membership changes. + fromEvent(this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged), + // Handle room membership changes (and displayname updates) + fromEvent(this.matrixRTCSession.room, RoomStateEvent.Members), + ).pipe( + startWith(this.matrixRTCSession.memberships), + map(() => { + return this.matrixRTCSession.memberships; + }), + ); /** * Displaynames for each member of the call. This will disambiguate * any displaynames that clashes with another member. Only members * joined to the call are considered here. */ - public readonly memberDisplaynames$ = merge( - // Handle call membership changes. - fromEvent(this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged), - // Handle room membership changes (and displayname updates) - fromEvent(this.matrixRTCSession.room, RoomStateEvent.Members), - ).pipe( - startWith(null), - map(() => { + public readonly memberDisplaynames$ = this.memberships$.pipe( + map((memberships) => { const displaynameMap = new Map(); - const { room, memberships } = this.matrixRTCSession; + const { room } = this.matrixRTCSession; // We only consider RTC members for disambiguation as they are the only visible members. for (const rtcMember of memberships) { @@ -482,261 +503,341 @@ export class CallViewModel extends ViewModel { return displaynameMap; }), // It turns out that doing the disambiguation above is rather expensive on Safari (10x slower - // than on Chrome/Firefox). This means it is important that we share() the result so that we - // don't do this work more times than we need to. This is achieve through the state() operator: - this.scope.state(), + // than on Chrome/Firefox). This means it is important that we multicast the result so that we + // don't do this work more times than we need to. This is achieved by converting to a behavior: + ); + + public readonly handsRaised$ = this.scope.behavior(this.handsRaisedSubject$); + + public readonly reactions$ = this.scope.behavior( + this.reactionsSubject$.pipe( + map((v) => + Object.fromEntries( + Object.entries(v).map(([a, { reactionOption }]) => [ + a, + reactionOption, + ]), + ), + ), + ), ); /** * List of MediaItems that we want to display */ - private readonly mediaItems$: Observable = combineLatest([ - this.remoteParticipants$, - observeParticipantMedia(this.livekitRoom.localParticipant), - duplicateTiles.value$, - // Also react to changes in the MatrixRTC session list. - // The session list will also be update if a room membership changes. - // No additional RoomState event listener needs to be set up. - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe(startWith(null)), - showNonMemberTiles.value$, - ]).pipe( - scan( - ( - prevItems, - [ - remoteParticipants, - { participant: localParticipant }, - duplicateTiles, - _membershipsChanged, - showNonMemberTiles, - ], - ) => { - const newItems = new Map( - function* (this: CallViewModel): Iterable<[string, MediaItem]> { - const room = this.matrixRTCSession.room; - // m.rtc.members are the basis for calculating what is visible in the call - for (const rtcMember of this.matrixRTCSession.memberships) { - const { member, id: livekitParticipantId } = - getRoomMemberFromRtcMember(rtcMember, room); - const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`; + private readonly mediaItems$ = this.scope.behavior( + combineLatest([ + this.remoteParticipants$, + observeParticipantMedia(this.livekitRoom.localParticipant), + duplicateTiles.value$, + // Also react to changes in the MatrixRTC session list. + // The session list will also be update if a room membership changes. + // No additional RoomState event listener needs to be set up. + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe(startWith(null)), + showNonMemberTiles.value$, + ]).pipe( + scan( + ( + prevItems, + [ + remoteParticipants, + { participant: localParticipant }, + duplicateTiles, + _membershipsChanged, + showNonMemberTiles, + ], + ) => { + const newItems = new Map( + function* (this: CallViewModel): Iterable<[string, MediaItem]> { + const room = this.matrixRTCSession.room; + // m.rtc.members are the basis for calculating what is visible in the call + for (const rtcMember of this.matrixRTCSession.memberships) { + const { member, id: livekitParticipantId } = + getRoomMemberFromRtcMember(rtcMember, room); + const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`; - let participant: - | LocalParticipant - | RemoteParticipant - | undefined = undefined; - if (livekitParticipantId === "local") { - participant = localParticipant; - } else { - participant = remoteParticipants.find( - (p) => p.identity === livekitParticipantId, - ); - } - - if (!member) { - logger.error( - "Could not find member for media id: ", - livekitParticipantId, - ); - } - for (let i = 0; i < 1 + duplicateTiles; i++) { - const indexedMediaId = `${livekitParticipantId}:${i}`; - let prevMedia = prevItems.get(indexedMediaId); - if (prevMedia && prevMedia instanceof UserMedia) { - prevMedia.updateParticipant(participant); - if (prevMedia.vm.member === undefined) { - // We have a previous media created because of the `debugShowNonMember` flag. - // In this case we actually replace the media item. - // This "hack" never occurs if we do not use the `debugShowNonMember` debugging - // option and if we always find a room member for each rtc member (which also - // only fails if we have a fundamental problem) - prevMedia = undefined; - } + let participant: + | LocalParticipant + | RemoteParticipant + | undefined = undefined; + if (livekitParticipantId === "local") { + participant = localParticipant; + } else { + participant = remoteParticipants.find( + (p) => p.identity === livekitParticipantId, + ); } - yield [ - indexedMediaId, - // We create UserMedia with or without a participant. - // This will be the initial value of a BehaviourSubject. - // Once a participant appears we will update the BehaviourSubject. (see above) - prevMedia ?? - new UserMedia( - indexedMediaId, - member, - participant, - this.encryptionSystem, - this.livekitRoom, - this.mediaDevices, - this.memberDisplaynames$.pipe( - map((m) => m.get(matrixIdentifier) ?? "[👻]"), - ), - this.handsRaised$.pipe( - map((v) => v[matrixIdentifier]?.time ?? null), - ), - this.reactions$.pipe( - map((v) => v[matrixIdentifier] ?? undefined), - ), - ), - ]; - if (participant?.isScreenShareEnabled) { - const screenShareId = `${indexedMediaId}:screen-share`; + if (!member) { + logger.error( + "Could not find member for media id: ", + livekitParticipantId, + ); + } + for (let i = 0; i < 1 + duplicateTiles; i++) { + const indexedMediaId = `${livekitParticipantId}:${i}`; + let prevMedia = prevItems.get(indexedMediaId); + if (prevMedia && prevMedia instanceof UserMedia) { + prevMedia.updateParticipant(participant); + if (prevMedia.vm.member === undefined) { + // We have a previous media created because of the `debugShowNonMember` flag. + // In this case we actually replace the media item. + // This "hack" never occurs if we do not use the `debugShowNonMember` debugging + // option and if we always find a room member for each rtc member (which also + // only fails if we have a fundamental problem) + prevMedia = undefined; + } + } yield [ - screenShareId, - prevItems.get(screenShareId) ?? - new ScreenShare( - screenShareId, + indexedMediaId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevMedia ?? + new UserMedia( + indexedMediaId, member, participant, - this.encryptionSystem, + this.options.encryptionSystem, this.livekitRoom, + this.mediaDevices, this.memberDisplaynames$.pipe( map((m) => m.get(matrixIdentifier) ?? "[👻]"), ), + this.handsRaised$.pipe( + map((v) => v[matrixIdentifier]?.time ?? null), + ), + this.reactions$.pipe( + map((v) => v[matrixIdentifier] ?? undefined), + ), ), ]; - } - } - } - }.bind(this)(), - ); - // Generate non member items (items without a corresponding MatrixRTC member) - // Those items should not be rendered, they are participants in LiveKit that do not have a corresponding - // MatrixRTC members. This cannot be any good: - // - A malicious user impersonates someone - // - Someone injects abusive content - // - The user cannot have encryption keys so it makes no sense to participate - // We can only trust users that have a MatrixRTC member event. - // - // This is still available as a debug option. This can be useful - // - If one wants to test scalability using the LiveKit CLI. - // - If an experimental project does not yet do the MatrixRTC bits. - // - If someone wants to debug if the LiveKit connection works but MatrixRTC room state failed to arrive. - const newNonMemberItems = showNonMemberTiles - ? new Map( - function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const participant of remoteParticipants) { - for (let i = 0; i < 1 + duplicateTiles; i++) { - const maybeNonMemberParticipantId = - participant.identity + ":" + i; - if (!newItems.has(maybeNonMemberParticipantId)) { - const nonMemberId = maybeNonMemberParticipantId; - yield [ - nonMemberId, - prevItems.get(nonMemberId) ?? - new UserMedia( - nonMemberId, - undefined, - participant, - this.encryptionSystem, - this.livekitRoom, - this.mediaDevices, - this.memberDisplaynames$.pipe( - map((m) => m.get(participant.identity) ?? "[👻]"), - ), - of(null), - of(null), + if (participant?.isScreenShareEnabled) { + const screenShareId = `${indexedMediaId}:screen-share`; + yield [ + screenShareId, + prevItems.get(screenShareId) ?? + new ScreenShare( + screenShareId, + member, + participant, + this.options.encryptionSystem, + this.livekitRoom, + this.memberDisplaynames$.pipe( + map((m) => m.get(matrixIdentifier) ?? "[👻]"), ), - ]; - } + ), + ]; } } - }.bind(this)(), - ) - : new Map(); - if (newNonMemberItems.size > 0) { - logger.debug("Added NonMember items: ", newNonMemberItems); - } + } + }.bind(this)(), + ); - const combinedNew = new Map([ - ...newNonMemberItems.entries(), - ...newItems.entries(), - ]); + // Generate non member items (items without a corresponding MatrixRTC member) + // Those items should not be rendered, they are participants in LiveKit that do not have a corresponding + // MatrixRTC members. This cannot be any good: + // - A malicious user impersonates someone + // - Someone injects abusive content + // - The user cannot have encryption keys so it makes no sense to participate + // We can only trust users that have a MatrixRTC member event. + // + // This is still available as a debug option. This can be useful + // - If one wants to test scalability using the LiveKit CLI. + // - If an experimental project does not yet do the MatrixRTC bits. + // - If someone wants to debug if the LiveKit connection works but MatrixRTC room state failed to arrive. + const newNonMemberItems = showNonMemberTiles + ? new Map( + function* (this: CallViewModel): Iterable<[string, MediaItem]> { + for (const participant of remoteParticipants) { + for (let i = 0; i < 1 + duplicateTiles; i++) { + const maybeNonMemberParticipantId = + participant.identity + ":" + i; + if (!newItems.has(maybeNonMemberParticipantId)) { + const nonMemberId = maybeNonMemberParticipantId; + yield [ + nonMemberId, + prevItems.get(nonMemberId) ?? + new UserMedia( + nonMemberId, + undefined, + participant, + this.options.encryptionSystem, + this.livekitRoom, + this.mediaDevices, + this.memberDisplaynames$.pipe( + map( + (m) => m.get(participant.identity) ?? "[👻]", + ), + ), + of(null), + of(null), + ), + ]; + } + } + } + }.bind(this)(), + ) + : new Map(); + if (newNonMemberItems.size > 0) { + logger.debug("Added NonMember items: ", newNonMemberItems); + } - for (const [id, t] of prevItems) if (!combinedNew.has(id)) t.destroy(); - return combinedNew; - }, - new Map(), + const combinedNew = new Map([ + ...newNonMemberItems.entries(), + ...newItems.entries(), + ]); + + for (const [id, t] of prevItems) + if (!combinedNew.has(id)) t.destroy(); + return combinedNew; + }, + new Map(), + ), + map((mediaItems) => [...mediaItems.values()]), + finalizeValue((ts) => { + for (const t of ts) t.destroy(); + }), ), - map((mediaItems) => [...mediaItems.values()]), - finalizeValue((ts) => { - for (const t of ts) t.destroy(); - }), - this.scope.state(), ); /** * List of MediaItems that we want to display, that are of type UserMedia */ - private readonly userMedia$: Observable = this.mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is UserMedia => m instanceof UserMedia), + private readonly userMedia$ = this.scope.behavior( + this.mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is UserMedia => m instanceof UserMedia), + ), ), ); - public readonly memberChanges$ = this.userMedia$ - .pipe(map((mediaItems) => mediaItems.map((m) => m.id))) - .pipe( - scan( - (prev, ids) => { - const left = prev.ids.filter((id) => !ids.includes(id)); - const joined = ids.filter((id) => !prev.ids.includes(id)); - return { ids, joined, left }; - }, - { ids: [], joined: [], left: [] }, - ), - ); + /** + * This observable tracks the currently connected participants. + * + * - Each participant has one livekit connection + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one matrix user. + */ + public readonly participantChanges$ = this.userMedia$.pipe( + map((mediaItems) => mediaItems.map((m) => m.id)), + scan( + (prev, ids) => { + const left = prev.ids.filter((id) => !ids.includes(id)); + const joined = ids.filter((id) => !prev.ids.includes(id)); + return { ids, joined, left }; + }, + { ids: [], joined: [], left: [] }, + ), + ); + + /** + * This observable tracks the matrix users that are currently in the call. + * There can be just one matrix user with multiple participants (see also participantChanges$) + */ + public readonly matrixUserChanges$ = this.userMedia$.pipe( + map( + (mediaItems) => + new Set( + mediaItems + .map((m) => m.vm.member?.userId) + .filter((id) => id !== undefined), + ), + ), + scan< + Set, + { + userIds: Set; + joinedUserIds: Set; + leftUserIds: Set; + } + >( + (prevState, userIds) => { + const left = new Set( + [...prevState.userIds].filter((id) => !userIds.has(id)), + ); + const joined = new Set( + [...userIds].filter((id) => !prevState.userIds.has(id)), + ); + return { userIds: userIds, joinedUserIds: joined, leftUserIds: left }; + }, + { userIds: new Set(), joinedUserIds: new Set(), leftUserIds: new Set() }, + ), + ); + + public readonly allOthersLeft$ = this.matrixUserChanges$.pipe( + map(({ userIds, leftUserIds }) => { + const userId = this.matrixRTCSession.room.client.getUserId(); + if (!userId) { + logger.warn("Could access client.getUserId to compute allOthersLeft"); + return false; + } + return userIds.size === 1 && userIds.has(userId) && leftUserIds.size > 0; + }), + startWith(false), + distinctUntilChanged(), + ); + + public readonly autoLeaveWhenOthersLeft$ = this.allOthersLeft$.pipe( + distinctUntilChanged(), + filter((leave) => (leave && this.options.autoLeaveWhenOthersLeft) ?? false), + map(() => {}), + ); /** * List of MediaItems that we want to display, that are of type ScreenShare */ - private readonly screenShares$: Observable = + private readonly screenShares$ = this.scope.behavior( this.mediaItems$.pipe( map((mediaItems) => mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), ), - this.scope.state(), - ); + ), + ); - private readonly spotlightSpeaker$: Observable = - this.userMedia$.pipe( - switchMap((mediaItems) => - mediaItems.length === 0 - ? of([]) - : combineLatest( - mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), + private readonly spotlightSpeaker$ = + this.scope.behavior( + this.userMedia$.pipe( + switchMap((mediaItems) => + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.vm.speaking$.pipe(map((s) => [m, s] as const)), + ), ), - ), + ), + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( + (prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.vm.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.vm.local)?.[0]); + }, + null, + ), + map((speaker) => speaker?.vm ?? null), ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), - this.scope.state(), ); - private readonly grid$: Observable = + private readonly grid$ = this.scope.behavior( this.userMedia$.pipe( switchMap((mediaItems) => { const bins = mediaItems.map((m) => @@ -774,10 +875,10 @@ export class CallViewModel extends ViewModel { ); }), distinctUntilChanged(shallowEquals), - this.scope.state(), - ); + ), + ); - private readonly spotlight$: Observable = + private readonly spotlight$ = this.scope.behavior( this.screenShares$.pipe( switchMap((screenShares) => { if (screenShares.length > 0) { @@ -789,44 +890,45 @@ export class CallViewModel extends ViewModel { ); }), distinctUntilChanged(shallowEquals), - this.scope.state(), - ); + ), + ); - private readonly pip$: Observable = combineLatest([ - this.screenShares$, - this.spotlightSpeaker$, - this.mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return this.spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } + private readonly pip$ = this.scope.behavior( + combineLatest([ + this.screenShares$, + this.spotlightSpeaker$, + this.mediaItems$, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return this.spotlightSpeaker$; + } + if (!spotlight || spotlight.local) { + return of(null); + } - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow$.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } - return null; - }), - ); - }), - this.scope.state(), + return null; + }), + ); + }), + ), ); private readonly hasRemoteScreenShares$: Observable = @@ -837,67 +939,71 @@ export class CallViewModel extends ViewModel { distinctUntilChanged(), ); - private readonly pipEnabled$: Observable = setPipEnabled$.pipe( - startWith(false), - ); + private readonly pipEnabled$ = this.scope.behavior(setPipEnabled$, false); - private readonly naturalWindowMode$: Observable = fromEvent( - window, - "resize", - ).pipe( - startWith(null), - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - this.scope.state(), + private readonly naturalWindowMode$ = this.scope.behavior( + fromEvent(window, "resize").pipe( + startWith(null), + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + // Our layouts for flat windows are better at adapting to a small width + // than our layouts for narrow windows are at adapting to a small height, + // so we give "flat" precedence here + if (height <= 600) return "flat"; + if (width <= 600) return "narrow"; + return "normal"; + }), + ), ); /** * The general shape of the window. */ - public readonly windowMode$: Observable = this.pipEnabled$.pipe( - switchMap((pip) => (pip ? of("pip") : this.naturalWindowMode$)), + public readonly windowMode$ = this.scope.behavior( + this.pipEnabled$.pipe( + switchMap((pip) => + pip ? of("pip") : this.naturalWindowMode$, + ), + ), ); private readonly spotlightExpandedToggle$ = new Subject(); - public readonly spotlightExpanded$: Observable = + public readonly spotlightExpanded$ = this.scope.behavior( this.spotlightExpandedToggle$.pipe( accumulate(false, (expanded) => !expanded), - this.scope.state(), - ); + ), + ); private readonly gridModeUserSelection$ = new Subject(); /** * The layout mode of the media tile grid. */ - public readonly gridMode$: Observable = + public readonly gridMode$ = // If the user hasn't selected spotlight and somebody starts screen sharing, // automatically switch to spotlight mode and reset when screen sharing ends - this.gridModeUserSelection$.pipe( - startWith(null), - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([this.hasRemoteScreenShares$, this.windowMode$]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), + this.scope.behavior( + this.gridModeUserSelection$.pipe( + startWith(null), + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([ + this.hasRemoteScreenShares$, + this.windowMode$, + ]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", + ), + ) + ).pipe(startWith(userSelection ?? "grid")), + ), ), - this.scope.state(), ); public setGridMode(value: GridMode): void { @@ -959,7 +1065,7 @@ export class CallViewModel extends ViewModel { /** * The media to be used to produce a layout. */ - private readonly layoutMedia$: Observable = + private readonly layoutMedia$ = this.scope.behavior( this.windowMode$.pipe( switchMap((windowMode) => { switch (windowMode) { @@ -1022,8 +1128,8 @@ export class CallViewModel extends ViewModel { return this.pipLayoutMedia$; } }), - this.scope.state(), - ); + ), + ); // There is a cyclical dependency here: the layout algorithms want to know // which tiles are on screen, but to know which tiles are on screen we have to @@ -1033,99 +1139,99 @@ export class CallViewModel extends ViewModel { private readonly setVisibleTiles = (value: number): void => this.visibleTiles$.next(value); - public readonly layoutInternals$: Observable< + private readonly layoutInternals$ = this.scope.behavior< LayoutScanState & { layout: Layout } - > = combineLatest([ - this.layoutMedia$, - this.visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": - case "spotlight-landscape": - case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - this.setVisibleTiles, - prevTiles, - ); - break; - case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; - case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; - } + >( + combineLatest([ + this.layoutMedia$, + this.visibleTiles$.pipe(startWith(0), distinctUntilChanged()), + ]).pipe( + scan< + [LayoutMedia, number], + LayoutScanState & { layout: Layout }, + LayoutScanState + >( + ({ tiles: prevTiles }, [media, visibleTiles]) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": + case "spotlight-landscape": + case "spotlight-portrait": + [layout, newTiles] = gridLikeLayout( + media, + visibleTiles, + this.setVisibleTiles, + prevTiles, + ); + break; + case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); + break; + case "one-on-one": + [layout, newTiles] = oneOnOneLayout(media, prevTiles); + break; + case "pip": + [layout, newTiles] = pipLayout(media, prevTiles); + break; + } - return { layout, tiles: newTiles }; - }, - { layout: null, tiles: TileStore.empty() }, + return { layout, tiles: newTiles }; + }, + { layout: null, tiles: TileStore.empty() }, + ), ), - this.scope.state(), ); /** * The layout of tiles in the call interface. */ - public readonly layout$: Observable = this.layoutInternals$.pipe( - map(({ layout }) => layout), - this.scope.state(), + public readonly layout$ = this.scope.behavior( + this.layoutInternals$.pipe(map(({ layout }) => layout)), ); /** * The current generation of the tile store, exposed for debugging purposes. */ - public readonly tileStoreGeneration$: Observable = - this.layoutInternals$.pipe( - map(({ tiles }) => tiles.generation), - this.scope.state(), - ); - - public showSpotlightIndicators$: Observable = this.layout$.pipe( - map((l) => l.type !== "grid"), - this.scope.state(), + public readonly tileStoreGeneration$ = this.scope.behavior( + this.layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), ); - public showSpeakingIndicators$: Observable = this.layout$.pipe( - switchMap((l) => { - switch (l.type) { - case "spotlight-landscape": - case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), - ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. - case "spotlight-expanded": - case "one-on-one": - return of(false); - default: - return of(true); - } - }), - this.scope.state(), + public showSpotlightIndicators$ = this.scope.behavior( + this.layout$.pipe(map((l) => l.type !== "grid")), ); - public readonly toggleSpotlightExpanded$: Observable<(() => void) | null> = + public showSpeakingIndicators$ = this.scope.behavior( + this.layout$.pipe( + switchMap((l) => { + switch (l.type) { + case "spotlight-landscape": + case "spotlight-portrait": + // If the spotlight is showing the active speaker, we can do without + // speaking indicators as they're a redundant visual cue. But if + // screen sharing feeds are in the spotlight we still need them. + return l.spotlight.media$.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), + ); + // In expanded spotlight layout, the active speaker is always shown in + // the picture-in-picture tile so there is no need for speaking + // indicators. And in one-on-one layout there's no question as to who is + // speaking. + case "spotlight-expanded": + case "one-on-one": + return of(false); + default: + return of(true); + } + }), + ), + ); + + public readonly toggleSpotlightExpanded$ = this.scope.behavior< + (() => void) | null + >( this.windowMode$.pipe( switchMap((mode) => mode === "normal" @@ -1142,8 +1248,8 @@ export class CallViewModel extends ViewModel { map((enabled) => enabled ? (): void => this.spotlightExpandedToggle$.next() : null, ), - this.scope.state(), - ); + ), + ); private readonly screenTap$ = new Subject(); private readonly controlsTap$ = new Subject(); @@ -1178,76 +1284,79 @@ export class CallViewModel extends ViewModel { this.screenUnhover$.next(); } - public readonly showHeader$: Observable = this.windowMode$.pipe( - map((mode) => mode !== "pip" && mode !== "flat"), - this.scope.state(), + public readonly showHeader$ = this.scope.behavior( + this.windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), ); - public readonly showFooter$: Observable = this.windowMode$.pipe( - switchMap((mode) => { - switch (mode) { - case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - this.screenTap$.pipe(map(() => "tap screen" as const)), - this.controlsTap$.pipe(map(() => "tap controls" as const)), - this.screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - this.screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); - } - }), - this.scope.state(), + public readonly showFooter$ = this.scope.behavior( + this.windowMode$.pipe( + switchMap((mode) => { + switch (mode) { + case "pip": + return of(false); + case "normal": + case "narrow": + return of(true); + case "flat": + // Sadly Firefox has some layering glitches that prevent the footer + // from appearing properly. They happen less often if we never hide + // the footer. + if (isFirefox()) return of(true); + // Show/hide the footer in response to interactions + return merge( + this.screenTap$.pipe(map(() => "tap screen" as const)), + this.controlsTap$.pipe(map(() => "tap controls" as const)), + this.screenHover$.pipe(map(() => "hover" as const)), + ).pipe( + switchScan((state, interaction) => { + switch (interaction) { + case "tap screen": + return state + ? // Toggle visibility on tap + of(false) + : // Hide after a timeout + timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "tap controls": + // The user is interacting with things, so reset the timeout + return timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "hover": + // Show on hover and hide after a timeout + return race( + timer(showFooterMs), + this.screenUnhover$.pipe(take(1)), + ).pipe( + map(() => false), + startWith(true), + ); + } + }, false), + startWith(false), + ); + } + }), + ), ); /** * Whether audio is currently being output through the earpiece. */ - public readonly earpieceMode$: Observable = combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => - selected !== undefined && available.get(selected.id)?.type === "earpiece", - ).pipe(this.scope.state()); + public readonly earpieceMode$ = this.scope.behavior( + combineLatest( + [ + this.mediaDevices.audioOutput.available$, + this.mediaDevices.audioOutput.selected$, + ], + (available, selected) => + selected !== undefined && + available.get(selected.id)?.type === "earpiece", + ), + ); /** * Callback to toggle between the earpiece and the loudspeaker. @@ -1255,61 +1364,55 @@ export class CallViewModel extends ViewModel { * This will be `null` in case the target does not exist in the list * of available audio outputs. */ - public readonly audioOutputSwitcher$: Observable<{ + public readonly audioOutputSwitcher$ = this.scope.behavior<{ targetOutput: "earpiece" | "speaker"; switch: () => void; - } | null> = combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; + } | null>( + combineLatest( + [ + this.mediaDevices.audioOutput.available$, + this.mediaDevices.audioOutput.selected$, + ], + (available, selected) => { + const selectionType = selected && available.get(selected.id)?.type; - // If we are in any output mode other than spaeker switch to speaker. - const newSelectionType = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; + // If we are in any output mode other than speaker switch to speaker. + const newSelectionType: "earpiece" | "speaker" = + selectionType === "speaker" ? "earpiece" : "speaker"; + const newSelection = [...available].find( + ([, d]) => d.type === newSelectionType, + ); + if (newSelection === undefined) return null; - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: () => this.mediaDevices.audioOutput.select(id), - }; - }, - ); - - public readonly reactions$ = this.reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [a, reactionOption]), - ), + const [id] = newSelection; + return { + targetOutput: newSelectionType, + switch: (): void => this.mediaDevices.audioOutput.select(id), + }; + }, ), ); - public readonly handsRaised$ = this.handsRaisedSubject$.pipe(); - /** * Emits an array of reactions that should be visible on the screen. */ - public readonly visibleReactions$ = showReactions.value$.pipe( - switchMap((show) => (show ? this.reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), + public readonly visibleReactions$ = this.scope.behavior( + showReactions.value$.pipe( + switchMap((show) => (show ? this.reactions$ : of({}))), + scan< + Record, + { sender: string; emoji: string; startX: number }[] + >((acc, latest) => { + const newSet: { sender: string; emoji: string; startX: number }[] = []; + for (const [sender, reaction] of Object.entries(latest)) { + const startX = + acc.find((v) => v.sender === sender && v.emoji)?.startX ?? + Math.ceil(Math.random() * 80) + 10; + newSet.push({ sender, emoji: reaction.emoji, startX }); + } + return newSet; + }, []), + ), ); /** @@ -1373,7 +1476,7 @@ export class CallViewModel extends ViewModel { private readonly matrixRTCSession: MatrixRTCSession, private readonly livekitRoom: LivekitRoom, private readonly mediaDevices: MediaDevices, - private readonly encryptionSystem: EncryptionSystem, + private readonly options: CallViewModelOptions, private readonly connectionState$: Observable, private readonly handsRaisedSubject$: Observable< Record diff --git a/src/state/MediaDevices.ts b/src/state/MediaDevices.ts index f251e759..2349e361 100644 --- a/src/state/MediaDevices.ts +++ b/src/state/MediaDevices.ts @@ -10,7 +10,6 @@ import { filter, map, merge, - of, pairwise, startWith, Subject, @@ -18,7 +17,7 @@ import { type Observable, } from "rxjs"; import { createMediaDeviceObserver } from "@livekit/components-core"; -import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { audioInput as audioInputSetting, @@ -34,11 +33,11 @@ import { import { getUrlParams } from "../UrlParams"; import { platform } from "../Platform"; import { switchWhen } from "../utils/observable"; +import { type Behavior, constant } from "./Behavior"; // This hardcoded id is used in EX ios! It can only be changed in coordination with // the ios swift team. const EARPIECE_CONFIG_ID = "earpiece-id"; -const logger = rootLogger.getChild("[MediaDevices]"); export type DeviceLabel = | { type: "name"; name: string } @@ -74,11 +73,11 @@ export interface MediaDevice { /** * A map from available device IDs to labels. */ - available$: Observable>; + available$: Behavior>; /** * The selected device. */ - selected$: Observable; + selected$: Behavior; /** * Selects a new device. */ @@ -94,35 +93,37 @@ export interface MediaDevice { * `availableOutputDevices$.includes((d)=>d.forEarpiece)` */ export const iosDeviceMenu$ = - platform === "ios" ? of(true) : alwaysShowIphoneEarpieceSetting.value$; + platform === "ios" ? constant(true) : alwaysShowIphoneEarpieceSetting.value$; function availableRawDevices$( kind: MediaDeviceKind, - usingNames$: Observable, + usingNames$: Behavior, scope: ObservableScope, -): Observable { + logger: Logger, +): Behavior { const logError = (e: Error): void => logger.error("Error creating MediaDeviceObserver", e); const devices$ = createMediaDeviceObserver(kind, logError, false); const devicesWithNames$ = createMediaDeviceObserver(kind, logError, true); - return usingNames$.pipe( - switchMap((withNames) => - withNames - ? // It might be that there is already a media stream running somewhere, - // and so we can do without requesting a second one. Only switch to the - // device observer that explicitly requests the names if we see that - // names are in fact missing from the initial device enumeration. - devices$.pipe( - switchWhen( - (devices, i) => i === 0 && devices.every((d) => !d.label), - devicesWithNames$, - ), - ) - : devices$, + return scope.behavior( + usingNames$.pipe( + switchMap((withNames) => + withNames + ? // It might be that there is already a media stream running somewhere, + // and so we can do without requesting a second one. Only switch to the + // device observer that explicitly requests the names if we see that + // names are in fact missing from the initial device enumeration. + devices$.pipe( + switchWhen( + (devices, i) => i === 0 && devices.every((d) => !d.label), + devicesWithNames$, + ), + ) + : devices$, + ), ), - startWith([]), - scope.state(), + [], ); } @@ -161,34 +162,40 @@ function selectDevice$
- {onToggleExpanded && ( +
- )} + + {onToggleExpanded && ( + + )} +
+ {canGoToNext && (