diff --git a/.github/workflows/publish-embedded-packages.yaml b/.github/workflows/publish-embedded-packages.yaml index 34e5885d0..2f1ef2974 100644 --- a/.github/workflows/publish-embedded-packages.yaml +++ b/.github/workflows/publish-embedded-packages.yaml @@ -9,7 +9,7 @@ on: - opened - labeled push: - branches: [livekit] + branches: [main] jobs: versioning: diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index ec3639365..d50216307 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -4,12 +4,12 @@ on: push: branches: - - livekit + - main paths: - .github/labels.yml - .github/workflows/sync-labels.yml -permissions: {} # We use ELEMENT_BOT_TOKEN instead +permissions: {} # We use LABEL_SYNC_GITHUB_TOKEN instead jobs: sync-labels: @@ -20,4 +20,4 @@ jobs: DELETE: true WET: true secrets: - ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }} + ELEMENT_BOT_TOKEN: ${{ secrets.LABEL_SYNC_GITHUB_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bf1894eff..3f86f093d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,7 +2,7 @@ name: Test on: pull_request: {} push: - branches: [livekit] + branches: [main] jobs: vitest: name: Run unit tests diff --git a/.github/workflows/translations-upload.yaml b/.github/workflows/translations-upload.yaml index daf968958..06d9b8721 100644 --- a/.github/workflows/translations-upload.yaml +++ b/.github/workflows/translations-upload.yaml @@ -2,7 +2,7 @@ name: Upload translation files to Localazy on: push: branches: - - livekit + - main paths-ignore: - ".github/**" diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index caeac8286..35ab5cb36 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -2,7 +2,7 @@ name: GitHub Actions Security Analysis with zizmor 🌈 on: push: - branches: ["livekit", "full-mesh"] + branches: ["main", "full-mesh"] pull_request: {} permissions: {} diff --git a/config/config.devenv.json b/config/config.devenv.json index df0ff4c18..48602406b 100644 --- a/config/config.devenv.json +++ b/config/config.devenv.json @@ -5,9 +5,6 @@ "server_name": "synapse.m.localhost" } }, - "features": { - "feature_use_device_session_member_events": true - }, "ssla": "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf", "matrix_rtc_session": { "wait_for_key_rotation_ms": 3000, diff --git a/config/config.sample.json b/config/config.sample.json index 78f9536da..0baa522e4 100644 --- a/config/config.sample.json +++ b/config/config.sample.json @@ -8,9 +8,6 @@ "livekit": { "livekit_service_url": "https://livekit-jwt.mydomain.com" }, - "features": { - "feature_use_device_session_member_events": true - }, "ssla": "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf", "matrix_rtc_mode": "compatibility", "matrix_rtc_session": { diff --git a/docker-compose-playwright.yml b/docker-compose-playwright.yml index 4dac164ff..3ed066063 100644 --- a/docker-compose-playwright.yml +++ b/docker-compose-playwright.yml @@ -13,7 +13,7 @@ services: - ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z element-web: # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates. - image: ghcr.io/element-hq/element-web:develop@sha256:af44e1e18bdd363a3332a577c3af66a9dde4ad352508a42ee5012c4a9528b30d + image: ghcr.io/element-hq/element-web:develop@sha256:133160c3c1e506276145cbb8ccaf48253f0a96af635129d785826265e4f3ca10 element-web-1: # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates. - image: ghcr.io/element-hq/element-web:develop@sha256:af44e1e18bdd363a3332a577c3af66a9dde4ad352508a42ee5012c4a9528b30d + image: ghcr.io/element-hq/element-web:develop@sha256:133160c3c1e506276145cbb8ccaf48253f0a96af635129d785826265e4f3ca10 diff --git a/docs/README.md b/docs/README.md index e5a5d08a3..56c96a6ca 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,5 +5,6 @@ This folder contains documentation for setup, usage, and development of Element - [Embedded vs standalone mode](./embedded_standalone.md) - [Url format and parameters](./url_params.md) - [Global JS controls](./controls.md) +- [MatrixRTC modes](./matrix_rtc_modes.md) - [Self-Hosting](./self_hosting.md) - [Developing with linked packages](./linking.md) diff --git a/docs/matrix_rtc_modes.md b/docs/matrix_rtc_modes.md new file mode 100644 index 000000000..a30248ba0 --- /dev/null +++ b/docs/matrix_rtc_modes.md @@ -0,0 +1,48 @@ +# MatrixRTC modes + +Element Call is in the middle of a transition of how a call session is +represented: from room _state_ events (`org.matrix.msc3401.call.member`) to +_sticky_ events +([MSC4354](https://github.com/matrix-org/matrix-spec-proposals/pull/4354)), +which are a much better fit for the short lived, per-device nature of call +memberships. + +Not every homeserver supports sticky events yet. The two MatrixRTC modes +controls whether Element Call uses them. + +## The modes + +| Mode | Membership events | SFU selection | JWT endpoint | +| --------------- | ----------------- | ------------- | ---------------------------- | +| `compatibility` | state events | multi SFU | legacy | +| `matrix_2_0` | sticky events | multi SFU | Matrix 2.0 (hashed identity) | + +**`compatibility`** β€” multi SFU, but still state events. Use it when the +homeserver does not support sticky events. This is the default. + +**`matrix_2_0`** β€” the target state. Requires a homeserver that advertises +MSC4354 and all clients on v0.17.0 or later. The local membership requests its +token from the Matrix 2.0 JWT endpoint of the +[MatrixRTC Authorization Service](https://github.com/element-hq/lk-jwt-service) +and identifies the room by a hashed identity instead of a `livekit_alias`. +(Remote memberships always try the new endpoint first and fall back to the +legacy one, so remote participants can be on either.) + +## Selecting a mode + +Users can choose a mode under **Settings β†’ Developer β†’ MatrixRTC mode**. The +Matrix 2.0 option is disabled if the homeserver does not support sticky events. + +A deployment can pin the mode for all its clients in `config.json`, which +disables the Developer Settings choice: + +```json +{ + "matrix_rtc_mode": "compatibility" +} +``` + +Valid values are `compatibility` and `matrix_2_0`; an invalid value is +ignored (with a warning) and the user's choice applies. Pinning `matrix_2_0` on a +homeserver without sticky event support makes joining fail with a "sticky events +required" error. diff --git a/docs/self_hosting.md b/docs/self_hosting.md index 0bdfe6284..e80fd4218 100644 --- a/docs/self_hosting.md +++ b/docs/self_hosting.md @@ -16,7 +16,7 @@ The following [MSCs](https://github.com/matrix-org/matrix-spec-proposals) are required for Element Call to work properly: -- **[MSC3266](https://github.com/deepbluev7/matrix-doc/blob/room-summaries/proposals/3266-room-summary.md): +- **[MSC3266](https://github.com/deepbluev7/matrix-doc/blob/room-summaries/proposals/3266-room-summary.md) Room Summary API**: In Standalone mode Element Call is able to join rooms over federation using knocking. In this context MSC3266 is required as it allows to request a room summary of rooms you are not joined. The summary @@ -29,19 +29,26 @@ required for Element Call to work properly: signalling. If disabled it is very likely that you end up with stuck calls in Matrix rooms. +- **[MSC4519](https://github.com/matrix-org/matrix-spec-proposals/blob/travis/msc/voip-transports-registry/proposals/4519-rtc-transports-registry.md) + MatrixRTC Transports Registry**: Defines an endpoint that clients can use to + query the available MatrixRTC transports (i.e. find your LiveKit SFU). + - **[MSC4222](https://github.com/matrix-org/matrix-spec-proposals/blob/erikj/sync_v2_state_after/proposals/4222-sync-v2-state-after.md) Adding `state_after` to sync v2**: Allow clients to opt-in to a change of the sync v2 API that allows them to correctly track the state of the room. This is required by Element Call to track room state reliably. If you're using [Synapse](https://github.com/element-hq/synapse/) as your -homeserver, you'll need to additionally add the following config items to -`homeserver.yaml` to comply with Element Call: +homeserver, you can configure these features by adding the following entries to +`homeserver.yaml`: ```yaml experimental_features: # MSC3266: Room summary API. Used for knocking over federation msc3266_enabled: true + # MSC4143: MatrixRTC. For historical reasons this flag enables the transports + # endpoint defined in MSC4519. + msc4143_enabled: true # MSC4222 needed for syncv2 state_after. This allow clients to # correctly track the state of the room. msc4222_enabled: true @@ -61,6 +68,15 @@ rc_delayed_event_mgmt: # Currently the heart-beat is every 5 seconds which translates into a rate of 0.2Hz per_second: 1 burst_count: 20 + +matrix_rtc: + transports: + # The transport you specify will be made available to clients over the + # /_matrix/client/unstable/org.matrix.msc4143/rtc/transports endpoint as + # defined in MSC4519. + - type: livekit + # Replace this with the actual URL of your MatrixRTC Authorization Service + livekit_service_url: https://matrix-rtc.example.com/livekit/jwt ``` As a prerequisite for the @@ -187,23 +203,6 @@ backend mxrtc_auth_backend ``` -#### MatrixRTC transport announcement - -Enable the unstable feature flag `msc4143_enabled`, and update the -[`matrix_rtc` section](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#matrix_rtc) -of your Synapse config file: - -```yaml -matrix_rtc: - transports: - - type: livekit - livekit_service_url: https://matrix-rtc.example.com/livekit/jwt -``` - -The transport you specify will be made available to clients over the -`/_matrix/client/unstable/org.matrix.msc4143/rtc/transports` endpoint as defined -in [MSC4143](https://github.com/matrix-org/matrix-spec-proposals/pull/4143). - ## Building Element Call > [!NOTE] diff --git a/locales/en/app.json b/locales/en/app.json index 2b3e08358..543942e2e 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -80,10 +80,6 @@ "description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)", "label": "Compatibility: state events & multi SFU" }, - "Legacy": { - "description": "Compatible with old versions of EC that do not support multi SFU", - "label": "Legacy: state events & oldest membership SFU" - }, "Matrix_2_0": { "description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later", "label": "Matrix 2.0: sticky events & multi SFU" diff --git a/playwright/spa-helpers.ts b/playwright/spa-helpers.ts index 24869141f..5f99e32d3 100644 --- a/playwright/spa-helpers.ts +++ b/playwright/spa-helpers.ts @@ -101,9 +101,7 @@ async function setRtcModeFromSettings( // Move to Developer tab now await page.getByRole("tab", { name: "Developer" }).click(); - if (mode == "legacy") { - await page.getByText("Legacy: state events").click(); - } else if (mode == "2_0") { + if (mode == "2_0") { await page.getByText("Matrix 2.0").click(); } else { // compat diff --git a/playwright/widget/federated-call.test.ts b/playwright/widget/federated-call.test.ts index 560636a5d..61f4750b3 100644 --- a/playwright/widget/federated-call.test.ts +++ b/playwright/widget/federated-call.test.ts @@ -12,9 +12,7 @@ import { HOST1, HOST2, type RtcMode, TestHelpers } from "./test-helpers"; const modePairs: [RtcMode, RtcMode][] = [ ["compat", "compat"], - ["legacy", "legacy"], - ["legacy", "compat"], - ["compat", "legacy"], + // TODO: Compatibility + Matrix 2.0? ]; modePairs.forEach(([rtcMode1, rtcMode2]) => { diff --git a/playwright/widget/federation-oldest-membership-bug.spec.ts b/playwright/widget/federation-oldest-membership-bug.spec.ts deleted file mode 100644 index ab5c70fc8..000000000 --- a/playwright/widget/federation-oldest-membership-bug.spec.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { expect, test } from "@playwright/test"; - -import { widgetTest } from "../fixtures/widget-user"; -import { HOST1, HOST2, TestHelpers } from "./test-helpers"; - -widgetTest( - "Bug new joiner was not publishing on correct SFU", - async ({ addUser, browserName }) => { - test.skip( - browserName === "firefox", - "This is a bug in the old widget, not a browser problem.", - ); - - test.slow(); - - // 2 users in federation - const florian = await addUser("floriant", HOST1); - const timo = await addUser("timo", HOST2); - - // Florian creates a room and invites Timo to it - const roomName = "Call Room"; - await TestHelpers.createRoom(roomName, florian.page, [timo.mxId]); - - // Timo joins the room - await TestHelpers.acceptRoomInvite(roomName, timo.page); - - // Ensure we are in legacy mode (should be the default) - await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget( - florian.page, - "legacy", - ); - await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget( - timo.page, - "legacy", - ); - - // Let timo create a call - await TestHelpers.startCallInCurrentRoom(timo.page, false); - await TestHelpers.joinCallFromLobby(timo.page); - - // We want to simulate that the oldest membership authentication is way slower than - // the preffered auth. - // In this setup, timo advertised$ transport will be it's own, and the active will be the one from florian - await florian.page.route( - "**/matrix-rtc.othersite.m.localhost/livekit/jwt/**", - async (route) => { - await new Promise((resolve) => setTimeout(resolve, 2000)); // 5 second delay - await route.continue(); - }, - ); - - // Florian joins the call - await expect(florian.page.getByTestId("join-call-button")).toBeVisible(); - await florian.page.getByTestId("join-call-button").click(); - await TestHelpers.joinCallFromLobby(florian.page); - - await florian.page.waitForTimeout(3000); - await timo.page.waitForTimeout(3000); - - // We should see 2 video tiles everywhere now - for (const user of [timo, florian]) { - const frame = user.page - .locator('iframe[title="Element Call"]') - .contentFrame(); - await expect(frame.getByTestId("videoTile")).toHaveCount(2); - - // No one should be waiting for media - await expect(frame.getByText("Waiting for media...")).not.toBeVisible(); - - // There should be 2 video elements, visible and autoplaying - await expect(frame.locator("video")).toHaveCount(2, { - timeout: 10000, - }); - - await TestHelpers.expectVisibleVideoCount(frame, 2); - } - }, -); diff --git a/playwright/widget/hotswap-legacy-compat.test.ts b/playwright/widget/hotswap-legacy-compat.test.ts deleted file mode 100644 index ed6f15083..000000000 --- a/playwright/widget/hotswap-legacy-compat.test.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { expect, test } from "@playwright/test"; - -import { widgetTest } from "../fixtures/widget-user"; -import { HOST1, HOST2, TestHelpers } from "./test-helpers"; - -// ## Issue -// This test reproduces an issue with the publisher. -// When switching local focus, we need to recreate the publisher. -// This failed because of a dead lock in the old publishers destruction. -// -// There are numerus ways to enforece this situation: -// - oldest member swap (manually set the oldest member focus and leave with the prev oldest member) -// This almost never happens in the real worls since clients will set their preferredFoci list to what the oldest member is. -// - switch from oldest member to multi sfu as the NOT the first joiner + the first joiner is on a different sfu than your preferred sfu. -// -// This test uses the "switch from oldest member to multi sfu" approach. -// -// It is a copy of federated-call.test.ts in the `["legacy", "legacy"]` setup, -// which once connected will make the second user switch to multi sfu. -widgetTest( - `Test swapping publisher from ${HOST1} to ${HOST2}`, - async ({ addUser, browserName }) => { - test.slow(); - test.skip( - browserName === "firefox", - "The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled", - ); - - const florian = await addUser("floriant", HOST1); - const timo = await addUser("timo", HOST2); - - const roomName = "Call Room"; - - await TestHelpers.createRoom(roomName, florian.page, [timo.mxId]); - - await TestHelpers.acceptRoomInvite(roomName, timo.page); - - await florian.page.pause(); - - await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget( - florian.page, - "legacy", - ); - await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget( - timo.page, - "legacy", - ); - - await TestHelpers.startCallInCurrentRoom(florian.page, false); - await TestHelpers.joinCallFromLobby(florian.page); - - // timo joins - await TestHelpers.joinCallInCurrentRoom(timo.page); - - // We should see 2 video tiles everywhere now - for (const user of [timo, florian]) { - const frame = user.page - .locator('iframe[title="Element Call"]') - .contentFrame(); - await expect(frame.getByTestId("videoTile")).toHaveCount(2); - - // Wait for "Waiting for media..." to disappear (with timeout) - await expect(frame.getByText("Waiting for media...")).not.toBeVisible({ - timeout: 10000, // Maximum time to wait - }); - - // There should be 2 video elements, visible and autoplaying - await expect(frame.locator("video")).toHaveCount(2, { - timeout: 10000, - }); - - await TestHelpers.expectVisibleVideoCount(frame, 2); - } - - // now we switch the mode for timo (second joiner on multi-sfu HOST2 but currently HOST1) - await TestHelpers.setEmbeddedElementCallRtcMode(timo.page, "compat"); - await timo.page.waitForTimeout(3000); - - await TestHelpers.expectVisibleVideoCount( - timo.page.locator('iframe[title="Element Call"]').contentFrame(), - 2, - ); - }, -); diff --git a/playwright/widget/test-helpers.ts b/playwright/widget/test-helpers.ts index 632b2592b..0322596aa 100644 --- a/playwright/widget/test-helpers.ts +++ b/playwright/widget/test-helpers.ts @@ -21,7 +21,7 @@ const PASSWORD = "foobarbaz1!"; export const HOST1 = "https://app.m.localhost/#/welcome"; export const HOST2 = "https://app.othersite.m.localhost/#/welcome"; -export type RtcMode = "legacy" | "compat" | "2_0"; +export type RtcMode = "compat" | "2_0"; export class TestHelpers { public static async startCallInCurrentRoom( @@ -309,9 +309,7 @@ export class TestHelpers { // Move to Developer tab now await iframe.getByRole("tab", { name: "Developer" }).click(); - if (mode == "legacy") { - await iframe.getByText("Legacy: state events").click(); - } else if (mode == "2_0") { + if (mode == "2_0") { await iframe.getByText("Matrix 2.0").click(); } else { // compat diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 04d1ca4a6..5d8f89cab 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -39,13 +39,13 @@ importers: version: 11.7.12 '@livekit/components-core': specifier: ^0.12.0 - version: 0.12.15(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1) + version: 0.12.15(livekit-client@2.22.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1) '@livekit/components-react': specifier: ^2.0.0 - version: 2.9.24(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(tslib@2.8.1) + version: 2.9.24(livekit-client@2.22.0(@types/dom-mediacapture-record@1.0.22))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(tslib@2.8.1) '@livekit/track-processors': specifier: ^0.7.1 - version: 0.7.2(@types/dom-mediacapture-transform@0.1.11)(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22)) + version: 0.7.2(@types/dom-mediacapture-transform@0.1.11)(livekit-client@2.22.0(@types/dom-mediacapture-record@1.0.22)) '@mediapipe/tasks-vision': specifier: ^0.10.18 version: 0.10.35 @@ -132,10 +132,10 @@ importers: version: 10.3.1(react@19.2.8) '@vector-im/compound-design-tokens': specifier: ^10.0.0 - version: 10.2.3(@types/react@19.2.17)(react@19.2.8) + version: 10.2.4(@types/react@19.2.17)(react@19.2.8) '@vector-im/compound-web': specifier: ^10.0.0 - version: 10.0.1(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(@vector-im/compound-design-tokens@10.2.3(@types/react@19.2.17)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8) + version: 10.0.1(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(@vector-im/compound-design-tokens@10.2.4(@types/react@19.2.17)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8) '@vitejs/plugin-react': specifier: ^6.0.2 version: 6.0.5(@rolldown/plugin-babel@0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.0)(rolldown@1.1.5)(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(sass@1.102.0)(terser@5.46.1)(yaml@2.9.0)))(babel-plugin-react-compiler@1.0.0)(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(sass@1.102.0)(terser@5.46.1)(yaml@2.9.0)) @@ -183,7 +183,7 @@ importers: version: 5.88.1(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2)(@types/node@24.13.3)(typescript@5.9.3) livekit-client: specifier: ^2.18.1 - version: 2.21.0(@types/dom-mediacapture-record@1.0.22) + version: 2.22.0(@types/dom-mediacapture-record@1.0.22) lodash-es: specifier: ^4.17.21 version: 4.18.1 @@ -3240,14 +3240,20 @@ packages: peerDependencies: react: '>= 16.8.0' - '@vector-im/compound-design-tokens@10.2.3': - resolution: {integrity: sha512-K2xNkjOzYn2Ya0kUqCy7JIYyx9jbkNIWpex2t9m3FUG4wzkNscvaiRJ/a9Wse1aJmwDDARcHU6mUjtHf5os5MA==} + '@vector-im/compound-design-tokens@10.2.4': + resolution: {integrity: sha512-2aCjhSvJxktPWnvh5s9Xdc0Rd76AfHqbDbN4QjOxo2VIac6zYdLyQbxMUQ5ZeoR9G3vAxILd0H1UvApRfMilpQ==} peerDependencies: + '@adobe/leonardo-contrast-colors': ^1.0.0 '@types/react': '*' + chroma-js: ^3.0.0 react: ^17 || ^18 || ^19.0.0 peerDependenciesMeta: + '@adobe/leonardo-contrast-colors': + optional: true '@types/react': optional: true + chroma-js: + optional: true react: optional: true @@ -4459,8 +4465,8 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true - jose@6.2.4: - resolution: {integrity: sha512-N8acGzVsQy6M/fjFcxtysNc4Q379TcM5dM/qKkNtsHFji88yANnXTr7BLeP75iPnFwBfQzM/jg2BZ9+HZrHCZA==} + jose@6.2.10: + resolution: {integrity: sha512-iiW7J9qRFlGxvCOIBDBDxFePQSn7ZMAnrYGhrrOo6siO/MIqwfyilLR27pkfDgUk+raLuzADS8A3S/KLBisc0g==} jose@6.2.9: resolution: {integrity: sha512-XrchZOFZUl/T3vTwRe8XK+cJrGtMF4th1ARnDfwbBXFKThGhlsxEE4Zu03AD/bjJSt/9jT/mxrOCkJWOg77aPA==} @@ -4608,8 +4614,8 @@ packages: lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} - livekit-client@2.21.0: - resolution: {integrity: sha512-RBUhPkV/sl1nzl8lokVlK5uATPwn0AlsudCBZXissw/kDl9yz8ac4pNJ43iPpMVoHOeBYH/BZ3vUC1adqa/zFQ==} + livekit-client@2.22.0: + resolution: {integrity: sha512-GLtYQfRh/RsvXaOX1x609bFZ17yyKmKWDqu9JmkMKn9vIFLi2GsapRv9gT8OJO/1R2dsitrkbGmloyUfxWQsaA==} peerDependencies: '@types/dom-mediacapture-record': ^1 @@ -7064,21 +7070,21 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@livekit/components-core@0.12.15(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)': + '@livekit/components-core@0.12.15(livekit-client@2.22.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)': dependencies: '@floating-ui/dom': 1.7.6 - livekit-client: 2.21.0(@types/dom-mediacapture-record@1.0.22) + livekit-client: 2.22.0(@types/dom-mediacapture-record@1.0.22) loglevel: 1.9.1 rxjs: 7.8.2 tslib: 2.8.1 - '@livekit/components-react@2.9.24(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(tslib@2.8.1)': + '@livekit/components-react@2.9.24(livekit-client@2.22.0(@types/dom-mediacapture-record@1.0.22))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(tslib@2.8.1)': dependencies: - '@livekit/components-core': 0.12.15(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1) + '@livekit/components-core': 0.12.15(livekit-client@2.22.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1) clsx: 2.1.1 events: 3.3.0 jose: 6.2.9 - livekit-client: 2.21.0(@types/dom-mediacapture-record@1.0.22) + livekit-client: 2.22.0(@types/dom-mediacapture-record@1.0.22) react: 19.2.8 react-dom: 19.2.8(react@19.2.8) tslib: 2.8.1 @@ -7090,11 +7096,11 @@ snapshots: dependencies: '@bufbuild/protobuf': 1.10.1 - '@livekit/track-processors@0.7.2(@types/dom-mediacapture-transform@0.1.11)(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))': + '@livekit/track-processors@0.7.2(@types/dom-mediacapture-transform@0.1.11)(livekit-client@2.22.0(@types/dom-mediacapture-record@1.0.22))': dependencies: '@mediapipe/tasks-vision': 0.10.35 '@types/dom-mediacapture-transform': 0.1.11 - livekit-client: 2.21.0(@types/dom-mediacapture-record@1.0.22) + livekit-client: 2.22.0(@types/dom-mediacapture-record@1.0.22) '@matrix-org/matrix-sdk-crypto-wasm@18.5.0': {} @@ -8711,12 +8717,12 @@ snapshots: '@use-gesture/core': 10.3.1 react: 19.2.8 - '@vector-im/compound-design-tokens@10.2.3(@types/react@19.2.17)(react@19.2.8)': + '@vector-im/compound-design-tokens@10.2.4(@types/react@19.2.17)(react@19.2.8)': optionalDependencies: '@types/react': 19.2.17 react: 19.2.8 - '@vector-im/compound-web@10.0.1(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(@vector-im/compound-design-tokens@10.2.3(@types/react@19.2.17)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)': + '@vector-im/compound-web@10.0.1(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(@vector-im/compound-design-tokens@10.2.4(@types/react@19.2.17)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)': dependencies: '@floating-ui/react': 0.27.20(react-dom@19.2.8(react@19.2.8))(react@19.2.8) '@floating-ui/react-dom': 2.1.9(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -8728,7 +8734,7 @@ snapshots: '@radix-ui/react-progress': 1.1.16(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) '@radix-ui/react-separator': 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) '@radix-ui/react-slot': 1.3.3(@types/react@19.2.17)(react@19.2.8) - '@vector-im/compound-design-tokens': 10.2.3(@types/react@19.2.17)(react@19.2.8) + '@vector-im/compound-design-tokens': 10.2.4(@types/react@19.2.17)(react@19.2.8) classnames: 2.5.1 react: 19.2.8 vaul: 1.1.2(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -10053,7 +10059,7 @@ snapshots: jiti@2.7.0: {} - jose@6.2.4: {} + jose@6.2.10: {} jose@6.2.9: {} @@ -10197,13 +10203,13 @@ snapshots: lines-and-columns@1.2.4: {} - livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22): + livekit-client@2.22.0(@types/dom-mediacapture-record@1.0.22): dependencies: '@livekit/mutex': 1.1.1 '@livekit/protocol': 1.50.4 '@types/dom-mediacapture-record': 1.0.22 events: 3.3.0 - jose: 6.2.4 + jose: 6.2.10 loglevel: 1.9.2 sdp-transform: 2.15.0 tslib: 2.8.1 diff --git a/src/components/CallFooter.stories.tsx b/src/components/CallFooter.stories.tsx index f95751094..667cb6070 100644 --- a/src/components/CallFooter.stories.tsx +++ b/src/components/CallFooter.stories.tsx @@ -127,6 +127,7 @@ export const Default: Story = { showFooter: true, hideControls: false, asOverlay: false, + showModals: true, sharingScreen: false, audioOutputSwitcher: undefined, reactionIdentifier: undefined, diff --git a/src/components/CallFooter.tsx b/src/components/CallFooter.tsx index f5c54ee38..4f79f236c 100644 --- a/src/components/CallFooter.tsx +++ b/src/components/CallFooter.tsx @@ -26,6 +26,7 @@ import { MediaMuteAndSwitchButton, type MenuOptions, } from "./MediaMuteAndSwitchButton"; +import { type Behavior } from "../state/Behavior"; import { type ViewModel } from "../state/ViewModel"; import { useBehavior } from "../useBehavior"; import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel"; @@ -76,6 +77,7 @@ export interface FooterState { /** The footer should be used as an overlay. * (Over the Call Grid) This saves spaces on small screens. */ asOverlay: boolean; + showModals: boolean; buttonSize: "md" | "lg"; showLogo: boolean; @@ -120,6 +122,7 @@ export const CallFooter: FC = ({ const asOverlay = useBehavior(vm.asOverlay$); const showFooter = useBehavior(vm.showFooter$); const hideControls = useBehavior(vm.hideControls$); + const showModals = useBehavior(vm.showModals$); const layoutSwitchVm = useBehavior(vm.layoutSwitchVm$); const openSettings = useBehavior(vm.openSettings$); const audioEnabled = useBehavior(vm.audioEnabled$); @@ -135,7 +138,6 @@ export const CallFooter: FC = ({ const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); const hangup = useBehavior(vm.hangup$); const debugTileLayout = useBehavior(vm.debugTileLayout$); - const tileStoreGeneration = useBehavior(vm.tileStoreGeneration$); const videoOptions = useBehavior(vm.videoOptions$); const selectedVideo = useBehavior(vm.selectedVideo$); const audioOptions = useBehavior(vm.audioOptions$); @@ -235,7 +237,8 @@ export const CallFooter: FC = ({ ); } - if (reactionIdentifier && reactionData) { + // Reaction button contains a pretty large menu, so treat it like a modal + if (reactionIdentifier && reactionData && showModals) { buttons.push( = ({ /> )} - {debugTileLayout ? `Tiles generation: ${tileStoreGeneration}` : undefined} + {debugTileLayout ? ( + + ) : undefined} ); @@ -316,3 +321,14 @@ export const CallFooter: FC = ({ ); }; + +interface TilesDebugInfoProps { + generation$: Behavior; +} + +// Isolated in its own component since the layout generation updates frequently +// and we can avoid re-rendering the footer this way +const TilesDebugInfo: FC = ({ generation$ }) => { + const generation = useBehavior(generation$); + return `Tiles generation: ${generation}`; +}; diff --git a/src/components/CallFooterViewModel.test.ts b/src/components/CallFooterViewModel.test.ts index ef3b756e6..1fc9187ad 100644 --- a/src/components/CallFooterViewModel.test.ts +++ b/src/components/CallFooterViewModel.test.ts @@ -49,6 +49,7 @@ function buildMinimalCallViewModel(layout: Layout): CallViewModel { reactions$: constant({}), tileStoreGeneration$: constant(0), showFooter$: constant(true), + showModals$: constant(true), settingsOpen$: constant(false), setSettingsOpen$: constant(() => {}), } as unknown as CallViewModel; diff --git a/src/components/CallFooterViewModel.tsx b/src/components/CallFooterViewModel.tsx index a374a0258..7e391b169 100644 --- a/src/components/CallFooterViewModel.tsx +++ b/src/components/CallFooterViewModel.tsx @@ -161,6 +161,7 @@ export function createCallFooterViewModel( // candidat to move into the FooterViewModel showFooter$: callModel.showFooter$, hideControls$: constant(!showControls), + showModals$: callModel.showModals$, asOverlay$: callModel.edgeToEdge$, buttonSize$: scope.behavior( isPip$.pipe(map((pip) => (pip ? "md" : "lg"))), @@ -168,12 +169,12 @@ export function createCallFooterViewModel( openSettings$: scope.behavior( combineLatest([ - isPip$, + callModel.showModals$, callModel.showHeader$, callModel.setSettingsOpen$, ]).pipe( - map(([isPip, showHeader, setSettingsOpen]) => - !isPip && headerStyle !== HeaderStyle.AppBar && showControls + map(([showModals, showHeader, setSettingsOpen]) => + showModals && headerStyle !== HeaderStyle.AppBar && showControls ? (): void => setSettingsOpen(true) : undefined, ), @@ -239,6 +240,7 @@ export function createLobbyFooterViewModel( showLogo, hideControls: false, asOverlay: false, + showModals: true, buttonSize: "lg", openSettings, hangup, diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index c7909659d..536966e00 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -12,9 +12,7 @@ Please see LICENSE in the repository root for full details. * Settings, or pinned for a deployment via `matrix_rtc_mode` in config.json. */ export enum MatrixRTCMode { - /** Legacy single-SFU + user-keyed memberships + legacy JWT endpoint. */ - Legacy = "legacy", - /** Multi-SFU transport, legacy JWT endpoint, no sticky events. */ + /** Multi-SFU transport, legacy JWT endpoint, state events. */ Compatibility = "compatibility", /** * Multi-SFU transport with: @@ -86,15 +84,6 @@ export interface ConfigOptions { * Allow to join group calls without audio and video. */ feature_group_calls_without_video_and_audio?: boolean; - - /** - * Send device-specific call session membership state events instead of - * legacy user-specific call membership state events. - * This setting has no effect when the user joins an active call with - * legacy state events. For compatibility, Element Call will always join - * active legacy calls with legacy state events. - */ - feature_use_device_session_member_events?: boolean; }; /** @@ -179,7 +168,8 @@ export interface ConfigOptions { /** * Pins the {@link MatrixRTCMode} for all clients on this deployment, * overriding any per-user choice from the Developer Settings. If unset, - * the user's Developer Settings choice (or its default of `Legacy`) wins. + * the user's Developer Settings choice (or its default of `Compatibility`) + * wins. */ matrix_rtc_mode?: MatrixRTCMode; @@ -279,9 +269,6 @@ export interface ResolvedConfigOptions extends ConfigOptions { } export const DEFAULT_CONFIG: ResolvedConfigOptions = { - features: { - feature_use_device_session_member_events: true, - }, sync_disconnect_grace_period_ms: 10000, ssla: "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf", media_quality: { diff --git a/src/graphics/desktop-gradient.png b/src/graphics/desktop-gradient.png new file mode 100644 index 000000000..84b414cfc Binary files /dev/null and b/src/graphics/desktop-gradient.png differ diff --git a/src/graphics/desktop-gradient.svg b/src/graphics/desktop-gradient.svg deleted file mode 100644 index 088244d80..000000000 --- a/src/graphics/desktop-gradient.svg +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/src/graphics/mobile-gradient.png b/src/graphics/mobile-gradient.png new file mode 100644 index 000000000..ab6d1ae43 Binary files /dev/null and b/src/graphics/mobile-gradient.png differ diff --git a/src/graphics/mobile-gradient.svg b/src/graphics/mobile-gradient.svg deleted file mode 100644 index e401846ed..000000000 --- a/src/graphics/mobile-gradient.svg +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/index.css b/src/index.css index d32f1fcfa..77db3394c 100644 --- a/src/index.css +++ b/src/index.css @@ -78,16 +78,19 @@ body { body[data-background="gradient"]::before { content: ""; position: fixed; - inset: 0; - background-image: url("graphics/mobile-gradient.svg"); - background-size: auto; + /* Chromium abruptly fades our images to fully transparent at the edge of + the element. If we just make the element a little bigger than the viewport, + this is no longer visible. */ + inset: -20px; + background-image: url("graphics/mobile-gradient.png"); + background-size: 1400px 305px; background-position: bottom; background-repeat: no-repeat; } body[data-background="gradient"][data-platform="desktop"]::before { - background-image: url("graphics/desktop-gradient.svg"); - background-size: calc(max(1440px, 100vw)) calc(max(800px, 100vh)); + background-image: url("graphics/desktop-gradient.png"); + background-size: max(1440px, 100vw) max(1440px, 100vh); background-position: center; } } diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 831ee9194..fbd589e78 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -555,6 +555,9 @@ export const GroupCallView: FC = ({ }} onError={(_error) => { if (rtcSession.isJoined()) onLeft("error"); + // If there is an error we need to be able to close the widget. This is done in `onLeft` as well + // We need it here explicitly in case rtcSession.isJoined is false. + void widget?.api.setAlwaysOnScreen(false); }} > {body} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index f8a8f3896..6945b8b6b 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -276,6 +276,7 @@ export const InCallView: FC = ({ const overflowing = useBehavior(vm.overflowing$); const showNameTags = useBehavior(vm.showNameTags$); const showHeader = useBehavior(vm.showHeader$); + const showModals = useBehavior(vm.showModals$); const settingsOpen = useBehavior(vm.settingsOpen$); const setSettingsOpen = useBehavior(vm.setSettingsOpen$); const earpieceMode = useBehavior(vm.earpieceMode$); @@ -637,7 +638,7 @@ export const InCallView: FC = ({ {earpieceOverlay} {footer} - {layout.type !== "pip" && ( + {showModals && ( <> { describe("matrix rtc mode", () => { afterEach(() => { - matrixRTCModeSetting.setValue(MatrixRTCMode.Legacy); + matrixRTCModeSetting.setValue(MatrixRTCMode.Compatibility); vi.restoreAllMocks(); }); function getModeRadios(): { - legacy: HTMLInputElement; compatibility: HTMLInputElement; matrix20: HTMLInputElement; } { return { - legacy: screen.getByDisplayValue( - MatrixRTCMode.Legacy, - ) as HTMLInputElement, compatibility: screen.getByDisplayValue( MatrixRTCMode.Compatibility, ) as HTMLInputElement, @@ -373,27 +369,21 @@ describe("DeveloperSettingsTab", () => { const radios = getModeRadios(); expect(radios.compatibility).toBeChecked(); - expect(radios.legacy).not.toBeChecked(); expect(radios.matrix20).not.toBeChecked(); // None are disabled by config; only Matrix_2_0 may be disabled by sticky-events support. - expect(radios.legacy).not.toBeDisabled(); expect(radios.compatibility).not.toBeDisabled(); }); - it.each([ - MatrixRTCMode.Legacy, - MatrixRTCMode.Compatibility, - MatrixRTCMode.Matrix_2_0, - ])( + it.each([MatrixRTCMode.Compatibility, MatrixRTCMode.Matrix_2_0])( "disables all radios and shows the config value (%s) as checked when matrix_rtc_mode is set", async (configMode) => { mockConfig({ matrix_rtc_mode: configMode }); // Local setting is intentionally different from the config value to // prove config wins. matrixRTCModeSetting.setValue( - configMode === MatrixRTCMode.Legacy - ? MatrixRTCMode.Compatibility - : MatrixRTCMode.Legacy, + configMode === MatrixRTCMode.Compatibility + ? MatrixRTCMode.Matrix_2_0 + : MatrixRTCMode.Compatibility, ); const client = createMockMatrixClient(); @@ -412,13 +402,11 @@ describe("DeveloperSettingsTab", () => { ); const radios = getModeRadios(); - expect(radios.legacy).toBeDisabled(); expect(radios.compatibility).toBeDisabled(); expect(radios.matrix20).toBeDisabled(); const checkedValue = ( { - [MatrixRTCMode.Legacy]: radios.legacy, [MatrixRTCMode.Compatibility]: radios.compatibility, [MatrixRTCMode.Matrix_2_0]: radios.matrix20, } as const diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 915052bd1..25b3c81ce 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -557,22 +557,6 @@ export const DeveloperSettingsTab: FC = ({ {matrixRTCModeForced &&

Your deployment overrides the mode.

}
- - } - > - - - {t("developer_mode.matrixRTCMode.Legacy.description")} - - renders and matches snapshot 1`] = ` class="_container_1ug7n_10" > -
-
- -
- - - Compatible with old versions of EC that do not support multi SFU - -
- -
-
-
- renders and matches snapshot 1`] = ` > Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later) @@ -366,9 +326,9 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_container_1ug7n_10" > renders and matches snapshot 1`] = ` > Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later @@ -491,7 +451,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -521,7 +481,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `

Configure resolution, framerate, bitrate, and codec for camera video. Changes apply on next call join.

@@ -543,7 +503,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > @@ -573,7 +533,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `

Configure resolution, framerate, bitrate, and codec for screen sharing

@@ -598,7 +558,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > renders and matches snapshot 1`] = ` class="_field_1bd8c0 _checkboxField_1bd8c0" > { +const modes = [[MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0]]; + +describe.each(modes)("CallViewModel (%s mode)", (mode) => { const withCallViewModel = withCallViewModelInMode(mode); test("participants are retained during a focus switch", () => { diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 7ef09ae63..09f73d1a6 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -79,7 +79,7 @@ import { type ReactionInfo, type ReactionOption, } from "../../reactions"; -import { shallowEquals } from "../../utils/array"; +import { shallowEquals as shallowArrayEquals } from "../../utils/array"; import { type MediaDevices } from "../MediaDevices"; import { constant, type Behavior } from "../Behavior"; import { E2eeType } from "../../e2ee/e2eeType"; @@ -89,6 +89,7 @@ import { getUrlParams, HeaderStyle } from "../../UrlParams"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../../widget"; import { + layoutShallowEquals, type Alignment, type GridLayoutMedia, type Layout, @@ -372,6 +373,11 @@ export interface CallViewModel { */ overflowing$: Behavior; + /** + * Whether modals such as settings and reactions should be accessible at all. + */ + showModals$: Behavior; + settingsOpen$: Behavior; setSettingsOpen$: Behavior<(open: boolean) => void>; @@ -442,7 +448,7 @@ export function createCallViewModel$( const matrixRTCMode$ = configMatrixRTCMode !== undefined ? constant(configMatrixRTCMode) - : (options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy)); + : (options.matrixRTCMode$ ?? constant(MatrixRTCMode.Compatibility)); // Each hbar seperates a block of input variables required for the CallViewModel to function. // The outputs of this block is written under the hbar. @@ -504,7 +510,6 @@ export function createCallViewModel$( mode === MatrixRTCMode.Matrix_2_0 ? JwtEndpointVersion.Matrix_2_0 : JwtEndpointVersion.Legacy, - useOldestMember: mode === MatrixRTCMode.Legacy, }), ), ), @@ -953,7 +958,7 @@ export function createCallViewModel$( bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m), ); }), - distinctUntilChanged(shallowEquals), + distinctUntilChanged(shallowArrayEquals), ), ); @@ -1012,7 +1017,7 @@ export function createCallViewModel$( const spotlight$ = scope.behavior( spotlightAndPip$.pipe( map(({ spotlight }) => spotlight), - distinctUntilChanged(shallowEquals), + distinctUntilChanged(shallowArrayEquals), ), ); @@ -1221,6 +1226,7 @@ export function createCallViewModel$( } return layout; }), + distinctUntilChanged(), scope.bind(), ) .subscribe((orientation) => { @@ -1460,6 +1466,11 @@ export function createCallViewModel$( map((naturallyShowFooter) => naturallyShowFooter && showFooterUrlParams), ), ); + + const showModals$ = scope.behavior( + windowMode$.pipe(map((mode) => mode !== "pip")), + ); + const settingsOpen$ = new BehaviorSubject(false); const setSettingsOpen$ = constant((open: boolean) => { settingsOpen$.next(open); @@ -1586,7 +1597,11 @@ export function createCallViewModel$( * The layout of tiles in the call interface. */ const layout$ = scope.behavior( - layoutInternals$.pipe(map(({ layout }) => layout)), + layoutInternals$.pipe( + map(({ layout }) => layout), + // Drop redundant layout updates before they would hit React. + distinctUntilChanged(layoutShallowEquals), + ), ); const overflowing$ = scope.behavior( @@ -1833,6 +1848,7 @@ export function createCallViewModel$( showNameTags$, showHeader$: showHeader$, showFooter$: showFooter$, + showModals$, settingsOpen$: settingsOpen$, setSettingsOpen$: setSettingsOpen$, edgeToEdge$, diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index fb93a62b8..4a408f114 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -59,7 +59,7 @@ import { initializeWidget(); -const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; +const MATRIX_RTC_MODE = MatrixRTCMode.Compatibility; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); vi.mock("../../../UrlParams", () => ({ getUrlParams })); vi.mock("@livekit/components-core", () => ({ @@ -71,12 +71,6 @@ vi.mock("@livekit/components-core", () => ({ describe("LocalMembership", () => { describe("enterRTCSession", () => { it("It joins the correct Session", () => { - const focusFromOlderMembership = { - type: "livekit", - livekit_service_url: "http://my-oldest-member-service-url.com", - livekit_alias: "my-oldest-member-service-alias", - }; - mockConfig({ livekit: { livekit_service_url: "http://my-default-service-url.com" }, }); @@ -95,10 +89,6 @@ describe("LocalMembership", () => { }, }, memberships: [], - getFocusInUse: vi.fn().mockReturnValue(focusFromOlderMembership), - getOldestMembership: vi.fn().mockReturnValue({ - getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]), - }), joinRTCSession: vi.fn(), }) as unknown as MatrixRTCSession; @@ -122,18 +112,13 @@ describe("LocalMembership", () => { memberId: "@alice:example.org:DEVICE", userId: "@alice:example.org", }, - [ - { - livekit_alias: "roomId", - livekit_service_url: "http://my-livekit-service-url.com", - type: "livekit", - }, - ], - undefined, - expect.objectContaining({ - manageMediaKeys: true, - useLegacyMemberEvents: false, - }), + [], + { + livekit_alias: "roomId", + livekit_service_url: "http://my-livekit-service-url.com", + type: "livekit", + }, + expect.objectContaining({ manageMediaKeys: true }), ); }); diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 078c5a036..2f4fde26b 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -117,7 +117,6 @@ export type LocalMemberState = }; /* - * - get oldest membership * - get transport to use * - get openId + jwt token * - wait for createTrack() call @@ -851,9 +850,7 @@ export function enterRTCSession( // have started tracking by the time calls start getting created. // groupCallOTelMembership?.onJoinCall(); - const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); - const useDeviceSessionMemberEvents = - features?.feature_use_device_session_member_events; + const { matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); const { sendNotificationType: notificationType, callIntent } = getUrlParams(); const multiSFU = matrixRTCMode === MatrixRTCMode.Compatibility || @@ -895,9 +892,6 @@ export function enterRTCSession( notificationType, callIntent, manageMediaKeys: encryptMedia, - ...(useDeviceSessionMemberEvents !== undefined && { - useLegacyMemberEvents: !useDeviceSessionMemberEvents, - }), delayedLeaveEventRestartMs: matrixRtcSessionConfig?.delayed_leave_event_restart_ms, delayedLeaveEventDelayMs: diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 89cb831da..09f6ecec0 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -13,7 +13,6 @@ import { it, type MockedObject, vi, - type MockInstance, } from "vitest"; import { type CallMembership, @@ -26,7 +25,6 @@ import { mockConfig, flushPromises, ownMemberMock, - mockRtcMembership, testScope, } from "../../../utils/test"; import { @@ -35,7 +33,7 @@ import { type LocalTransportWithSFUConfig, } from "./LocalTransport"; import { constant } from "../../Behavior"; -import { Epoch, ObservableScope, trackEpoch } from "../../ObservableScope"; +import { Epoch, ObservableScope } from "../../ObservableScope"; import { MatrixRTCTransportMissingError, FailToGetOpenIdToken, @@ -58,7 +56,6 @@ describe("LocalTransport", () => { const { advertised$, active$ } = createLocalTransport$({ scope: testScope(), roomId: "!room:example.org", - useOldestMember: false, memberships$: constant(new Epoch([])), client: { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -101,7 +98,6 @@ describe("LocalTransport", () => { const { advertised$, active$ } = createLocalTransport$({ scope, roomId: "!example_room_id", - useOldestMember: false, memberships$: constant(new Epoch([])), client: { baseUrl: "https://example.org", @@ -144,7 +140,6 @@ describe("LocalTransport", () => { const { advertised$, active$ } = createLocalTransport$({ scope: testScope(), roomId: "!room:example.org", - useOldestMember: false, memberships$: constant(new Epoch([])), client: { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -185,127 +180,6 @@ describe("LocalTransport", () => { }); }); - describe("oldest member mode", () => { - const aliceTransport: LivekitTransportConfig = { - type: "livekit", - livekit_service_url: "https://alice.example.org", - }; - const bobTransport: LivekitTransportConfig = { - type: "livekit", - livekit_service_url: "https://bob.example.org", - }; - const aliceMembership = mockRtcMembership("@alice:example.org", "AAA", { - fociPreferred: [aliceTransport], - }); - const bobMembership = mockRtcMembership("@bob:example.org", "BBB", { - fociPreferred: [bobTransport], - }); - - let openIdSpy: MockInstance<(typeof openIDSFU)["getSFUConfigWithOpenID"]>; - beforeEach(() => { - openIdSpy = vi - .spyOn(openIDSFU, "getSFUConfigWithOpenID") - .mockResolvedValue(openIdResponse); - }); - - it("updates active transport when oldest member changes", async () => { - // Initially, Alice is the only member - const memberships$ = new BehaviorSubject([aliceMembership]); - - const scope = testScope(); - const { advertised$, active$ } = createLocalTransport$({ - scope, - roomId: "!example_room_id", - useOldestMember: true, - memberships$: scope.behavior(memberships$.pipe(trackEpoch())), - client: { - getDomain: () => "example.org", - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - baseUrl: "https://example.org", - }, - ownMembershipIdentity: ownMemberMock, - forceJwtEndpoint: JwtEndpointVersion.Legacy, - delayId$: constant("delay_id_mock"), - }); - - expect(active$.value).toBe(null); - await flushPromises(); - // SFU config should've been fetched - expect(openIdSpy).toHaveBeenCalled(); - // Alice's transport should be active and advertised - expect(active$.value?.transport).toStrictEqual(aliceTransport); - expect(advertised$.value).toStrictEqual(aliceTransport); - - // Now Bob joins the call, but Alice is still the oldest member - openIdSpy.mockClear(); - memberships$.next([aliceMembership, bobMembership]); - await flushPromises(); - // No new SFU config should've been fetched - expect(openIdSpy).not.toHaveBeenCalled(); - // Alice's transport should still be active and advertised - expect(active$.value?.transport).toStrictEqual(aliceTransport); - expect(advertised$.value).toStrictEqual(aliceTransport); - - // Now Bob takes Alice's place as the oldest member - openIdSpy.mockClear(); - memberships$.next([bobMembership, aliceMembership]); - // Active transport should reset to null until we have Bob's SFU config - expect(active$.value).toStrictEqual(null); - await flushPromises(); - // Bob's SFU config should've been fetched - expect(openIdSpy).toHaveBeenCalled(); - // Bob's transport should be active, but Alice's should remain advertised - // (since we don't want the change in oldest member to cause a wave of new - // state events) - expect(active$.value?.transport).toStrictEqual(bobTransport); - expect(advertised$.value).toStrictEqual(aliceTransport); - }); - - it("advertises preferred transport when no other member exists", async () => { - // Initially, there are no members - const memberships$ = new BehaviorSubject([]); - - const scope = testScope(); - const { advertised$, active$ } = createLocalTransport$({ - scope, - roomId: "!example_room_id", - useOldestMember: true, - memberships$: scope.behavior(memberships$.pipe(trackEpoch())), - client: { - getDomain: () => "example.org", - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => - Promise.resolve([aliceTransport]), - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - baseUrl: "https://example.org", - }, - ownMembershipIdentity: ownMemberMock, - forceJwtEndpoint: JwtEndpointVersion.Legacy, - delayId$: constant("delay_id_mock"), - }); - - expect(active$.value).toBe(null); - await flushPromises(); - // Our own preferred transport should be advertised - expect(advertised$.value).toStrictEqual(aliceTransport); - // No transport should be active however (there is still no oldest member) - expect(active$.value).toBe(null); - - // Now Bob joins the call and becomes the oldest member - memberships$.next([bobMembership]); - await flushPromises(); - // We should still advertise our own preferred transport (to avoid - // unnecessary state changes) - expect(advertised$.value).toStrictEqual(aliceTransport); - // Bob's transport should become active - expect(active$.value?.transport).toBe(bobTransport); - }); - }); - type LocalTransportProps = Parameters[0]; describe("transport configuration mechanisms", () => { @@ -320,7 +194,6 @@ describe("LocalTransport", () => { ownMembershipIdentity: ownMemberMock, scope: testScope(), roomId: "!example_room_id", - useOldestMember: false, forceJwtEndpoint: JwtEndpointVersion.Legacy, delayId$: constant(null), memberships$: constant(new Epoch([])), @@ -433,7 +306,6 @@ describe("LocalTransport", () => { scope: testScope(), ownMembershipIdentity: ownMemberMock, roomId: "!example_room_id", - useOldestMember: false, forceJwtEndpoint: JwtEndpointVersion.Legacy, delayId$: constant(null), memberships$: constant(new Epoch([])), @@ -473,7 +345,6 @@ describe("LocalTransport", () => { ownMembershipIdentity: ownMemberMock, roomId: "!example_room_id", // We want multi-sdu - useOldestMember: false, forceJwtEndpoint: JwtEndpointVersion.Legacy, delayId$: delayId$, memberships$: constant(new Epoch([])), diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 1a6dddc1f..f98a266fd 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -7,23 +7,16 @@ Please see LICENSE in the repository root for full details. import { type CallMembership, - isLivekitTransportConfig, type LivekitTransportConfig, } from "matrix-js-sdk/lib/matrixrtc"; import { type MatrixClient } from "matrix-js-sdk"; import { - catchError, combineLatest, distinctUntilChanged, - first, from, map, - merge, - type Observable, of, - startWith, switchMap, - tap, } from "rxjs"; import { logger as rootLogger, type Logger } from "matrix-js-sdk/lib/logger"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; @@ -47,8 +40,7 @@ import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts"; /* * It figures out β€œwhich LiveKit focus URL/alias the local user should use,” - * optionally aligning with the oldest member, and ensures the SFU path is primed - * before advertising that choice. + * and ensures the SFU path is primed before advertising that choice. */ interface Props { scope: ObservableScope; @@ -61,7 +53,6 @@ interface Props { OpenIDClientParts; // Used by the jwt service to create the livekit room and compute the livekit alias. roomId: string; - useOldestMember: boolean; forceJwtEndpoint: JwtEndpointVersion; delayId$: Behavior; } @@ -119,8 +110,6 @@ export interface LocalTransport { /** * Connects to the JWT service and determines the transports that the local member should use. * - * @prop useOldestMember Whether to use the same transport as the oldest member. - * This will only update once the first oldest member appears. Will not recompute if the oldest member leaves. * @prop useOldJwtEndpoint Whether to set forceOldJwtEndpoint on the returned transport and to use the old JWT endpoint. * This is used when the connection manager needs to know if it has to use the legacy endpoint which implies a string concatenated rtcBackendIdentity. * (which is expected for non sticky event based rtc member events) @@ -133,18 +122,10 @@ export const createLocalTransport$ = ({ ownMembershipIdentity, client, roomId, - useOldestMember, forceJwtEndpoint, delayId$, }: Props): LocalTransport => { const logger = rootLogger.getChild("[LocalTransport]"); - // The LiveKit transport in use by the oldest RTC membership. `null` when the - // oldest member has no such transport. - const oldestMemberTransport$ = observerOldestMembership$( - scope, - memberships$, - logger, - ); const transportDiscovery = new RtcTransportAutoDiscovery({ client: client, @@ -203,19 +184,6 @@ export const createLocalTransport$ = ({ }), ); - if (useOldestMember) { - return observeLocalTransportForOldestMembership( - scope, - oldestMemberTransport$, - preferredTransport$, - client, - ownMembershipIdentity, - roomId, - logger, - ); - } - - // --- Multi-SFU mode --- // Always publish on and advertise the preferred transport. return { advertised$: scope.behavior( @@ -243,47 +211,6 @@ export const createLocalTransport$ = ({ }; }; -/** - * Observes the oldest member in the room and returns the transport that it uses if it is a livekit transport. - * @param scope - The observable scope. - * @param memberships$ - The observable of the call's memberships.' - */ -function observerOldestMembership$( - scope: ObservableScope, - memberships$: Behavior>, - logger: Logger, -): Behavior { - return scope.behavior( - memberships$.pipe( - map((memberships) => { - const oldestMember = memberships.value[0]; - if (oldestMember === undefined) { - logger.info("Oldest member: not found"); - return null; - } - const transport = oldestMember.getTransport(oldestMember); - if (transport === undefined) { - logger.warn( - `Oldest member: ${oldestMember.userId}|${oldestMember.deviceId}|${oldestMember.memberId} has no transport`, - ); - return null; - } - if (!isLivekitTransportConfig(transport)) { - logger.warn( - `Oldest member: ${oldestMember.userId}|${oldestMember.deviceId}|${oldestMember.memberId} has invalid transport`, - ); - return null; - } - logger.info( - "Oldest member: ${oldestMember.userId}|${oldestMember.deviceId}|${oldestMember.memberId} has valid transport", - ); - return transport; - }), - distinctUntilChanged(areLivekitTransportsEqual), - ), - ); -} - /** * Utility to ensure the user can authenticate with the SFU. * We will call `getSFUConfigWithOpenID` once per transport here as it's our @@ -331,85 +258,6 @@ async function doOpenIdAndJWTFromUrl( }; } -function observeLocalTransportForOldestMembership( - scope: ObservableScope, - oldestMemberTransport$: Behavior, - preferredTransport$: Observable, - client: Pick< - MatrixClient, - "getDomain" | "baseUrl" | "_unstable_getRTCTransports" - > & - OpenIDClientParts, - ownMembershipIdentity: CallMembershipIdentityParts, - roomId: string, - logger: Logger, -): LocalTransport { - // Ensure we can authenticate with the SFU. - const authenticatedOldestMemberTransport$ = oldestMemberTransport$.pipe( - switchMap((transport) => { - // Oldest member not available -we are first- (or invalid SFU config). - if (transport === null) return of(null); - - // Whenever there is transport change we want to revert - // to no transport while we do the authentication. - // So do a from(promise) here to be able to startWith(null) - return from( - doOpenIdAndJWTFromUrl( - transport, - JwtEndpointVersion.Legacy, - ownMembershipIdentity, - roomId, - client, - undefined, - logger, - ), - ).pipe( - catchError((e: unknown) => { - logger.error( - `Failed to authenticate to transport ${transport.livekit_service_url}`, - e, - ); - throw mapAuthErrorToUserFriendlyError(e); - }), - startWith(null), - ); - }), - ); - - // --- Oldest member mode --- - return { - // Never update the transport that we advertise in our membership. Just - // take the first valid oldest member or preferred transport that we learn - // about, and stick with that. This avoids unnecessary SFU hops and room - // state changes. - advertised$: scope.behavior( - merge( - authenticatedOldestMemberTransport$.pipe( - map((t) => t?.transport ?? null), - ), - preferredTransport$.pipe(map((t) => t.transport)), - ).pipe( - first((t) => t !== null), - tap((t) => - logger.info(`Advertise transport: ${t.livekit_service_url}`), - ), - ), - null, - ), - // Publish on the transport used by the oldest member. - active$: scope.behavior( - authenticatedOldestMemberTransport$.pipe( - tap((t) => - logger.info( - `Publish on transport: ${t?.transport.livekit_service_url}`, - ), - ), - ), - null, - ), - }; -} - function mapAuthErrorToUserFriendlyError(e: unknown): Error { if ( e instanceof FailToGetOpenIdToken || diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 013bd96c7..f320e6303 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -36,7 +36,6 @@ import { SFURoomCreationRestrictedError, UnknownCallError, } from "../../../utils/errors.ts"; -import { type JwtEndpointVersion } from "../localMember/LocalTransport.ts"; export interface ConnectionOpts { /** @@ -44,11 +43,6 @@ export interface ConnectionOpts { * On top the local transport will send additional data to the jwt server to use delayed event delegation. */ existingSFUConfig?: SFUConfig; - /** - * For local connections that use the oldest member pattern. here we have not prefetched the sfuConfig - * and hence we need to let the connection do the jwt token fetching. - */ - forceJwtEndpoint?: JwtEndpointVersion; /** The identity parts to use on this connection */ ownMembershipIdentity: CallMembershipIdentityParts; /** The media transport to connect to. */ diff --git a/src/state/CallViewModelWidget.test.ts b/src/state/CallViewModelWidget.test.ts index 2e4ef39dd..2f331bd32 100644 --- a/src/state/CallViewModelWidget.test.ts +++ b/src/state/CallViewModelWidget.test.ts @@ -35,11 +35,7 @@ vi.mock("../widget", () => ({ }, })); -it.each([ - [MatrixRTCMode.Legacy], - [MatrixRTCMode.Compatibility], - [MatrixRTCMode.Matrix_2_0], -])( +it.each([[MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0]])( "expect leave when ElementWidgetActions.HangupCall is called (%s mode)", async (mode) => { const pr = Promise.withResolvers(); diff --git a/src/state/layout-types.test.ts b/src/state/layout-types.test.ts new file mode 100644 index 000000000..468c15bb5 --- /dev/null +++ b/src/state/layout-types.test.ts @@ -0,0 +1,74 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { test, expect } from "vitest"; +import { + type Alignment, + layoutShallowEquals, + type Layout, +} from "./layout-types"; +import { + type SpotlightTileViewModel, + type GridTileViewModel, +} from "./TileViewModel"; +import { BehaviorSubject } from "rxjs"; + +const spotlightTile = {} as unknown as SpotlightTileViewModel; +const gridTile = {} as unknown as GridTileViewModel; +const pipAlignment$ = new BehaviorSubject({ + inline: "end", + block: "end", +}); + +const spotlightExpanded: Layout = { + type: "spotlight-expanded", + spotlight: spotlightTile, + pipAlignment$, +}; + +const spotlightPortrait: Layout = { + type: "spotlight-portrait", + spotlight: spotlightTile, + grid: [gridTile], + setVisibleTiles: () => {}, +}; + +test("layoutShallowEquals considers a layout to be equal to its shallow clone", () => + expect(layoutShallowEquals(spotlightExpanded, { ...spotlightExpanded })).toBe( + true, + )); + +test("layoutShallowEquals detects a missing key", () => { + expect( + layoutShallowEquals(spotlightExpanded, { + ...spotlightExpanded, + pip: gridTile, + }), + ).toBe(false); + expect( + layoutShallowEquals( + { ...spotlightExpanded, pip: gridTile }, + spotlightExpanded, + ), + ).toBe(false); +}); + +test("layoutShallowEquals considers grid arrays with equal contents to be equal", () => + expect( + layoutShallowEquals(spotlightPortrait, { + ...spotlightPortrait, + grid: [...spotlightPortrait.grid], + }), + ).toBe(true)); + +test("layoutShallowEquals detects grid arrays with different contents", () => + expect( + layoutShallowEquals(spotlightPortrait, { + ...spotlightPortrait, + grid: [...spotlightPortrait.grid, gridTile], + }), + ).toBe(false)); diff --git a/src/state/layout-types.ts b/src/state/layout-types.ts index 5c63a9c90..be86f2a36 100644 --- a/src/state/layout-types.ts +++ b/src/state/layout-types.ts @@ -16,6 +16,7 @@ import { type SpotlightTileViewModel, } from "./TileViewModel.ts"; import { type Behavior } from "./Behavior.ts"; +import { shallowEquals as arrayShallowEquals } from "../utils/array.ts"; export interface GridLayoutMedia { type: "grid"; @@ -140,3 +141,30 @@ export type Layout = | OneOnOneDesktopLayout | OneOnOneMobileLayout | PipLayout; + +/** + * Tests whether the top-level properties and array elements of layout `a` are + * equal to those of layout `b`. Useful for deduping redundant layout updates. + */ +export function layoutShallowEquals(a: Layout, b: Layout): boolean { + // If a and b have the same number of keys and every key in a is also in b, + // then they have the same keys. + const aKeys = Object.keys(a); + const bKeys = Object.keys(b); + if (aKeys.length !== bKeys.length) return false; + + for (const key of aKeys) { + if (!(key in b)) return false; + + // Now check that they have the same values. + const aValue = (a as any)[key]; + const bValue = (b as any)[key]; + if (Array.isArray(aValue) && Array.isArray(bValue)) { + // Special case for arrays so we can detect when the grid tiles arrays are + // essentially the same. + if (!arrayShallowEquals(aValue, bValue)) return false; + } else if (aValue !== bValue) return false; + } + + return true; +} diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts index ea0331030..61d534656 100644 --- a/src/state/media/UserMediaViewModel.ts +++ b/src/state/media/UserMediaViewModel.ts @@ -33,7 +33,6 @@ import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel"; import { type ObservableScope } from "../ObservableScope"; import { showConnectionStats } from "../../settings/settings"; import { observeRtpStreamStats$ } from "./observeRtpStreamStats"; -import { videoFit$, videoSizeFromParticipant$ } from "../../utils/videoFit.ts"; /** * A participant's user media (i.e. their microphone and camera feed). @@ -47,7 +46,6 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel { speaking$: Behavior; audioEnabled$: Behavior; videoEnabled$: Behavior; - videoFit$: Behavior<"cover" | "contain">; videoOrientation$: Behavior<"landscape" | "portrait">; toggleCropVideo: () => void; /** @@ -63,12 +61,9 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel { RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined >; /** - * Set the target dimensions of the HTML element (final dimension after anim). - * This can be used to determine the best video fit (fit to frame / keep ratio). - * @param targetWidth - The target width of the HTML element displaying the video. - * @param targetHeight - The target height of the HTML element displaying the video. + * Set the aspect ratio of the video track to determine the orientation. */ - setTargetDimensions: (targetWidth: number, targetHeight: number) => void; + setVideoAspectRatio: (ratio: number) => void; } export interface BaseUserMediaInputs extends Omit< @@ -98,14 +93,8 @@ export function createBaseUserMedia( ), ); const toggleCropVideo$ = new Subject(); + const videoAspectRatio$ = new BehaviorSubject(NaN); - // The target size of the video element, used to determine the best video fit. - // The target size is the final size of the HTML element after any animations have completed. - const targetSize$ = new BehaviorSubject< - { width: number; height: number } | undefined - >(undefined); - - const videoSize$ = videoSizeFromParticipant$(participant$); return { ...createMemberMedia(scope, { ...inputs, @@ -132,13 +121,11 @@ export function createBaseUserMedia( media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), ), videoOrientation$: scope.behavior( - videoSize$.pipe( - map((s) => (s ? s.width / s.height : 1)), + videoAspectRatio$.pipe( map((aspect) => (aspect > 1 ? "landscape" : "portrait")), ), "portrait", ), - videoFit$: videoFit$(scope, videoSize$, targetSize$), toggleCropVideo: () => toggleCropVideo$.next(), rtcBackendIdentity, handRaised$, @@ -162,8 +149,6 @@ export function createBaseUserMedia( return observeRtpStreamStats$(p, Track.Source.Camera, statsType); }), ), - setTargetDimensions: (targetWidth: number, targetHeight: number): void => { - targetSize$.next({ width: targetWidth, height: targetHeight }); - }, + setVideoAspectRatio: (ratio) => videoAspectRatio$.next(ratio), }; } diff --git a/src/state/media/observeRtpStreamStats.ts b/src/state/media/observeRtpStreamStats.ts index 44181f857..afb0da62d 100644 --- a/src/state/media/observeRtpStreamStats.ts +++ b/src/state/media/observeRtpStreamStats.ts @@ -19,10 +19,15 @@ import { startWith, switchMap, map, + share, } from "rxjs"; import { observeTrackReference$ } from "../observeTrackReference"; +// Use a shared timer for all the stats observers so that we don't clog up the +// event loop with hundreds of timers in large calls +const refreshStats$ = interval(1000).pipe(share()); + export function observeRtpStreamStats$( participant: Participant, source: Track.Source, @@ -32,9 +37,7 @@ export function observeRtpStreamStats$( > { return combineLatest([ observeTrackReference$(participant, source), - // The update frequency is high because we use this value to update the PiP orientation and the fit/fill video tile props based on that - // We want it to be responsive. For just the debug tools 1s would be sufficient. - interval(350).pipe(startWith(0)), + refreshStats$.pipe(startWith(0)), ]).pipe( switchMap(async ([trackReference]) => { const track = trackReference?.publication?.track; @@ -69,12 +72,3 @@ export function observeInboundRtpStreamStats$( map((x) => x as RTCInboundRtpStreamStats | undefined), ); } - -export function observeOutboundRtpStreamStats$( - participant: Participant, - source: Track.Source, -): Observable { - return observeRtpStreamStats$(participant, source, "outbound-rtp").pipe( - map((x) => x as RTCOutboundRtpStreamStats | undefined), - ); -} diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 01d59f9d1..1334e210c 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -11,7 +11,6 @@ import { type ReactNode, type Ref, useCallback, - useEffect, useRef, useState, useMemo, @@ -92,7 +91,6 @@ const RingingMediaTile: FC = ({ } avatarStyle="translucent" videoEnabled={false} - videoFit="cover" mirror={false} {...props} /> @@ -144,19 +142,11 @@ const UserMediaTileInner: FC = ({ const audioEnabled = useBehavior(vm.audioEnabled$); const videoEnabled = useBehavior(vm.videoEnabled$); const speaking = useBehavior(vm.speaking$); - const videoFit = useBehavior(vm.videoFit$); const rtcBackendIdentity = vm.rtcBackendIdentity; const handRaised = useBehavior(vm.handRaised$); const reaction = useBehavior(vm.reaction$); - // Whenever bounds change, inform the viewModel - useEffect(() => { - if (targetWidth > 0 && targetHeight > 0) { - vm.setTargetDimensions(targetWidth, targetHeight); - } - }, [targetWidth, targetHeight, vm]); - const AudioIcon = playbackMuted ? VolumeOffSolidIcon : audioEnabled @@ -195,7 +185,6 @@ const UserMediaTileInner: FC = ({ userId={vm.userId} unencryptedWarning={unencryptedWarning} videoEnabled={videoEnabled} - videoFit={videoFit} className={classNames(className, styles.tile, { [styles.speaking]: showSpeaking, [styles.handRaised]: !showSpeaking && handRaised, @@ -231,6 +220,7 @@ const UserMediaTileInner: FC = ({ raisedHandOnClick={raisedHandOnClick} waitingForMedia={waitingForMedia} focusUrl={focusUrl} + setVideoAspectRatio={vm.setVideoAspectRatio} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} rtcBackendIdentity={rtcBackendIdentity} diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 099cbaa22..660d37a27 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -33,7 +33,6 @@ describe("MediaView", () => { const baseProps: ComponentProps = { displayName: "some name", videoEnabled: true, - videoFit: "contain", targetWidth: 300, targetHeight: 200, mirror: false, diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index a00fb6cba..d57441687 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -7,7 +7,13 @@ Please see LICENSE in the repository root for full details. import { type TrackReferenceOrPlaceholder } from "@livekit/components-core"; import { animated } from "@react-spring/web"; -import { type FC, type ComponentProps, type ReactNode } from "react"; +import { + type FC, + type ComponentProps, + type ReactNode, + type SyntheticEvent, + useState, +} from "react"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; import { VideoTrack } from "@livekit/components-react"; @@ -26,6 +32,7 @@ import { type ReactionOption } from "../reactions"; import { ReactionIndicator } from "../reactions/ReactionIndicator"; import { RTCConnectionStats } from "../RTCConnectionStats"; import videoPlaceholder from "../graphics/video-placeholder.gif"; +import { autoVideoFit } from "../utils/videoFit"; interface Props extends ComponentProps { className?: string; @@ -33,7 +40,11 @@ interface Props extends ComponentProps { targetWidth: number; targetHeight: number; video: TrackReferenceOrPlaceholder | undefined; - videoFit: "cover" | "contain"; + /** + * How to fit the video content inside the tile. When undefined, MediaView + * chooses a smart default based on the aspect ratios of the tile and video. + */ + videoFit?: "cover" | "contain"; mirror: boolean; soundWaves?: boolean; userId: string; @@ -55,8 +66,15 @@ interface Props extends ComponentProps { audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; rtcBackendIdentity?: string; - // The focus url, mainly for debugging purposes + /** + * The focus url, mainly for debugging purposes. + */ focusUrl?: string; + /** + * Called whenever the aspect ratio of the video content becomes known or + * otherwise changes. + */ + setVideoAspectRatio?: (ratio: number) => void; } export const MediaView: FC = ({ @@ -89,6 +107,7 @@ export const MediaView: FC = ({ videoStreamStats, rtcBackendIdentity, focusUrl, + setVideoAspectRatio: setTheirVideoAspectRatio, ...props }) => { const { t } = useTranslation(); @@ -100,6 +119,22 @@ export const MediaView: FC = ({ (soundWaves === undefined ? 0.5 : 0.38), ); + const [videoAspectRatio, setOurVideoAspectRatio] = useState(NaN); + const tileAspectRatio = targetWidth / targetHeight; + + // Propagate video dimensions + const setVideoAspectRatio = (ratio: number) => { + setOurVideoAspectRatio(ratio); + setTheirVideoAspectRatio?.(ratio); + }; + const videoRef = (el: HTMLVideoElement | null) => { + if (el !== null) setVideoAspectRatio(el.videoWidth / el.videoHeight); + }; + const onResize = (ev: SyntheticEvent) => + setVideoAspectRatio( + ev.currentTarget.videoWidth / ev.currentTarget.videoHeight, + ); + const warnings = unencryptedWarning && ( = ({ ref={ref} data-testid="videoTile" data-video-enabled={video && videoEnabled} - data-video-fit={videoFit} + data-video-fit={ + videoFit ?? autoVideoFit(videoAspectRatio, tileAspectRatio) + } data-background={background} {...props} > @@ -158,6 +195,8 @@ export const MediaView: FC = ({ // Set the placeholder to a small transparent image. (On Android web // views the default poster image is particularly ugly.) poster={videoPlaceholder} + ref={videoRef} + onResize={onResize} /> )}
diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index d21a7f5f5..036e044fe 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -68,6 +68,7 @@ interface SpotlightItemBaseProps { background: "solid" | "transparent"; focusable: boolean; "aria-hidden"?: boolean; + setVideoAspectRatio?: (ratio: number) => void; } interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps { @@ -77,7 +78,6 @@ interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps { } interface SpotlightUserMediaItemBaseProps extends SpotlightMemberMediaItemBaseProps { - videoFit: "contain" | "cover"; videoEnabled: boolean; soundWaves: boolean | undefined; } @@ -120,20 +120,12 @@ const SpotlightUserMediaItem: FC = ({ targetHeight, ...props }) => { - const videoFit = useBehavior(vm.videoFit$); const videoEnabled = useBehavior(vm.videoEnabled$); const speaking = useBehavior(vm.speaking$); - // Whenever target bounds change, inform the viewModel - useEffect(() => { - if (targetWidth > 0 && targetHeight > 0) { - vm.setTargetDimensions(targetWidth, targetHeight); - } - }, [targetWidth, targetHeight, vm]); - const baseProps: SpotlightUserMediaItemBaseProps & RefAttributes = { - videoFit, + setVideoAspectRatio: vm.setVideoAspectRatio, videoEnabled, soundWaves: props.background === "transparent" ? speaking : undefined, targetWidth, @@ -227,7 +219,6 @@ const SpotlightRingingMediaItem: FC = ({ } avatarStyle="translucent" videoEnabled={false} - videoFit="cover" mirror={false} {...props} /> diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index fde9aac59..f53910024 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -174,7 +174,7 @@ export function getBasicCallViewModelEnvironment( setE2EEEnabled: async () => Promise.resolve(), }), connectionState$: constant(ConnectionState.Connected), - matrixRTCMode$: constant(MatrixRTCMode.Legacy), + matrixRTCMode$: constant(MatrixRTCMode.Compatibility), ...callViewModelOptions, }, handRaisedSubject$, diff --git a/src/utils/test.ts b/src/utils/test.ts index 38a1bb4a5..de74ac53e 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -237,7 +237,7 @@ export function mockRtcMembership( fociPreferred: [exampleTransport], focusActive: { type: "livekit" as const, - focus_selection: "oldest_membership" as const, + focus_selection: "multi_sfu" as const, }, callId: "", membership: {}, @@ -463,9 +463,6 @@ export class MockRTCSession extends TypedEventEmitter< session.reemitEncryptionKeys = vi .fn<() => void>() .mockReturnValue(undefined); - session.getOldestMembership = vi - .fn<() => CallMembership | undefined>() - .mockReturnValue(this.memberships[0]); return session; } diff --git a/src/utils/videoFit.test.ts b/src/utils/videoFit.test.ts index 5068526ba..ba46b6d84 100644 --- a/src/utils/videoFit.test.ts +++ b/src/utils/videoFit.test.ts @@ -5,259 +5,106 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, test, vi } from "vitest"; -import { - LocalTrack, - type LocalTrackPublication, - type RemoteTrackPublication, - Track, -} from "livekit-client"; +import { describe, expect, test } from "vitest"; -import { ObservableScope } from "../state/ObservableScope"; -import { videoFit$, videoSizeFromParticipant$ } from "./videoFit"; -import { constant } from "../state/Behavior"; -import { - flushPromises, - mockLocalParticipant, - mockRemoteParticipant, -} from "./test"; +import { autoVideoFit } from "./videoFit"; describe("videoFit$ defaults", () => { test.each([ { - videoSize: { width: 1920, height: 1080 }, - tileSize: undefined, + videoAspectRatio: 1920 / 1080, + tileAspectRatio: NaN, }, { - videoSize: { width: 1080, height: 1920 }, - tileSize: undefined, + videoAspectRatio: 1080 / 1920, + tileAspectRatio: NaN, }, { - videoSize: undefined, - tileSize: { width: 1920, height: 1080 }, + videoAspectRatio: NaN, + tileAspectRatio: 1920 / 1080, }, { - videoSize: undefined, - tileSize: { width: 1080, height: 1920 }, + videoAspectRatio: NaN, + tileAspectRatio: 1080 / 1920, }, ])( - "videoFit$ returns `cover` when videoSize is $videoSize and tileSize is $tileSize", - ({ videoSize, tileSize }) => { - const scope = new ObservableScope(); - const videoSize$ = constant(videoSize); - const tileSize$ = constant(tileSize); - - const fit = videoFit$(scope, videoSize$, tileSize$); - expect(fit.value).toBe("cover"); - }, + "videoFit$ returns `cover` when videoAspectRatio is $videoAspectRatio and tileAspectRatio is $tileAspectRatio", + ({ videoAspectRatio, tileAspectRatio }) => + expect(autoVideoFit(videoAspectRatio, tileAspectRatio)).toBe("cover"), ); }); -const VIDEO_480_L = { width: 640, height: 480 }; -const VIDEO_720_L = { width: 1280, height: 720 }; -const VIDEO_1080_L = { width: 1920, height: 1080 }; +const VIDEO_480_L = 640 / 480; +const VIDEO_720_L = 1280 / 720; +const VIDEO_1080_L = 1920 / 1080; // Some sizes from real world testing, which don't match the standard video sizes exactly -const TILE_SIZE_1_L = { width: 180, height: 135 }; -const TILE_SIZE_3_P = { width: 379, height: 542 }; -const TILE_SIZE_4_L = { width: 957, height: 542 }; +const TILE_SIZE_1_L = 180 / 135; +const TILE_SIZE_3_P = 379 / 542; +const TILE_SIZE_4_L = 957 / 542; // This is the size of an iPhone Xr in portrait mode -const TILE_SIZE_5_P = { width: 414, height: 896 }; +const TILE_SIZE_5_P = 414 / 896; -export function invertSize(size: { width: number; height: number }): { - width: number; - height: number; -} { - return { - width: size.height, - height: size.width, - }; +function inverse(ratio: number): number { + return 1 / ratio; } test.each([ { - videoSize: VIDEO_480_L, - tileSize: TILE_SIZE_1_L, + videoAspectRatio: VIDEO_480_L, + tileAspectRatio: TILE_SIZE_1_L, expected: "cover", }, { - videoSize: invertSize(VIDEO_480_L), - tileSize: TILE_SIZE_1_L, + videoAspectRatio: inverse(VIDEO_480_L), + tileAspectRatio: TILE_SIZE_1_L, expected: "contain", }, { - videoSize: VIDEO_720_L, - tileSize: TILE_SIZE_4_L, + videoAspectRatio: VIDEO_720_L, + tileAspectRatio: TILE_SIZE_4_L, expected: "cover", }, { - videoSize: invertSize(VIDEO_720_L), - tileSize: TILE_SIZE_4_L, + videoAspectRatio: inverse(VIDEO_720_L), + tileAspectRatio: TILE_SIZE_4_L, expected: "contain", }, { - videoSize: invertSize(VIDEO_1080_L), - tileSize: TILE_SIZE_3_P, + videoAspectRatio: inverse(VIDEO_1080_L), + tileAspectRatio: TILE_SIZE_3_P, expected: "cover", }, { - videoSize: VIDEO_1080_L, - tileSize: TILE_SIZE_5_P, + videoAspectRatio: VIDEO_1080_L, + tileAspectRatio: TILE_SIZE_5_P, expected: "contain", }, { - videoSize: invertSize(VIDEO_1080_L), - tileSize: TILE_SIZE_5_P, + videoAspectRatio: inverse(VIDEO_1080_L), + tileAspectRatio: TILE_SIZE_5_P, expected: "cover", }, { // square video - videoSize: { width: 400, height: 400 }, - tileSize: VIDEO_480_L, + videoAspectRatio: 400 / 400, + tileAspectRatio: VIDEO_480_L, expected: "contain", }, { // Should default to cover if the initial size is 0:0. // Or else it will cause a flash of "contain" mode until the real size is loaded, which can be jarring. - videoSize: VIDEO_480_L, - tileSize: { width: 0, height: 0 }, + videoAspectRatio: VIDEO_480_L, + tileAspectRatio: 0 / 0, expected: "cover", }, { - videoSize: { width: 0, height: 0 }, - tileSize: VIDEO_480_L, + videoAspectRatio: 0 / 0, + tileAspectRatio: VIDEO_480_L, expected: "cover", }, ])( - "videoFit$ returns $expected when videoSize is $videoSize and tileSize is $tileSize", - ({ videoSize, tileSize, expected }) => { - const scope = new ObservableScope(); - const videoSize$ = constant(videoSize); - const tileSize$ = constant(tileSize); - - const fit = videoFit$(scope, videoSize$, tileSize$); - expect(fit.value).toBe(expected); - }, + "videoFit$ returns $expected when videoAspectRatio is $videoAspectRatio and tileAspectRatio is $tileAspectRatio", + ({ videoAspectRatio, tileAspectRatio, expected }) => + expect(autoVideoFit(videoAspectRatio, tileAspectRatio)).toBe(expected), ); - -describe("extracting video size from participant stats", () => { - function createMockRtpStats( - isInbound: boolean, - props: Partial = {}, - ): RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats { - const baseStats = { - id: "mock-stats-id", - timestamp: Date.now(), - type: isInbound ? "inbound-rtp" : "outbound-rtp", - kind: "video", - ...props, - }; - - return baseStats as RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; - } - - test("get stats for local user", async () => { - const localParticipant = mockLocalParticipant({ - identity: "@local:example.org:AAAAAA", - }); - - const mockReport: RTCStatsReport = new Map([ - [ - "OT01V639885149", - createMockRtpStats(false, { - frameWidth: 1280, - frameHeight: 720, - }), - ], - ]); - - const track = { - source: Track.Source.Camera, - getRTCStatsReport: vi - .fn() - .mockImplementation(async () => Promise.resolve(mockReport)), - } as Partial as LocalTrack; - - // Set up the prototype chain (there is an instanceof check in getRTCStatsReport) - Object.setPrototypeOf(track, LocalTrack.prototype); - - localParticipant.getTrackPublication = vi - .fn() - .mockImplementation((source: Track.Source) => { - if (source === Track.Source.Camera) { - return { - track, - } as unknown as LocalTrackPublication; - } else { - return undefined; - } - }); - - const videoDimensions$ = videoSizeFromParticipant$( - constant(localParticipant), - ); - - const publishedDimensions: { width: number; height: number }[] = []; - videoDimensions$.subscribe((dimensions) => { - if (dimensions) publishedDimensions.push(dimensions); - }); - - await flushPromises(); - - const dimension = publishedDimensions.pop(); - expect(dimension).toEqual({ width: 1280, height: 720 }); - }); - - test("get stats for remote user", async () => { - // vi.useFakeTimers() - const remoteParticipant = mockRemoteParticipant({ - identity: "@bob:example.org:AAAAAA", - }); - - const mockReport: RTCStatsReport = new Map([ - [ - "OT01V639885149", - createMockRtpStats(true, { - frameWidth: 480, - frameHeight: 640, - }), - ], - ]); - - const track = { - source: Track.Source.Camera, - getRTCStatsReport: vi - .fn() - .mockImplementation(async () => Promise.resolve(mockReport)), - } as Partial as LocalTrack; - - // Set up the prototype chain (there is an instanceof check in getRTCStatsReport) - Object.setPrototypeOf(track, LocalTrack.prototype); - - remoteParticipant.getTrackPublication = vi - .fn() - .mockImplementation((source: Track.Source) => { - if (source === Track.Source.Camera) { - return { - track, - } as unknown as RemoteTrackPublication; - } else { - return undefined; - } - }); - - const videoDimensions$ = videoSizeFromParticipant$( - constant(remoteParticipant), - ); - - const publishedDimensions: { width: number; height: number }[] = []; - videoDimensions$.subscribe((dimensions) => { - if (dimensions) publishedDimensions.push(dimensions); - }); - - await flushPromises(); - - const dimension = publishedDimensions.pop(); - expect(dimension).toEqual({ width: 480, height: 640 }); - }); -}); diff --git a/src/utils/videoFit.ts b/src/utils/videoFit.ts index 39dc28c9f..16d7cb843 100644 --- a/src/utils/videoFit.ts +++ b/src/utils/videoFit.ts @@ -5,107 +5,27 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { combineLatest, map, type Observable, of, switchMap } from "rxjs"; -import { - type LocalParticipant, - type RemoteParticipant, - Track, -} from "livekit-client"; - -import { type ObservableScope } from "../state/ObservableScope.ts"; -import { type Behavior } from "../state/Behavior.ts"; -import { - observeInboundRtpStreamStats$, - observeOutboundRtpStreamStats$, -} from "../state/media/observeRtpStreamStats"; - -type Size = { - width: number; - height: number; -}; - /** * Computes the appropriate video fit mode ("cover" or "contain") based on the aspect ratios of the video and the tile. * - If the video and tile have the same orientation (both landscape or both portrait), we use "cover" to fill the tile, even if it means cropping. * - If the video and tile have different orientations, we use "contain" to ensure the entire video is visible, even if it means letterboxing (black bars). - * @param scope - the ObservableScope to create the Behavior in - * @param videoSize$ - an Observable of the video size (width and height) or undefined if the size is not yet known (no data yet received). - * @param tileSize$ - an Observable of the tile size (width and height) or undefined if the size is not yet known (not yet rendered). */ -export function videoFit$( - scope: ObservableScope, - videoSize$: Observable, - tileSize$: Observable, -): Behavior<"cover" | "contain"> { - const fit$ = combineLatest([videoSize$, tileSize$]).pipe( - map(([videoSize, tileSize]) => { - if (!videoSize || !tileSize) { - // If we don't have the sizes, default to cover to avoid black bars. - // This is a reasonable default as it will ensure the video fills the tile, even if it means cropping. - return "cover"; - } - if ( - videoSize.width === 0 || - videoSize.height === 0 || - tileSize.width === 0 || - tileSize.height === 0 - ) { - // If we have invalid sizes (e.g. width or height is 0), default to cover to avoid black bars. - return "cover"; - } - const videoAspectRatio = videoSize.width / videoSize.height; - const tileAspectRatio = tileSize.width / tileSize.height; +export function autoVideoFit( + videoAspectRatio: number, + tileAspectRatio: number, +): "cover" | "contain" { + if (Number.isNaN(videoAspectRatio) || Number.isNaN(tileAspectRatio)) { + // If we have invalid sizes (e.g. useMeasure returns 0Γ—0 on an initial render), + // default to cover to avoid black bars. + return "cover"; + } - // If video is landscape (ratio > 1) and tile is portrait (ratio < 1) or vice versa, - // we want to use "contain" (fit) mode to avoid excessive cropping - const videoIsLandscape = videoAspectRatio > 1; - const tileIsLandscape = tileAspectRatio > 1; + // If video is landscape (ratio > 1) and tile is portrait (ratio < 1) or vice versa, + // we want to use "contain" (fit) mode to avoid excessive cropping + const videoIsLandscape = videoAspectRatio > 1; + const tileIsLandscape = tileAspectRatio > 1; - // If the orientations are the same, use the cover mode (Preserves the aspect ratio, and the image fills the container.) - // If they're not the same orientation, use the contain mode (Preserves the aspect ratio, but the image is letterboxed - black bars- to fit within the container.) - return videoIsLandscape === tileIsLandscape ? "cover" : "contain"; - }), - ); - - return scope.behavior(fit$, "cover"); -} - -/** - * Helper function to get the video size from a participant. - * It observes the participant's video track stats and extracts the frame width and height. - * @param participant$ - an Observable of a LocalParticipant or RemoteParticipant, or null if no participant is selected. - * @returns an Observable of the video size (width and height) or undefined if the size cannot be determined. - */ -export function videoSizeFromParticipant$( - participant$: Observable, -): Observable<{ width: number; height: number } | undefined> { - return participant$ - .pipe( - // If we have a participant, observe their video track stats. If not, return undefined. - switchMap((p) => { - if (!p) return of(undefined); - if (p.isLocal) { - return observeOutboundRtpStreamStats$(p, Track.Source.Camera); - } else { - return observeInboundRtpStreamStats$(p, Track.Source.Camera); - } - }), - ) - .pipe( - // Extract the frame width and height from the stats. If we don't have valid stats, return undefined. - map((stats) => { - if (!stats) return undefined; - if ( - // For video tracks, frameWidth and frameHeight should be numbers. If they're not, we can't determine the size. - typeof stats.frameWidth !== "number" || - typeof stats.frameHeight !== "number" - ) { - return undefined; - } - return { - width: stats.frameWidth, - height: stats.frameHeight, - }; - }), - ); + // If the orientations are the same, use the cover mode (Preserves the aspect ratio, and the image fills the container.) + // If they're not the same orientation, use the contain mode (Preserves the aspect ratio, but the image is letterboxed - black bars- to fit within the container.) + return videoIsLandscape === tileIsLandscape ? "cover" : "contain"; }