diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 067c5246..11116ed2 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. `; module.exports = { - plugins: ["matrix-org", "rxjs"], + plugins: ["matrix-org", "rxjs", "jsdoc"], extends: [ "plugin:matrix-org/react", "plugin:matrix-org/a11y", @@ -26,6 +26,13 @@ module.exports = { node: true, }, rules: { + "jsdoc/no-types": "error", + "jsdoc/empty-tags": "error", + "jsdoc/check-property-names": "error", + "jsdoc/check-values": "error", + "jsdoc/check-param-names": "warn", + // "jsdoc/require-param": "warn", + "jsdoc/require-param-description": "warn", "matrix-org/require-copyright-header": ["error", COPYRIGHT_HEADER], "jsx-a11y/media-has-caption": "off", "react/display-name": "error", @@ -75,6 +82,23 @@ module.exports = { "no-console": ["error"], }, }, + { + files: [ + "**/*.test.ts", + "**/*.test.tsx", + "**/test.ts", + "**/test.tsx", + "**/test-**", + ], + rules: { + "jsdoc/no-types": "off", + "jsdoc/empty-tags": "off", + "jsdoc/check-property-names": "off", + "jsdoc/check-values": "off", + "jsdoc/check-param-names": "off", + "jsdoc/require-param-description": "off", + }, + }, ], settings: { react: { diff --git a/.github/workflows/build-and-publish-docker.yaml b/.github/workflows/build-and-publish-docker.yaml index a50fca48..4ad1a551 100644 --- a/.github/workflows/build-and-publish-docker.yaml +++ b/.github/workflows/build-and-publish-docker.yaml @@ -23,7 +23,7 @@ jobs: packages: write steps: - name: Check it out - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: 📥 Download artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 @@ -42,7 +42,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v5.9.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: ${{ inputs.docker_tags}} diff --git a/.github/workflows/build-element-call.yaml b/.github/workflows/build-element-call.yaml index 214c78d6..01553fec 100644 --- a/.github/workflows/build-element-call.yaml +++ b/.github/workflows/build-element-call.yaml @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Enable Corepack run: corepack enable - name: Yarn cache diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index e0271231..32dde869 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Enable Corepack run: corepack enable - name: Yarn cache diff --git a/.github/workflows/publish-embedded-packages.yaml b/.github/workflows/publish-embedded-packages.yaml index 546191ab..275397b5 100644 --- a/.github/workflows/publish-embedded-packages.yaml +++ b/.github/workflows/publish-embedded-packages.yaml @@ -85,7 +85,7 @@ jobs: run: find ${{ env.FILENAME_PREFIX }} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${{ env.FILENAME_PREFIX }}.sha256 - name: Upload if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }} - uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 with: files: | ${{ env.FILENAME_PREFIX }}.tar.gz @@ -103,7 +103,7 @@ jobs: id-token: write # Allow npm to authenticate as a trusted publisher steps: - name: Checkout - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: 📥 Download built element-call artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 @@ -142,7 +142,7 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: 📥 Download built element-call artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 @@ -197,7 +197,7 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: path: element-call @@ -210,7 +210,7 @@ jobs: path: element-call/embedded/ios/Sources/dist - name: Checkout element-call-swift - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: repository: element-hq/element-call-swift path: element-call-swift @@ -262,7 +262,7 @@ jobs: echo "iOS: ${{ needs.publish_ios.outputs.ARTIFACT_VERSION }}" - name: Add release notes if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }} - uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 with: append_body: true body: | diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 34835635..7f2c58fe 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -42,7 +42,7 @@ jobs: - name: Create Checksum run: find ${{ env.FILENAME_PREFIX }} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${{ env.FILENAME_PREFIX }}.sha256 - name: Upload - uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 with: files: | ${{ env.FILENAME_PREFIX }}.tar.gz @@ -61,6 +61,7 @@ jobs: docker_tags: | type=sha,format=short,event=branch type=raw,value=${{ github.event.release.tag_name }} + type=raw,value=latest # Like before, using ${{ env.VERSION }} above doesn't work add_docker_release_note: needs: publish_docker @@ -68,7 +69,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Add release note - uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 with: append_body: true body: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 54035ea4..3251f50e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Enable Corepack run: corepack enable - name: Yarn cache @@ -33,7 +33,7 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-latest steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Enable Corepack run: corepack enable - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 diff --git a/.github/workflows/translations-download.yaml b/.github/workflows/translations-download.yaml index 39e68ec3..76fe418c 100644 --- a/.github/workflows/translations-download.yaml +++ b/.github/workflows/translations-download.yaml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout the code - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Enable Corepack run: corepack enable @@ -42,7 +42,7 @@ jobs: - name: Create Pull Request id: cpr - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 with: token: ${{ secrets.ELEMENT_BOT_TOKEN }} branch: actions/localazy-download diff --git a/.github/workflows/translations-upload.yaml b/.github/workflows/translations-upload.yaml index e7c3ee3d..4c062513 100644 --- a/.github/workflows/translations-upload.yaml +++ b/.github/workflows/translations-upload.yaml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout the code - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Upload uses: localazy/upload@27e6b5c0fddf4551596b42226b1c24124335d24a # v1 diff --git a/codecov.yaml b/codecov.yaml index e1289344..f08dc9b2 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -13,7 +13,6 @@ coverage: informational: true patch: default: - # Encourage (but don't enforce) 80% coverage on all lines that a PR + # Enforce 80% coverage on all lines that a PR # touches target: 80% - informational: true diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index 50498c7a..c7591847 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -47,7 +47,7 @@ services: - ecbackend livekit: - image: livekit/livekit-server:latest + image: livekit/livekit-server:v1.9.4 pull_policy: always hostname: livekit-sfu command: --dev --config /etc/livekit.yaml @@ -67,7 +67,7 @@ services: - ecbackend livekit-1: - image: livekit/livekit-server:latest + image: livekit/livekit-server:v1.9.4 pull_policy: always hostname: livekit-sfu-1 command: --dev --config /etc/livekit.yaml diff --git a/embedded/android/gradle/libs.versions.toml b/embedded/android/gradle/libs.versions.toml index 8ec7801a..5a91e19e 100644 --- a/embedded/android/gradle/libs.versions.toml +++ b/embedded/android/gradle/libs.versions.toml @@ -2,11 +2,11 @@ # https://docs.gradle.org/current/userguide/platforms.html#sub::toml-dependencies-format [versions] -android_gradle_plugin = "8.13.0" +android_gradle_plugin = "8.13.1" [libraries] android_gradle_plugin = { module = "com.android.tools.build:gradle", version.ref = "android_gradle_plugin" } [plugins] android_library = { id = "com.android.library", version.ref = "android_gradle_plugin" } -maven_publish = { id = "com.vanniktech.maven.publish", version = "0.34.0" } \ No newline at end of file +maven_publish = { id = "com.vanniktech.maven.publish", version = "0.35.0" } \ No newline at end of file diff --git a/knip.ts b/knip.ts index 6b378e29..7edfaf65 100644 --- a/knip.ts +++ b/knip.ts @@ -9,7 +9,7 @@ import { type KnipConfig } from "knip"; export default { vite: { - config: ["vite.config.ts", "vite-embedded.config.ts"], + config: ["vite.config.ts", "vite-embedded.config.ts", "vite-sdk.config.ts"], }, entry: ["src/main.tsx", "i18next-parser.config.ts"], ignoreBinaries: [ diff --git a/locales/en/app.json b/locales/en/app.json index 9e8fbbd3..1ff066ea 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -108,11 +108,14 @@ "connection_lost_description": "You were disconnected from the call.", "e2ee_unsupported": "Incompatible browser", "e2ee_unsupported_description": "Your web browser does not support encrypted calls. Supported browsers include Chrome, Safari, and Firefox 117+.", + "failed_to_start_livekit": "Failed to start Livekit connection", "generic": "Something went wrong", "generic_description": "Submitting debug logs will help us track down the problem.", "insufficient_capacity": "Insufficient capacity", "insufficient_capacity_description": "The server has reached its maximum capacity and you cannot join the call at this time. Try again later, or contact your server admin if the problem persists.", "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).", + "membership_manager": "Membership Manager Error", + "membership_manager_description": "The Membership Manager had to shut down. This is caused by many consequtive failed network requests.", "open_elsewhere": "Opened in another tab", "open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.", "room_creation_restricted": "Failed to create call", diff --git a/package.json b/package.json index 75ae74e5..336dd5d7 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "build:embedded": "yarn build:full --config vite-embedded.config.js", "build:embedded:production": "yarn build:embedded", "build:embedded:development": "yarn build:embedded --mode development", + "build:sdk": "yarn build:full --config vite-sdk.config.js", + "build:sdk:development": "yarn build:sdk --mode development", "serve": "vite preview", "prettier:check": "prettier -c .", "prettier:format": "prettier -w .", @@ -47,7 +49,14 @@ "@livekit/protocol": "^1.42.2", "@livekit/track-processors": "^0.5.5", "@mediapipe/tasks-vision": "^0.10.18", - "@playwright/test": "^1.56.1", + "@opentelemetry/api": "^1.4.0", + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.203.0", + "@opentelemetry/resources": "^2.0.0", + "@opentelemetry/sdk-trace-base": "^2.0.0", + "@opentelemetry/sdk-trace-web": "^2.0.0", + "@opentelemetry/semantic-conventions": "^1.25.1", + "@playwright/test": "^1.57.0", "@radix-ui/react-dialog": "^1.0.4", "@radix-ui/react-slider": "^1.1.2", "@radix-ui/react-visually-hidden": "^1.0.3", @@ -86,6 +95,7 @@ "eslint-config-prettier": "^10.0.0", "eslint-plugin-deprecate": "^0.8.2", "eslint-plugin-import": "^2.26.0", + "eslint-plugin-jsdoc": "^61.5.0", "eslint-plugin-jsx-a11y": "^6.5.1", "eslint-plugin-matrix-org": "2.1.0", "eslint-plugin-react": "^7.29.4", @@ -102,8 +112,9 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21", - "matrix-widget-api": "^1.13.0", + "matrix-js-sdk": "matrix-org/matrix-js-sdk#2218ec4e3102e841ba3e794e1c492c0a5aa6c1c3", + "matrix-widget-api": "^1.14.0", + "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", "pako": "^2.0.4", @@ -126,6 +137,7 @@ "vite": "^7.0.0", "vite-plugin-generate-file": "^0.3.0", "vite-plugin-html": "^3.2.2", + "vite-plugin-node-stdlib-browser": "^0.2.1", "vite-plugin-svgr": "^4.0.0", "vitest": "^3.0.0", "vitest-axe": "^1.0.0-pre.3" diff --git a/playwright.config.ts b/playwright.config.ts index 7a8ee530..391e746f 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -38,6 +38,7 @@ export default defineConfig({ projects: [ { name: "chromium", + testIgnore: "**/mobile/**", use: { ...devices["Desktop Chrome"], permissions: [ @@ -56,9 +57,9 @@ export default defineConfig({ }, }, }, - { name: "firefox", + testIgnore: "**/mobile/**", use: { ...devices["Desktop Firefox"], ignoreHTTPSErrors: true, @@ -70,6 +71,27 @@ export default defineConfig({ }, }, }, + { + name: "mobile", + testMatch: "**/mobile/**", + use: { + ...devices["Pixel 7"], + ignoreHTTPSErrors: true, + permissions: [ + "clipboard-write", + "clipboard-read", + "microphone", + "camera", + ], + launchOptions: { + args: [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + "--mute-audio", + ], + }, + }, + }, // No safari for now, until I find a solution to fix `Not allowed to request resource` due to calling // clear http to the homeserver diff --git a/playwright/errors.spec.ts b/playwright/errors.spec.ts index 851e448d..085fb0b4 100644 --- a/playwright/errors.spec.ts +++ b/playwright/errors.spec.ts @@ -7,6 +7,8 @@ Please see LICENSE in the repository root for full details. import { expect, test } from "@playwright/test"; +import { createJTWToken } from "./fixtures/jwt-token"; + test("Should show error screen if fails to get JWT token", async ({ page }) => { await page.goto("/"); @@ -75,7 +77,12 @@ test("Should automatically retry non fatal JWT errors", async ({ test("Should show error screen if call creation is restricted", async ({ page, + browserName, }) => { + test.skip( + browserName === "firefox", + "The is test is not working on firefox CI environment.", + ); await page.goto("/"); // We need the socket connection to fail, but this cannot be done by using the websocket route. @@ -88,7 +95,7 @@ test("Should show error screen if call creation is restricted", async ({ contentType: "application/json", body: JSON.stringify({ url: "wss://badurltotricktest/livekit/sfu", - jwt: "FAKE", + jwt: createJTWToken("@fake:user", "!fake:room"), }), }), ); diff --git a/playwright/fixtures/fixture-mobile-create.ts b/playwright/fixtures/fixture-mobile-create.ts new file mode 100644 index 00000000..3920c978 --- /dev/null +++ b/playwright/fixtures/fixture-mobile-create.ts @@ -0,0 +1,73 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type Browser, type Page, test, expect } from "@playwright/test"; + +export interface MobileCreateFixtures { + asMobile: { + creatorPage: Page; + inviteLink: string; + }; +} + +export const mobileTest = test.extend({ + asMobile: async ({ browser }, pUse) => { + const fixtures = await createCallAndInvite(browser); + await pUse({ + creatorPage: fixtures.page, + inviteLink: fixtures.inviteLink, + }); + }, +}); + +/** + * Create a call and generate an invite link + */ +async function createCallAndInvite( + browser: Browser, +): Promise<{ page: Page; inviteLink: string }> { + const creatorContext = await browser.newContext({ reducedMotion: "reduce" }); + const creatorPage = await creatorContext.newPage(); + + await creatorPage.goto("/"); + + // ======== + // ARRANGE: The first user creates a call as guest, join it, then click the invite button to copy the invite link + // ======== + await creatorPage.getByTestId("home_callName").click(); + await creatorPage.getByTestId("home_callName").fill("Welcome"); + await creatorPage.getByTestId("home_displayName").click(); + await creatorPage.getByTestId("home_displayName").fill("Inviter"); + await creatorPage.getByTestId("home_go").click(); + await expect(creatorPage.locator("video")).toBeVisible(); + + await creatorPage + .getByRole("button", { name: "Continue in browser" }) + .click(); + // join + await creatorPage.getByTestId("lobby_joinCall").click(); + + // Get the invite link + await creatorPage.getByRole("button", { name: "Invite" }).click(); + await expect( + creatorPage.getByRole("heading", { name: "Invite to this call" }), + ).toBeVisible(); + await expect(creatorPage.getByRole("img", { name: "QR Code" })).toBeVisible(); + await expect(creatorPage.getByTestId("modal_inviteLink")).toBeVisible(); + await expect(creatorPage.getByTestId("modal_inviteLink")).toBeVisible(); + await creatorPage.getByTestId("modal_inviteLink").click(); + + const inviteLink = (await creatorPage.evaluate( + "navigator.clipboard.readText()", + )) as string; + expect(inviteLink).toContain("room/#/"); + + return { + page: creatorPage, + inviteLink, + }; +} diff --git a/playwright/fixtures/jwt-token.ts b/playwright/fixtures/jwt-token.ts new file mode 100644 index 00000000..18119c7e --- /dev/null +++ b/playwright/fixtures/jwt-token.ts @@ -0,0 +1,22 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +export function createJTWToken(sub: string, room: string): string { + return [ + {}, // header + { + // payload + sub, + video: { + room, + }, + }, + {}, // signature + ] + .map((d) => global.btoa(JSON.stringify(d))) + .join("."); +} diff --git a/playwright/fixtures/widget-user.ts b/playwright/fixtures/widget-user.ts index 433c960b..f1f738b7 100644 --- a/playwright/fixtures/widget-user.ts +++ b/playwright/fixtures/widget-user.ts @@ -67,7 +67,6 @@ const CONFIG_JSON = { /** * Set the Element Call URL in the dev tool settings using `window.mxSettingsStore` via `page.evaluate`. - * @param page */ const setDevToolElementCallDevUrl = process.env.USE_DOCKER ? async (page: Page): Promise => { @@ -111,19 +110,27 @@ async function registerUser( await page.getByRole("textbox", { name: "Confirm password" }).click(); await page.getByRole("textbox", { name: "Confirm password" }).fill(PASSWORD); await page.getByRole("button", { name: "Register" }).click(); - const continueButton = page.getByRole("button", { name: "Continue" }); - try { - await expect(continueButton).toBeVisible({ timeout: 5000 }); - await page - .getByRole("textbox", { name: "Password", exact: true }) - .fill(PASSWORD); - await continueButton.click(); - } catch { - // continueButton not visible, continue as normal - } + await expect( page.getByRole("heading", { name: `Welcome ${username}` }), ).toBeVisible(); + + const browserUnsupportedToast = page + .getByText("Element does not support this browser") + .locator("..") + .locator(".."); + + // Dismiss incompatible browser toast + const dismissButton = browserUnsupportedToast.getByRole("button", { + name: "Dismiss", + }); + try { + await expect(dismissButton).toBeVisible({ timeout: 700 }); + await dismissButton.click(); + } catch { + // dismissButton not visible, continue as normal + } + await setDevToolElementCallDevUrl(page); const clientHandle = await page.evaluateHandle(() => diff --git a/playwright/mobile/create-call-mobile.spec.ts b/playwright/mobile/create-call-mobile.spec.ts new file mode 100644 index 00000000..141ffaae --- /dev/null +++ b/playwright/mobile/create-call-mobile.spec.ts @@ -0,0 +1,115 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "@playwright/test"; + +import { mobileTest } from "../fixtures/fixture-mobile-create.ts"; + +test("@mobile Start a new call then leave and show the feedback screen", async ({ + page, +}) => { + await page.goto("/"); + + await page.getByTestId("home_callName").click(); + await page.getByTestId("home_callName").fill("HelloCall"); + await page.getByTestId("home_displayName").click(); + await page.getByTestId("home_displayName").fill("John Doe"); + await page.getByTestId("home_go").click(); + + // await page.pause(); + await expect(page.locator("video")).toBeVisible(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + + await page.getByRole("button", { name: "Continue in browser" }).click(); + // Join the call + await page.getByTestId("lobby_joinCall").click(); + + // Ensure that the call is connected + await page + .locator("div") + .filter({ hasText: /^HelloCall$/ }) + .click(); + // Check the number of participants + await expect(page.locator("div").filter({ hasText: /^1$/ })).toBeVisible(); + // The tooltip with the name should be visible + await expect(page.getByTestId("name_tag")).toContainText("John Doe"); + + // leave the call + await page.getByTestId("incall_leave").click(); + await expect(page.getByRole("heading")).toContainText( + "John Doe, your call has ended. How did it go?", + ); + await expect(page.getByRole("main")).toContainText( + "Why not finish by setting up a password to keep your account?", + ); + + await expect( + page.getByRole("link", { name: "Not now, return to home screen" }), + ).toBeVisible(); +}); + +mobileTest( + "Test earpiece overlay in controlledAudioDevices mode", + async ({ asMobile, browser }) => { + test.slow(); // Triples the timeout + const { creatorPage, inviteLink } = asMobile; + + // ======== + // ACT: The other user use the invite link to join the call as a guest + // ======== + const guestInviteeContext = await browser.newContext({ + reducedMotion: "reduce", + }); + const guestPage = await guestInviteeContext.newPage(); + await guestPage.goto(inviteLink + "&controlledAudioDevices=true"); + + await guestPage + .getByRole("button", { name: "Continue in browser" }) + .click(); + + await guestPage.getByTestId("joincall_displayName").fill("Invitee"); + await expect(guestPage.getByTestId("joincall_joincall")).toBeVisible(); + await guestPage.getByTestId("joincall_joincall").click(); + await guestPage.getByTestId("lobby_joinCall").click(); + + // ======== + // ASSERT: check that there are two members in the call + // ======== + + // There should be two participants now + await expect( + guestPage.getByTestId("roomHeader_participants_count"), + ).toContainText("2"); + expect(await guestPage.getByTestId("videoTile").count()).toBe(2); + + // Same in creator page + await expect( + creatorPage.getByTestId("roomHeader_participants_count"), + ).toContainText("2"); + expect(await creatorPage.getByTestId("videoTile").count()).toBe(2); + + // TEST: control audio devices from the invitee page + + await guestPage.evaluate(() => { + window.controls.setAvailableAudioDevices([ + { id: "speaker", name: "Speaker", isSpeaker: true }, + { id: "earpiece", name: "Handset", isEarpiece: true }, + { id: "headphones", name: "Headphones" }, + ]); + window.controls.setAudioDevice("earpiece"); + }); + await expect( + guestPage.getByRole("heading", { name: "Handset Mode" }), + ).toBeVisible(); + await expect( + guestPage.getByRole("button", { name: "Back to Speaker Mode" }), + ).toBeVisible(); + + // Should auto-mute the video when earpiece is selected + await expect(guestPage.getByTestId("incall_videomute")).toBeDisabled(); + }, +); diff --git a/playwright/sfu-reconnect-bug.spec.ts b/playwright/sfu-reconnect-bug.spec.ts index 6138eb78..9f666f0f 100644 --- a/playwright/sfu-reconnect-bug.spec.ts +++ b/playwright/sfu-reconnect-bug.spec.ts @@ -68,11 +68,6 @@ test("When creator left, avoid reconnect to the same SFU", async ({ reducedMotion: "reduce", }); const guestCPage = await guestC.newPage(); - let sfuGetCallCount = 0; - await guestCPage.route("**/livekit/jwt/sfu/get", async (route) => { - sfuGetCallCount++; - await route.continue(); - }); // Track WebSocket connections let wsConnectionCount = 0; await guestCPage.routeWebSocket("**", (ws) => { @@ -100,5 +95,4 @@ test("When creator left, avoid reconnect to the same SFU", async ({ // https://github.com/element-hq/element-call/issues/3344 // The app used to request a new jwt token then to reconnect to the SFU expect(wsConnectionCount).toBe(1); - expect(sfuGetCallCount).toBe(2 /* the first one is for the warmup */); }); diff --git a/sdk/README.md b/sdk/README.md new file mode 100644 index 00000000..03801b83 --- /dev/null +++ b/sdk/README.md @@ -0,0 +1,35 @@ +# SDK mode + +EC can be build in sdk mode. This will result in a compiled js file that can be imported in very simple webapps. + +It allows to use matrixRTC in combination with livekit without relying on element call. + +This is done by instantiating the call view model and exposing some useful behaviors (observables) and methods. + +This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver ellowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template. + +## Widgets + +The sdk mode is particularly interesting to be used in widgets where you do not need to pay attention to matrix login/cs api ... +To create a widget see the example index.html file in this folder. And add it to EW via: +`/addwidget ` (see **url parameters** for more details on ``) + +### url parameters + +``` +widgetId = $matrix_widget_id +perParticipantE2EE = true +userId = $matrix_user_id +deviceId = $org.matrix.msc3819.matrix_device_id +baseUrl = $org.matrix.msc4039.matrix_base_url +``` + +`parentUrl = // will be inserted automatically` + +Full template use as ``: + +``` +http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id +``` + +the `$` prefixed variables will be replaced by EW on widget instantiation. (e.g. `$matrix_user_id` -> `@user:example.com` (url encoding will also be applied automatically by EW) -> `%40user%3Aexample.com`) diff --git a/sdk/helper.ts b/sdk/helper.ts new file mode 100644 index 00000000..a3d597be --- /dev/null +++ b/sdk/helper.ts @@ -0,0 +1,55 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * This file contains helper functions and types for the MatrixRTC SDK. + */ + +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { scan } from "rxjs"; + +import { widget as _widget } from "../src/widget"; +import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel"; + +export const logger = rootLogger.getChild("[MatrixRTCSdk]"); + +if (!_widget) throw Error("No widget. This webapp can only start as a widget"); +export const widget = _widget; + +export const tryMakeSticky = (): void => { + logger.info("try making sticky MatrixRTCSdk"); + void widget.api + .setAlwaysOnScreen(true) + .then(() => { + logger.info("sticky MatrixRTCSdk"); + }) + .catch((error) => { + logger.error("failed to make sticky MatrixRTCSdk", error); + }); +}; +export const TEXT_LK_TOPIC = "matrixRTC"; +/** + * simple helper operator to combine the last emitted and the current emitted value of a rxjs observable + * + * I think there should be a builtin for this but i did not find it... + */ +export const currentAndPrev = scan< + LivekitRoomItem[], + { + prev: LivekitRoomItem[]; + current: LivekitRoomItem[]; + } +>( + ({ current: lastCurrentVal }, items) => ({ + prev: lastCurrentVal, + current: items, + }), + { + prev: [], + current: [], + }, +); diff --git a/sdk/index.html b/sdk/index.html new file mode 100644 index 00000000..51110ebd --- /dev/null +++ b/sdk/index.html @@ -0,0 +1,87 @@ + + + + Godot MatrixRTC Widget + + + + + + + +
+
+ + +
+
+
+ + diff --git a/sdk/main.ts b/sdk/main.ts new file mode 100644 index 00000000..376674a4 --- /dev/null +++ b/sdk/main.ts @@ -0,0 +1,308 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * This file is the entrypoint for the sdk build of element call: `yarn build:sdk` + * use in widgets. + * It exposes the `createMatrixRTCSdk` which creates the `MatrixRTCSdk` interface (see below) that + * can be used to join a rtc session and exchange realtime data. + * It takes care of all the tricky bits: + * - sending delayed events + * - finding the right sfu + * - handling the media stream + * - sending join/leave state or sticky events + * - setting up encryption and scharing keys + */ + +import { + combineLatest, + map, + type Observable, + of, + shareReplay, + Subject, + switchMap, + tap, +} from "rxjs"; +import { + type CallMembership, + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/lib/matrixrtc"; +import { + type Room as LivekitRoom, + type TextStreamReader, + type LocalParticipant, + type RemoteParticipant, +} from "livekit-client"; + +// TODO how can this get fixed? to just be part of `livekit-client` +// Can this be done in the tsconfig.json +import { type TextStreamInfo } from "../node_modules/livekit-client/dist/src/room/types"; +import { type Behavior, constant } from "../src/state/Behavior"; +import { createCallViewModel$ } from "../src/state/CallViewModel/CallViewModel"; +import { ObservableScope } from "../src/state/ObservableScope"; +import { getUrlParams } from "../src/UrlParams"; +import { MuteStates } from "../src/state/MuteStates"; +import { MediaDevices } from "../src/state/MediaDevices"; +import { E2eeType } from "../src/e2ee/e2eeType"; +import { + currentAndPrev, + logger, + TEXT_LK_TOPIC, + tryMakeSticky, + widget, +} from "./helper"; +import { ElementWidgetActions } from "../src/widget"; +import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection"; + +interface MatrixRTCSdk { + /** + * observe connected$ to track the state. + * @returns + */ + join: () => void; + /** @throws on leave errors */ + leave: () => void; + data$: Observable<{ sender: string; data: string }>; + /** + * flattened list of members + */ + members$: Behavior< + { + connection: Connection | null; + membership: CallMembership; + participant: LocalParticipant | RemoteParticipant | null; + }[] + >; + /** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state */ + connected$: Behavior; + sendData?: (data: unknown) => Promise; +} + +export async function createMatrixRTCSdk( + application: string = "m.call", + id: string = "", +): Promise { + logger.info("Hello"); + const client = await widget.client; + logger.info("client created"); + const scope = new ObservableScope(); + const { roomId } = getUrlParams(); + if (roomId === null) throw Error("could not get roomId from url params"); + + const room = client.getRoom(roomId); + if (room === null) throw Error("could not get room from client"); + + const mediaDevices = new MediaDevices(scope); + const muteStates = new MuteStates(scope, mediaDevices, constant(true)); + const slot = { application, id }; + const rtcSession = new MatrixRTCSession( + client, + room, + MatrixRTCSession.sessionMembershipsForSlot(room, slot), + slot, + ); + const callViewModel = createCallViewModel$( + scope, + rtcSession, + room, + mediaDevices, + muteStates, + { encryptionSystem: { kind: E2eeType.PER_PARTICIPANT } }, + of({}), + of({}), + constant({ supported: false, processor: undefined }), + ); + logger.info("CallViewModelCreated"); + // create data listener + const data$ = new Subject<{ sender: string; data: string }>(); + + const lkTextStreamHandlerFunction = async ( + reader: TextStreamReader, + participantInfo: { identity: string }, + livekitRoom: LivekitRoom, + ): Promise => { + const info = reader.info; + logger.info( + `Received text stream from ${participantInfo.identity}\n` + + ` Topic: ${info.topic}\n` + + ` Timestamp: ${info.timestamp}\n` + + ` ID: ${info.id}\n` + + ` Size: ${info.size}`, // Optional, only available if the stream was sent with `sendText` + ); + + const participants = callViewModel.livekitRoomItems$.value.find( + (i) => i.livekitRoom === livekitRoom, + )?.participants; + if (participants && participants.includes(participantInfo.identity)) { + const text = await reader.readAll(); + logger.info(`Received text: ${text}`); + data$.next({ sender: participantInfo.identity, data: text }); + } else { + logger.warn( + "Received text from unknown participant", + participantInfo.identity, + ); + } + }; + + const livekitRoomItemsSub = callViewModel.livekitRoomItems$ + .pipe( + tap((beforecurrentAndPrev) => { + logger.info( + `LiveKit room items updated: ${beforecurrentAndPrev.length}`, + beforecurrentAndPrev, + ); + }), + currentAndPrev, + tap((aftercurrentAndPrev) => { + logger.info( + `LiveKit room items updated: ${aftercurrentAndPrev.current.length}, ${aftercurrentAndPrev.prev.length}`, + aftercurrentAndPrev, + ); + }), + ) + .subscribe({ + next: ({ prev, current }) => { + const prevRooms = prev.map((i) => i.livekitRoom); + const currentRooms = current.map((i) => i.livekitRoom); + const addedRooms = currentRooms.filter((r) => !prevRooms.includes(r)); + const removedRooms = prevRooms.filter((r) => !currentRooms.includes(r)); + addedRooms.forEach((r) => { + logger.info(`Registering text stream handler for room `); + r.registerTextStreamHandler( + TEXT_LK_TOPIC, + (reader, participantInfo) => + void lkTextStreamHandlerFunction(reader, participantInfo, r), + ); + }); + removedRooms.forEach((r) => { + logger.info(`Unregistering text stream handler for room `); + r.unregisterTextStreamHandler(TEXT_LK_TOPIC); + }); + }, + complete: () => { + logger.info("Livekit room items subscription completed"); + for (const item of callViewModel.livekitRoomItems$.value) { + logger.info("unregistering room item from room", item.url); + item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC); + } + }, + }); + + // create sendData function + const sendFn: Behavior<(data: string) => Promise> = + scope.behavior( + callViewModel.localMatrixLivekitMember$.pipe( + switchMap((m) => { + if (!m) + return of((data: string): never => { + throw Error("local membership not yet ready."); + }); + return m.participant.value$.pipe( + map((p) => { + if (p === null) { + return (data: string): never => { + throw Error("local participant not yet ready to send data."); + }; + } else { + return async (data: string): Promise => + p.sendText(data, { topic: TEXT_LK_TOPIC }); + } + }), + ); + }), + ), + ); + + const sendData = async (data: unknown): Promise => { + const dataString = JSON.stringify(data); + logger.info("try sending: ", dataString); + try { + await Promise.resolve(); + const info = await sendFn.value(dataString); + logger.info(`Sent text with stream ID: ${info.id}`); + } catch (e) { + logger.error("failed sending: ", dataString, e); + } + }; + + // after hangup gets called + const leaveSubs = callViewModel.leave$.subscribe(() => { + const scheduleWidgetCloseOnLeave = async (): Promise => { + const leaveResolver = Promise.withResolvers(); + logger.info("waiting for RTC leave"); + rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, (isJoined) => { + logger.info("received RTC join update: ", isJoined); + if (!isJoined) leaveResolver.resolve(); + }); + await leaveResolver.promise; + logger.info("send Unstick"); + await widget.api + .setAlwaysOnScreen(false) + .catch((e) => + logger.error( + "Failed to set call widget `alwaysOnScreen` to false", + e, + ), + ); + logger.info("send Close"); + await widget.api.transport + .send(ElementWidgetActions.Close, {}) + .catch((e) => logger.error("Failed to send close action", e)); + }; + + // schedule close first and then leave (scope.end) + void scheduleWidgetCloseOnLeave(); + + // actual hangup (ending scope will send the leave event.. its kinda odd. since you might end up closing the widget too fast) + scope.end(); + }); + + logger.info("createMatrixRTCSdk done"); + + return { + join: (): void => { + // first lets try making the widget sticky + tryMakeSticky(); + callViewModel.join(); + }, + leave: (): void => { + callViewModel.hangup(); + leaveSubs.unsubscribe(); + livekitRoomItemsSub.unsubscribe(); + }, + data$, + connected$: callViewModel.connected$, + members$: scope.behavior( + callViewModel.matrixLivekitMembers$.pipe( + switchMap((members) => { + const listOfMemberObservables = members.map((member) => + combineLatest([ + member.connection$, + member.membership$, + member.participant.value$, + ]).pipe( + map(([connection, membership, participant]) => ({ + connection, + membership, + participant, + })), + // using shareReplay instead of a Behavior here because the behavior would need + // a tricky scope.end() setup. + shareReplay({ bufferSize: 1, refCount: true }), + ), + ); + return combineLatest(listOfMemberObservables); + }), + ), + [], + ), + sendData, + }; +} diff --git a/src/UrlParams.test.ts b/src/UrlParams.test.ts index cd8fc6d5..faba394f 100644 --- a/src/UrlParams.test.ts +++ b/src/UrlParams.test.ts @@ -332,6 +332,42 @@ describe("UrlParams", () => { expect(computeUrlParams("?intent=join_existing").skipLobby).toBe(false); }); }); + + describe("noiseSuppression", () => { + it("defaults to true", () => { + expect(computeUrlParams().noiseSuppression).toBe(true); + }); + + it("is parsed", () => { + expect( + computeUrlParams("?intent=start_call&noiseSuppression=true") + .noiseSuppression, + ).toBe(true); + expect( + computeUrlParams("?intent=start_call&noiseSuppression&bar=foo") + .noiseSuppression, + ).toBe(true); + expect(computeUrlParams("?noiseSuppression=false").noiseSuppression).toBe( + false, + ); + }); + }); + + describe("echoCancellation", () => { + it("defaults to true", () => { + expect(computeUrlParams().echoCancellation).toBe(true); + }); + + it("is parsed", () => { + expect(computeUrlParams("?echoCancellation=true").echoCancellation).toBe( + true, + ); + expect(computeUrlParams("?echoCancellation=false").echoCancellation).toBe( + false, + ); + }); + }); + describe("header", () => { it("uses header if provided", () => { expect(computeUrlParams("?header=app_bar&hideHeader=true").header).toBe( diff --git a/src/UrlParams.ts b/src/UrlParams.ts index 4eb69298..f78841fb 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -233,6 +233,17 @@ export interface UrlConfiguration { */ waitForCallPickup: boolean; + /** + * Whether to enable echo cancellation for audio capture. + * Defaults to true. + */ + echoCancellation?: boolean; + /** + * Whether to enable noise suppression for audio capture. + * Defaults to true. + */ + noiseSuppression?: boolean; + callIntent?: RTCCallIntent; } interface IntentAndPlatformDerivedConfiguration { @@ -525,6 +536,8 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => { ]), waitForCallPickup: parser.getFlag("waitForCallPickup"), autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"), + noiseSuppression: parser.getFlagParam("noiseSuppression", true), + echoCancellation: parser.getFlagParam("echoCancellation", true), }; // Log the final configuration for debugging purposes. diff --git a/src/analytics/PosthogAnalytics.ts b/src/analytics/PosthogAnalytics.ts index 8b2aa91d..46223afe 100644 --- a/src/analytics/PosthogAnalytics.ts +++ b/src/analytics/PosthogAnalytics.ts @@ -247,9 +247,8 @@ export class PosthogAnalytics { // wins, and the first writer will send tracking with an ID that doesn't match the one on the server // until the next time account data is refreshed and this function is called (most likely on next // page load). This will happen pretty infrequently, so we can tolerate the possibility. - const accountDataAnalyticsId = analyticsIdGenerator(); - await this.setAccountAnalyticsId(accountDataAnalyticsId); - analyticsID = await this.hashedEcAnalyticsId(accountDataAnalyticsId); + analyticsID = analyticsIdGenerator(); + await this.setAccountAnalyticsId(analyticsID); } } catch (e) { // The above could fail due to network requests, but not essential to starting the application, @@ -270,37 +269,14 @@ export class PosthogAnalytics { private async getAnalyticsId(): Promise { const client: MatrixClient = window.matrixclient; - let accountAnalyticsId: string | null; if (widget) { - accountAnalyticsId = getUrlParams().posthogUserId; + return getUrlParams().posthogUserId; } else { const accountData = await client.getAccountDataFromServer( PosthogAnalytics.ANALYTICS_EVENT_TYPE, ); - accountAnalyticsId = accountData?.id ?? null; + return accountData?.id ?? null; } - if (accountAnalyticsId) { - // we dont just use the element web analytics ID because that would allow to associate - // users between the two posthog instances. By using a hash from the username and the element web analytics id - // it is not possible to conclude the element web posthog user id from the element call user id and vice versa. - return await this.hashedEcAnalyticsId(accountAnalyticsId); - } - return null; - } - - private async hashedEcAnalyticsId( - accountAnalyticsId: string, - ): Promise { - const client: MatrixClient = window.matrixclient; - const posthogIdMaterial = "ec" + accountAnalyticsId + client.getUserId(); - const bufferForPosthogId = await crypto.subtle.digest( - "sha-256", - new TextEncoder().encode(posthogIdMaterial), - ); - const view = new Int32Array(bufferForPosthogId); - return Array.from(view) - .map((b) => Math.abs(b).toString(16).padStart(2, "0")) - .join(""); } private async setAccountAnalyticsId(analyticsID: string): Promise { diff --git a/src/e2ee/sharedKeyManagement.ts b/src/e2ee/sharedKeyManagement.ts index c68ba453..18d007e2 100644 --- a/src/e2ee/sharedKeyManagement.ts +++ b/src/e2ee/sharedKeyManagement.ts @@ -34,8 +34,8 @@ const getRoomSharedKeyLocalStorageKey = (roomId: string): string => `room-shared-key-${roomId}`; /** - * An upto-date shared key for the room. Either from local storage or the value from `setInitialValue`. - * @param roomId + * An up-to-date shared key for the room. Either from local storage or the value from `setInitialValue`. + * @param roomId The room ID we want the shared key for. * @param setInitialValue The value we get from the URL. The hook will overwrite the local storage value with this. * @returns [roomSharedKey, setRoomSharedKey] like a react useState hook. */ diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 5b1149e9..741529b8 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -166,7 +166,11 @@ interface StereoPanAudioTrackProps { * It main purpose is to remount the AudioTrack component when switching from * audioContext to normal audio playback. * As of now the AudioTrack component does not support adding audio nodes while being mounted. - * @param param0 + * @param props The component props + * @param props.trackRef The track reference + * @param props.muted If the track should be muted + * @param props.audioContext The audio context to use + * @param props.audioNodes The audio nodes to use * @returns */ function AudioTrackWithAudioNodes({ diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts new file mode 100644 index 00000000..2a260b01 --- /dev/null +++ b/src/livekit/openIDSFU.test.ts @@ -0,0 +1,112 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + beforeEach, + afterEach, + describe, + expect, + it, + type MockedObject, + vitest, +} from "vitest"; +import fetchMock from "fetch-mock"; + +import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU"; +import { testJWTToken } from "../utils/test-fixtures"; + +const sfuUrl = "https://sfu.example.org"; + +describe("getSFUConfigWithOpenID", () => { + let matrixClient: MockedObject; + beforeEach(() => { + matrixClient = { + getOpenIdToken: vitest.fn(), + getDeviceId: vitest.fn(), + }; + }); + afterEach(() => { + vitest.clearAllMocks(); + fetchMock.reset(); + }); + it("should handle fetching a token", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: sfuUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfigWithOpenID( + matrixClient, + "https://sfu.example.org", + "!example_room_id", + ); + expect(config).toEqual({ + jwt: testJWTToken, + url: sfuUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + void (await fetchMock.flush()); + }); + it("should fail if the SFU errors", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { error: "Test failure" }, + }; + }); + try { + await getSFUConfigWithOpenID( + matrixClient, + "https://sfu.example.org", + "!example_room_id", + ); + } catch (ex) { + expect(((ex as Error).cause as Error).message).toEqual( + "SFU Config fetch failed with status code 500", + ); + void (await fetchMock.flush()); + return; + } + expect.fail("Expected test to throw;"); + }); + + it("should retry fetching the openid token", async () => { + let count = 0; + matrixClient.getOpenIdToken.mockImplementation(async () => { + count++; + if (count < 2) { + throw Error("Test failure"); + } + return Promise.resolve({ + token_type: "Bearer", + access_token: "foobar", + matrix_server_name: "example.org", + expires_in: 30, + }); + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: sfuUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfigWithOpenID( + matrixClient, + "https://sfu.example.org", + "!example_room_id", + ); + expect(config).toEqual({ + jwt: testJWTToken, + url: sfuUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + void (await fetchMock.flush()); + }); +}); diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 3ae003fb..34c98a88 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -11,9 +11,47 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { FailToGetOpenIdToken } from "../utils/errors"; import { doNetworkOperationWithRetry } from "../utils/matrix"; +/** + * Configuration and access tokens provided by the SFU on successful authentication. + */ export interface SFUConfig { url: string; jwt: string; + livekitAlias: string; + livekitIdentity: string; +} + +/** + * Decoded details from the JWT. + */ +interface SFUJWTPayload { + /** + * Expiration time for the JWT. + * Note: This value is in seconds since Unix epoch. + */ + exp: number; + /** + * Name of the instance which authored the JWT + */ + iss: string; + /** + * Time at which the JWT can start to be used. + * Note: This value is in seconds since Unix epoch. + */ + nbf: number; + /** + * Subject. The Livekit alias in this context. + */ + sub: string; + /** + * The set of permissions for the user. + */ + video: { + canPublish: boolean; + canSubscribe: boolean; + room: string; + roomJoin: boolean; + }; } // The bits we need from MatrixClient @@ -25,9 +63,9 @@ export type OpenIDClientParts = Pick< * Gets a bearer token from the homeserver and then use it to authenticate * to the matrix RTC backend in order to get acces to the SFU. * It has built-in retry for calls to the homeserver with a backoff policy. - * @param client - * @param serviceUrl - * @param matrixRoomId + * @param client The Matrix client + * @param serviceUrl The URL of the livekit SFU service + * @param matrixRoomId The Matrix room ID for which to get the SFU config * @returns Object containing the token information * @throws FailToGetOpenIdToken */ @@ -57,7 +95,17 @@ export async function getSFUConfigWithOpenID( ); logger.info(`Got JWT from call's active focus URL.`); - return sfuConfig; + // Pull the details from the JWT + const [, payloadStr] = sfuConfig.jwt.split("."); + // TODO: Prefer Uint8Array.fromBase64 when widely available + const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; + return { + jwt: sfuConfig.jwt, + url: sfuConfig.url, + livekitAlias: payload.video.room, + // NOTE: Currently unused. + livekitIdentity: payload.sub, + }; } async function getLiveKitJWT( @@ -65,7 +113,7 @@ async function getLiveKitJWT( livekitServiceURL: string, roomName: string, openIDToken: IOpenIDToken, -): Promise { +): Promise<{ url: string; jwt: string }> { try { const res = await fetch(livekitServiceURL + "/sfu/get", { method: "POST", @@ -83,6 +131,6 @@ async function getLiveKitJWT( } return await res.json(); } catch (e) { - throw new Error("SFU Config fetch failed with exception " + e); + throw new Error("SFU Config fetch failed with exception", { cause: e }); } } diff --git a/src/reactions/RaisedHandIndicator.test.tsx b/src/reactions/RaisedHandIndicator.test.tsx index fedd8ec2..62e3ffb5 100644 --- a/src/reactions/RaisedHandIndicator.test.tsx +++ b/src/reactions/RaisedHandIndicator.test.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, test } from "vitest"; +import { beforeEach, describe, expect, test, vi } from "vitest"; import { render, configure } from "@testing-library/react"; import { RaisedHandIndicator } from "./RaisedHandIndicator"; @@ -15,6 +15,13 @@ configure({ }); describe("RaisedHandIndicator", () => { + const fixedTime = new Date("2025-01-01T12:00:00.000Z"); + + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(fixedTime); + }); + test("renders nothing when no hand has been raised", () => { const { container } = render(); expect(container.firstChild).toBeNull(); diff --git a/src/reactions/ReactionsReader.ts b/src/reactions/ReactionsReader.ts index 74b47c77..7ce59812 100644 --- a/src/reactions/ReactionsReader.ts +++ b/src/reactions/ReactionsReader.ts @@ -135,10 +135,10 @@ export class ReactionsReader { } /** - * Fetchest any hand wave reactions by the given sender on the given + * Fetches any hand wave reactions by the given sender on the given * membership event. - * @param membershipEventId - * @param expectedSender + * @param membershipEventId - The user membership event id. + * @param expectedSender - The expected sender of the reaction. * @returns A MatrixEvent if one was found. */ private getLastReactionEvent( diff --git a/src/reactions/__snapshots__/RaisedHandIndicator.test.tsx.snap b/src/reactions/__snapshots__/RaisedHandIndicator.test.tsx.snap index ab6fafa3..43c3f928 100644 --- a/src/reactions/__snapshots__/RaisedHandIndicator.test.tsx.snap +++ b/src/reactions/__snapshots__/RaisedHandIndicator.test.tsx.snap @@ -15,7 +15,7 @@ exports[`RaisedHandIndicator > renders a smaller indicator when miniature is spe

- 00:01 + 00:00

`; @@ -35,7 +35,7 @@ exports[`RaisedHandIndicator > renders an indicator when a hand has been raised

- 00:01 + 00:00

`; @@ -55,7 +55,7 @@ exports[`RaisedHandIndicator > renders an indicator when a hand has been raised

- 01:01 + 01:00

`; diff --git a/src/room/EarpieceOverlay.module.css b/src/room/EarpieceOverlay.module.css index d0757cdb..e53a1974 100644 --- a/src/room/EarpieceOverlay.module.css +++ b/src/room/EarpieceOverlay.module.css @@ -7,34 +7,17 @@ align-items: center; justify-content: center; gap: var(--cpd-space-2x); -} - -@keyframes fade-in { - from { - opacity: 0; - } - to { - opacity: 1; - } + transition: opacity 200ms; } .overlay[data-show="true"] { - animation: fade-in 200ms; -} - -@keyframes fade-out { - from { - opacity: 1; - } - to { - opacity: 0; - display: none; - } + opacity: 1; } .overlay[data-show="false"] { - animation: fade-out 130ms forwards; + opacity: 0; pointer-events: none; + transition-duration: 130ms; } .overlay::before { diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 75438f7f..dfd11ff3 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -160,6 +160,7 @@ export const GroupCallView: FC = ({ }, [rtcSession]); // TODO move this into the callViewModel LocalMembership.ts + // We might actually not need this at all. Since we get into fatalError on those errors already? useTypedEventEmitter( rtcSession, MatrixRTCSessionEvent.MembershipManagerError, @@ -313,6 +314,7 @@ export const GroupCallView: FC = ({ const navigate = useNavigate(); + // TODO split this into leave and onDisconnect const onLeft = useCallback( ( reason: "timeout" | "user" | "allOthersLeft" | "decline" | "error", diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 72a35da6..c6f22b43 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -24,7 +24,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import classNames from "classnames"; import { BehaviorSubject, map } from "rxjs"; import { useObservable } from "observable-hooks"; -import { logger } from "matrix-js-sdk/lib/logger"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { VoiceCallSolidIcon, VolumeOnSolidIcon, @@ -87,6 +87,7 @@ import { ReactionsOverlay } from "./ReactionsOverlay"; import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { debugTileLayout as debugTileLayoutSetting, + matrixRTCMode as matrixRTCModeSetting, useSetting, } from "../settings/settings"; import { ReactionsReader } from "../reactions/ReactionsReader"; @@ -108,6 +109,8 @@ import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.t import { type Layout } from "../state/layout-types.ts"; import { ObservableScope } from "../state/ObservableScope.ts"; +const logger = rootLogger.getChild("[InCallView]"); + const maxTapDurationMs = 400; export interface ActiveCallProps @@ -126,6 +129,7 @@ export const ActiveCall: FC = (props) => { const mediaDevices = useMediaDevices(); const trackProcessorState$ = useTrackProcessorObservable$(); useEffect(() => { + logger.info("START CALL VIEW SCOPE"); const scope = new ObservableScope(); const reactionsReader = new ReactionsReader(scope, props.rtcSession); const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = @@ -140,15 +144,20 @@ export const ActiveCall: FC = (props) => { encryptionSystem: props.e2eeSystem, autoLeaveWhenOthersLeft, waitForCallPickup: waitForCallPickup && sendNotificationType === "ring", + matrixRTCMode$: matrixRTCModeSetting.value$, }, reactionsReader.raisedHands$, reactionsReader.reactions$, scope.behavior(trackProcessorState$), ); + // TODO move this somewhere else once we use the callViewModel in the lobby as well! + vm.join(); setVm(vm); vm.leave$.pipe(scope.bind()).subscribe(props.onLeft); + return (): void => { + logger.info("END CALL VIEW SCOPE"); scope.end(); }; }, [ @@ -249,7 +258,7 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); - const audioParticipants = useBehavior(vm.audioParticipants$); + const audioParticipants = useBehavior(vm.livekitRoomItems$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); const windowMode = useBehavior(vm.windowMode$); @@ -266,7 +275,10 @@ export const InCallView: FC = ({ const ringOverlay = useBehavior(vm.ringOverlay$); const fatalCallError = useBehavior(vm.fatalError$); // Stop the rendering and throw for the error boundary - if (fatalCallError) throw fatalCallError; + if (fatalCallError) { + logger.debug("fatalCallError stop rendering", fatalCallError); + throw fatalCallError; + } // We need to set the proper timings on the animation based upon the sound length. const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1; diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index ad4f30b3..10e098f1 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -79,9 +79,9 @@ export const LobbyView: FC = ({ waitingForInvite, }) => { useEffect(() => { - logger.info("[Lifecycle] GroupCallView Component mounted"); + logger.info("[Lifecycle] LobbyView Component mounted"); return (): void => { - logger.info("[Lifecycle] GroupCallView Component unmounted"); + logger.info("[Lifecycle] LobbyView Component unmounted"); }; }, []); diff --git a/src/room/useLoadGroupCall.ts b/src/room/useLoadGroupCall.ts index ab6ccf64..2cd0d40b 100644 --- a/src/room/useLoadGroupCall.ts +++ b/src/room/useLoadGroupCall.ts @@ -106,22 +106,18 @@ async function joinRoomAfterInvite( export class CallTerminatedMessage extends Error { /** + * Creates a new CallTerminatedMessage. + * + * @param icon The icon to display with the message * @param messageTitle The title of the call ended screen message (translated) + * @param messageBody The message explaining the kind of termination + * (kick, ban, knock reject, etc.) (translated) + * @param reason The user-provided reason for the termination (kick/ban) */ public constructor( - /** - * The icon to display with the message. - */ public readonly icon: ComponentType>, messageTitle: string, - /** - * The message explaining the kind of termination (kick, ban, knock reject, - * etc.) (translated) - */ public readonly messageBody: string, - /** - * The user-provided reason for the termination (kick/ban) - */ public readonly reason?: string, ) { super(messageTitle); diff --git a/src/settings/rageshake.ts b/src/settings/rageshake.ts index 6c1a0f61..26d0839b 100644 --- a/src/settings/rageshake.ts +++ b/src/settings/rageshake.ts @@ -99,7 +99,7 @@ class ConsoleLogger extends EventEmitter { /** * Returns the log lines to flush to disk and empties the internal log buffer - * @return {string} \n delimited log lines + * @return \n delimited log lines */ public popLogs(): string { const logsToFlush = this.logs; @@ -109,7 +109,7 @@ class ConsoleLogger extends EventEmitter { /** * Returns lines currently in the log buffer without removing them - * @return {string} \n delimited log lines + * @return \n delimited log lines */ public peekLogs(): string { return this.logs; @@ -139,7 +139,7 @@ class IndexedDBLogStore { } /** - * @return {Promise} Resolves when the store is ready. + * @return Resolves when the store is ready. */ public async connect(): Promise { const req = this.indexedDB.open("logs"); @@ -219,7 +219,7 @@ class IndexedDBLogStore { * This guarantees that we will always eventually do a flush when flush() is * called. * - * @return {Promise} Resolved when the logs have been flushed. + * @return Resolved when the logs have been flushed. */ public flush = async (): Promise => { // check if a flush() operation is ongoing @@ -270,7 +270,7 @@ class IndexedDBLogStore { * returned are deleted at the same time, so this can be called at startup * to do house-keeping to keep the logs from growing too large. * - * @return {Promise} Resolves to an array of objects. The array is + * @return Resolves to an array of objects. The array is * sorted in time (oldest first) based on when the log file was created (the * log ID). The objects have said log ID in an "id" field and "lines" which * is a big string with all the new-line delimited logs. @@ -421,12 +421,12 @@ class IndexedDBLogStore { /** * Helper method to collect results from a Cursor and promiseify it. - * @param {ObjectStore|Index} store The store to perform openCursor on. - * @param {IDBKeyRange=} keyRange Optional key range to apply on the cursor. - * @param {Function} resultMapper A function which is repeatedly called with a + * @param store - The store to perform openCursor on. + * @param keyRange - Optional key range to apply on the cursor. + * @param resultMapper - A function which is repeatedly called with a * Cursor. * Return the data you want to keep. - * @return {Promise} Resolves to an array of whatever you returned from + * @return Resolves to an array of whatever you returned from * resultMapper. */ async function selectQuery( @@ -464,9 +464,7 @@ declare global { /** * Configure rage shaking support for sending bug reports. * Modifies globals. - * @param {boolean} setUpPersistence When true (default), the persistence will - * be set up immediately for the logs. - * @return {Promise} Resolves when set up. + * @return Resolves when set up. */ export async function init(): Promise { global.mx_rage_logger = new ConsoleLogger(); @@ -503,7 +501,7 @@ export async function init(): Promise { /** * Try to start up the rageshake storage for logs. If not possible (client unsupported) * then this no-ops. - * @return {Promise} Resolves when complete. + * @return Resolves when complete. */ async function tryInitStorage(): Promise { if (global.mx_rage_initStoragePromise) { @@ -536,7 +534,7 @@ async function tryInitStorage(): Promise { /** * Get a recent snapshot of the logs, ready for attaching to a bug report * - * @return {LogEntry[]} list of log data + * @return list of log data */ export async function getLogsForReport(): Promise { if (!global.mx_rage_logger) { diff --git a/src/state/CallViewModel/CallNotificationLifecycle.ts b/src/state/CallViewModel/CallNotificationLifecycle.ts index 2a0bf2f1..d90f35ba 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.ts @@ -81,7 +81,7 @@ export interface Props { localUser: { deviceId: string; userId: string }; } /** - * @returns {callPickupState$, autoLeave$} + * @returns two observables: * `callPickupState$` The current call pickup state of the call. * - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership. * Then we can conclude if we were the first one to join or not. diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 76be5f65..3205c07f 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -60,7 +60,8 @@ import { import { MediaDevices } from "../MediaDevices.ts"; import { getValue } from "../../utils/observable.ts"; import { type Behavior, constant } from "../Behavior.ts"; -import { withCallViewModel } from "./CallViewModelTestUtils.ts"; +import { withCallViewModel as withCallViewModelInMode } from "./CallViewModelTestUtils.ts"; +import { MatrixRTCMode } from "../../settings/settings.ts"; vi.mock("rxjs", async (importOriginal) => ({ ...(await importOriginal()), @@ -229,7 +230,13 @@ function mockRingEvent( // need a value to fill in for them when emitting notifications const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; -describe("CallViewModel", () => { +describe.each([ + [MatrixRTCMode.Legacy], + [MatrixRTCMode.Compatibil], + [MatrixRTCMode.Matrix_2_0], +])("CallViewModel (%s mode)", (mode) => { + const withCallViewModel = withCallViewModelInMode(mode); + test("participants are retained during a focus switch", () => { withTestScheduler(({ behavior, expectObservable }) => { // Participants disappear on frame 2 and come back on frame 3 @@ -267,7 +274,7 @@ describe("CallViewModel", () => { }); }); - it.skip("screen sharing activates spotlight layout", () => { + test("screen sharing activates spotlight layout", () => { withTestScheduler(({ behavior, schedule, expectObservable }) => { // Start with no screen shares, then have Alice and Bob share their screens, // then return to no screen shares, then have just Alice share for a bit @@ -502,6 +509,48 @@ describe("CallViewModel", () => { }); }); + test("layout reacts to window size", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const windowSizeInputMarbles = "abc"; + const expectedLayoutMarbles = " abc"; + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember]), + windowSize$: behavior(windowSizeInputMarbles, { + a: { width: 300, height: 600 }, // Start very narrow, like a phone + b: { width: 1000, height: 800 }, // Go to normal desktop window size + c: { width: 200, height: 180 }, // Go to PiP size + }), + }, + (vm) => { + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + // This is the expected one-on-one layout for a narrow window + type: "spotlight-expanded", + spotlight: [`${aliceId}:0`], + pip: `${localId}:0`, + }, + b: { + // In a larger window, expect the normal one-on-one layout + type: "one-on-one", + local: `${localId}:0`, + remote: `${aliceId}:0`, + }, + c: { + // In a PiP-sized window, we of course expect a PiP layout + type: "pip", + spotlight: [`${aliceId}:0`], + }, + }, + ); + }, + ); + }); + }); + test("spotlight speakers swap places", () => { withTestScheduler(({ behavior, schedule, expectObservable }) => { // Go immediately into spotlight mode for the test @@ -1207,7 +1256,9 @@ describe("CallViewModel", () => { rtcSession.membershipStatus = Status.Connected; }, n: () => { - rtcSession.membershipStatus = Status.Reconnecting; + // NOTE: This was removed in https://github.com/matrix-org/matrix-js-sdk/pull/5103 accidentally. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + rtcSession.membershipStatus = "Reconnecting" as any; }, }); schedule(probablyLeftMarbles, { diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 506eca1b..5324c65d 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -15,9 +15,9 @@ import { } from "livekit-client"; import { type Room as MatrixRoom } from "matrix-js-sdk"; import { + catchError, combineLatest, distinctUntilChanged, - EMPTY, filter, fromEvent, map, @@ -28,7 +28,6 @@ import { pairwise, race, scan, - skip, skipWhile, startWith, Subject, @@ -54,11 +53,15 @@ import { ScreenShareViewModel, type UserMediaViewModel, } from "../MediaViewModel"; -import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; +import { + accumulate, + filterBehavior, + generateItems, + pauseWhen, +} from "../../utils/observable"; import { duplicateTiles, MatrixRTCMode, - matrixRTCMode, playReactionsSound, showReactions, } from "../../settings/settings"; @@ -77,7 +80,7 @@ import { } from "../../reactions"; import { shallowEquals } from "../../utils/array"; import { type MediaDevices } from "../MediaDevices"; -import { type Behavior } from "../Behavior"; +import { constant, type Behavior } from "../Behavior"; import { E2eeType } from "../../e2ee/e2eeType"; import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; import { type MuteStates } from "../MuteStates"; @@ -95,15 +98,14 @@ import { type SpotlightLandscapeLayoutMedia, type SpotlightPortraitLayoutMedia, } from "../layout-types.ts"; -import { type ElementCallError } from "../../utils/errors.ts"; +import { ElementCallError } from "../../utils/errors.ts"; import { type ObservableScope } from "../ObservableScope.ts"; import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts"; import { createLocalMembership$, enterRTCSession, - LivekitState, - type LocalMemberConnectionState, -} from "./localMember/LocalMembership.ts"; + TransportState, +} from "./localMember/LocalMember.ts"; import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; import { createMemberships$, @@ -113,7 +115,9 @@ import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; import { createMatrixLivekitMembers$, - type MatrixLivekitMember, + type TaggedParticipant, + type LocalMatrixLivekitMember, + type RemoteMatrixLivekitMember, } from "./remoteMembers/MatrixLivekitMembers.ts"; import { type AutoLeaveReason, @@ -128,6 +132,7 @@ import { } from "./remoteMembers/MatrixMemberMetadata.ts"; import { Publisher } from "./localMember/Publisher.ts"; import { type Connection } from "./remoteMembers/Connection.ts"; +import { createLayoutModeSwitch } from "./LayoutSwitch.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO @@ -149,6 +154,10 @@ export interface CallViewModelOptions { livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; /** Optional behavior overriding the local connection state, mainly for testing purposes. */ connectionState$?: Behavior; + /** Optional behavior overriding the computed window size, mainly for testing purposes. */ + windowSize$?: Behavior<{ width: number; height: number }>; + /** The version & compatibility mode of MatrixRTC that we should use. */ + matrixRTCMode$?: Behavior; } // Do not play any sounds if the participant count has exceeded this @@ -174,7 +183,7 @@ interface LayoutScanState { } type MediaItem = UserMedia | ScreenShare; -type AudioLivekitItem = { +export type LivekitRoomItem = { livekitRoom: LivekitRoom; participants: string[]; url: string; @@ -197,12 +206,15 @@ export interface CallViewModel { callPickupState$: Behavior< "unknown" | "ringing" | "timeout" | "decline" | "success" | null >; + /** Observable that emits when the user should leave the call (hangup pressed, widget action, error). + * THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is by ending the scope. + */ leave$: Observable<"user" | AutoLeaveReason>; - /** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */ + /** Call to initiate hangup. Use in conbination with reconnectino state track the async hangup process. */ hangup: () => void; // joining - join: () => LocalMemberConnectionState; + join: () => void; // screen sharing /** @@ -250,7 +262,11 @@ export interface CallViewModel { */ participantCount$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ - audioParticipants$: Behavior; + livekitRoomItems$: Behavior; + userMedia$: Behavior; + /** use the layout instead, this is just for the sdk export. */ + matrixLivekitMembers$: Behavior; + localMatrixLivekitMember$: Behavior; /** List of participants raising their hand */ handsRaised$: Behavior>; /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ @@ -333,18 +349,17 @@ export interface CallViewModel { switch: () => void; } | null>; - // connection state /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. + * Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state. */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis reconnecting$: Behavior; + + /** + * Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit + */ + connected$: Behavior; } + /** * A view model providing all the application logic needed to show the in-call * UI (may eventually be expanded to cover the lobby and feedback screens in the @@ -372,6 +387,8 @@ export function createCallViewModel$( options.encryptionSystem, matrixRTCSession, ); + const matrixRTCMode$ = + options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy); // Each hbar seperates a block of input variables required for the CallViewModel to function. // The outputs of this block is written under the hbar. @@ -404,7 +421,7 @@ export function createCallViewModel$( client, roomId: matrixRoom.roomId, useOldestMember$: scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), }); @@ -415,6 +432,8 @@ export function createCallViewModel$( livekitKeyProvider, getUrlParams().controlledAudioDevices, options.livekitRoomFactory, + getUrlParams().echoCancellation, + getUrlParams().noiseSuppression, ); const connectionManager = createConnectionManager$({ @@ -422,7 +441,18 @@ export function createCallViewModel$( connectionFactory: connectionFactory, inputTransports$: scope.behavior( combineLatest( - [localTransport$, membershipsAndTransports.transports$], + [ + localTransport$.pipe( + catchError((e: unknown) => { + logger.info( + "dont pass local transport to createConnectionManager$. localTransport$ threw an error", + e, + ); + return of(null); + }), + ), + membershipsAndTransports.transports$, + ], (localTransport, transports) => { const localTransportAsArray = localTransport ? [localTransport] : []; return transports.mapInner((transports) => [ @@ -432,7 +462,7 @@ export function createCallViewModel$( }, ), ), - logger: logger, + logger, }); const matrixLivekitMembers$ = createMatrixLivekitMembers$({ @@ -443,7 +473,7 @@ export function createCallViewModel$( }); const connectOptions$ = scope.behavior( - matrixRTCMode.value$.pipe( + matrixRTCMode$.pipe( map((mode) => ({ encryptMedia: livekitKeyProvider !== undefined, // TODO. This might need to get called again on each change of matrixRTCMode... @@ -454,13 +484,13 @@ export function createCallViewModel$( const localMembership = createLocalMembership$({ scope: scope, - homeserverConnected$: createHomeserverConnected$( + homeserverConnected: createHomeserverConnected$( scope, client, matrixRTCSession, ), muteStates: muteStates, - joinMatrixRTC: async (transport: LivekitTransport) => { + joinMatrixRTC: (transport: LivekitTransport) => { return enterRTCSession( matrixRTCSession, transport, @@ -474,6 +504,9 @@ export function createCallViewModel$( mediaDevices, muteStates, trackProcessorState$, + logger.getChild( + "[Publisher " + connection.transport.livekit_service_url + "]", + ), ); }, connectionManager: connectionManager, @@ -494,22 +527,21 @@ export function createCallViewModel$( ), ); - const localMatrixLivekitMemberUninitialized = { - membership$: localRtcMembership$, - participant$: localMembership.participant$, - connection$: localMembership.connection$, - userId: userId, - }; - - const localMatrixLivekitMember$: Behavior = + const localMatrixLivekitMember$: Behavior = scope.behavior( localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - localMatrixLivekitMemberUninitialized as MatrixLivekitMember, - ); + filterBehavior((membership) => membership !== null), + map((membership$) => { + if (membership$ === null) return null; + return { + membership$, + participant: { + type: "local" as const, + value$: localMembership.participant$, + }, + connection$: localMembership.connection$, + userId, + }; }), ), ); @@ -572,34 +604,16 @@ export function createCallViewModel$( ), ); - // CODESMELL? - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - const leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - const reconnecting$ = localMembership.reconnecting$; - const pretendToBeDisconnected$ = reconnecting$; - - const audioParticipants$ = scope.behavior( + const livekitRoomItems$ = scope.behavior( matrixLivekitMembers$.pipe( + tap((val) => { + logger.debug("matrixLivekitMembers$ updated", val.value); + }), switchMap((membersWithEpoch) => { const members = membersWithEpoch.value; const a$ = combineLatest( members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( + combineLatest([member.connection$, member.participant.value$]).pipe( map(([connection, participant]) => { // do not render audio for local participant if (!connection || !participant || participant.isLocal) @@ -619,7 +633,7 @@ export function createCallViewModel$( return a$; }), map((members) => - members.reduce((acc, curr) => { + members.reduce((acc, curr) => { if (!curr) return acc; const existing = acc.find((item) => item.url === curr.url); @@ -640,7 +654,7 @@ export function createCallViewModel$( ); const handsRaised$ = scope.behavior( - handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), + handsRaisedSubject$.pipe(pauseWhen(localMembership.reconnecting$)), ); const reactions$ = scope.behavior( @@ -653,7 +667,7 @@ export function createCallViewModel$( ]), ), ), - pauseWhen(pretendToBeDisconnected$), + pauseWhen(localMembership.reconnecting$), ), ); @@ -674,10 +688,10 @@ export function createCallViewModel$( { value: matrixLivekitMembers }, duplicateTiles, ]) { - let localParticipantId = undefined; + let localParticipantId: string | undefined = undefined; // add local member if available if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = + const { userId, participant, connection$, membership$ } = localMatrixLivekitMember; localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional // const participantId = membership$.value.membershipID; @@ -688,7 +702,7 @@ export function createCallViewModel$( dup, localParticipantId, userId, - participant$, + participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely connection$, ], data: undefined, @@ -699,7 +713,7 @@ export function createCallViewModel$( // add remote members that are available for (const { userId, - participant$, + participant, connection$, membership$, } of matrixLivekitMembers) { @@ -708,7 +722,7 @@ export function createCallViewModel$( // const participantId = membership$.value?.identity; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [dup, participantId, userId, participant$, connection$], + keys: [dup, participantId, userId, participant, connection$], data: undefined, }; } @@ -720,7 +734,7 @@ export function createCallViewModel$( dup, participantId, userId, - participant$, + participant, connection$, ) => { const livekitRoom$ = scope.behavior( @@ -739,12 +753,12 @@ export function createCallViewModel$( scope, `${participantId}:${dup}`, userId, - participant$, + participant, options.encryptionSystem, livekitRoom$, focusUrl$, mediaDevices, - pretendToBeDisconnected$, + localMembership.reconnecting$, displayName$, matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), @@ -840,10 +854,7 @@ export function createCallViewModel$( merge( autoLeave$, merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), - ).pipe( - scope.share, - tap((reason) => leaveHoisted$.next(reason)), - ); + ).pipe(scope.share); const spotlightSpeaker$ = scope.behavior( userMedia$.pipe( @@ -952,20 +963,29 @@ export function createCallViewModel$( ), ); - const hasRemoteScreenShares$: Observable = spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + const hasRemoteScreenShares$ = scope.behavior( + spotlight$.pipe( + map((spotlight) => + spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + ), ), - distinctUntilChanged(), ); const pipEnabled$ = scope.behavior(setPipEnabled$, false); + const windowSize$ = + options.windowSize$ ?? + scope.behavior<{ width: number; height: number }>( + fromEvent(window, "resize").pipe( + startWith(null), + map(() => ({ width: window.innerWidth, height: window.innerHeight })), + ), + ); + + // A guess at what the window's mode should be based on its size and shape. const naturalWindowMode$ = scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; + windowSize$.pipe( + map(({ width, height }) => { if (height <= 400 && width <= 340) return "pip"; // Our layouts for flat windows are better at adapting to a small width // than our layouts for narrow windows are at adapting to a small height, @@ -975,7 +995,6 @@ export function createCallViewModel$( return "normal"; }), ), - "normal", ); /** @@ -992,36 +1011,11 @@ export function createCallViewModel$( spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), ); - const gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - const gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - scope.behavior( - gridModeUserSelection$.pipe( - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), - ), - ), - "grid", - ); - - const setGridMode = (value: GridMode): void => { - gridModeUserSelection$.next(value); - }; + const { setGridMode, gridMode$ } = createLayoutModeSwitch( + scope, + windowMode$, + hasRemoteScreenShares$, + ); const gridLayoutMedia$: Observable = combineLatest( [grid$, spotlight$], @@ -1448,16 +1442,44 @@ export function createCallViewModel$( // reassigned here to make it publicly accessible const toggleScreenSharing = localMembership.toggleScreenSharing; - const join = localMembership.requestConnect; - // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - join(); + const errors$ = scope.behavior<{ + transportError?: ElementCallError; + matrixError?: ElementCallError; + connectionError?: ElementCallError; + publishError?: ElementCallError; + } | null>( + localMembership.localMemberState$.pipe( + map((value) => { + const returnObject: { + transportError?: ElementCallError; + matrixError?: ElementCallError; + connectionError?: ElementCallError; + publishError?: ElementCallError; + } = {}; + if (value instanceof ElementCallError) return { transportError: value }; + if (value === TransportState.Waiting) return null; + if (value.matrix instanceof ElementCallError) + returnObject.matrixError = value.matrix; + if (value.media instanceof ElementCallError) + returnObject.publishError = value.media; + else if ( + typeof value.media === "object" && + value.media.connection instanceof ElementCallError + ) + returnObject.connectionError = value.media.connection; + return returnObject; + }), + ), + null, + ); + return { autoLeave$: autoLeave$, callPickupState$: callPickupState$, ringOverlay$: ringOverlay$, leave$: leave$, hangup: (): void => userHangup$.next(), - join: join, + join: localMembership.requestJoinAndPublish, toggleScreenSharing: toggleScreenSharing, sharingScreen$: sharingScreen$, @@ -1467,16 +1489,21 @@ export function createCallViewModel$( unhoverScreen: (): void => screenUnhover$.next(), fatalError$: scope.behavior( - localMembership.connectionState.livekit$.pipe( - filter((v) => v.state === LivekitState.Error), - map((s) => s.error), + errors$.pipe( + map((errors) => { + logger.debug("errors$ to compute any fatal errors:", errors); + return ( + errors?.transportError ?? + errors?.matrixError ?? + errors?.connectionError ?? + null + ); + }), + filter((error) => error !== null), ), null, ), - participantCount$: participantCount$, - audioParticipants$: audioParticipants$, - handsRaised$: handsRaised$, reactions$: reactions$, joinSoundEffect$: joinSoundEffect$, @@ -1495,6 +1522,16 @@ export function createCallViewModel$( spotlight$: spotlight$, pip$: pip$, layout$: layout$, + userMedia$, + localMatrixLivekitMember$, + matrixLivekitMembers$: scope.behavior( + matrixLivekitMembers$.pipe( + map((members) => members.value), + tap((v) => { + logger.debug("matrixLivekitMembers$ updated (exported)", v); + }), + ), + ), tileStoreGeneration$: tileStoreGeneration$, showSpotlightIndicators$: showSpotlightIndicators$, showSpeakingIndicators$: showSpeakingIndicators$, @@ -1502,7 +1539,9 @@ export function createCallViewModel$( showFooter$: showFooter$, earpieceMode$: earpieceMode$, audioOutputSwitcher$: audioOutputSwitcher$, - reconnecting$: reconnecting$, + reconnecting$: localMembership.reconnecting$, + livekitRoomItems$, + connected$: localMembership.connected$, }; } diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index f86921c5..b6f53275 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -53,6 +53,7 @@ import { import { type Behavior, constant } from "../Behavior"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { type MediaDevices } from "../MediaDevices"; +import { type MatrixRTCMode } from "../../settings/settings"; mockConfig({ livekit: { livekit_service_url: "http://my-default-service-url.com" }, @@ -75,119 +76,130 @@ export interface CallViewModelInputs { speaking: Map>; mediaDevices: MediaDevices; initialSyncState: SyncState; + windowSize$: Behavior<{ width: number; height: number }>; } const localParticipant = mockLocalParticipant({ identity: "" }); -export function withCallViewModel( - { - remoteParticipants$ = constant([]), - rtcMembers$ = constant([localRtcMember]), - livekitConnectionState$: connectionState$ = constant( - ConnectionState.Connected, - ), - speaking = new Map(), - mediaDevices = mockMediaDevices({}), - initialSyncState = SyncState.Syncing, - }: Partial = {}, - continuation: ( - vm: CallViewModel, - rtcSession: MockRTCSession, - subjects: { raisedHands$: BehaviorSubject> }, - setSyncState: (value: SyncState) => void, - ) => void, - options: CallViewModelOptions = { - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - autoLeaveWhenOthersLeft: false, - }, -): void { - let syncState = initialSyncState; - const setSyncState = (value: SyncState): void => { - const prev = syncState; - syncState = value; - room.client.emit(ClientEvent.Sync, value, prev); - }; - const room = mockMatrixRoom({ - client: new (class extends EventEmitter { - public getUserId(): string | undefined { - return localRtcMember.userId; - } +export function withCallViewModel(mode: MatrixRTCMode) { + return ( + { + remoteParticipants$ = constant([]), + rtcMembers$ = constant([localRtcMember]), + livekitConnectionState$: connectionState$ = constant( + ConnectionState.Connected, + ), + speaking = new Map(), + mediaDevices = mockMediaDevices({}), + initialSyncState = SyncState.Syncing, + windowSize$ = constant({ width: 1000, height: 800 }), + }: Partial = {}, + continuation: ( + vm: CallViewModel, + rtcSession: MockRTCSession, + subjects: { + raisedHands$: BehaviorSubject>; + }, + setSyncState: (value: SyncState) => void, + ) => void, + options: Partial = {}, + ): void => { + let syncState = initialSyncState; + const setSyncState = (value: SyncState): void => { + const prev = syncState; + syncState = value; + room.client.emit(ClientEvent.Sync, value, prev); + }; + const room = mockMatrixRoom({ + client: new (class extends EventEmitter { + public getUserId(): string | undefined { + return localRtcMember.userId; + } - public getDeviceId(): string { - return localRtcMember.deviceId; - } + public getDeviceId(): string { + return localRtcMember.deviceId; + } - public getDomain(): string { - return "example.com"; - } + public getDomain(): string { + return "example.com"; + } - public getSyncState(): SyncState { - return syncState; - } - })() as Partial as MatrixClient, - getMembers: () => Array.from(roomMembers.values()), - getMembersWithMembership: () => Array.from(roomMembers.values()), - }); - const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$); - const participantsSpy = vi - .spyOn(ComponentsCore, "connectedParticipantsObserver") - .mockReturnValue(remoteParticipants$); - const mediaSpy = vi - .spyOn(ComponentsCore, "observeParticipantMedia") - .mockImplementation((p) => - of({ participant: p } as Partial< - ComponentsCore.ParticipantMedia - > as ComponentsCore.ParticipantMedia), + public getSyncState(): SyncState { + return syncState; + } + })() as Partial as MatrixClient, + getMembers: () => Array.from(roomMembers.values()), + getMembersWithMembership: () => Array.from(roomMembers.values()), + }); + const rtcSession = new MockRTCSession(room, []).withMemberships( + rtcMembers$, ); - const eventsSpy = vi - .spyOn(ComponentsCore, "observeParticipantEvents") - .mockImplementation((p, ...eventTypes) => { - if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) { - return (speaking.get(p) ?? of(false)).pipe( - map((s): Participant => ({ ...p, isSpeaking: s }) as Participant), - ); - } else { - return of(p); - } + const participantsSpy = vi + .spyOn(ComponentsCore, "connectedParticipantsObserver") + .mockReturnValue(remoteParticipants$); + const mediaSpy = vi + .spyOn(ComponentsCore, "observeParticipantMedia") + .mockImplementation((p) => + of({ participant: p } as Partial< + ComponentsCore.ParticipantMedia + > as ComponentsCore.ParticipantMedia), + ); + const eventsSpy = vi + .spyOn(ComponentsCore, "observeParticipantEvents") + .mockImplementation((p, ...eventTypes) => { + if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) { + return (speaking.get(p) ?? of(false)).pipe( + map((s): Participant => ({ ...p, isSpeaking: s }) as Participant), + ); + } else { + return of(p); + } + }); + + const roomEventSelectorSpy = vi + .spyOn(ComponentsCore, "roomEventSelector") + .mockImplementation((_room, _eventType) => of()); + const muteStates = mockMuteStates(); + const raisedHands$ = new BehaviorSubject>( + {}, + ); + const reactions$ = new BehaviorSubject>({}); + + const vm = createCallViewModel$( + testScope(), + rtcSession.asMockedSession(), + room, + mediaDevices, + muteStates, + { + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + autoLeaveWhenOthersLeft: false, + livekitRoomFactory: (): LivekitRoom => + mockLivekitRoom({ + localParticipant, + disconnect: async () => Promise.resolve(), + setE2EEEnabled: async () => Promise.resolve(), + }), + connectionState$, + windowSize$, + matrixRTCMode$: constant(mode), + ...options, + }, + raisedHands$, + reactions$, + new BehaviorSubject({ + processor: undefined, + supported: undefined, + }), + ); + + onTestFinished(() => { + participantsSpy.mockRestore(); + mediaSpy.mockRestore(); + eventsSpy.mockRestore(); + roomEventSelectorSpy.mockRestore(); }); - const roomEventSelectorSpy = vi - .spyOn(ComponentsCore, "roomEventSelector") - .mockImplementation((_room, _eventType) => of()); - const muteStates = mockMuteStates(); - const raisedHands$ = new BehaviorSubject>({}); - const reactions$ = new BehaviorSubject>({}); - - const vm = createCallViewModel$( - testScope(), - rtcSession.asMockedSession(), - room, - mediaDevices, - muteStates, - { - ...options, - livekitRoomFactory: (): LivekitRoom => - mockLivekitRoom({ - localParticipant, - disconnect: async () => Promise.resolve(), - setE2EEEnabled: async () => Promise.resolve(), - }), - connectionState$, - }, - raisedHands$, - reactions$, - new BehaviorSubject({ - processor: undefined, - supported: undefined, - }), - ); - - onTestFinished(() => { - participantsSpy.mockRestore(); - mediaSpy.mockRestore(); - eventsSpy.mockRestore(); - roomEventSelectorSpy.mockRestore(); - }); - - continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); + continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); + }; } diff --git a/src/state/CallViewModel/LayoutSwitch.test.ts b/src/state/CallViewModel/LayoutSwitch.test.ts new file mode 100644 index 00000000..0d184017 --- /dev/null +++ b/src/state/CallViewModel/LayoutSwitch.test.ts @@ -0,0 +1,132 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, test } from "vitest"; + +import { createLayoutModeSwitch } from "./LayoutSwitch"; +import { testScope, withTestScheduler } from "../../utils/test"; + +function testLayoutSwitch({ + windowMode = "n", + hasScreenShares = "n", + userSelection = "", + expectedGridMode, +}: { + windowMode?: string; + hasScreenShares?: string; + userSelection?: string; + expectedGridMode: string; +}): void { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const { gridMode$, setGridMode } = createLayoutModeSwitch( + testScope(), + behavior(windowMode, { n: "normal", N: "narrow", f: "flat" }), + behavior(hasScreenShares, { y: true, n: false }), + ); + schedule(userSelection, { + g: () => setGridMode("grid"), + s: () => setGridMode("spotlight"), + }); + expectObservable(gridMode$).toBe(expectedGridMode, { + g: "grid", + s: "spotlight", + }); + }); +} + +describe("default mode", () => { + test("uses grid layout by default", () => + testLayoutSwitch({ + expectedGridMode: "g", + })); + + test("uses spotlight mode when window mode is flat", () => + testLayoutSwitch({ + windowMode: " f", + expectedGridMode: "s", + })); +}); + +test("allows switching modes manually", () => + testLayoutSwitch({ + userSelection: " --sgs", + expectedGridMode: "g-sgs", + })); + +test("switches to spotlight mode when there is a remote screen share", () => + testLayoutSwitch({ + hasScreenShares: " n--y", + expectedGridMode: "g--s", + })); + +test("can manually switch to grid when there is a screenshare", () => + testLayoutSwitch({ + hasScreenShares: " n-y", + userSelection: " ---g", + expectedGridMode: "g-sg", + })); + +test("auto-switches after manually selecting grid", () => + testLayoutSwitch({ + // Two screenshares will happen in sequence. There is a screen share that + // forces spotlight, then the user manually switches back to grid. + hasScreenShares: " n-y-ny", + userSelection: " ---g", + expectedGridMode: "g-sg-s", + // If we did want to respect manual selection, the expectation would be: g-sg + })); + +test("switches back to grid mode when the remote screen share ends", () => + testLayoutSwitch({ + hasScreenShares: " n--y--n", + expectedGridMode: "g--s--g", + })); + +test("auto-switches to spotlight again after first screen share ends", () => + testLayoutSwitch({ + hasScreenShares: " nyny", + expectedGridMode: "gsgs", + })); + +test("switches manually to grid after screen share while manually in spotlight", () => + testLayoutSwitch({ + // Initially, no one is sharing. Then the user manually switches to spotlight. + // After a screen share starts, the user manually switches to grid. + hasScreenShares: " n-y", + userSelection: " -s-g", + expectedGridMode: "gs-g", + })); + +test("auto-switches to spotlight when in flat window mode", () => + testLayoutSwitch({ + // First normal, then narrow, then flat. + windowMode: " nNf", + expectedGridMode: "g-s", + })); + +test("allows switching modes manually when in flat window mode", () => + testLayoutSwitch({ + // Window becomes flat, then user switches to grid and back. + // Finally the window returns to a normal shape. + windowMode: " nf--n", + userSelection: " --gs", + expectedGridMode: "gsgsg", + })); + +test("stays in spotlight while there are screen shares even when window mode changes", () => + testLayoutSwitch({ + windowMode: " nfn", + hasScreenShares: " y", + expectedGridMode: "s", + })); + +test("ignores end of screen share until window mode returns to normal", () => + testLayoutSwitch({ + windowMode: " nf-n", + hasScreenShares: " y-n", + expectedGridMode: "s--g", + })); diff --git a/src/state/CallViewModel/LayoutSwitch.ts b/src/state/CallViewModel/LayoutSwitch.ts new file mode 100644 index 00000000..97a4ee6f --- /dev/null +++ b/src/state/CallViewModel/LayoutSwitch.ts @@ -0,0 +1,93 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + combineLatest, + map, + Subject, + startWith, + skipWhile, + switchMap, +} from "rxjs"; + +import { type GridMode, type WindowMode } from "./CallViewModel.ts"; +import { constant, type Behavior } from "../Behavior.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; + +/** + * Creates a layout mode switch that allows switching between grid and spotlight modes. + * The actual layout mode might switch automatically to spotlight if there is a + * remote screen share active or if the window mode is flat. + * + * @param scope - The observable scope to manage subscriptions. + * @param windowMode$ - The current window mode. + * @param hasRemoteScreenShares$ - A behavior indicating if there are remote screen shares active. + */ +export function createLayoutModeSwitch( + scope: ObservableScope, + windowMode$: Behavior, + hasRemoteScreenShares$: Behavior, +): { + gridMode$: Behavior; + setGridMode: (value: GridMode) => void; +} { + const userSelection$ = new Subject(); + // Callback to set the grid mode desired by the user. + // Notice that this is only a preference, the actual grid mode can be overridden + // if there is a remote screen share active. + const setGridMode = (value: GridMode): void => userSelection$.next(value); + + /** + * The natural grid mode - the mode that the grid would prefer to be in, + * not accounting for the user's manual selections. + */ + const naturalGridMode$ = scope.behavior( + combineLatest( + [hasRemoteScreenShares$, windowMode$], + (hasRemoteScreenShares, windowMode) => + // When there are screen shares or the window is flat (as with a phone + // in landscape orientation), spotlight is a better experience. + // We want screen shares to be big and readable, and we want flipping + // your phone into landscape to be a quick way of maximising the + // spotlight tile. + hasRemoteScreenShares || windowMode === "flat" ? "spotlight" : "grid", + ), + ); + + /** + * The layout mode of the media tile grid. + */ + const gridMode$ = scope.behavior( + // Whenever the user makes a selection, we enter a new mode of behavior: + userSelection$.pipe( + map((selection) => { + if (selection === "grid") + // The user has selected grid mode. Start by respecting their choice, + // but then follow the natural mode again as soon as it matches. + return naturalGridMode$.pipe( + skipWhile((naturalMode) => naturalMode !== selection), + startWith(selection), + ); + + // The user has selected spotlight mode. If this matches the natural + // mode, then follow the natural mode going forward. + return selection === naturalGridMode$.value + ? naturalGridMode$ + : constant(selection); + }), + // Initially the mode of behavior is to just follow the natural grid mode. + startWith(naturalGridMode$), + // Switch between each mode of behavior. + switchMap((mode$) => mode$), + ), + ); + + return { + gridMode$, + setGridMode, + }; +} diff --git a/src/state/CallViewModel/localMember/HomeserverConnected.test.ts b/src/state/CallViewModel/localMember/HomeserverConnected.test.ts index 1f61e533..87ca35d0 100644 --- a/src/state/CallViewModel/localMember/HomeserverConnected.test.ts +++ b/src/state/CallViewModel/localMember/HomeserverConnected.test.ts @@ -97,106 +97,106 @@ describe("createHomeserverConnected$", () => { // LLM generated test cases. They are a bit overkill but I improved the mocking so it is // easy enough to read them so I think they can stay. it("is false when sync state is not Syncing", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); - expect(hsConnected$.value).toBe(false); + const hsConnected = createHomeserverConnected$(scope, client, session); + expect(hsConnected.combined$.value).toBe(false); }); it("remains false while membership status is not Connected even if sync is Syncing", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); client.setSyncState(SyncState.Syncing); - expect(hsConnected$.value).toBe(false); // membership still disconnected + expect(hsConnected.combined$.value).toBe(false); // membership still disconnected }); it("is false when membership status transitions to Connected but ProbablyLeft is true", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); // Make sync loop OK client.setSyncState(SyncState.Syncing); // Indicate probable leave before connection session.setProbablyLeft(true); session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); }); it("becomes true only when all three conditions are satisfied", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); // 1. Sync loop connected client.setSyncState(SyncState.Syncing); - expect(hsConnected$.value).toBe(false); // not yet membership connected + expect(hsConnected.combined$.value).toBe(false); // not yet membership connected // 2. Membership connected session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); // probablyLeft is false + expect(hsConnected.combined$.value).toBe(true); // probablyLeft is false }); it("drops back to false when sync loop leaves Syncing", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); // Reach connected state client.setSyncState(SyncState.Syncing); session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); // Sync loop error => should flip false client.setSyncState(SyncState.Error); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); }); it("drops back to false when membership status becomes disconnected", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); client.setSyncState(SyncState.Syncing); session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); session.setMembershipStatus(Status.Disconnected); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); }); it("drops to false when ProbablyLeft is emitted after being true", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); client.setSyncState(SyncState.Syncing); session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); session.setProbablyLeft(true); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); }); it("recovers to true if ProbablyLeft becomes false again while other conditions remain true", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); client.setSyncState(SyncState.Syncing); session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); session.setProbablyLeft(true); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); // Simulate clearing the flag (in realistic scenario membership manager would update) session.setProbablyLeft(false); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); }); it("composite sequence reflects each individual failure reason", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); // Initially false (sync error + disconnected + not probably left) - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); // Fix sync only client.setSyncState(SyncState.Syncing); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); // Fix membership session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); // Introduce probablyLeft -> false session.setProbablyLeft(true); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); // Restore notProbablyLeft -> true again session.setProbablyLeft(false); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); // Drop sync -> false client.setSyncState(SyncState.Error); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); }); }); diff --git a/src/state/CallViewModel/localMember/HomeserverConnected.ts b/src/state/CallViewModel/localMember/HomeserverConnected.ts index e1c28078..c8bcd021 100644 --- a/src/state/CallViewModel/localMember/HomeserverConnected.ts +++ b/src/state/CallViewModel/localMember/HomeserverConnected.ts @@ -25,6 +25,11 @@ import { type NodeStyleEventEmitter } from "../../../utils/test"; */ const logger = rootLogger.getChild("[HomeserverConnected]"); +export interface HomeserverConnected { + combined$: Behavior; + rtsSession$: Behavior; +} + /** * Behavior representing whether we consider ourselves connected to the Matrix homeserver * for the purposes of a MatrixRTC session. @@ -39,7 +44,7 @@ export function createHomeserverConnected$( client: NodeStyleEventEmitter & Pick, matrixRTCSession: NodeStyleEventEmitter & Pick, -): Behavior { +): HomeserverConnected { const syncing$ = ( fromEvent(client, ClientEvent.Sync) as Observable<[SyncState]> ).pipe( @@ -47,12 +52,15 @@ export function createHomeserverConnected$( map(([state]) => state === SyncState.Syncing), ); - const membershipConnected$ = fromEvent( - matrixRTCSession, - MembershipManagerEvent.StatusChanged, - ).pipe( - startWith(null), - map(() => matrixRTCSession.membershipStatus === Status.Connected), + const rtsSession$ = scope.behavior( + fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe( + map(() => matrixRTCSession.membershipStatus ?? Status.Unknown), + ), + Status.Unknown, + ); + + const membershipConnected$ = rtsSession$.pipe( + map((status) => status === Status.Connected), ); // This is basically notProbablyLeft$ @@ -71,15 +79,13 @@ export function createHomeserverConnected$( map(() => matrixRTCSession.probablyLeft !== true), ); - const connectedCombined$ = and$( - syncing$, - membershipConnected$, - certainlyConnected$, - ).pipe( - tap((connected) => { - logger.info(`Homeserver connected update: ${connected}`); - }), + const combined$ = scope.behavior( + and$(syncing$, membershipConnected$, certainlyConnected$).pipe( + tap((connected) => { + logger.info(`Homeserver connected update: ${connected}`); + }), + ), ); - return scope.behavior(connectedCombined$); + return { combined$, rtsSession$ }; } diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts new file mode 100644 index 00000000..0d77611b --- /dev/null +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -0,0 +1,527 @@ +/* +Copyright 2025 Element Creations Ltd. +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + Status as RTCMemberStatus, + type LivekitTransport, + type MatrixRTCSession, +} from "matrix-js-sdk/lib/matrixrtc"; +import { describe, expect, it, vi } from "vitest"; +import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; +import { BehaviorSubject, map, of } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { type LocalParticipant, type LocalTrack } from "livekit-client"; + +import { MatrixRTCMode } from "../../../settings/settings"; +import { + flushPromises, + mockConfig, + mockLivekitRoom, + mockMuteStates, + withTestScheduler, +} from "../../../utils/test"; +import { + TransportState, + createLocalMembership$, + enterRTCSession, + PublishState, + TrackState, +} from "./LocalMember"; +import { MatrixRTCTransportMissingError } from "../../../utils/errors"; +import { Epoch, ObservableScope } from "../../ObservableScope"; +import { constant } from "../../Behavior"; +import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; +import { ConnectionState, type Connection } from "../remoteMembers/Connection"; +import { type Publisher } from "./Publisher"; + +const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; +const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); +vi.mock("../../../UrlParams", () => ({ getUrlParams })); +vi.mock("@livekit/components-core", () => ({ + observeParticipantEvents: vi + .fn() + .mockReturnValue(of({ isScreenShareEnabled: false })), +})); + +describe("LocalMembership", () => { + describe("enterRTCSession", () => { + it("It joins the correct Session", () => { + const focusFromOlderMembership = { + type: "livekit", + livekit_service_url: "http://my-oldest-member-service-url.com", + livekit_alias: "my-oldest-member-service-alias", + }; + + const focusConfigFromWellKnown = { + type: "livekit", + livekit_service_url: "http://my-well-known-service-url.com", + }; + const focusConfigFromWellKnown2 = { + type: "livekit", + livekit_service_url: "http://my-well-known-service-url2.com", + }; + const clientWellKnown = { + "org.matrix.msc4143.rtc_foci": [ + focusConfigFromWellKnown, + focusConfigFromWellKnown2, + ], + }; + + mockConfig({ + livekit: { livekit_service_url: "http://my-default-service-url.com" }, + }); + + vi.spyOn(AutoDiscovery, "getRawClientConfig").mockImplementation( + async (domain) => { + if (domain === "example.org") { + return Promise.resolve(clientWellKnown); + } + return Promise.resolve({}); + }, + ); + + const mockedSession = vi.mocked({ + room: { + roomId: "roomId", + client: { + getDomain: vi.fn().mockReturnValue("example.org"), + getOpenIdToken: vi.fn().mockResolvedValue({ + access_token: "ACCCESS_TOKEN", + token_type: "Bearer", + matrix_server_name: "localhost", + expires_in: 10000, + }), + }, + }, + memberships: [], + getFocusInUse: vi.fn().mockReturnValue(focusFromOlderMembership), + getOldestMembership: vi.fn().mockReturnValue({ + getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]), + }), + joinRoomSession: vi.fn(), + }) as unknown as MatrixRTCSession; + + enterRTCSession( + mockedSession, + { + livekit_alias: "roomId", + livekit_service_url: "http://my-well-known-service-url.com", + type: "livekit", + }, + { + encryptMedia: true, + matrixRTCMode: MATRIX_RTC_MODE, + }, + ); + + expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith( + [ + { + livekit_alias: "roomId", + livekit_service_url: "http://my-well-known-service-url.com", + type: "livekit", + }, + ], + undefined, + expect.objectContaining({ + manageMediaKeys: true, + useLegacyMemberEvents: false, + }), + ); + }); + + it("It should not fail with configuration error if homeserver config has livekit url but not fallback", () => { + mockConfig({}); + vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({ + "org.matrix.msc4143.rtc_foci": [ + { + type: "livekit", + livekit_service_url: "http://my-well-known-service-url.com", + }, + ], + }); + + const mockedSession = vi.mocked({ + room: { + roomId: "roomId", + client: { + getDomain: vi.fn().mockReturnValue("example.org"), + getOpenIdToken: vi.fn().mockResolvedValue({ + access_token: "ACCCESS_TOKEN", + token_type: "Bearer", + matrix_server_name: "localhost", + expires_in: 10000, + }), + }, + }, + memberships: [], + getFocusInUse: vi.fn(), + joinRoomSession: vi.fn(), + }) as unknown as MatrixRTCSession; + + enterRTCSession( + mockedSession, + { + livekit_alias: "roomId", + livekit_service_url: "http://my-well-known-service-url.com", + type: "livekit", + }, + { + encryptMedia: true, + matrixRTCMode: MATRIX_RTC_MODE, + }, + ); + }); + }); + + const defaultCreateLocalMemberValues = { + options: constant({ + encryptMedia: false, + matrixRTCMode: MatrixRTCMode.Matrix_2_0, + }), + matrixRTCSession: { + updateCallIntent: () => {}, + leaveRoomSession: () => {}, + } as unknown as MatrixRTCSession, + muteStates: mockMuteStates(), + trackProcessorState$: constant({ + supported: false, + processor: undefined, + }), + logger: logger, + createPublisherFactory: vi.fn(), + joinMatrixRTC: async (): Promise => {}, + homeserverConnected: { + combined$: constant(true), + rtsSession$: constant(RTCMemberStatus.Connected), + }, + }; + + it("throws error on missing RTC config error", () => { + withTestScheduler(({ scope, hot, expectObservable }) => { + const localTransport$ = scope.behavior( + hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), + null, + ); + + // we do not need any connection data since we want to fail before reaching that. + const mockConnectionManager = { + transports$: scope.behavior( + localTransport$.pipe(map((t) => new Epoch([t]))), + ), + connectionManagerData$: constant( + new Epoch(new ConnectionManagerData()), + ), + }; + + const localMembership = createLocalMembership$({ + scope, + ...defaultCreateLocalMemberValues, + connectionManager: mockConnectionManager, + localTransport$, + }); + localMembership.requestJoinAndPublish(); + + expectObservable(localMembership.localMemberState$).toBe("ne", { + n: TransportState.Waiting, + e: expect.toSatisfy((e) => e instanceof MatrixRTCTransportMissingError), + }); + }); + }); + + const aTransport = { + livekit_service_url: "a", + } as LivekitTransport; + const bTransport = { + livekit_service_url: "b", + } as LivekitTransport; + + const connectionTransportAConnected = { + livekitRoom: mockLivekitRoom({ + localParticipant: { + isScreenShareEnabled: false, + trackPublications: [], + } as unknown as LocalParticipant, + }), + state$: constant(ConnectionState.LivekitConnected), + transport: aTransport, + } as unknown as Connection; + const connectionTransportAConnecting = { + ...connectionTransportAConnected, + state$: constant(ConnectionState.LivekitConnecting), + livekitRoom: mockLivekitRoom({}), + } as unknown as Connection; + const connectionTransportBConnected = { + state$: constant(ConnectionState.LivekitConnected), + transport: bTransport, + livekitRoom: mockLivekitRoom({}), + } as unknown as Connection; + + it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => { + const scope = new ObservableScope(); + + const localTransport$ = new BehaviorSubject(aTransport); + + const publishers: Publisher[] = []; + let seed = 0; + defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation( + () => { + const a = seed; + seed += 1; + logger.info(`creating [${a}]`); + const p = { + stopPublishing: vi.fn().mockImplementation(() => { + logger.info(`stopPublishing [${a}]`); + }), + stopTracks: vi.fn(), + }; + publishers.push(p as unknown as Publisher); + return p; + }, + ); + const publisherFactory = + defaultCreateLocalMemberValues.createPublisherFactory as ReturnType< + typeof vi.fn + >; + + const connectionManagerData = new ConnectionManagerData(); + connectionManagerData.add(connectionTransportAConnected, []); + connectionManagerData.add(connectionTransportBConnected, []); + createLocalMembership$({ + scope, + ...defaultCreateLocalMemberValues, + connectionManager: { + connectionManagerData$: constant(new Epoch(connectionManagerData)), + }, + localTransport$, + }); + await flushPromises(); + localTransport$.next(bTransport); + await flushPromises(); + expect(publisherFactory).toHaveBeenCalledTimes(2); + expect(publishers.length).toBe(2); + // stop the first Publisher and let the second one life. + expect(publishers[0].stopTracks).toHaveBeenCalled(); + expect(publishers[1].stopTracks).not.toHaveBeenCalled(); + expect(publishers[0].stopPublishing).toHaveBeenCalled(); + expect(publishers[1].stopPublishing).not.toHaveBeenCalled(); + expect(publisherFactory.mock.calls[0][0].transport).toBe(aTransport); + expect(publisherFactory.mock.calls[1][0].transport).toBe(bTransport); + scope.end(); + await flushPromises(); + // stop all tracks after ending scopes + expect(publishers[1].stopPublishing).toHaveBeenCalled(); + // expect(publishers[1].stopTracks).toHaveBeenCalled(); + + defaultCreateLocalMemberValues.createPublisherFactory.mockReset(); + }); + + it("only start tracks if requested", async () => { + const scope = new ObservableScope(); + + const localTransport$ = new BehaviorSubject(aTransport); + + const publishers: Publisher[] = []; + + const tracks$ = new BehaviorSubject([]); + const publishing$ = new BehaviorSubject(false); + defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation( + () => { + const p = { + stopPublishing: vi.fn(), + stopTracks: vi.fn(), + createAndSetupTracks: vi.fn().mockImplementation(async () => { + tracks$.next([{}, {}] as LocalTrack[]); + return Promise.resolve(); + }), + tracks$, + publishing$, + }; + publishers.push(p as unknown as Publisher); + return p; + }, + ); + const publisherFactory = + defaultCreateLocalMemberValues.createPublisherFactory as ReturnType< + typeof vi.fn + >; + + const connectionManagerData = new ConnectionManagerData(); + connectionManagerData.add(connectionTransportAConnected, []); + // connectionManagerData.add(connectionTransportB, []); + const localMembership = createLocalMembership$({ + scope, + ...defaultCreateLocalMemberValues, + connectionManager: { + connectionManagerData$: constant(new Epoch(connectionManagerData)), + }, + localTransport$, + }); + await flushPromises(); + expect(publisherFactory).toHaveBeenCalledOnce(); + // expect(localMembership.tracks$.value.length).toBe(0); + expect(publishers[0].createAndSetupTracks).not.toHaveBeenCalled(); + localMembership.startTracks(); + await flushPromises(); + expect(publishers[0].createAndSetupTracks).toHaveBeenCalled(); + // expect(localMembership.tracks$.value.length).toBe(2); + scope.end(); + await flushPromises(); + // stop all tracks after ending scopes + expect(publishers[0].stopPublishing).toHaveBeenCalled(); + // expect(publishers[0].stopTracks).toHaveBeenCalled(); + publisherFactory.mockClear(); + }); + // TODO add an integration test combining publisher and localMembership + // + it("tracks livekit state correctly", async () => { + const scope = new ObservableScope(); + + const connectionManagerData = new ConnectionManagerData(); + const localTransport$ = new BehaviorSubject(null); + const connectionManagerData$ = new BehaviorSubject( + new Epoch(connectionManagerData), + ); + const publishers: Publisher[] = []; + + const tracks$ = new BehaviorSubject([]); + const publishing$ = new BehaviorSubject(false); + const createTrackResolver = Promise.withResolvers(); + const publishResolver = Promise.withResolvers(); + defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation( + () => { + const p = { + stopPublishing: vi.fn(), + stopTracks: vi.fn().mockImplementation(() => { + logger.info("stopTracks"); + tracks$.next([]); + }), + createAndSetupTracks: vi.fn().mockImplementation(async () => { + await createTrackResolver.promise; + tracks$.next([{}, {}] as LocalTrack[]); + }), + startPublishing: vi.fn().mockImplementation(async () => { + await publishResolver.promise; + publishing$.next(true); + }), + tracks$, + publishing$, + }; + publishers.push(p as unknown as Publisher); + return p; + }, + ); + + const publisherFactory = + defaultCreateLocalMemberValues.createPublisherFactory as ReturnType< + typeof vi.fn + >; + + const localMembership = createLocalMembership$({ + scope, + ...defaultCreateLocalMemberValues, + connectionManager: { + connectionManagerData$, + }, + localTransport$, + }); + + await flushPromises(); + expect(localMembership.localMemberState$.value).toStrictEqual( + TransportState.Waiting, + ); + localTransport$.next(aTransport); + await flushPromises(); + expect(localMembership.localMemberState$.value).toStrictEqual({ + matrix: RTCMemberStatus.Connected, + media: { connection: null, tracks: TrackState.WaitingForUser }, + }); + + const connectionManagerData2 = new ConnectionManagerData(); + connectionManagerData2.add( + // clone because we will mutate this later. + { ...connectionTransportAConnecting } as unknown as Connection, + [], + ); + + connectionManagerData$.next(new Epoch(connectionManagerData2)); + await flushPromises(); + expect(localMembership.localMemberState$.value).toStrictEqual({ + matrix: RTCMemberStatus.Connected, + media: { + connection: ConnectionState.LivekitConnecting, + tracks: TrackState.WaitingForUser, + }, + }); + + ( + connectionManagerData2.getConnectionForTransport(aTransport)! + .state$ as BehaviorSubject + ).next(ConnectionState.LivekitConnected); + expect(localMembership.localMemberState$.value).toStrictEqual({ + matrix: RTCMemberStatus.Connected, + media: { + connection: ConnectionState.LivekitConnected, + tracks: TrackState.WaitingForUser, + }, + }); + + expect(publisherFactory).toHaveBeenCalledOnce(); + // expect(localMembership.tracks$.value.length).toBe(0); + + // ------- + localMembership.startTracks(); + // ------- + + await flushPromises(); + // expect(localMembership.localMemberState$.value).toStrictEqual({ + // matrix: RTCMemberStatus.Connected, + // media: { + // tracks: TrackState.Creating, + // connection: ConnectionState.LivekitConnected, + // }, + // }); + createTrackResolver.resolve(); + await flushPromises(); + expect( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (localMembership.localMemberState$.value as any).media, + ).toStrictEqual(PublishState.WaitingForUser); + + // ------- + localMembership.requestJoinAndPublish(); + // ------- + + expect( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (localMembership.localMemberState$.value as any).media, + ).toStrictEqual(PublishState.Publishing); + + publishResolver.resolve(); + await flushPromises(); + expect( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (localMembership.localMemberState$.value as any).media, + ).toStrictEqual(PublishState.Publishing); + + expect(publishers[0].stopPublishing).not.toHaveBeenCalled(); + + expect(localMembership.localMemberState$.isStopped).toBe(false); + scope.end(); + await flushPromises(); + // stays in connected state because it is stopped before the update to tracks update the state. + expect( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (localMembership.localMemberState$.value as any).media, + ).toStrictEqual(PublishState.Publishing); + // stop all tracks after ending scopes + expect(publishers[0].stopPublishing).toHaveBeenCalled(); + // expect(publishers[0].stopTracks).toHaveBeenCalled(); + }); + // TODO add tests for matrix local matrix participation. +}); diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts new file mode 100644 index 00000000..dc22db23 --- /dev/null +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -0,0 +1,739 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type Participant, + ParticipantEvent, + type LocalParticipant, + type ScreenShareCaptureOptions, + RoomEvent, + MediaDeviceFailure, +} from "livekit-client"; +import { observeParticipantEvents } from "@livekit/components-core"; +import { + Status as RTCSessionStatus, + type LivekitTransport, + type MatrixRTCSession, +} from "matrix-js-sdk/lib/matrixrtc"; +import { + BehaviorSubject, + catchError, + combineLatest, + distinctUntilChanged, + from, + fromEvent, + map, + type Observable, + of, + pairwise, + startWith, + switchMap, + tap, +} from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; + +import { type Behavior } from "../../Behavior.ts"; +import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts"; +import { type ObservableScope } from "../../ObservableScope.ts"; +import { type Publisher } from "./Publisher.ts"; +import { type MuteStates } from "../../MuteStates.ts"; +import { + ElementCallError, + FailToStartLivekitConnection, + MembershipManagerError, + UnknownCallError, +} from "../../../utils/errors.ts"; +import { ElementWidgetActions, widget } from "../../../widget.ts"; +import { getUrlParams } from "../../../UrlParams.ts"; +import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; +import { MatrixRTCMode } from "../../../settings/settings.ts"; +import { Config } from "../../../config/Config.ts"; +import { + ConnectionState, + type Connection, + type FailedToStartError, +} from "../remoteMembers/Connection.ts"; +import { type HomeserverConnected } from "./HomeserverConnected.ts"; +import { and$ } from "../../../utils/observable.ts"; + +export enum TransportState { + /** Not even a transport is available to the LocalMembership */ + Waiting = "transport_waiting", +} + +export enum PublishState { + WaitingForUser = "publish_waiting_for_user", + // XXX: This state is removed for now since we do not have full control over + // track publication anymore with the publisher abstraction, might come back in the future? + // /** Implies lk connection is connected */ + // Starting = "publish_start_publishing", + /** Implies lk connection is connected */ + Publishing = "publish_publishing", +} + +// TODO not sure how to map that correctly with the +// new publisher that does not manage tracks itself anymore +export enum TrackState { + /** The track is waiting for user input to create tracks (waiting to call `startTracks()`) */ + WaitingForUser = "tracks_waiting_for_user", + // XXX: This state is removed for now since we do not have full control over + // track creation anymore with the publisher abstraction, might come back in the future? + // /** Implies lk connection is connected */ + // Creating = "tracks_creating", + /** Implies lk connection is connected */ + Ready = "tracks_ready", +} + +export type LocalMemberMediaState = + | { + tracks: TrackState; + connection: ConnectionState | FailedToStartError; + } + | PublishState + | ElementCallError; +export type LocalMemberState = + | ElementCallError + | TransportState.Waiting + | { + media: LocalMemberMediaState; + matrix: ElementCallError | RTCSessionStatus; + }; + +/* + * - get well known + * - get oldest membership + * - get transport to use + * - get openId + jwt token + * - wait for createTrack() call + * - create tracks + * - wait for join() call + * - Publisher.publishTracks() + * - send join state/sticky event + */ + +interface Props { + // TODO add a comment into some code style readme or file header callviewmodel + // that the inputs for those createSomething$() functions should NOT contain any js-sdk objectes + scope: ObservableScope; + muteStates: MuteStates; + connectionManager: IConnectionManager; + createPublisherFactory: (connection: Connection) => Publisher; + joinMatrixRTC: (transport: LivekitTransport) => void; + homeserverConnected: HomeserverConnected; + localTransport$: Behavior; + matrixRTCSession: Pick< + MatrixRTCSession, + "updateCallIntent" | "leaveRoomSession" + >; + logger: Logger; +} + +/** + * This class is responsible for managing the own membership in a room. + * We want + * - a publisher + * - + * @param props The properties required to create the local membership. + * @param props.scope The observable scope to use. + * @param props.connectionManager The connection manager to get connections from. + * @param props.createPublisherFactory Factory to create a publisher once we have a connection. + * @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport. + * @param props.homeserverConnected The homeserver connected state. + * @param props.localTransport$ The local transport to use for publishing. + * @param props.logger The logger to use. + * @param props.muteStates The mute states for video and audio. + * @param props.matrixRTCSession The matrix RTC session to join. + * @returns + * - publisher: The handle to create tracks and publish them to the room. + * - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication) + * - transport$: the transport object the ownMembership$ ended up using. + * - connectionState: the current connection state. Including matrix server and livekit server connection. + * - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen. + */ +export const createLocalMembership$ = ({ + scope, + connectionManager, + localTransport$: localTransportCanThrow$, + homeserverConnected, + createPublisherFactory, + joinMatrixRTC, + logger: parentLogger, + muteStates, + matrixRTCSession, +}: Props): { + /** + * This request to start audio and video tracks. + * Can be called early to pre-emptively get media permissions and start devices. + */ + startTracks: () => void; + /** + * This sets a inner state (shouldPublish) to true and instructs the js-sdk and livekit to keep the user + * connected to matrix and livekit. + */ + requestJoinAndPublish: () => void; + requestDisconnect: () => void; + localMemberState$: Behavior; + sharingScreen$: Behavior; + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + toggleScreenSharing: (() => void) | null; + // tracks$: Behavior; + participant$: Behavior; + connection$: Behavior; + /** + * Tracks the homserver and livekit connected state and based on that computes reconnecting. + */ + reconnecting$: Behavior; + /** Shorthand for homeserverConnected.rtcSession === Status.Disconnected + * Direct translation to the js-sdk membership manager connection `Status`. + */ + disconnected$: Behavior; + /** + * Fully connected + */ + connected$: Behavior; +} => { + const logger = parentLogger.getChild("[LocalMembership]"); + logger.debug(`Creating local membership..`); + + // Unwrap the local transport and set the state of the LocalMembership to error in case the transport is an error. + const localTransport$ = scope.behavior( + localTransportCanThrow$.pipe( + catchError((e: unknown) => { + let error: ElementCallError; + if (e instanceof ElementCallError) { + error = e; + } else { + error = new UnknownCallError( + e instanceof Error + ? e + : new Error("Unknown error from localTransport"), + ); + } + setTransportError(error); + return of(null); + }), + ), + ); + + // Drop Epoch data here since we will not combine this anymore + const localConnection$ = scope.behavior( + combineLatest([ + connectionManager.connectionManagerData$, + localTransport$, + ]).pipe( + map(([{ value: connectionData }, localTransport]) => { + if (localTransport === null) { + return null; + } + + return connectionData.getConnectionForTransport(localTransport); + }), + tap((connection) => { + logger.info( + `Local connection updated: ${connection?.transport?.livekit_service_url}`, + ); + }), + ), + ); + + // Tracks error that happen when creating the local tracks. + const mediaErrors$ = localConnection$.pipe( + switchMap((connection) => { + if (!connection) { + return of(null); + } else { + return fromEvent( + connection.livekitRoom, + RoomEvent.MediaDevicesError, + (error: Error) => { + return MediaDeviceFailure.getFailure(error) ?? null; + }, + ); + } + }), + ); + + mediaErrors$.pipe(scope.bind()).subscribe((error) => { + if (error) { + logger.error(`Failed to create local tracks:`, error); + setMatrixError( + // TODO is it fatal? Do we need to create a new Specialized Error? + new UnknownCallError(new Error(`Media device error: ${error}`)), + ); + } + }); + // MATRIX RELATED + + // This should be used in a combineLatest with publisher$ to connect. + // to make it possible to call startTracks before the preferredTransport$ has resolved. + const trackStartRequested = Promise.withResolvers(); + + // This should be used in a combineLatest with publisher$ to connect. + // to make it possible to call startTracks before the preferredTransport$ has resolved. + const joinAndPublishRequested$ = new BehaviorSubject(false); + + /** + * The publisher is stored in here an abstracts creating and publishing tracks. + */ + const publisher$ = new BehaviorSubject(null); + + const startTracks = (): void => { + trackStartRequested.resolve(); + // This used to return the tracks, but now they are only accessible via the publisher. + }; + + const requestJoinAndPublish = (): void => { + trackStartRequested.resolve(); + joinAndPublishRequested$.next(true); + }; + + const requestDisconnect = (): void => { + joinAndPublishRequested$.next(false); + }; + + // Take care of the publisher$ + // create a new one as soon as a local Connection is available + // + // Recreate a new one once the local connection changes + // - stop publishing + // - destruct all current streams + // - overwrite current publisher + scope.reconcile(localConnection$, async (connection) => { + if (connection !== null) { + const publisher = createPublisherFactory(connection); + publisher$.next(publisher); + // Clean-up callback + return Promise.resolve(async (): Promise => { + await publisher.stopPublishing(); + await publisher.stopTracks(); + }); + } + }); + + // Use reconcile here to not run concurrent createAndSetupTracks calls + // `tracks$` will update once they are ready. + scope.reconcile( + scope.behavior( + combineLatest([ + publisher$ /*, tracks$*/, + from(trackStartRequested.promise), + ]), + null, + ), + async (valueIfReady) => { + if (!valueIfReady) return; + const [publisher] = valueIfReady; + if (publisher) { + await publisher.createAndSetupTracks().catch((e) => logger.error(e)); + } + }, + ); + + // Based on `connectRequested$` we start publishing tracks. (once they are there!) + scope.reconcile( + scope.behavior(combineLatest([publisher$, joinAndPublishRequested$])), + async ([publisher, shouldJoinAndPublish]) => { + // Get the current publishing state to avoid redundant calls. + const isPublishing = publisher?.shouldPublish === true; + if (shouldJoinAndPublish && !isPublishing) { + try { + await publisher?.startPublishing(); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + setPublishError(new FailToStartLivekitConnection(message)); + } + } else if (isPublishing) { + try { + await publisher?.stopPublishing(); + } catch (error) { + setPublishError(new UnknownCallError(error as Error)); + } + } + }, + ); + + // STATE COMPUTATION + + // These are non fatal since we can join a room and concume media even though publishing failed. + const publishError$ = new BehaviorSubject(null); + const setPublishError = (e: ElementCallError): void => { + if (publishError$.value !== null) { + logger.error("Multiple Media Errors:", e); + } else { + publishError$.next(e); + } + }; + + const fatalTransportError$ = new BehaviorSubject( + null, + ); + + const setTransportError = (e: ElementCallError): void => { + if (fatalTransportError$.value !== null) { + logger.error("Multiple Transport Errors:", e); + } else { + fatalTransportError$.next(e); + } + }; + + const localConnectionState$ = localConnection$.pipe( + switchMap((connection) => (connection ? connection.state$ : of(null))), + ); + + const mediaState$: Behavior = scope.behavior( + combineLatest([ + localConnectionState$, + localTransport$, + joinAndPublishRequested$, + from(trackStartRequested.promise).pipe( + map(() => true), + startWith(false), + ), + ]).pipe( + map( + ([ + localConnectionState, + localTransport, + shouldPublish, + shouldStartTracks, + ]) => { + if (!localTransport) return null; + const trackState: TrackState = shouldStartTracks + ? TrackState.Ready + : TrackState.WaitingForUser; + + if ( + localConnectionState !== ConnectionState.LivekitConnected || + trackState !== TrackState.Ready + ) + return { + connection: localConnectionState, + tracks: trackState, + }; + if (!shouldPublish) return PublishState.WaitingForUser; + // if (!publishing) return PublishState.Starting; + return PublishState.Publishing; + }, + ), + distinctUntilChanged(deepCompare), + ), + ); + const fatalMatrixError$ = new BehaviorSubject(null); + const setMatrixError = (e: ElementCallError): void => { + if (fatalMatrixError$.value !== null) { + logger.error("Multiple Matrix Errors:", e); + } else { + fatalMatrixError$.next(e); + } + }; + + const localMemberState$ = scope.behavior( + combineLatest([ + mediaState$, + homeserverConnected.rtsSession$, + fatalMatrixError$, + fatalTransportError$, + publishError$, + ]).pipe( + map( + ([ + mediaState, + rtcSessionStatus, + fatalMatrixError, + fatalTransportError, + publishError, + ]) => { + if (fatalTransportError !== null) return fatalTransportError; + // `mediaState` will be 'null' until the transport/connection appears. + if (mediaState && rtcSessionStatus) + return { + matrix: fatalMatrixError ?? rtcSessionStatus, + media: publishError ?? mediaState, + }; + return TransportState.Waiting; + }, + ), + ), + ); + + /** + * Whether we are "fully" connected to the call. Accounts for both the + * connection to the MatrixRTC session and the LiveKit publish connection. + */ + const matrixAndLivekitConnected$ = scope.behavior( + and$( + homeserverConnected.combined$, + localConnectionState$.pipe( + map((state) => state === ConnectionState.LivekitConnected), + ), + ).pipe( + tap((v) => logger.debug("livekit+matrix: Connected state changed", v)), + ), + ); + + /** + * Whether we should tell the user that we're reconnecting to the call. + */ + const reconnecting$ = scope.behavior( + matrixAndLivekitConnected$.pipe( + pairwise(), + map(([prev, current]) => prev === true && current === false), + ), + false, + ); + + // inform the widget about the connect and disconnect intent from the user. + scope + .behavior(joinAndPublishRequested$.pipe(pairwise(), scope.bind()), [ + undefined, + joinAndPublishRequested$.value, + ]) + .subscribe(([prev, current]) => { + if (!widget) return; + // JOIN prev=false (was left) => current-true (now joiend) + if (!prev && current) { + widget.api.transport + .send(ElementWidgetActions.JoinCall, {}) + .catch((e) => { + logger.error("Failed to send join action", e); + }); + } + // LEAVE prev=false (was joined) => current-true (now left) + if (prev && !current) { + widget.api.transport + .send(ElementWidgetActions.HangupCall, {}) + .catch((e) => { + logger.error("Failed to send hangup action", e); + }); + } + }); + + combineLatest([muteStates.video.enabled$, homeserverConnected.combined$]) + .pipe(scope.bind()) + .subscribe(([videoEnabled, connected]) => { + if (!connected) return; + void matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"); + }); + + // Keep matrix rtc session in sync with localTransport$, connectRequested$ + scope.reconcile( + scope.behavior(combineLatest([localTransport$, joinAndPublishRequested$])), + async ([transport, shouldConnect]) => { + if (!transport) return; + // if shouldConnect=false we will do the disconnect as the cleanup from the previous reconcile iteration. + if (!shouldConnect) return; + + try { + joinMatrixRTC(transport); + } catch (error) { + logger.error("Error entering RTC session", error); + if (error instanceof Error) + setMatrixError(new MembershipManagerError(error)); + } + + return Promise.resolve(async (): Promise => { + try { + // TODO Update matrixRTCSession to allow udpating the transport without leaving the session! + await matrixRTCSession.leaveRoomSession(1000); + } catch (e) { + logger.error("Error leaving RTC session", e); + } + }); + }, + ); + + const participant$ = scope.behavior( + localConnection$.pipe(map((c) => c?.livekitRoom?.localParticipant ?? null)), + ); + + // Pause upstream of all local media tracks when we're disconnected from + // MatrixRTC, because it can be an unpleasant surprise for the app to say + // 'reconnecting' and yet still be transmitting your media to others. + // We use matrixConnected$ rather than reconnecting$ because we want to + // pause tracks during the initial joining sequence too until we're sure + // that our own media is displayed on screen. + // TODO refactor this based no livekitState$ + combineLatest([participant$, homeserverConnected.combined$]) + .pipe(scope.bind()) + .subscribe(([participant, connected]) => { + if (!participant) return; + const publications = participant.trackPublications.values(); + if (connected) { + for (const p of publications) { + if (p.track?.isUpstreamPaused === true) { + const kind = p.track.kind; + logger.info( + `Resuming ${kind} track (MatrixRTC connection present)`, + ); + p.track + .resumeUpstream() + .catch((e) => + logger.error( + `Failed to resume ${kind} track after MatrixRTC reconnection`, + e, + ), + ); + } + } + } else { + for (const p of publications) { + if (p.track?.isUpstreamPaused === false) { + const kind = p.track.kind; + logger.info( + `Pausing ${kind} track (uncertain MatrixRTC connection)`, + ); + p.track + .pauseUpstream() + .catch((e) => + logger.error( + `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`, + e, + ), + ); + } + } + } + }); + + /** + * Whether the user is currently sharing their screen. + */ + const sharingScreen$ = scope.behavior( + participant$.pipe( + switchMap((p) => (p !== null ? observeSharingScreen$(p) : of(false))), + ), + ); + + let toggleScreenSharing: (() => void) | null = null; + if ( + "getDisplayMedia" in (navigator.mediaDevices ?? {}) && + !getUrlParams().hideScreensharing + ) { + toggleScreenSharing = (): void => { + const screenshareSettings: ScreenShareCaptureOptions = { + audio: true, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + }; + const targetScreenshareState = !sharingScreen$.value; + logger.info( + `toggleScreenSharing called. Switching ${ + targetScreenshareState ? "On" : "Off" + }`, + ); + // If a connection is ready, toggle screen sharing. + // We deliberately do nothing in the case of a null connection because + // it looks nice for the call control buttons to all become available + // at once upon joining the call, rather than introducing a disabled + // state. The user can just click again. + // We also allow screen sharing to be toggled even if the connection + // is still initializing or publishing tracks, because there's no + // technical reason to disallow this. LiveKit will publish if it can. + participant$.value + ?.setScreenShareEnabled(targetScreenshareState, screenshareSettings) + .catch(logger.error); + }; + } + + return { + startTracks, + requestJoinAndPublish, + requestDisconnect, + localMemberState$, + participant$, + reconnecting$, + connected$: matrixAndLivekitConnected$, + disconnected$: scope.behavior( + homeserverConnected.rtsSession$.pipe( + map((state) => state === RTCSessionStatus.Disconnected), + ), + ), + sharingScreen$, + toggleScreenSharing, + connection$: localConnection$, + }; +}; + +export function observeSharingScreen$(p: Participant): Observable { + return observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled)); +} + +interface EnterRTCSessionOptions { + encryptMedia: boolean; + matrixRTCMode: MatrixRTCMode; +} + +/** + * Does the necessary steps to enter the RTC session on the matrix side: + * - Preparing the membership info (FOCUS to use, options) + * - Sends the matrix event to join the call, and starts the membership manager: + * - Delay events management + * - Handles retries (fails only after several attempts) + * + * @param rtcSession - The MatrixRTCSession to join. + * @param transport - The LivekitTransport to use for this session. + * @param options - Options for entering the RTC session. + * @param options.encryptMedia - Whether to encrypt media. + * @param options.matrixRTCMode - The Matrix RTC mode to use. + * @throws If the widget could not send ElementWidgetActions.JoinCall action. + */ +// Exported for unit testing +export function enterRTCSession( + rtcSession: MatrixRTCSession, + transport: LivekitTransport, + { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions, +): void { + PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); + PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); + + // This must be called before we start trying to join the call, as we need to + // have started tracking by the time calls start getting created. + // groupCallOTelMembership?.onJoinCall(); + + const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); + const useDeviceSessionMemberEvents = + features?.feature_use_device_session_member_events; + const { sendNotificationType: notificationType, callIntent } = getUrlParams(); + const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy; + // Multi-sfu does not need a preferred foci list. just the focus that is actually used. + // TODO where/how do we track errors originating from the ongoing rtcSession? + rtcSession.joinRoomSession( + multiSFU ? [] : [transport], + multiSFU ? transport : undefined, + { + notificationType, + callIntent, + manageMediaKeys: encryptMedia, + ...(useDeviceSessionMemberEvents !== undefined && { + useLegacyMemberEvents: !useDeviceSessionMemberEvents, + }), + delayedLeaveEventRestartMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_ms, + delayedLeaveEventDelayMs: + matrixRtcSessionConfig?.delayed_leave_event_delay_ms, + delayedLeaveEventRestartLocalTimeoutMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, + networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, + makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, + membershipEventExpiryMs: + matrixRtcSessionConfig?.membership_event_expiry_ms, + useExperimentalToDeviceTransport: true, + unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0, + }, + ); +} diff --git a/src/state/CallViewModel/localMember/LocalMembership.test.ts b/src/state/CallViewModel/localMember/LocalMembership.test.ts deleted file mode 100644 index 9459d419..00000000 --- a/src/state/CallViewModel/localMember/LocalMembership.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. -Copyright 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type LivekitTransport, - type MatrixRTCSession, -} from "matrix-js-sdk/lib/matrixrtc"; -import { describe, expect, it, vi } from "vitest"; -import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; -import { map } from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; - -import { MatrixRTCMode } from "../../../settings/settings"; -import { - mockConfig, - mockMuteStates, - withTestScheduler, -} from "../../../utils/test"; -import { - createLocalMembership$, - enterRTCSession, - LivekitState, -} from "./LocalMembership"; -import { MatrixRTCTransportMissingError } from "../../../utils/errors"; -import { Epoch } from "../../ObservableScope"; -import { constant } from "../../Behavior"; -import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; -import { type Publisher } from "./Publisher"; - -const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; -const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); -vi.mock("../../../UrlParams", () => ({ getUrlParams })); - -describe("LocalMembership", () => { - describe("enterRTCSession", () => { - it("It joins the correct Session", async () => { - const focusFromOlderMembership = { - type: "livekit", - livekit_service_url: "http://my-oldest-member-service-url.com", - livekit_alias: "my-oldest-member-service-alias", - }; - - const focusConfigFromWellKnown = { - type: "livekit", - livekit_service_url: "http://my-well-known-service-url.com", - }; - const focusConfigFromWellKnown2 = { - type: "livekit", - livekit_service_url: "http://my-well-known-service-url2.com", - }; - const clientWellKnown = { - "org.matrix.msc4143.rtc_foci": [ - focusConfigFromWellKnown, - focusConfigFromWellKnown2, - ], - }; - - mockConfig({ - livekit: { livekit_service_url: "http://my-default-service-url.com" }, - }); - - vi.spyOn(AutoDiscovery, "getRawClientConfig").mockImplementation( - async (domain) => { - if (domain === "example.org") { - return Promise.resolve(clientWellKnown); - } - return Promise.resolve({}); - }, - ); - - const mockedSession = vi.mocked({ - room: { - roomId: "roomId", - client: { - getDomain: vi.fn().mockReturnValue("example.org"), - getOpenIdToken: vi.fn().mockResolvedValue({ - access_token: "ACCCESS_TOKEN", - token_type: "Bearer", - matrix_server_name: "localhost", - expires_in: 10000, - }), - }, - }, - memberships: [], - getFocusInUse: vi.fn().mockReturnValue(focusFromOlderMembership), - getOldestMembership: vi.fn().mockReturnValue({ - getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]), - }), - joinRoomSession: vi.fn(), - }) as unknown as MatrixRTCSession; - - await enterRTCSession( - mockedSession, - { - livekit_alias: "roomId", - livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit", - }, - { - encryptMedia: true, - matrixRTCMode: MATRIX_RTC_MODE, - }, - ); - - expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith( - [ - { - livekit_alias: "roomId", - livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit", - }, - ], - undefined, - expect.objectContaining({ - manageMediaKeys: true, - useLegacyMemberEvents: false, - }), - ); - }); - - it("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => { - mockConfig({}); - vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({ - "org.matrix.msc4143.rtc_foci": [ - { - type: "livekit", - livekit_service_url: "http://my-well-known-service-url.com", - }, - ], - }); - - const mockedSession = vi.mocked({ - room: { - roomId: "roomId", - client: { - getDomain: vi.fn().mockReturnValue("example.org"), - getOpenIdToken: vi.fn().mockResolvedValue({ - access_token: "ACCCESS_TOKEN", - token_type: "Bearer", - matrix_server_name: "localhost", - expires_in: 10000, - }), - }, - }, - memberships: [], - getFocusInUse: vi.fn(), - joinRoomSession: vi.fn(), - }) as unknown as MatrixRTCSession; - - await enterRTCSession( - mockedSession, - { - livekit_alias: "roomId", - livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit", - }, - { - encryptMedia: true, - matrixRTCMode: MATRIX_RTC_MODE, - }, - ); - }); - }); - - const defaultCreateLocalMemberValues = { - options: constant({ - encryptMedia: false, - matrixRTCMode: MatrixRTCMode.Matrix_2_0, - }), - matrixRTCSession: { - updateCallIntent: () => {}, - leaveRoomSession: () => {}, - } as unknown as MatrixRTCSession, - muteStates: mockMuteStates(), - isHomeserverConnected: constant(true), - trackProcessorState$: constant({ - supported: false, - processor: undefined, - }), - logger: logger, - createPublisherFactory: (): Publisher => ({}) as unknown as Publisher, - joinMatrixRTC: async (): Promise => {}, - homeserverConnected$: constant(true), - }; - - it("throws error on missing RTC config error", () => { - withTestScheduler(({ scope, hot, expectObservable }) => { - const goodTransport = { - livekit_service_url: "other", - } as LivekitTransport; - - const localTransport$ = scope.behavior( - hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), - goodTransport, - ); - - const mockConnectionManager = { - transports$: scope.behavior( - localTransport$.pipe(map((t) => new Epoch([t]))), - ), - connectionManagerData$: constant( - new Epoch(new ConnectionManagerData()), - ), - }; - - const localMembership = createLocalMembership$({ - scope, - ...defaultCreateLocalMemberValues, - connectionManager: mockConnectionManager, - localTransport$, - }); - - expectObservable(localMembership.connectionState.livekit$).toBe("ne", { - n: { state: LivekitState.Uninitialized }, - e: { - state: LivekitState.Error, - error: expect.toSatisfy( - (e) => e instanceof MatrixRTCTransportMissingError, - ), - }, - }); - }); - }); -}); diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts deleted file mode 100644 index 36952c5a..00000000 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ /dev/null @@ -1,629 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type LocalTrack, - type Participant, - ParticipantEvent, - type LocalParticipant, - type ScreenShareCaptureOptions, -} from "livekit-client"; -import { observeParticipantEvents } from "@livekit/components-core"; -import { - type LivekitTransport, - type MatrixRTCSession, -} from "matrix-js-sdk/lib/matrixrtc"; -import { - BehaviorSubject, - catchError, - combineLatest, - distinctUntilChanged, - map, - type Observable, - of, - scan, - switchMap, - tap, -} from "rxjs"; -import { type Logger } from "matrix-js-sdk/lib/logger"; - -import { type Behavior } from "../../Behavior"; -import { type IConnectionManager } from "../remoteMembers/ConnectionManager"; -import { ObservableScope } from "../../ObservableScope"; -import { type Publisher } from "./Publisher"; -import { type MuteStates } from "../../MuteStates"; -import { and$ } from "../../../utils/observable"; -import { ElementCallError, UnknownCallError } from "../../../utils/errors"; -import { ElementWidgetActions, widget } from "../../../widget"; -import { getUrlParams } from "../../../UrlParams.ts"; -import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; -import { MatrixRTCMode } from "../../../settings/settings.ts"; -import { Config } from "../../../config/Config.ts"; -import { - type Connection, - type ConnectionState, -} from "../remoteMembers/Connection.ts"; - -export enum LivekitState { - Uninitialized = "uninitialized", - Connecting = "connecting", - Connected = "connected", - Error = "error", - Disconnected = "disconnected", - Disconnecting = "disconnecting", -} - -type LocalMemberLivekitState = - | { state: LivekitState.Error; error: ElementCallError } - | { state: LivekitState.Connected } - | { state: LivekitState.Connecting } - | { state: LivekitState.Uninitialized } - | { state: LivekitState.Disconnected } - | { state: LivekitState.Disconnecting }; - -export enum MatrixState { - Connected = "connected", - Disconnected = "disconnected", - Connecting = "connecting", - Error = "Error", -} - -type LocalMemberMatrixState = - | { state: MatrixState.Connected } - | { state: MatrixState.Connecting } - | { state: MatrixState.Disconnected } - | { state: MatrixState.Error; error: Error }; - -export interface LocalMemberConnectionState { - livekit$: Behavior; - matrix$: Behavior; -} - -/* - * - get well known - * - get oldest membership - * - get transport to use - * - get openId + jwt token - * - wait for createTrack() call - * - create tracks - * - wait for join() call - * - Publisher.publishTracks() - * - send join state/sticky event - */ - -interface Props { - // TODO add a comment into some code style readme or file header callviewmodel - // that the inputs for those createSomething$() functions should NOT contain any js-sdk objectes - scope: ObservableScope; - muteStates: MuteStates; - connectionManager: IConnectionManager; - createPublisherFactory: (connection: Connection) => Publisher; - joinMatrixRTC: (trasnport: LivekitTransport) => Promise; - homeserverConnected$: Behavior; - localTransport$: Behavior; - matrixRTCSession: Pick< - MatrixRTCSession, - "updateCallIntent" | "leaveRoomSession" - >; - logger: Logger; -} - -/** - * This class is responsible for managing the own membership in a room. - * We want - * - a publisher - * - - * @param param0 - * @returns - * - publisher: The handle to create tracks and publish them to the room. - * - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication) - * - transport$: the transport object the ownMembership$ ended up using. - * - connectionState: the current connection state. Including matrix server and livekit server connection. - * - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen. - */ -export const createLocalMembership$ = ({ - scope, - connectionManager, - localTransport$: localTransportCanThrow$, - homeserverConnected$, - createPublisherFactory, - joinMatrixRTC, - logger: parentLogger, - muteStates, - matrixRTCSession, -}: Props): { - requestConnect: () => LocalMemberConnectionState; - startTracks: () => Behavior; - requestDisconnect: () => Observable | null; - connectionState: LocalMemberConnectionState; - sharingScreen$: Behavior; - /** - * Callback to toggle screen sharing. If null, screen sharing is not possible. - */ - toggleScreenSharing: (() => void) | null; - participant$: Behavior; - connection$: Behavior; - homeserverConnected$: Behavior; - // deprecated fields - /** @deprecated use state instead*/ - connected$: Behavior; - // this needs to be discussed - /** @deprecated use state instead*/ - reconnecting$: Behavior; -} => { - const logger = parentLogger.getChild("[LocalMembership]"); - logger.debug(`Creating local membership..`); - const state = { - livekit$: new BehaviorSubject({ - state: LivekitState.Uninitialized, - }), - matrix$: new BehaviorSubject({ - state: MatrixState.Disconnected, - }), - }; - - // This should be used in a combineLatest with publisher$ to connect. - // to make it possible to call startTracks before the preferredTransport$ has resolved. - const trackStartRequested$ = new BehaviorSubject(false); - - // This should be used in a combineLatest with publisher$ to connect. - // to make it possible to call startTracks before the preferredTransport$ has resolved. - const connectRequested$ = new BehaviorSubject(false); - - // This should be used in a combineLatest with publisher$ to connect. - const tracks$ = new BehaviorSubject([]); - - // unwrap the local transport and set the state of the LocalMembership to error in case the transport is an error. - const localTransport$ = scope.behavior( - localTransportCanThrow$.pipe( - catchError((e: unknown) => { - let error: ElementCallError; - if (e instanceof ElementCallError) { - error = e; - } else { - error = new UnknownCallError( - e instanceof Error - ? e - : new Error("Unknown error from localTransport"), - ); - } - state.livekit$.next({ state: LivekitState.Error, error }); - return of(null); - }), - ), - ); - - // Drop Epoch data here since we will not combine this anymore - const localConnection$ = scope.behavior( - combineLatest([ - connectionManager.connectionManagerData$, - localTransport$, - ]).pipe( - map(([connectionData, localTransport]) => { - if (localTransport === null) { - return null; - } - - return connectionData.value.getConnectionForTransport(localTransport); - }), - tap((connection) => { - logger.info( - `Local connection updated: ${connection?.transport?.livekit_service_url}`, - ); - }), - ), - ); - - // /** - // * Whether we are "fully" connected to the call. Accounts for both the - // * connection to the MatrixRTC session and the LiveKit publish connection. - // */ - // // TODO use this in combination with the MemberState. - const connected$ = scope.behavior( - and$( - homeserverConnected$, - localConnection$.pipe( - switchMap((c) => - c - ? c.state$.pipe(map((state) => state.state === "ConnectedToLkRoom")) - : of(false), - ), - ), - ), - ); - - const publisher$ = new BehaviorSubject(null); - localConnection$.pipe(scope.bind()).subscribe((connection) => { - if (connection !== null && publisher$.value === null) { - // TODO looks strange to not change publisher if connection changes. - // @toger5 will take care of this! - publisher$.next(createPublisherFactory(connection)); - } - }); - - // const mutestate= publisher$.pipe(switchMap((publisher) => { - // return publisher.muteState$ - // }); - - combineLatest([publisher$, trackStartRequested$]).subscribe( - ([publisher, shouldStartTracks]) => { - if (publisher && shouldStartTracks) { - publisher - .createAndSetupTracks() - .then((tracks) => { - tracks$.next(tracks); - }) - .catch((error) => { - logger.error("Error creating tracks:", error); - }); - } - }, - ); - - // MATRIX RELATED - - // /** - // * Whether we should tell the user that we're reconnecting to the call. - // */ - // DISCUSSION is there a better way to do this? - // sth that is more deriectly implied from the membership manager of the js sdk. (fromEvent(matrixRTCSession, Reconnecting)) ??? or similar - const reconnecting$ = scope.behavior( - connected$.pipe( - // We are reconnecting if we previously had some successful initial - // connection but are now disconnected - scan( - ({ connectedPreviously }, connectedNow) => ({ - connectedPreviously: connectedPreviously || connectedNow, - reconnecting: connectedPreviously && !connectedNow, - }), - { connectedPreviously: false, reconnecting: false }, - ), - map(({ reconnecting }) => reconnecting), - ), - ); - - const startTracks = (): Behavior => { - trackStartRequested$.next(true); - return tracks$; - }; - - combineLatest([publisher$, tracks$]).subscribe(([publisher, tracks]) => { - if ( - tracks.length === 0 || - // change this to !== Publishing - state.livekit$.value.state !== LivekitState.Uninitialized - ) { - return; - } - state.livekit$.next({ state: LivekitState.Connecting }); - publisher - ?.startPublishing() - .then(() => { - state.livekit$.next({ state: LivekitState.Connected }); - }) - .catch((error) => { - state.livekit$.next({ state: LivekitState.Error, error }); - }); - }); - - combineLatest([localTransport$, connectRequested$]).subscribe( - // TODO reconnect when transport changes => create test. - ([transport, connectRequested]) => { - if ( - transport === null || - !connectRequested || - state.matrix$.value.state !== MatrixState.Disconnected - ) { - logger.info( - "Not yet connecting because: ", - "transport === null:", - transport === null, - "!connectRequested:", - !connectRequested, - "state.matrix$.value.state !== MatrixState.Disconnected:", - state.matrix$.value.state !== MatrixState.Disconnected, - ); - return; - } - state.matrix$.next({ state: MatrixState.Connecting }); - logger.info("Matrix State connecting"); - - joinMatrixRTC(transport).catch((error) => { - logger.error(error); - state.matrix$.next({ state: MatrixState.Error, error }); - }); - }, - ); - - // TODO add this and update `state.matrix$` based on it. - // useTypedEventEmitter( - // rtcSession, - // MatrixRTCSessionEvent.MembershipManagerError, - // (error) => setExternalError(new ConnectionLostError()), - // ); - - const requestConnect = (): LocalMemberConnectionState => { - trackStartRequested$.next(true); - connectRequested$.next(true); - - return state; - }; - - const requestDisconnect = (): Behavior | null => { - if (state.livekit$.value.state !== LivekitState.Connected) return null; - state.livekit$.next({ state: LivekitState.Disconnecting }); - combineLatest([publisher$, tracks$], (publisher, tracks) => { - publisher - ?.stopPublishing() - .then(() => { - tracks.forEach((track) => track.stop()); - state.livekit$.next({ state: LivekitState.Disconnected }); - }) - .catch((error) => { - state.livekit$.next({ state: LivekitState.Error, error }); - }); - }); - - return state.livekit$; - }; - - // Pause upstream of all local media tracks when we're disconnected from - // MatrixRTC, because it can be an unpleasant surprise for the app to say - // 'reconnecting' and yet still be transmitting your media to others. - // We use matrixConnected$ rather than reconnecting$ because we want to - // pause tracks during the initial joining sequence too until we're sure - // that our own media is displayed on screen. - combineLatest([localConnection$, homeserverConnected$]) - .pipe(scope.bind()) - .subscribe(([connection, connected]) => { - if (connection?.state$.value.state !== "ConnectedToLkRoom") return; - const publications = - connection.livekitRoom.localParticipant.trackPublications.values(); - if (connected) { - for (const p of publications) { - if (p.track?.isUpstreamPaused === true) { - const kind = p.track.kind; - logger.info( - `Resuming ${kind} track (MatrixRTC connection present)`, - ); - p.track - .resumeUpstream() - .catch((e) => - logger.error( - `Failed to resume ${kind} track after MatrixRTC reconnection`, - e, - ), - ); - } - } - } else { - for (const p of publications) { - if (p.track?.isUpstreamPaused === false) { - const kind = p.track.kind; - logger.info( - `Pausing ${kind} track (uncertain MatrixRTC connection)`, - ); - p.track - .pauseUpstream() - .catch((e) => - logger.error( - `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`, - e, - ), - ); - } - } - } - }); - // TODO: Refactor updateCallIntent to sth like this: - // combineLatest([muteStates.video.enabled$,localTransport$, state.matrix$]).pipe(map(()=>{ - // matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"), - // })) - // - - // TODO I do not fully understand what this does. - // Is it needed? - // Is this at the right place? - // Can this be simplified? - // Start and stop session membership as needed - // Discussed in statndup -> It seems we can remove this (there is another call to enterRTCSession in this file) - // MAKE SURE TO UNDERSTAND why reconcile is needed and what is potentially missing from the alternative enterRTCSession block. - // @toger5 will try to take care of this. - scope.reconcile(localTransport$, async (transport) => { - if (transport !== null && transport !== undefined) { - try { - state.matrix$.next({ state: MatrixState.Connecting }); - await joinMatrixRTC(transport); - } catch (e) { - logger.error("Error entering RTC session", e); - } - - // Update our member event when our mute state changes. - const intentScope = new ObservableScope(); - intentScope.reconcile(muteStates.video.enabled$, async (videoEnabled) => - matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"), - ); - - return async (): Promise => { - intentScope.end(); - // Only sends Matrix leave event. The LiveKit session will disconnect - // as soon as either the stopConnection$ handler above gets to it or - // the view model is destroyed. - try { - await matrixRTCSession.leaveRoomSession(); - } catch (e) { - logger.error("Error leaving RTC session", e); - } - try { - await widget?.api.transport.send(ElementWidgetActions.HangupCall, {}); - } catch (e) { - logger.error("Failed to send hangup action", e); - } - }; - } - }); - - localConnection$ - .pipe( - distinctUntilChanged(), - switchMap((c) => - c === null ? of({ state: "Initialized" } as ConnectionState) : c.state$, - ), - map((s) => { - logger.trace(`Local connection state update: ${s.state}`); - if (s.state == "FailedToStart") { - return s.error instanceof ElementCallError - ? s.error - : new UnknownCallError(s.error); - } - }), - scope.bind(), - ) - .subscribe((error) => { - if (error !== undefined) - state.livekit$.next({ state: LivekitState.Error, error }); - }); - - /** - * Whether the user is currently sharing their screen. - */ - const sharingScreen$ = scope.behavior( - localConnection$.pipe( - switchMap((c) => - c !== null - ? observeSharingScreen$(c.livekitRoom.localParticipant) - : of(false), - ), - ), - ); - - let toggleScreenSharing = null; - if ( - "getDisplayMedia" in (navigator.mediaDevices ?? {}) && - !getUrlParams().hideScreensharing - ) { - toggleScreenSharing = (): void => { - const screenshareSettings: ScreenShareCaptureOptions = { - audio: true, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }; - const targetScreenshareState = !sharingScreen$.value; - logger.info( - `toggleScreenSharing called. Switching ${ - targetScreenshareState ? "On" : "Off" - }`, - ); - // If a connection is ready, toggle screen sharing. - // We deliberately do nothing in the case of a null connection because - // it looks nice for the call control buttons to all become available - // at once upon joining the call, rather than introducing a disabled - // state. The user can just click again. - // We also allow screen sharing to be toggled even if the connection - // is still initializing or publishing tracks, because there's no - // technical reason to disallow this. LiveKit will publish if it can. - localConnection$.value?.livekitRoom.localParticipant - .setScreenShareEnabled(targetScreenshareState, screenshareSettings) - .catch(logger.error); - }; - } - - const participant$ = scope.behavior( - localConnection$.pipe(map((c) => c?.livekitRoom?.localParticipant ?? null)), - ); - return { - startTracks, - requestConnect, - requestDisconnect, - connectionState: state, - homeserverConnected$, - connected$, - reconnecting$, - - sharingScreen$, - toggleScreenSharing, - participant$, - connection$: localConnection$, - }; -}; - -export function observeSharingScreen$(p: Participant): Observable { - return observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled)); -} - -interface EnterRTCSessionOptions { - encryptMedia: boolean; - matrixRTCMode: MatrixRTCMode; -} - -/** - * Does the necessary steps to enter the RTC session on the matrix side: - * - Preparing the membership info (FOCUS to use, options) - * - Sends the matrix event to join the call, and starts the membership manager: - * - Delay events management - * - Handles retries (fails only after several attempts) - * - * @param rtcSession - * @param transport - * @param options - * @throws If the widget could not send ElementWidgetActions.JoinCall action. - */ -// Exported for unit testing -export async function enterRTCSession( - rtcSession: MatrixRTCSession, - transport: LivekitTransport, - { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions, -): Promise { - PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); - PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); - - // This must be called before we start trying to join the call, as we need to - // have started tracking by the time calls start getting created. - // groupCallOTelMembership?.onJoinCall(); - - const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); - const useDeviceSessionMemberEvents = - features?.feature_use_device_session_member_events; - const { sendNotificationType: notificationType, callIntent } = getUrlParams(); - const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy; - // Multi-sfu does not need a preferred foci list. just the focus that is actually used. - // TODO where/how do we track errors originating from the ongoing rtcSession? - rtcSession.joinRoomSession( - multiSFU ? [] : [transport], - multiSFU ? transport : undefined, - { - notificationType, - callIntent, - manageMediaKeys: encryptMedia, - ...(useDeviceSessionMemberEvents !== undefined && { - useLegacyMemberEvents: !useDeviceSessionMemberEvents, - }), - delayedLeaveEventRestartMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_ms, - delayedLeaveEventDelayMs: - matrixRtcSessionConfig?.delayed_leave_event_delay_ms, - delayedLeaveEventRestartLocalTimeoutMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, - networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, - makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, - membershipEventExpiryMs: - matrixRtcSessionConfig?.membership_event_expiry_ms, - useExperimentalToDeviceTransport: true, - unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0, - }, - ); - if (widget) { - await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); - } -} diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index d543f97a..3e69bf2c 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -5,8 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + afterEach, + beforeEach, + describe, + expect, + it, + type MockedObject, + vi, +} from "vitest"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject, lastValueFrom } from "rxjs"; +import fetchMock from "fetch-mock"; import { mockConfig, flushPromises } from "../../../utils/test"; import { createLocalTransport$ } from "./LocalTransport"; @@ -17,31 +27,22 @@ import { FailToGetOpenIdToken, } from "../../../utils/errors"; import * as openIDSFU from "../../../livekit/openIDSFU"; +import { customLivekitUrl } from "../../../settings/settings"; +import { testJWTToken } from "../../../utils/test-fixtures"; describe("LocalTransport", () => { + const openIdResponse: openIDSFU.SFUConfig = { + url: "https://lk.example.org", + jwt: testJWTToken, + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + }; + let scope: ObservableScope; - beforeEach(() => (scope = new ObservableScope())); - afterEach(() => scope.end()); - - it("throws if config is missing", async () => { - const localTransport$ = createLocalTransport$({ - scope, - roomId: "!room:example.org", - useOldestMember$: constant(false), - memberships$: constant(new Epoch([])), - client: { - getDomain: () => "", - // These won't be called in this error path but satisfy the type - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, - }); - await flushPromises(); - - expect(() => localTransport$.value).toThrow( - new MatrixRTCTransportMissingError(""), - ); + beforeEach(() => { + scope = new ObservableScope(); }); + afterEach(() => scope.end()); it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => { // Provide a valid config so makeTransportInternal resolves a transport @@ -60,12 +61,14 @@ describe("LocalTransport", () => { const errors: Error[] = []; const localTransport$ = createLocalTransport$({ scope, - roomId: "!room:example.org", + roomId: "!example_room_id", useOldestMember$: constant(false), memberships$: constant(new Epoch([])), client: { // Use empty domain to skip .well-known and use config directly getDomain: () => "", + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: async () => Promise.resolve([]), getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), }, @@ -83,12 +86,12 @@ describe("LocalTransport", () => { expect(() => localTransport$.value).toThrow(expectedError); }); - it("emits preferred transport after OpenID resolves", async () => { + it("updates local transport when oldest member changes", async () => { // Use config so transport discovery succeeds, but delay OpenID JWT fetch mockConfig({ livekit: { livekit_service_url: "https://lk.example.org" }, }); - + const memberships$ = new BehaviorSubject(new Epoch([])); const openIdResolver = Promise.withResolvers(); vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue( @@ -97,24 +100,171 @@ describe("LocalTransport", () => { const localTransport$ = createLocalTransport$({ scope, - roomId: "!room:example.org", - useOldestMember$: constant(false), - memberships$: constant(new Epoch([])), + roomId: "!example_room_id", + useOldestMember$: constant(true), + memberships$, client: { getDomain: () => "", + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: async () => Promise.resolve([]), getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), }, }); - openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" }); + openIdResolver.resolve?.(openIdResponse); expect(localTransport$.value).toBe(null); await flushPromises(); // final expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!room:example.org", + livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", }); }); + + type LocalTransportProps = Parameters[0]; + + describe("transport configuration mechanisms", () => { + let localTransportOpts: LocalTransportProps & { + client: MockedObject; + }; + let openIdResolver: PromiseWithResolvers; + beforeEach(() => { + mockConfig({}); + customLivekitUrl.setValue(customLivekitUrl.defaultValue); + localTransportOpts = { + scope, + roomId: "!example_room_id", + useOldestMember$: constant(false), + memberships$: constant(new Epoch([])), + client: { + getDomain: vi.fn().mockReturnValue(""), + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), + getOpenIdToken: vi.fn(), + getDeviceId: vi.fn(), + }, + }; + openIdResolver = Promise.withResolvers(); + vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue( + openIdResolver.promise, + ); + }); + + afterEach(() => { + fetchMock.reset(); + }); + + it("supports getting transport via application config", async () => { + mockConfig({ + livekit: { livekit_service_url: "https://lk.example.org" }, + }); + const localTransport$ = createLocalTransport$(localTransportOpts); + openIdResolver.resolve?.(openIdResponse); + expect(localTransport$.value).toBe(null); + await flushPromises(); + expect(localTransport$.value).toStrictEqual({ + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }); + }); + it("supports getting transport via user settings", async () => { + customLivekitUrl.setValue("https://lk.example.org"); + const localTransport$ = createLocalTransport$(localTransportOpts); + openIdResolver.resolve?.(openIdResponse); + expect(localTransport$.value).toBe(null); + await flushPromises(); + expect(localTransport$.value).toStrictEqual({ + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }); + }); + it("supports getting transport via backend", async () => { + localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ + { type: "livekit", livekit_service_url: "https://lk.example.org" }, + ]); + const localTransport$ = createLocalTransport$(localTransportOpts); + openIdResolver.resolve?.(openIdResponse); + expect(localTransport$.value).toBe(null); + await flushPromises(); + expect(localTransport$.value).toStrictEqual({ + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }); + }); + it("fails fast if the openID request fails for backend config", async () => { + localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ + { type: "livekit", livekit_service_url: "https://lk.example.org" }, + ]); + openIdResolver.reject( + new FailToGetOpenIdToken(new Error("Test driven error")), + ); + try { + await lastValueFrom(createLocalTransport$(localTransportOpts)); + throw Error("Expected test to throw"); + } catch (ex) { + expect(ex).toBeInstanceOf(FailToGetOpenIdToken); + } + }); + it("supports getting transport via well-known", async () => { + localTransportOpts.client.getDomain.mockReturnValue("example.org"); + fetchMock.getOnce("https://example.org/.well-known/matrix/client", { + "org.matrix.msc4143.rtc_foci": [ + { type: "livekit", livekit_service_url: "https://lk.example.org" }, + ], + }); + const localTransport$ = createLocalTransport$(localTransportOpts); + openIdResolver.resolve?.(openIdResponse); + expect(localTransport$.value).toBe(null); + await flushPromises(); + expect(localTransport$.value).toStrictEqual({ + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }); + expect(fetchMock.done()).toEqual(true); + }); + it("fails fast if the openId request fails for the well-known config", async () => { + localTransportOpts.client.getDomain.mockReturnValue("example.org"); + fetchMock.getOnce("https://example.org/.well-known/matrix/client", { + "org.matrix.msc4143.rtc_foci": [ + { type: "livekit", livekit_service_url: "https://lk.example.org" }, + ], + }); + openIdResolver.reject( + new FailToGetOpenIdToken(new Error("Test driven error")), + ); + try { + await lastValueFrom(createLocalTransport$(localTransportOpts)); + throw Error("Expected test to throw"); + } catch (ex) { + expect(ex).toBeInstanceOf(FailToGetOpenIdToken); + } + }); + it("throws if no options are available", async () => { + const localTransport$ = createLocalTransport$({ + scope, + roomId: "!example_room_id", + useOldestMember$: constant(false), + memberships$: constant(new Epoch([])), + client: { + getDomain: () => "", + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: async () => Promise.resolve([]), + // These won't be called in this error path but satisfy the type + getOpenIdToken: vi.fn(), + getDeviceId: vi.fn(), + }, + }); + await flushPromises(); + + expect(() => localTransport$.value).toThrow( + new MatrixRTCTransportMissingError(""), + ); + }); + }); }); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 0a85bbc1..e72b076f 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -8,11 +8,11 @@ Please see LICENSE in the repository root for full details. import { type CallMembership, isLivekitTransport, - type LivekitTransportConfig, type LivekitTransport, isLivekitTransportConfig, + type Transport, } from "matrix-js-sdk/lib/matrixrtc"; -import { type MatrixClient } from "matrix-js-sdk"; +import { MatrixError, type MatrixClient } from "matrix-js-sdk"; import { combineLatest, distinctUntilChanged, @@ -27,7 +27,10 @@ import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { type Behavior } from "../../Behavior.ts"; import { type Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { Config } from "../../../config/Config.ts"; -import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; +import { + FailToGetOpenIdToken, + MatrixRTCTransportMissingError, +} from "../../../utils/errors.ts"; import { getSFUConfigWithOpenID, type OpenIDClientParts, @@ -45,7 +48,8 @@ const logger = rootLogger.getChild("[LocalTransport]"); interface Props { scope: ObservableScope; memberships$: Behavior>; - client: Pick & OpenIDClientParts; + client: Pick & + OpenIDClientParts; roomId: string; useOldestMember$: Behavior; } @@ -85,7 +89,7 @@ export const createLocalTransport$ = ({ * The transport that we would personally prefer to publish on (if not for the * transport preferences of others, perhaps). * - * @throws + * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ const preferredTransport$: Behavior = scope.behavior( customLivekitUrl.value$.pipe( @@ -116,73 +120,150 @@ export const createLocalTransport$ = ({ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; /** + * Determine the correct Transport for the current session, including + * validating auth against the service to ensure it's correct. + * Prefers in order: * - * @param client - * @param roomId - * @returns + * 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw. + * 2. The transports returned via the homeserver. + * 3. The transports returned via .well-known. + * 4. The transport configured in Element Call's config. + * + * @param client The authenticated Matrix client for the current user + * @param roomId The ID of the room to be connected to. + * @param urlFromDevSettings Override URL provided by the user's local config. + * @returns A fully validated transport config. * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ async function makeTransport( - client: Pick & OpenIDClientParts, + client: Pick & + OpenIDClientParts, roomId: string, urlFromDevSettings: string | null, ): Promise { - let transport: LivekitTransport | undefined; logger.trace("Searching for a preferred transport"); - //TODO refactor this to use the jwt service returned alias. - const livekitAlias = roomId; + + // We will call `getSFUConfigWithOpenID` once per transport here as it's our + // only mechanism of valiation. This means we will also ask the + // homeserver for a OpenID token a few times. Since OpenID tokens are single + // use we don't want to risk any issues by re-using a token. + // + // If the OpenID request were to fail then it's acceptable for us to fail + // this function early, as we assume the homeserver has got some problems. // DEVTOOL: Highest priority: Load from devtool setting if (urlFromDevSettings !== null) { - const transportFromStorage: LivekitTransport = { + logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings); + // Validate that the SFU is up. Otherwise, we want to fail on this + // as we don't permit other SFUs. + const config = await getSFUConfigWithOpenID( + client, + urlFromDevSettings, + roomId, + ); + return { type: "livekit", livekit_service_url: urlFromDevSettings, - livekit_alias: livekitAlias, + livekit_alias: config.livekitAlias, }; - logger.info( - "Using LiveKit transport from dev tools: ", - transportFromStorage, - ); - transport = transportFromStorage; } - // WELL_KNOWN: Prioritize the .well-known/matrix/client, if available, over the configured SFU + async function getFirstUsableTransport( + transports: Transport[], + ): Promise { + for (const potentialTransport of transports) { + if (isLivekitTransportConfig(potentialTransport)) { + try { + const { livekitAlias } = await getSFUConfigWithOpenID( + client, + potentialTransport.livekit_service_url, + roomId, + ); + return { + ...potentialTransport, + livekit_alias: livekitAlias, + }; + } catch (ex) { + if (ex instanceof FailToGetOpenIdToken) { + // Explictly throw these + throw ex; + } + logger.debug( + `Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`, + ex, + ); + } + } + } + return null; + } + + // MSC4143: Attempt to fetch transports from backend. + if ("_unstable_getRTCTransports" in client) { + try { + const selectedTransport = await getFirstUsableTransport( + await client._unstable_getRTCTransports(), + ); + if (selectedTransport) { + logger.info("Using backend-configured SFU", selectedTransport); + return selectedTransport; + } + } catch (ex) { + if (ex instanceof MatrixError && ex.httpStatus === 404) { + // Expected, this is an unstable endpoint and it's not required. + logger.debug("Backend does not provide any RTC transports", ex); + } else if (ex instanceof FailToGetOpenIdToken) { + throw ex; + } else { + // We got an error that wasn't just missing support for the feature, so log it loudly. + logger.error( + "Unexpected error fetching RTC transports from backend", + ex, + ); + } + } + } + + // Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available. const domain = client.getDomain(); - if (domain && transport === undefined) { + if (domain) { // we use AutoDiscovery instead of relying on the MatrixClient having already // been fully configured and started const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ FOCI_WK_KEY ]; - if (Array.isArray(wellKnownFoci)) { - const wellKnownTransport: LivekitTransportConfig | undefined = - wellKnownFoci.find((f) => f && isLivekitTransportConfig(f)); - if (wellKnownTransport !== undefined) { - logger.info("Using LiveKit transport from .well-known: ", transport); - transport = { ...wellKnownTransport, livekit_alias: livekitAlias }; - } + const selectedTransport = Array.isArray(wellKnownFoci) + ? await getFirstUsableTransport(wellKnownFoci) + : null; + if (selectedTransport) { + logger.info("Using .well-known SFU", selectedTransport); + return selectedTransport; } } // CONFIG: Least prioritized; Load from config file const urlFromConf = Config.get().livekit?.livekit_service_url; - if (urlFromConf && transport === undefined) { - const transportFromConf: LivekitTransport = { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; - logger.info("Using LiveKit transport from config: ", transportFromConf); - transport = transportFromConf; + if (urlFromConf) { + try { + const { livekitAlias } = await getSFUConfigWithOpenID( + client, + urlFromConf, + roomId, + ); + const selectedTransport: LivekitTransport = { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; + logger.info("Using config SFU", selectedTransport); + return selectedTransport; + } catch (ex) { + if (ex instanceof FailToGetOpenIdToken) { + throw ex; + } + logger.error("Failed to validate config SFU", ex); + } } - if (!transport) throw new MatrixRTCTransportMissingError(domain ?? ""); // this will call the jwt/sfu/get endpoint to pre create the livekit room. - - await getSFUConfigWithOpenID( - client, - transport.livekit_service_url, - transport.livekit_alias, - ); - - return transport; + throw new MatrixRTCTransportMissingError(domain ?? ""); } diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts new file mode 100644 index 00000000..38a80bed --- /dev/null +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -0,0 +1,360 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, it, test, vi } from "vitest"; +import { + ConnectionState as LivekitConnectionState, + LocalParticipant, + type LocalTrack, + type LocalTrackPublication, + ParticipantEvent, + Track, +} from "livekit-client"; +import { BehaviorSubject } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { ObservableScope } from "../../ObservableScope"; +import { constant } from "../../Behavior"; +import { + flushPromises, + mockLivekitRoom, + mockMediaDevices, +} from "../../../utils/test"; +import { Publisher } from "./Publisher"; +import { type Connection } from "../remoteMembers/Connection"; +import { type MuteStates } from "../../MuteStates"; + +let scope: ObservableScope; + +beforeEach(() => { + scope = new ObservableScope(); +}); + +afterEach(() => scope.end()); + +function createMockLocalTrack(source: Track.Source): LocalTrack { + const track = { + source, + isMuted: false, + isUpstreamPaused: false, + } as Partial as LocalTrack; + + vi.mocked(track).mute = vi.fn().mockImplementation(() => { + track.isMuted = true; + }); + vi.mocked(track).unmute = vi.fn().mockImplementation(() => { + track.isMuted = false; + }); + vi.mocked(track).pauseUpstream = vi.fn().mockImplementation(() => { + // @ts-expect-error - for that test we want to set isUpstreamPaused directly + track.isUpstreamPaused = true; + }); + vi.mocked(track).resumeUpstream = vi.fn().mockImplementation(() => { + // @ts-expect-error - for that test we want to set isUpstreamPaused directly + track.isUpstreamPaused = false; + }); + + return track; +} + +function createMockMuteState(enabled$: BehaviorSubject): { + enabled$: BehaviorSubject; + setHandler: (h: (enabled: boolean) => void) => void; + unsetHandler: () => void; +} { + let currentHandler = (enabled: boolean): void => {}; + + const ms = { + enabled$, + setHandler: vi.fn().mockImplementation((h: (enabled: boolean) => void) => { + currentHandler = h; + }), + unsetHandler: vi.fn().mockImplementation(() => { + currentHandler = (enabled: boolean): void => {}; + }), + }; + // forward enabled$ emissions to the current handler + enabled$.subscribe((enabled) => { + logger.info(`MockMuteState: enabled changed to ${enabled}`); + currentHandler(enabled); + }); + + return ms; +} + +let connection: Connection; +let muteStates: MuteStates; +let localParticipant: LocalParticipant; +let audioEnabled$: BehaviorSubject; +let videoEnabled$: BehaviorSubject; +let trackPublications: LocalTrackPublication[]; +// use it to control when track creation resolves, default to resolved +let createTrackLock: Promise; + +beforeEach(() => { + trackPublications = []; + audioEnabled$ = new BehaviorSubject(false); + videoEnabled$ = new BehaviorSubject(false); + createTrackLock = Promise.resolve(); + + muteStates = { + audio: createMockMuteState(audioEnabled$), + video: createMockMuteState(videoEnabled$), + } as unknown as MuteStates; + + const mockSendDataPacket = vi.fn(); + const mockEngine = { + client: { + sendUpdateLocalMetadata: vi.fn(), + }, + on: vi.fn().mockReturnThis(), + sendDataPacket: mockSendDataPacket, + }; + + localParticipant = new LocalParticipant( + "local-sid", + "local-identity", + // @ts-expect-error - for that test we want a real LocalParticipant to have the pending publications logic + mockEngine, + { + adaptiveStream: true, + dynacase: false, + audioCaptureDefaults: {}, + videoCaptureDefaults: {}, + stopLocalTrackOnUnpublish: true, + reconnectPolicy: "always", + disconnectOnPageLeave: true, + }, + new Map(), + {}, + ); + + vi.mocked(localParticipant).createTracks = vi + .fn() + .mockImplementation(async (opts) => { + const tracks: LocalTrack[] = []; + if (opts.audio) { + tracks.push(createMockLocalTrack(Track.Source.Microphone)); + } + if (opts.video) { + tracks.push(createMockLocalTrack(Track.Source.Camera)); + } + await createTrackLock; + return tracks; + }); + + vi.mocked(localParticipant).publishTrack = vi + .fn() + .mockImplementation(async (track: LocalTrack) => { + const pub = { + track, + source: track.source, + mute: track.mute, + unmute: track.unmute, + } as Partial as LocalTrackPublication; + trackPublications.push(pub); + localParticipant.emit(ParticipantEvent.LocalTrackPublished, pub); + return Promise.resolve(pub); + }); + + vi.mocked(localParticipant).getTrackPublication = vi + .fn() + .mockImplementation((source: Track.Source) => { + return trackPublications.find((pub) => pub.track?.source === source); + }); + + connection = { + state$: constant({ + state: "ConnectedToLkRoom", + livekitConnectionState$: constant(LivekitConnectionState.Connected), + }), + livekitRoom: mockLivekitRoom({ + localParticipant: localParticipant, + }), + } as unknown as Connection; +}); + +describe("Publisher", () => { + let publisher: Publisher; + + beforeEach(() => { + publisher = new Publisher( + scope, + connection, + mockMediaDevices({}), + muteStates, + constant({ supported: false, processor: undefined }), + logger, + ); + }); + + afterEach(() => {}); + + it("Should not create tracks if started muted to avoid unneeded permission requests", async () => { + const createTracksSpy = vi.spyOn( + connection.livekitRoom.localParticipant, + "createTracks", + ); + + audioEnabled$.next(false); + videoEnabled$.next(false); + await publisher.createAndSetupTracks(); + + expect(createTracksSpy).not.toHaveBeenCalled(); + }); + + it("Should minimize permission request by querying create at once", async () => { + const enableCameraAndMicrophoneSpy = vi.spyOn( + localParticipant, + "enableCameraAndMicrophone", + ); + const createTracksSpy = vi.spyOn(localParticipant, "createTracks"); + + audioEnabled$.next(true); + videoEnabled$.next(true); + await publisher.createAndSetupTracks(); + await flushPromises(); + + expect(enableCameraAndMicrophoneSpy).toHaveBeenCalled(); + + // It should create both at once + expect(createTracksSpy).toHaveBeenCalledWith({ + audio: true, + video: true, + }); + }); + + it("Ensure no data is streamed until publish has been called", async () => { + audioEnabled$.next(true); + await publisher.createAndSetupTracks(); + + // The track should be created and paused + expect(localParticipant.createTracks).toHaveBeenCalledWith({ + audio: true, + video: undefined, + }); + await flushPromises(); + expect(localParticipant.publishTrack).toHaveBeenCalled(); + + await flushPromises(); + const track = localParticipant.getTrackPublication( + Track.Source.Microphone, + )?.track; + expect(track).toBeDefined(); + expect(track!.pauseUpstream).toHaveBeenCalled(); + expect(track!.isUpstreamPaused).toBe(true); + }); + + it("Ensure resume upstream when published is called", async () => { + videoEnabled$.next(true); + await publisher.createAndSetupTracks(); + // await flushPromises(); + await publisher.startPublishing(); + + const track = localParticipant.getTrackPublication( + Track.Source.Camera, + )?.track; + expect(track).toBeDefined(); + // expect(track.pauseUpstream).toHaveBeenCalled(); + expect(track!.isUpstreamPaused).toBe(false); + }); + + describe("Mute states", () => { + let publisher: Publisher; + beforeEach(() => { + publisher = new Publisher( + scope, + connection, + mockMediaDevices({}), + muteStates, + constant({ supported: false, processor: undefined }), + logger, + ); + }); + + test.each([ + { mutes: { audioEnabled: true, videoEnabled: false } }, + { mutes: { audioEnabled: true, videoEnabled: false } }, + ])("only create the tracks that are unmuted $mutes", async ({ mutes }) => { + // Ensure all muted + audioEnabled$.next(mutes.audioEnabled); + videoEnabled$.next(mutes.videoEnabled); + + vi.mocked(connection.livekitRoom.localParticipant).createTracks = vi + .fn() + .mockResolvedValue([]); + + await publisher.createAndSetupTracks(); + + expect( + connection.livekitRoom.localParticipant.createTracks, + ).toHaveBeenCalledOnce(); + + expect( + connection.livekitRoom.localParticipant.createTracks, + ).toHaveBeenCalledWith({ + audio: mutes.audioEnabled ? true : undefined, + video: mutes.videoEnabled ? true : undefined, + }); + }); + }); + + it("does mute unmute audio", async () => {}); +}); + +describe("Bug fix", () => { + // There is a race condition when creating and publishing tracks while the mute state changes. + // This race condition could cause tracks to be published even though they are muted at the + // beginning of a call coming from lobby. + // This is caused by our stack using manually the low level API to create and publish tracks, + // but also using the higher level setMicrophoneEnabled and setCameraEnabled functions that also create + // and publish tracks, and managing pending publications. + // Race is as follow, on creation of the Publisher we create the tracks then publish them. + // If in the middle of that process the mute state changes: + // - the `setMicrophoneEnabled` will be no-op because it is not aware of our created track and can't see any pending publication + // - If start publication is requested it will publish the track even though there was a mute request. + it("wrongly publish tracks while muted", async () => { + // setLogLevel(`debug`); + const publisher = new Publisher( + scope, + connection, + mockMediaDevices({}), + muteStates, + constant({ supported: false, processor: undefined }), + logger, + ); + audioEnabled$.next(true); + + const resolvers = Promise.withResolvers(); + createTrackLock = resolvers.promise; + + // Initially the audio is unmuted, so creating tracks should publish the audio track + const createTracks = publisher.createAndSetupTracks(); + void publisher.startPublishing(); + void createTracks.then(() => { + void publisher.startPublishing(); + }); + // now mute the audio before allowing track creation to complete + audioEnabled$.next(false); + resolvers.resolve(undefined); + await createTracks; + + await flushPromises(); + + const track = localParticipant.getTrackPublication( + Track.Source.Microphone, + )?.track; + expect(track).toBeDefined(); + + try { + expect(localParticipant.publishTrack).not.toHaveBeenCalled(); + } catch { + expect(track!.mute).toHaveBeenCalled(); + expect(track!.isMuted).toBe(true); + } + }); +}); diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 11f35424..27c53726 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -6,12 +6,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { + ConnectionState as LivekitConnectionState, + type LocalTrackPublication, LocalVideoTrack, + ParticipantEvent, type Room as LivekitRoom, Track, - type LocalTrack, - type LocalTrackPublication, - ConnectionState as LivekitConnectionState, } from "livekit-client"; import { map, @@ -40,31 +40,36 @@ import { type ObservableScope } from "../../ObservableScope.ts"; * The Publisher is also responsible for creating the media tracks. */ export class Publisher { - public tracks: LocalTrack[] = []; + /** + * By default, livekit will start publishing tracks as soon as they are created. + * In the matrix RTC world, we want to control when tracks are published based + * on whether the user is part of the RTC session or not. + */ + public shouldPublish = false; + /** * Creates a new Publisher. * @param scope - The observable scope to use for managing the publisher. * @param connection - The connection to use for publishing. * @param devices - The media devices to use for audio and video input. * @param muteStates - The mute states for audio and video. - * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). + * @param logger - The logger to use for logging :D. */ public constructor( private scope: ObservableScope, - private connection: Connection, + private connection: Pick, //setE2EEEnabled, devices: MediaDevices, private readonly muteStates: MuteStates, trackerProcessorState$: Behavior, - private logger?: Logger, + private logger: Logger, ) { - this.logger?.info("[PublishConnection] Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); const room = connection.livekitRoom; room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => { - this.logger?.error("Failed to set E2EE enabled on room", e); + this.logger.error("Failed to set E2EE enabled on room", e); }); // Setup track processor syncing (blur) @@ -74,117 +79,187 @@ export class Publisher { this.workaroundRestartAudioInputTrackChrome(devices, scope); this.scope.onEnd(() => { - this.logger?.info( - "[PublishConnection] Scope ended -> stop publishing all tracks", - ); + this.logger.info("Scope ended -> stop publishing all tracks"); void this.stopPublishing(); + muteStates.audio.unsetHandler(); + muteStates.video.unsetHandler(); }); + + this.connection.livekitRoom.localParticipant.on( + ParticipantEvent.LocalTrackPublished, + this.onLocalTrackPublished.bind(this), + ); + } + + // LiveKit will publish the tracks as soon as they are created + // but we want to control when tracks are published. + // We cannot just mute the tracks, even if this will effectively stop the publishing, + // it would also prevent the user from seeing their own video/audio preview. + // So for that we use pauseUpStream(): Stops sending media to the server by replacing + // the sender track with null, but keeps the local MediaStreamTrack active. + // The user can still see/hear themselves locally, but remote participants see nothing. + private onLocalTrackPublished( + localTrackPublication: LocalTrackPublication, + ): void { + this.logger.info("Local track published", localTrackPublication); + const lkRoom = this.connection.livekitRoom; + if (!this.shouldPublish) { + this.pauseUpstreams(lkRoom, [localTrackPublication.source]).catch((e) => { + this.logger.error(`Failed to pause upstreams`, e); + }); + } + // also check the mute state and apply it + if (localTrackPublication.source === Track.Source.Microphone) { + const enabled = this.muteStates.audio.enabled$.value; + lkRoom.localParticipant.setMicrophoneEnabled(enabled).catch((e) => { + this.logger.error( + `Failed to enable microphone track, enabled:${enabled}`, + e, + ); + }); + } else if (localTrackPublication.source === Track.Source.Camera) { + const enabled = this.muteStates.video.enabled$.value; + lkRoom.localParticipant.setCameraEnabled(enabled).catch((e) => { + this.logger.error( + `Failed to enable camera track, enabled:${enabled}`, + e, + ); + }); + } + } + /** + * Create and setup local audio and video tracks based on the current mute states. + * It creates the tracks only if audio and/or video is enabled, to avoid unnecessary + * permission prompts. + * + * It also observes mute state changes to update LiveKit microphone/camera states accordingly. + * If a track is not created initially because disabled, it will be created when unmuting. + * + * This call is not blocking anymore, instead callers can listen to the + * `RoomEvent.MediaDevicesError` event in the LiveKit room to be notified of any errors. + * + */ + public async createAndSetupTracks(): Promise { + this.logger.debug("createAndSetupTracks called"); + const lkRoom = this.connection.livekitRoom; + // Observe mute state changes and update LiveKit microphone/camera states accordingly + this.observeMuteStates(); + + // Check if audio and/or video is enabled. We only create tracks if enabled, + // because it could prompt for permission, and we don't want to do that unnecessarily. + const audio = this.muteStates.audio.enabled$.value; + const video = this.muteStates.video.enabled$.value; + + // We don't await the creation, because livekit could block until the tracks + // are fully published, and not only that they are created. + // We don't have control on that, localParticipant creates and publishes the tracks + // asap. + // We are using the `ParticipantEvent.LocalTrackPublished` to be notified + // when tracks are actually published, and at that point + // we can pause upstream if needed (depending on if startPublishing has been called). + if (audio && video) { + // Enable both at once in order to have a single permission prompt! + void lkRoom.localParticipant.enableCameraAndMicrophone(); + } else if (audio) { + void lkRoom.localParticipant.setMicrophoneEnabled(true); + } else if (video) { + void lkRoom.localParticipant.setCameraEnabled(true); + } + + return Promise.resolve(); + } + + private async pauseUpstreams( + lkRoom: LivekitRoom, + sources: Track.Source[], + ): Promise { + for (const source of sources) { + const track = lkRoom.localParticipant.getTrackPublication(source)?.track; + if (track) { + await track.pauseUpstream(); + } else { + this.logger.warn( + `No track found for source ${source} to pause upstream`, + ); + } + } + } + + private async resumeUpstreams( + lkRoom: LivekitRoom, + sources: Track.Source[], + ): Promise { + for (const source of sources) { + const track = lkRoom.localParticipant.getTrackPublication(source)?.track; + if (track) { + await track.resumeUpstream(); + } else { + this.logger.warn( + `No track found for source ${source} to resume upstream`, + ); + } + } } /** - * Start the connection to LiveKit and publish local tracks. * - * This will: - * wait for the connection to be ready. - // * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) - // * 2. Use this token to request the SFU config to the MatrixRtc authentication service. - // * 3. Connect to the configured LiveKit room. - // * 4. Create local audio and video tracks based on the current mute states and publish them to the room. + * Request to publish local tracks to the LiveKit room. + * This will wait for the connection to be ready before publishing. + * Livekit also have some local retry logic for publishing tracks. + * Can be called multiple times, localparticipant manages the state of published tracks (or pending publications). * - * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. - * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. + * @returns */ - public async createAndSetupTracks(): Promise { - const lkRoom = this.connection.livekitRoom; - // Observe mute state changes and update LiveKit microphone/camera states accordingly - this.observeMuteStates(this.scope); - - // TODO: This should be an autostarted connection no need to start here. just check the connection state. - // TODO: This will fetch the JWT token. Perhaps we could keep it preloaded - // instead? This optimization would only be safe for a publish connection, - // because we don't want to leak the user's intent to perhaps join a call to - // remote servers before they actually commit to it. - // const { promise, resolve, reject } = Promise.withResolvers(); - // const sub = this.connection.state$.subscribe((s) => { - // if (s.state === "FailedToStart") { - // reject(new Error("Disconnected from LiveKit server")); - // } else if (s.state === "ConnectedToLkRoom") { - // resolve(); - // } - // }); - // try { - // await promise; - // } catch (e) { - // throw e; - // } finally { - // sub.unsubscribe(); - // } - // TODO-MULTI-SFU: Prepublish a microphone track - const audio = this.muteStates.audio.enabled$.value; - const video = this.muteStates.video.enabled$.value; - // createTracks throws if called with audio=false and video=false - if (audio || video) { - // TODO this can still throw errors? It will also prompt for permissions if not already granted - this.tracks = - (await lkRoom.localParticipant - .createTracks({ - audio, - video, - }) - .catch((error) => { - this.logger?.error("Failed to create tracks", error); - })) ?? []; + public async startPublishing(): Promise { + if (this.shouldPublish) { + this.logger.debug(`Already publishing, ignoring startPublishing call`); + return; } - return this.tracks; - } + this.shouldPublish = true; + this.logger.debug("startPublishing called"); - public async startPublishing(): Promise { const lkRoom = this.connection.livekitRoom; - const { promise, resolve, reject } = Promise.withResolvers(); - const sub = this.connection.state$.subscribe((s) => { - switch (s.state) { - case "ConnectedToLkRoom": - resolve(); - break; - case "FailedToStart": - reject(new Error("Failed to connect to LiveKit server")); - break; - default: - this.logger?.info("waiting for connection: ", s.state); - } - }); + + // Resume upstream for both audio and video tracks + // We need to call it explicitly because call setTrackEnabled does not always + // resume upstream. It will only if you switch the track from disabled to enabled, + // but if the track is already enabled but upstream is paused, it won't resume it. + // TODO what about screen share? try { - await promise; + await this.resumeUpstreams(lkRoom, [ + Track.Source.Microphone, + Track.Source.Camera, + ]); } catch (e) { - throw e; - } finally { - sub.unsubscribe(); + this.logger.error(`Failed to resume upstreams`, e); } - for (const track of this.tracks) { - // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally - // with a timeout. - await lkRoom.localParticipant.publishTrack(track).catch((error) => { - this.logger?.error("Failed to publish track", error); - }); - - // TODO: check if the connection is still active? and break the loop if not? - } - return this.tracks; } public async stopPublishing(): Promise { - // TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope - // actually has the right lifetime - this.muteStates.audio.unsetHandler(); - this.muteStates.video.unsetHandler(); + this.logger.debug("stopPublishing called"); + this.shouldPublish = false; + // Pause upstream will stop sending media to the server, while keeping + // the local MediaStreamTrack active, so the user can still see themselves. + await this.pauseUpstreams(this.connection.livekitRoom, [ + Track.Source.Microphone, + Track.Source.Camera, + Track.Source.ScreenShare, + ]); + } - const localParticipant = this.connection.livekitRoom.localParticipant; - const tracks: LocalTrack[] = []; - const addToTracksIfDefined = (p: LocalTrackPublication): void => { - if (p.track !== undefined) tracks.push(p.track); - }; - localParticipant.trackPublications.forEach(addToTracksIfDefined); - await localParticipant.unpublishTracks(tracks); + public async stopTracks(): Promise { + const lkRoom = this.connection.livekitRoom; + for (const source of [ + Track.Source.Microphone, + Track.Source.Camera, + Track.Source.ScreenShare, + ]) { + const localPub = lkRoom.localParticipant.getTrackPublication(source); + if (localPub?.track) { + // stops and unpublishes the track + await lkRoom.localParticipant.unpublishTrack(localPub!.track, true); + } + } } /// Private methods @@ -221,6 +296,9 @@ export class Publisher { // the process of being restarted. activeMicTrack.mediaStreamTrack.readyState !== "ended" ) { + this.logger?.info( + "Restarting audio device track due to active media device changed (workaroundRestartAudioInputTrackChrome)", + ); // Restart the track, which will cause Livekit to do another // getUserMedia() call with deviceId: default to get the *new* default device. // Note that room.switchActiveDevice() won't work: Livekit will ignore it because @@ -229,7 +307,7 @@ export class Publisher { .getTrackPublication(Track.Source.Microphone) ?.audioTrack?.restartTrack() .catch((e) => { - this.logger?.error(`Failed to restart audio device track`, e); + this.logger.error(`Failed to restart audio device track`, e); }); } }); @@ -249,7 +327,7 @@ export class Publisher { selected$.pipe(scope.bind()).subscribe((device) => { if (lkRoom.state != LivekitConnectionState.Connected) return; // if (this.connectionState$.value !== ConnectionState.Connected) return; - this.logger?.info( + this.logger.info( "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", lkRoom.getActiveDevice(kind), " !== ", @@ -262,7 +340,7 @@ export class Publisher { lkRoom .switchActiveDevice(kind, device.id) .catch((e: Error) => - this.logger?.error( + this.logger.error( `Failed to sync ${kind} device with LiveKit`, e, ), @@ -278,30 +356,37 @@ export class Publisher { /** * Observe changes in the mute states and update the LiveKit room accordingly. - * @param scope * @private */ - private observeMuteStates(scope: ObservableScope): void { + private observeMuteStates(): void { const lkRoom = this.connection.livekitRoom; - this.muteStates.audio.setHandler(async (desired) => { + this.muteStates.audio.setHandler(async (enable) => { try { - await lkRoom.localParticipant.setMicrophoneEnabled(desired); - } catch (e) { - this.logger?.error( - "Failed to update LiveKit audio input mute state", - e, + this.logger.debug( + `handler: Setting LiveKit microphone enabled: ${enable}`, ); + await lkRoom.localParticipant.setMicrophoneEnabled(enable); + // Unmute will restart the track if it was paused upstream, + // but until explicitly requested, we want to keep it paused. + if (!this.shouldPublish && enable) { + await this.pauseUpstreams(lkRoom, [Track.Source.Microphone]); + } + } catch (e) { + this.logger.error("Failed to update LiveKit audio input mute state", e); } return lkRoom.localParticipant.isMicrophoneEnabled; }); - this.muteStates.video.setHandler(async (desired) => { + this.muteStates.video.setHandler(async (enable) => { try { - await lkRoom.localParticipant.setCameraEnabled(desired); + this.logger.debug(`handler: Setting LiveKit camera enabled: ${enable}`); + await lkRoom.localParticipant.setCameraEnabled(enable); + // Unmute will restart the track if it was paused upstream, + // but until explicitly requested, we want to keep it paused. + if (!this.shouldPublish && enable) { + await this.pauseUpstreams(lkRoom, [Track.Source.Camera]); + } } catch (e) { - this.logger?.error( - "Failed to update LiveKit video input mute state", - e, - ); + this.logger.error("Failed to update LiveKit video input mute state", e); } return lkRoom.localParticipant.isCameraEnabled; }); @@ -315,7 +400,7 @@ export class Publisher { const track$ = scope.behavior( observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( map((trackRef) => { - const track = trackRef?.publication?.track; + const track = trackRef?.publication.track; return track instanceof LocalVideoTrack ? track : null; }), ), diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 3f58bcf6..c1e24eb4 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -30,13 +30,17 @@ import { logger } from "matrix-js-sdk/lib/logger"; import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { Connection, + ConnectionState, type ConnectionOpts, - type ConnectionState, - type PublishingParticipant, } from "./Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; -import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; +import { + ElementCallError, + FailToGetOpenIdToken, +} from "../../../utils/errors.ts"; +import { testJWTToken } from "../../../utils/test-fixtures.ts"; +import { mockRemoteParticipant } from "../../../utils/test.ts"; let testScope: ObservableScope; @@ -47,11 +51,6 @@ let fakeLivekitRoom: MockedObject; let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; -let fakeRoomEventEmiter: EventEmitter; -// let fakeMembershipsFocusMap$: BehaviorSubject< -// { membership: CallMembership; transport: LivekitTransport }[] -// >; - const livekitFocus: LivekitTransport = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", @@ -88,22 +87,25 @@ function setupTest(): void { localParticipantEventEmiter, ), } as unknown as LocalParticipant); - fakeRoomEventEmiter = new EventEmitter(); + const fakeRoomEventEmitter = new EventEmitter(); fakeLivekitRoom = vi.mocked({ connect: vi.fn(), disconnect: vi.fn(), remoteParticipants: new Map(), localParticipant: fakeLocalParticipant, state: LivekitConnectionState.Disconnected, - on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), - off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), - addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), + on: fakeRoomEventEmitter.on.bind(fakeRoomEventEmitter), + off: fakeRoomEventEmitter.off.bind(fakeRoomEventEmitter), + addListener: fakeRoomEventEmitter.addListener.bind(fakeRoomEventEmitter), removeListener: - fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), + fakeRoomEventEmitter.removeListener.bind(fakeRoomEventEmitter), removeAllListeners: - fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), + fakeRoomEventEmitter.removeAllListeners.bind(fakeRoomEventEmitter), setE2EEEnabled: vi.fn().mockResolvedValue(undefined), + emit: (eventName: string | symbol, ...args: unknown[]) => { + fakeRoomEventEmitter.emit(eventName, ...args); + }, } as unknown as LivekitRoom); } @@ -120,12 +122,21 @@ function setupRemoteConnection(): Connection { status: 200, body: { url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: "ATOKEN", + jwt: testJWTToken, }, }; }); - fakeLivekitRoom.connect.mockResolvedValue(undefined); + fakeLivekitRoom.connect.mockImplementation(async (): Promise => { + const changeEv = RoomEvent.ConnectionStateChanged; + + fakeLivekitRoom.state = LivekitConnectionState.Connecting; + fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state); + fakeLivekitRoom.state = LivekitConnectionState.Connected; + fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state); + + return Promise.resolve(); + }); return new Connection(opts, logger); } @@ -148,7 +159,7 @@ describe("Start connection states", () => { }; const connection = new Connection(opts, logger); - expect(connection.state$.getValue().state).toEqual("Initialized"); + expect(connection.state$.getValue()).toEqual("Initialized"); }); it("fail to getOpenId token then error state", async () => { @@ -164,7 +175,7 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: ConnectionState[] = []; + const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -184,22 +195,20 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState!.state).toEqual("FetchingConfig"); + expect(capturedState!).toEqual("FetchingConfig"); deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); await vi.runAllTimersAsync(); capturedState = capturedStates.pop(); - if (capturedState!.state === "FailedToStart") { - expect(capturedState!.error.message).toEqual("Something went wrong"); - expect(capturedState!.transport.livekit_alias).toEqual( + if (capturedState instanceof Error) { + expect(capturedState.message).toEqual("Something went wrong"); + expect(connection.transport.livekit_alias).toEqual( livekitFocus.livekit_alias, ); } else { - expect.fail( - "Expected FailedToStart state but got " + capturedState?.state, - ); + expect.fail("Expected FailedToStart state but got " + capturedState); } }); @@ -216,7 +225,7 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: ConnectionState[] = []; + const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -238,24 +247,25 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState?.state).toEqual("FetchingConfig"); + expect(capturedState).toEqual(ConnectionState.FetchingConfig); deferredSFU.resolve(); await vi.runAllTimersAsync(); capturedState = capturedStates.pop(); - if (capturedState?.state === "FailedToStart") { - expect(capturedState?.error.message).toContain( - "SFU Config fetch failed with exception Error", + if ( + capturedState instanceof ElementCallError && + capturedState.cause instanceof Error + ) { + expect(capturedState.cause.message).toContain( + "SFU Config fetch failed with exception", ); - expect(capturedState?.transport.livekit_alias).toEqual( + expect(connection.transport.livekit_alias).toEqual( livekitFocus.livekit_alias, ); } else { - expect.fail( - "Expected FailedToStart state but got " + capturedState?.state, - ); + expect.fail("Expected FailedToStart state but got " + capturedState); } }); @@ -272,7 +282,7 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: ConnectionState[] = []; + const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -285,7 +295,7 @@ describe("Start connection states", () => { status: 200, body: { url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: "ATOKEN", + jwt: testJWTToken, }, }; }); @@ -302,18 +312,21 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState?.state).toEqual("FetchingConfig"); + expect(capturedState).toEqual(ConnectionState.FetchingConfig); deferredSFU.resolve(); await vi.runAllTimersAsync(); capturedState = capturedStates.pop(); - if (capturedState && capturedState?.state === "FailedToStart") { - expect(capturedState.error.message).toContain( + if ( + capturedState instanceof ElementCallError && + capturedState.cause instanceof Error + ) { + expect(capturedState.cause.message).toContain( "Failed to connect to livekit", ); - expect(capturedState.transport.livekit_alias).toEqual( + expect(connection.transport.livekit_alias).toEqual( livekitFocus.livekit_alias, ); } else { @@ -329,7 +342,7 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); - const capturedStates: ConnectionState[] = []; + const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -339,13 +352,15 @@ describe("Start connection states", () => { await vi.runAllTimersAsync(); const initialState = capturedStates.shift(); - expect(initialState?.state).toEqual("Initialized"); + expect(initialState).toEqual(ConnectionState.Initialized); const fetchingState = capturedStates.shift(); - expect(fetchingState?.state).toEqual("FetchingConfig"); + expect(fetchingState).toEqual(ConnectionState.FetchingConfig); + const disconnectedState = capturedStates.shift(); + expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected); const connectingState = capturedStates.shift(); - expect(connectingState?.state).toEqual("ConnectingToLkRoom"); + expect(connectingState).toEqual(ConnectionState.LivekitConnecting); const connectedState = capturedStates.shift(); - expect(connectedState?.state).toEqual("ConnectedToLkRoom"); + expect(connectedState).toEqual(ConnectionState.LivekitConnected); }); it("shutting down the scope should stop the connection", async () => { @@ -363,46 +378,32 @@ describe("Start connection states", () => { }); }); -function fakeRemoteLivekitParticipant( - id: string, - publications: number = 1, -): RemoteParticipant { - return { - identity: id, - getTrackPublications: () => Array(publications), - } as unknown as RemoteParticipant; -} - -describe("Publishing participants observations", () => { - it("should emit the list of publishing participants", () => { +describe("remote participants", () => { + it("emits the list of remote participants", () => { setupTest(); const connection = setupRemoteConnection(); - const bobIsAPublisher = Promise.withResolvers(); - const danIsAPublisher = Promise.withResolvers(); - const observedPublishers: PublishingParticipant[][] = []; - const s = connection.remoteParticipantsWithTracks$.subscribe( - (publishers) => { - observedPublishers.push(publishers); - if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) { - bobIsAPublisher.resolve(); - } - if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) { - danIsAPublisher.resolve(); - } - }, - ); + const observedParticipants: RemoteParticipant[][] = []; + const s = connection.remoteParticipants$.subscribe((participants) => { + observedParticipants.push(participants); + }); onTestFinished(() => s.unsubscribe()); - // The publishingParticipants$ observable is derived from the current members of the + // The remoteParticipants$ observable is derived from the current members of the // livekitRoom and the rtc membership in order to publish the members that are publishing // on this connection. let participants: RemoteParticipant[] = [ - fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 0), - fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0), - fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 0), - fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 0), + mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }), + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), + mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }), + // Mock Dan to have no published tracks. We want him to still show show up + // in the participants list. + mockRemoteParticipant({ + identity: "@dan:example.org:DEV333", + getTrackPublication: () => undefined, + getTrackPublications: () => [], + }), ]; // Let's simulate 3 members on the livekitRoom @@ -411,24 +412,26 @@ describe("Publishing participants observations", () => { ); participants.forEach((p) => - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p), + fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p), ); - // At this point there should be no publishers - expect(observedPublishers.pop()!.length).toEqual(0); + // At this point there should be ~~no~~ publishers + // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks) + // The filtering we do is just based on the matrixRTC member events. + expect(observedParticipants.pop()!.length).toEqual(4); participants = [ - fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 1), - fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1), - fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 1), - fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 2), + mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }), + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), + mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }), + mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }), ]; participants.forEach((p) => - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p), + fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p), ); // At this point there should be no publishers - expect(observedPublishers.pop()!.length).toEqual(4); + expect(observedParticipants.pop()!.length).toEqual(4); }); it("should be scoped to parent scope", (): void => { @@ -436,16 +439,14 @@ describe("Publishing participants observations", () => { const connection = setupRemoteConnection(); - let observedPublishers: PublishingParticipant[][] = []; - const s = connection.remoteParticipantsWithTracks$.subscribe( - (publishers) => { - observedPublishers.push(publishers); - }, - ); + let observedParticipants: RemoteParticipant[][] = []; + const s = connection.remoteParticipants$.subscribe((participants) => { + observedParticipants.push(participants); + }); onTestFinished(() => s.unsubscribe()); let participants: RemoteParticipant[] = [ - fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0), + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), ]; // Let's simulate 3 members on the livekitRoom @@ -454,38 +455,29 @@ describe("Publishing participants observations", () => { ); for (const participant of participants) { - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); + fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, participant); } - // At this point there should be no publishers - expect(observedPublishers.pop()!.length).toEqual(0); - - participants = [fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1)]; - - for (const participant of participants) { - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); - } - - // We should have bob has a publisher now - const publishers = observedPublishers.pop(); - expect(publishers?.length).toEqual(1); - expect(publishers?.[0]?.identity).toEqual("@bob:example.org:DEV111"); + // We should have bob as a participant now + const ps = observedParticipants.pop(); + expect(ps?.length).toEqual(1); + expect(ps?.[0]?.identity).toEqual("@bob:example.org:DEV111"); // end the parent scope testScope.end(); - observedPublishers = []; + observedParticipants = []; - // SHOULD NOT emit any more publishers as the scope is ended + // SHOULD NOT emit any more participants as the scope is ended participants = participants.filter( (p) => p.identity !== "@bob:example.org:DEV111", ); - fakeRoomEventEmiter.emit( + fakeLivekitRoom.emit( RoomEvent.ParticipantDisconnected, - fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), ); - expect(observedPublishers.length).toEqual(0); + expect(observedParticipants.length).toEqual(0); }); }); diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index c17fae2b..41dfe665 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -12,14 +12,11 @@ import { } from "@livekit/components-core"; import { ConnectionError, - type ConnectionState as LivekitConenctionState, type Room as LivekitRoom, - type LocalParticipant, type RemoteParticipant, - RoomEvent, } from "livekit-client"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, map, type Observable } from "rxjs"; +import { BehaviorSubject, map } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { @@ -30,12 +27,12 @@ import { import { type Behavior } from "../../Behavior.ts"; import { type ObservableScope } from "../../ObservableScope.ts"; import { + ElementCallError, InsufficientCapacityError, SFURoomCreationRestrictedError, + UnknownCallError, } from "../../../utils/errors.ts"; -export type PublishingParticipant = LocalParticipant | RemoteParticipant; - export interface ConnectionOpts { /** The media transport to connect to. */ transport: LivekitTransport; @@ -47,19 +44,30 @@ export interface ConnectionOpts { /** Optional factory to create the LiveKit room, mainly for testing purposes. */ livekitRoomFactory: () => LivekitRoom; } +export class FailedToStartError extends Error { + public constructor(message: string) { + super(message); + this.name = "FailedToStartError"; + } +} -export type ConnectionState = - | { state: "Initialized" } - | { state: "FetchingConfig"; transport: LivekitTransport } - | { state: "ConnectingToLkRoom"; transport: LivekitTransport } - | { state: "PublishingTracks"; transport: LivekitTransport } - | { state: "FailedToStart"; error: Error; transport: LivekitTransport } - | { - state: "ConnectedToLkRoom"; - livekitConnectionState$: Observable; - transport: LivekitTransport; - } - | { state: "Stopped"; transport: LivekitTransport }; +export enum ConnectionState { + /** The start state of a connection. It has been created but nothing has loaded yet. */ + Initialized = "Initialized", + /** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room */ + FetchingConfig = "FetchingConfig", + Stopped = "Stopped", + /** The same as ConnectionState.Disconnected from `livekit-client` */ + LivekitDisconnected = "disconnected", + /** The same as ConnectionState.Connecting from `livekit-client` */ + LivekitConnecting = "connecting", + /** The same as ConnectionState.Connected from `livekit-client` */ + LivekitConnected = "connected", + /** The same as ConnectionState.Reconnecting from `livekit-client` */ + LivekitReconnecting = "reconnecting", + /** The same as ConnectionState.SignalReconnecting from `livekit-client` */ + LivekitSignalReconnecting = "signalReconnecting", +} /** * A connection to a Matrix RTC LiveKit backend. @@ -68,14 +76,32 @@ export type ConnectionState = */ export class Connection { // Private Behavior - private readonly _state$ = new BehaviorSubject({ - state: "Initialized", - }); + private readonly _state$ = new BehaviorSubject< + ConnectionState | ElementCallError + >(ConnectionState.Initialized); /** * The current state of the connection to the media transport. */ - public readonly state$: Behavior = this._state$; + public readonly state$: Behavior = this._state$; + + /** + * The media transport to connect to. + */ + public readonly transport: LivekitTransport; + + public readonly livekitRoom: LivekitRoom; + + private scope: ObservableScope; + + /** + * The remote LiveKit participants that are visible on this connection. + * + * Note that this may include participants that are connected only to + * subscribe, or publishers that are otherwise unattested in MatrixRTC state. + * It is therefore more low-level than what should be presented to the user. + */ + public readonly remoteParticipants$: Behavior; /** * Whether the connection has been stopped. @@ -96,24 +122,29 @@ export class Connection { * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. */ - // TODO dont make this throw and instead store a connection error state in this class? // TODO consider an autostart pattern... public async start(): Promise { this.logger.debug("Starting Connection"); this.stopped = false; try { - this._state$.next({ - state: "FetchingConfig", - transport: this.transport, - }); + this._state$.next(ConnectionState.FetchingConfig); + // We should already have this information after creating the localTransport. + // It would probably be better to forward this here. const { url, jwt } = await this.getSFUConfigWithOpenID(); // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - this._state$.next({ - state: "ConnectingToLkRoom", - transport: this.transport, - }); + // Setup observer once we are done with getSFUConfigWithOpenID + connectionStateObserver(this.livekitRoom) + .pipe( + this.scope.bind(), + map((s) => s as unknown as ConnectionState), + ) + .subscribe((lkState) => { + // It is save to cast lkState to ConnectionState as they are fully overlapping. + this._state$.next(lkState); + }); + try { await this.livekitRoom.connect(url, jwt); } catch (e) { @@ -128,7 +159,8 @@ export class Connection { throw new InsufficientCapacityError(); } if (e.status === 404) { - // error msg is "Could not establish signal connection: requested room does not exist" + // error msg is "Failed to create call" + // error description is "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists." // The room does not exist. There are two different modes of operation for the SFU: // - the room is created on the fly when connecting (livekit `auto_create` option) // - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service) @@ -140,19 +172,16 @@ export class Connection { } // If we were stopped while connecting, don't proceed to update state. if (this.stopped) return; - - this._state$.next({ - state: "ConnectedToLkRoom", - transport: this.transport, - livekitConnectionState$: connectionStateObserver(this.livekitRoom), - }); } catch (error) { this.logger.debug(`Failed to connect to LiveKit room: ${error}`); - this._state$.next({ - state: "FailedToStart", - error: error instanceof Error ? error : new Error(`${error}`), - transport: this.transport, - }); + this._state$.next( + error instanceof ElementCallError + ? error + : error instanceof Error + ? new UnknownCallError(error) + : new UnknownCallError(new Error(`${error}`)), + ); + // Its okay to ignore the throw. The error is part of the state. throw error; } } @@ -177,30 +206,11 @@ export class Connection { ); if (this.stopped) return; await this.livekitRoom.disconnect(); - this._state$.next({ - state: "Stopped", - transport: this.transport, - }); + this._state$.next(ConnectionState.Stopped); this.stopped = true; } - /** - * An observable of the participants that are publishing on this connection. (Excluding our local participant) - * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`. - * It filters the participants to only those that are associated with a membership that claims to publish on this connection. - */ - public readonly remoteParticipantsWithTracks$: Behavior< - PublishingParticipant[] - >; - - /** - * The media transport to connect to. - */ - public readonly transport: LivekitTransport; - private readonly client: OpenIDClientParts; - public readonly livekitRoom: LivekitRoom; - private readonly logger: Logger; /** @@ -208,36 +218,23 @@ export class Connection { * * @param opts - Connection options {@link ConnectionOpts}. * - * @param logger + * @param logger - The logger to use. */ public constructor(opts: ConnectionOpts, logger: Logger) { this.logger = logger.getChild("[Connection]"); this.logger.info( - `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, + `Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, ); const { transport, client, scope } = opts; + this.scope = scope; this.livekitRoom = opts.livekitRoomFactory(); this.transport = transport; this.client = client; - // REMOTE participants with track!!! - // this.remoteParticipantsWithTracks$ - this.remoteParticipantsWithTracks$ = scope.behavior( - // only tracks remote participants - connectedParticipantsObserver(this.livekitRoom, { - additionalRoomEvents: [ - RoomEvent.TrackPublished, - RoomEvent.TrackUnpublished, - ], - }).pipe( - map((participants) => { - return participants.filter( - (participant) => participant.getTrackPublications().length > 0, - ); - }), - ), - [], + this.remoteParticipants$ = scope.behavior( + // Only tracks remote participants + connectedParticipantsObserver(this.livekitRoom), ); scope.onEnd(() => { diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index f58fcb76..c3364059 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -7,13 +7,15 @@ Please see LICENSE in the repository root for full details. import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { - type E2EEOptions, Room as LivekitRoom, type RoomOptions, type BaseKeyProvider, + type E2EEManagerOptions, + type BaseE2EEManager, } from "livekit-client"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import E2EEWorker from "livekit-client/e2ee-worker?worker"; +// imported as inline to support worker when loaded from a cdn (cross domain) +import E2EEWorker from "livekit-client/e2ee-worker?worker&inline"; import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; @@ -41,9 +43,11 @@ export class ECConnectionFactory implements ConnectionFactory { * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens. * @param devices - Used for video/audio out/in capture options. * @param processorState$ - Effects like background blur (only for publishing connection?) - * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit Room. + * @param livekitKeyProvider - Optional key provider for end-to-end encryption. * @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app). * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used. + * @param echoCancellation - Whether to enable echo cancellation for audio capture. + * @param noiseSuppression - Whether to enable noise suppression for audio capture. */ public constructor( private client: OpenIDClientParts, @@ -52,20 +56,24 @@ export class ECConnectionFactory implements ConnectionFactory { livekitKeyProvider: BaseKeyProvider | undefined, private controlledAudioDevices: boolean, livekitRoomFactory?: () => LivekitRoom, + echoCancellation: boolean = true, + noiseSuppression: boolean = true, ) { const defaultFactory = (): LivekitRoom => new LivekitRoom( - generateRoomOption( - this.devices, - this.processorState$.value, - livekitKeyProvider && { + generateRoomOption({ + devices: this.devices, + processorState: this.processorState$.value, + e2eeLivekitOptions: livekitKeyProvider && { keyProvider: livekitKeyProvider, // It's important that every room use a separate E2EE worker. // They get confused if given streams from multiple rooms. worker: new E2EEWorker(), }, - this.controlledAudioDevices, - ), + controlledAudioDevices: this.controlledAudioDevices, + echoCancellation, + noiseSuppression, + }), ); this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; } @@ -90,12 +98,24 @@ export class ECConnectionFactory implements ConnectionFactory { /** * Generate the initial LiveKit RoomOptions based on the current media devices and processor state. */ -function generateRoomOption( - devices: MediaDevices, - processorState: ProcessorState, - e2eeLivekitOptions: E2EEOptions | undefined, - controlledAudioDevices: boolean, -): RoomOptions { +function generateRoomOption({ + devices, + processorState, + e2eeLivekitOptions, + controlledAudioDevices, + echoCancellation, + noiseSuppression, +}: { + devices: MediaDevices; + processorState: ProcessorState; + e2eeLivekitOptions: + | E2EEManagerOptions + | { e2eeManager: BaseE2EEManager } + | undefined; + controlledAudioDevices: boolean; + echoCancellation: boolean; + noiseSuppression: boolean; +}): RoomOptions { return { ...defaultLiveKitOptions, videoCaptureDefaults: { @@ -106,6 +126,8 @@ function generateRoomOption( audioCaptureDefaults: { ...defaultLiveKitOptions.audioCaptureDefaults, deviceId: devices.audioInput.selected$.value?.id, + echoCancellation, + noiseSuppression, }, audioOutput: { // When using controlled audio devices, we don't want to set the diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index 484a44e7..280d8ff7 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { type Participant as LivekitParticipant } from "livekit-client"; +import { type RemoteParticipant } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts"; @@ -52,7 +52,7 @@ beforeEach(() => { (transport: LivekitTransport, scope: ObservableScope) => { const mockConnection = { transport, - remoteParticipantsWithTracks$: new BehaviorSubject([]), + remoteParticipants$: new BehaviorSubject([]), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); @@ -200,24 +200,21 @@ describe("connections$ stream", () => { }); describe("connectionManagerData$ stream", () => { - // Used in test to control fake connections' remoteParticipantsWithTracks$ streams - let fakePublishingParticipantsStreams: Map< - string, - Behavior - >; + // Used in test to control fake connections' remoteParticipants$ streams + let fakeRemoteParticipantsStreams: Map>; function keyForTransport(transport: LivekitTransport): string { return `${transport.livekit_service_url}|${transport.livekit_alias}`; } beforeEach(() => { - fakePublishingParticipantsStreams = new Map(); + fakeRemoteParticipantsStreams = new Map(); - function getPublishingParticipantsFor( + function getRemoteParticipantsFor( transport: LivekitTransport, - ): Behavior { + ): Behavior { return ( - fakePublishingParticipantsStreams.get(keyForTransport(transport)) ?? + fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ?? new BehaviorSubject([]) ); } @@ -227,13 +224,12 @@ describe("connectionManagerData$ stream", () => { .fn() .mockImplementation( (transport: LivekitTransport, scope: ObservableScope) => { - const fakePublishingParticipants$ = new BehaviorSubject< - LivekitParticipant[] + const fakeRemoteParticipants$ = new BehaviorSubject< + RemoteParticipant[] >([]); const mockConnection = { transport, - remoteParticipantsWithTracks$: - getPublishingParticipantsFor(transport), + remoteParticipants$: getRemoteParticipantsFor(transport), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); @@ -242,36 +238,36 @@ describe("connectionManagerData$ stream", () => { void mockConnection.stop(); }); - fakePublishingParticipantsStreams.set( + fakeRemoteParticipantsStreams.set( keyForTransport(transport), - fakePublishingParticipants$, + fakeRemoteParticipants$, ); return mockConnection; }, ); }); - test("Should report connections with the publishing participants", () => { + test("Should report connections with the remote participants", () => { withTestScheduler(({ expectObservable, schedule, behavior }) => { // Setup the fake participants streams behavior // ============================== - fakePublishingParticipantsStreams.set( + fakeRemoteParticipantsStreams.set( keyForTransport(TRANSPORT_1), behavior("oa-b", { o: [], - a: [{ identity: "user1A" } as LivekitParticipant], + a: [{ identity: "user1A" } as RemoteParticipant], b: [ - { identity: "user1A" } as LivekitParticipant, - { identity: "user1B" } as LivekitParticipant, + { identity: "user1A" } as RemoteParticipant, + { identity: "user1B" } as RemoteParticipant, ], }), ); - fakePublishingParticipantsStreams.set( + fakeRemoteParticipantsStreams.set( keyForTransport(TRANSPORT_2), behavior("o-a", { o: [], - a: [{ identity: "user2A" } as LivekitParticipant], + a: [{ identity: "user2A" } as RemoteParticipant], }), ); // ============================== @@ -289,47 +285,47 @@ describe("connectionManagerData$ stream", () => { a: expect.toSatisfy((e) => { const data: ConnectionManagerData = e.value; expect(data.getConnections().length).toBe(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); return true; }), b: expect.toSatisfy((e) => { const data: ConnectionManagerData = e.value; expect(data.getConnections().length).toBe(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0); - expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( - "user1A", - ); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[0].identity, + ).toBe("user1A"); return true; }), c: expect.toSatisfy((e) => { const data: ConnectionManagerData = e.value; expect(data.getConnections().length).toBe(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1); - expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( - "user1A", - ); - expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe( - "user2A", - ); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[0].identity, + ).toBe("user1A"); + expect( + data.getParticipantsForTransport(TRANSPORT_2)[0].identity, + ).toBe("user2A"); return true; }), d: expect.toSatisfy((e) => { const data: ConnectionManagerData = e.value; expect(data.getConnections().length).toBe(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1); - expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( - "user1A", - ); - expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe( - "user1B", - ); - expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe( - "user2A", - ); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[0].identity, + ).toBe("user1A"); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[1].identity, + ).toBe("user1B"); + expect( + data.getParticipantsForTransport(TRANSPORT_2)[0].identity, + ).toBe("user2A"); return true; }), }); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index d9a0380e..101e34ed 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -6,13 +6,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - type LivekitTransport, - type ParticipantId, -} from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, combineLatest, map, of, switchMap, tap } from "rxjs"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { combineLatest, map, of, switchMap, tap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; +import { type RemoteParticipant } from "livekit-client"; import { type Behavior } from "../../Behavior.ts"; import { type Connection } from "./Connection.ts"; @@ -24,21 +21,18 @@ import { type ConnectionFactory } from "./ConnectionFactory.ts"; export class ConnectionManagerData { private readonly store: Map< string, - [Connection, (LocalParticipant | RemoteParticipant)[]] + { connection: Connection; participants: RemoteParticipant[] } > = new Map(); public constructor() {} - public add( - connection: Connection, - participants: (LocalParticipant | RemoteParticipant)[], - ): void { + public add(connection: Connection, participants: RemoteParticipant[]): void { const key = this.getKey(connection.transport); const existing = this.store.get(key); if (!existing) { - this.store.set(key, [connection, participants]); + this.store.set(key, { connection, participants }); } else { - existing[1].push(...participants); + existing.participants.push(...participants); } } @@ -47,59 +41,46 @@ export class ConnectionManagerData { } public getConnections(): Connection[] { - return Array.from(this.store.values()).map(([connection]) => connection); + return Array.from(this.store.values()).map(({ connection }) => connection); } public getConnectionForTransport( transport: LivekitTransport, ): Connection | null { - return this.store.get(this.getKey(transport))?.[0] ?? null; + return this.store.get(this.getKey(transport))?.connection ?? null; } - public getParticipantForTransport( + public getParticipantsForTransport( transport: LivekitTransport, - ): (LocalParticipant | RemoteParticipant)[] { + ): RemoteParticipant[] { const key = transport.livekit_service_url + "|" + transport.livekit_alias; const existing = this.store.get(key); if (existing) { - return existing[1]; + return existing.participants; } return []; } - /** - * Get all connections where the given participant is publishing. - * In theory, there could be several connections where the same participant is publishing but with - * only well behaving clients a participant should only be publishing on a single connection. - * @param participantId - */ - public getConnectionsForParticipant( - participantId: ParticipantId, - ): Connection[] { - const connections: Connection[] = []; - for (const [connection, participants] of this.store.values()) { - if (participants.some((p) => p.identity === participantId)) { - connections.push(connection); - } - } - return connections; - } } + interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; inputTransports$: Behavior>; logger: Logger; } + // TODO - write test for scopes (do we really need to bind scope) export interface IConnectionManager { - transports$: Behavior>; connectionManagerData$: Behavior>; } + /** * Crete a `ConnectionManager` - * @param scope the observable scope used by this object. - * @param connectionFactory used to create new connections. - * @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport. + * @param props - Configuration object + * @param props.scope - The observable scope used by this object + * @param props.connectionFactory - Used to create new connections + * @param props.inputTransports$ - A list of Behaviors each containing a LIST of LivekitTransport. + * @param props.logger - The logger to use * Each of these behaviors can be interpreted as subscribed list of transports. * * Using `registerTransports` independent external modules can control what connections @@ -116,9 +97,6 @@ export function createConnectionManager$({ logger: parentLogger, }: Props): IConnectionManager { const logger = parentLogger.getChild("[ConnectionManager]"); - - const running$ = new BehaviorSubject(true); - scope.onEnd(() => running$.next(false)); // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing /** @@ -130,10 +108,7 @@ export function createConnectionManager$({ * externally this is modified via `registerTransports()`. */ const transports$ = scope.behavior( - combineLatest([running$, inputTransports$]).pipe( - map(([running, transports]) => - transports.mapInner((transport) => (running ? transport : [])), - ), + inputTransports$.pipe( map((transports) => transports.mapInner(removeDuplicateTransports)), tap(({ value: transports }) => { logger.trace( @@ -183,23 +158,25 @@ export function createConnectionManager$({ const epoch = connections.epoch; // Map the connections to list of {connection, participants}[] - const listOfConnectionsWithPublishingParticipants = - connections.value.map((connection) => { - return connection.remoteParticipantsWithTracks$.pipe( + const listOfConnectionsWithRemoteParticipants = connections.value.map( + (connection) => { + return connection.remoteParticipants$.pipe( map((participants) => ({ connection, participants, })), ); - }); + }, + ); // probably not required - if (listOfConnectionsWithPublishingParticipants.length === 0) { + + if (listOfConnectionsWithRemoteParticipants.length === 0) { return of(new Epoch(new ConnectionManagerData(), epoch)); } // combineLatest the several streams into a single stream with the ConnectionManagerData - return combineLatest(listOfConnectionsWithPublishingParticipants).pipe( + return combineLatest(listOfConnectionsWithRemoteParticipants).pipe( map( (lists) => new Epoch( @@ -216,7 +193,7 @@ export function createConnectionManager$({ new Epoch(new ConnectionManagerData()), ); - return { transports$, connectionManagerData$ }; + return { connectionManagerData$ }; } function removeDuplicateTransports( diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts new file mode 100644 index 00000000..0c439a6b --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -0,0 +1,133 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { Room as LivekitRoom } from "livekit-client"; +import { BehaviorSubject } from "rxjs"; +import fetchMock from "fetch-mock"; +import { logger } from "matrix-js-sdk/lib/logger"; +import EventEmitter from "events"; + +import { ObservableScope } from "../../ObservableScope.ts"; +import { ECConnectionFactory } from "./ConnectionFactory.ts"; +import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import { exampleTransport, mockMediaDevices } from "../../../utils/test.ts"; +import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; +import { constant } from "../../Behavior"; + +// At the top of your test file, after imports +vi.mock("livekit-client", async (importOriginal) => { + return { + ...(await importOriginal()), + Room: vi.fn().mockImplementation(function (this: LivekitRoom, options) { + const emitter = new EventEmitter(); + return { + on: emitter.on.bind(emitter), + off: emitter.off.bind(emitter), + emit: emitter.emit.bind(emitter), + disconnect: vi.fn(), + remoteParticipants: new Map(), + } as unknown as LivekitRoom; + }), + }; +}); + +let testScope: ObservableScope; +let mockClient: OpenIDClientParts; + +beforeEach(() => { + testScope = new ObservableScope(); + mockClient = { + getOpenIdToken: vi.fn().mockReturnValue(""), + getDeviceId: vi.fn().mockReturnValue("DEV000"), + }; +}); + +describe("ECConnectionFactory - Audio inputs options", () => { + test.each([ + { echo: true, noise: true }, + { echo: true, noise: false }, + { echo: false, noise: true }, + { echo: false, noise: false }, + ])( + "it sets echoCancellation=$echo and noiseSuppression=$noise based on constructor parameters", + ({ echo, noise }) => { + // test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => { + const RoomConstructor = vi.mocked(LivekitRoom); + + const ecConnectionFactory = new ECConnectionFactory( + mockClient, + mockMediaDevices({}), + new BehaviorSubject({ + supported: true, + processor: undefined, + }), + undefined, + false, + undefined, + echo, + noise, + ); + ecConnectionFactory.createConnection(exampleTransport, testScope, logger); + + // Check if Room was constructed with expected options + expect(RoomConstructor).toHaveBeenCalledWith( + expect.objectContaining({ + audioCaptureDefaults: expect.objectContaining({ + echoCancellation: echo, + noiseSuppression: noise, + }), + }), + ); + }, + ); +}); + +describe("ECConnectionFactory - ControlledAudioDevice", () => { + test.each([{ controlled: true }, { controlled: false }])( + "it sets controlledAudioDevice=$controlled then uses deviceId accordingly", + ({ controlled }) => { + // test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => { + const RoomConstructor = vi.mocked(LivekitRoom); + + const ecConnectionFactory = new ECConnectionFactory( + mockClient, + mockMediaDevices({ + audioOutput: { + available$: constant(new Map()), + selected$: constant({ id: "DEV00", virtualEarpiece: false }), + select: () => {}, + }, + }), + new BehaviorSubject({ + supported: true, + processor: undefined, + }), + undefined, + controlled, + undefined, + false, + false, + ); + ecConnectionFactory.createConnection(exampleTransport, testScope, logger); + + // Check if Room was constructed with expected options + expect(RoomConstructor).toHaveBeenCalledWith( + expect.objectContaining({ + audioOutput: expect.objectContaining({ + deviceId: controlled ? undefined : "DEV00", + }), + }), + ); + }, + ); +}); + +afterEach(() => { + testScope.end(); + fetchMock.reset(); +}); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index e675f723..d26bac37 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -15,7 +15,7 @@ import { combineLatest, map, type Observable } from "rxjs"; import { type IConnectionManager } from "./ConnectionManager.ts"; import { - type MatrixLivekitMember, + type RemoteMatrixLivekitMember, createMatrixLivekitMembers$, } from "./MatrixLivekitMembers.ts"; import { @@ -91,7 +91,7 @@ test("should signal participant not yet connected to livekit", () => { }), ); - const matrixLivekitMember$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), connectionManager: { @@ -99,21 +99,24 @@ test("should signal participant not yet connected to livekit", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant$).toBe("a", { - a: null, - }); - expectObservable(data[0].connection$).toBe("a", { - a: null, - }); - return true; - }), - }); + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant.value$).toBe("a", { + a: null, + }); + expectObservable(data[0].connection$).toBe("a", { + a: null, + }); + return true; + }), + }, + ); }); }); @@ -171,7 +174,7 @@ test("should signal participant on a connection that is publishing", () => { }), ); - const matrixLivekitMember$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), connectionManager: { @@ -179,25 +182,28 @@ test("should signal participant on a connection that is publishing", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant$).toBe("a", { - a: expect.toSatisfy((participant) => { - expect(participant).toBeDefined(); - expect(participant!.identity).toEqual(bobParticipantId); - return true; - }), - }); - expectObservable(data[0].connection$).toBe("a", { - a: connection, - }); - return true; - }), - }); + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant.value$).toBe("a", { + a: expect.toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }), + }); + expectObservable(data[0].connection$).toBe("a", { + a: connection, + }); + return true; + }), + }, + ); }); }); @@ -222,7 +228,7 @@ test("should signal participant on a connection that is not publishing", () => { }), ); - const matrixLivekitMember$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), connectionManager: { @@ -230,21 +236,24 @@ test("should signal participant on a connection that is not publishing", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant$).toBe("a", { - a: null, - }); - expectObservable(data[0].connection$).toBe("a", { - a: connection, - }); - return true; - }), - }); + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant.value$).toBe("a", { + a: null, + }); + expectObservable(data[0].connection$).toBe("a", { + a: connection, + }); + return true; + }), + }, + ); }); }); @@ -283,7 +292,7 @@ describe("Publication edge case", () => { }), ); - const matrixLivekitMember$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior( membershipsWithTransport$, @@ -293,10 +302,10 @@ describe("Publication edge case", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( "a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { expect(data.length).toEqual(2); expectObservable(data[0].membership$).toBe("a", { a: bobMembership, @@ -305,7 +314,7 @@ describe("Publication edge case", () => { // The real connection should be from transportA as per the membership a: connectionA, }); - expectObservable(data[0].participant$).toBe("a", { + expectObservable(data[0].participant.value$).toBe("a", { a: expect.toSatisfy((participant) => { expect(participant).toBeDefined(); expect(participant!.identity).toEqual(bobParticipantId); @@ -349,7 +358,7 @@ describe("Publication edge case", () => { }), ); - const matrixLivekitMember$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior( membershipsWithTransport$, @@ -359,10 +368,10 @@ describe("Publication edge case", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( "a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { expect(data.length).toEqual(2); expectObservable(data[0].membership$).toBe("a", { a: bobMembership, @@ -371,7 +380,7 @@ describe("Publication edge case", () => { // The real connection should be from transportA as per the membership a: connectionA, }); - expectObservable(data[0].participant$).toBe("a", { + expectObservable(data[0].participant.value$).toBe("a", { // No participant as Bob is not publishing on his membership transport a: null, }); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 2f152630..6501adb4 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -5,10 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - type LocalParticipant as LocalLivekitParticipant, - type RemoteParticipant as RemoteLivekitParticipant, -} from "livekit-client"; +import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; import { type LivekitTransport, type CallMembership, @@ -24,22 +21,44 @@ import { generateItemsWithEpoch } from "../../../utils/observable"; const logger = rootLogger.getChild("[MatrixLivekitMembers]"); -/** - * Represents a Matrix call member and their associated LiveKit participation. - * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room - * or if it has no livekit transport at all. - */ -export interface MatrixLivekitMember { +interface LocalTaggedParticipant { + type: "local"; + value$: Behavior; +} +interface RemoteTaggedParticipant { + type: "remote"; + value$: Behavior; +} +export type TaggedParticipant = + | LocalTaggedParticipant + | RemoteTaggedParticipant; + +interface MatrixLivekitMember { membership$: Behavior; - participant$: Behavior< - LocalLivekitParticipant | RemoteLivekitParticipant | null - >; connection$: Behavior; // participantId: string; We do not want a participantId here since it will be generated by the jwt // TODO decide if we can also drop the userId. Its in the matrix membership anyways. userId: string; } +/** + * Represents the local Matrix call member and their associated LiveKit participation. + * `livekitParticipant` can be null if the member is not yet connected to the livekit room + * or if it has no livekit transport at all. + */ +export interface LocalMatrixLivekitMember extends MatrixLivekitMember { + participant: LocalTaggedParticipant; +} + +/** + * Represents a remote Matrix call member and their associated LiveKit participation. + * `livekitParticipant` can be null if the member is not yet connected to the livekit room + * or if it has no livekit transport at all. + */ +export interface RemoteMatrixLivekitMember extends MatrixLivekitMember { + participant: RemoteTaggedParticipant; +} + interface Props { scope: ObservableScope; membershipsWithTransport$: Behavior< @@ -61,7 +80,7 @@ export function createMatrixLivekitMembers$({ scope, membershipsWithTransport$, connectionManager, -}: Props): Behavior> { +}: Props): Behavior> { /** * Stream of all the call members and their associated livekit data (if available). */ @@ -91,7 +110,7 @@ export function createMatrixLivekitMembers$({ const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; const participants = transport - ? managerData.getParticipantForTransport(transport) + ? managerData.getParticipantsForTransport(transport) : []; const participant = participants.find((p) => p.identity == participantId) ?? null; @@ -108,14 +127,16 @@ export function createMatrixLivekitMembers$({ // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory. (scope, data$, participantId, userId) => { logger.debug( - `Updating data$ for participantId: ${participantId}, userId: ${userId}`, + `Generating member for participantId: ${participantId}, userId: ${userId}`, ); + const { participant$, ...rest } = scope.splitBehavior(data$); // will only get called once per `participantId, userId` pair. // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. return { participantId, userId, - ...scope.splitBehavior(data$), + participant: { type: "remote" as const, value$: participant$ }, + ...rest, }; }, ), diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index e3aa6be8..d885ddc6 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -29,10 +29,11 @@ import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx" import { areLivekitTransportsEqual, createMatrixLivekitMembers$, - type MatrixLivekitMember, + type RemoteMatrixLivekitMember, } from "./MatrixLivekitMembers.ts"; import { createConnectionManager$ } from "./ConnectionManager.ts"; import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; +import { testJWTToken } from "../../../utils/test-fixtures.ts"; // Test the integration of ConnectionManager and MatrixLivekitMerger @@ -85,7 +86,7 @@ beforeEach(() => { status: 200, body: { url: `wss://${domain}/livekit/sfu`, - jwt: "ATOKEN", + jwt: testJWTToken, }, }; }); @@ -124,15 +125,15 @@ test("bob, carl, then bob joining no tracks yet", () => { logger: logger, }); - const matrixLivekitItems$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, connectionManager, }); - expectObservable(matrixLivekitItems$).toBe(vMarble, { - a: expect.toSatisfy((e: Epoch) => { + expectObservable(matrixLivekitMembers$).toBe(vMarble, { + a: expect.toSatisfy((e: Epoch) => { const items = e.value; expect(items.length).toBe(1); const item = items[0]!; @@ -147,12 +148,12 @@ test("bob, carl, then bob joining no tracks yet", () => { ), ), }); - expectObservable(item.participant$).toBe("a", { + expectObservable(item.participant.value$).toBe("a", { a: null, }); return true; }), - b: expect.toSatisfy((e: Epoch) => { + b: expect.toSatisfy((e: Epoch) => { const items = e.value; expect(items.length).toBe(2); @@ -161,7 +162,7 @@ test("bob, carl, then bob joining no tracks yet", () => { expectObservable(item.membership$).toBe("a", { a: bobMembership, }); - expectObservable(item.participant$).toBe("a", { + expectObservable(item.participant.value$).toBe("a", { a: null, }); } @@ -172,7 +173,7 @@ test("bob, carl, then bob joining no tracks yet", () => { expectObservable(item.membership$).toBe("a", { a: carlMembership, }); - expectObservable(item.participant$).toBe("a", { + expectObservable(item.participant.value$).toBe("a", { a: null, }); expectObservable(item.connection$).toBe("a", { @@ -189,7 +190,7 @@ test("bob, carl, then bob joining no tracks yet", () => { } return true; }), - c: expect.toSatisfy((e: Epoch) => { + c: expect.toSatisfy((e: Epoch) => { const items = e.value; expect(items.length).toBe(3); @@ -216,7 +217,7 @@ test("bob, carl, then bob joining no tracks yet", () => { return true; }), }); - expectObservable(item.participant$).toBe("a", { + expectObservable(item.participant.value$).toBe("a", { a: null, }); } diff --git a/src/state/CallViewModelWidget.test.ts b/src/state/CallViewModelWidget.test.ts index afcf69ba..5d6442f1 100644 --- a/src/state/CallViewModelWidget.test.ts +++ b/src/state/CallViewModelWidget.test.ts @@ -15,6 +15,7 @@ import { constant } from "./Behavior.ts"; import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts"; import { ElementWidgetActions, widget } from "../widget.ts"; import { E2eeType } from "../e2ee/e2eeType.ts"; +import { MatrixRTCMode } from "../settings/settings.ts"; vi.mock("@livekit/components-core", { spy: true }); @@ -34,36 +35,43 @@ vi.mock("../widget", () => ({ }, })); -it("expect leave when ElementWidgetActions.HangupCall is called", async () => { - const pr = Promise.withResolvers(); - withCallViewModel( - { - remoteParticipants$: constant([aliceParticipant]), - rtcMembers$: constant([localRtcMember]), - }, - (vm: CallViewModel) => { - vm.leave$.subscribe((s: string) => { - pr.resolve(s); - }); +it.each([ + [MatrixRTCMode.Legacy], + [MatrixRTCMode.Compatibil], + [MatrixRTCMode.Matrix_2_0], +])( + "expect leave when ElementWidgetActions.HangupCall is called (%s mode)", + async (mode) => { + const pr = Promise.withResolvers(); + withCallViewModel(mode)( + { + remoteParticipants$: constant([aliceParticipant]), + rtcMembers$: constant([localRtcMember]), + }, + (vm: CallViewModel) => { + vm.leave$.subscribe((s: string) => { + pr.resolve(s); + }); - widget!.lazyActions!.emit( - ElementWidgetActions.HangupCall, - new CustomEvent(ElementWidgetActions.HangupCall, { - detail: { - action: "im.vector.hangup", - api: "toWidget", - data: {}, - requestId: "widgetapi-1761237395918", - widgetId: "mrUjS9T6uKUOWHMxXvLbSv0F", - }, - }), - ); - }, - { - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); + widget!.lazyActions!.emit( + ElementWidgetActions.HangupCall, + new CustomEvent(ElementWidgetActions.HangupCall, { + detail: { + action: "im.vector.hangup", + api: "toWidget", + data: {}, + requestId: "widgetapi-1761237395918", + widgetId: "mrUjS9T6uKUOWHMxXvLbSv0F", + }, + }), + ); + }, + { + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); - const source = await pr.promise; - expect(source).toBe("user"); -}); + const source = await pr.promise; + expect(source).toBe("user"); + }, +); diff --git a/src/state/MediaViewModel.test.ts b/src/state/MediaViewModel.test.ts index 61fa2d8c..92868216 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/MediaViewModel.test.ts @@ -20,6 +20,7 @@ import { createLocalMedia, createRemoteMedia, withTestScheduler, + mockRemoteParticipant, } from "../utils/test"; import { getValue } from "../utils/observable"; import { constant } from "./Behavior"; @@ -44,7 +45,11 @@ const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); test("control a participant's volume", () => { const setVolumeSpy = vi.fn(); - const vm = createRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy }); + const vm = createRemoteMedia( + rtcMembership, + {}, + mockRemoteParticipant({ setVolume: setVolumeSpy }), + ); withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab---c---d|", { a() { @@ -88,7 +93,7 @@ test("control a participant's volume", () => { }); test("toggle fit/contain for a participant's video", () => { - const vm = createRemoteMedia(rtcMembership, {}, {}); + const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab|", { a: () => vm.toggleFitContain(), @@ -199,3 +204,35 @@ test("switch cameras", async () => { }); expect(deviceId).toBe("front camera"); }); + +test("remote media is in waiting state when participant has not yet connected", () => { + const vm = createRemoteMedia(rtcMembership, {}, null); // null participant + expect(vm.waitingForMedia$.value).toBe(true); +}); + +test("remote media is not in waiting state when participant is connected", () => { + const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); + expect(vm.waitingForMedia$.value).toBe(false); +}); + +test("remote media is not in waiting state when participant is connected with no publications", () => { + const vm = createRemoteMedia( + rtcMembership, + {}, + mockRemoteParticipant({ + getTrackPublication: () => undefined, + getTrackPublications: () => [], + }), + ); + expect(vm.waitingForMedia$.value).toBe(false); +}); + +test("remote media is not in waiting state when user does not intend to publish anywhere", () => { + const vm = createRemoteMedia( + rtcMembership, + {}, + mockRemoteParticipant({}), + undefined, // No room (no advertised transport) + ); + expect(vm.waitingForMedia$.value).toBe(false); +}); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 74e64b93..9888d6bf 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -7,8 +7,8 @@ Please see LICENSE in the repository root for full details. import { type AudioSource, - type TrackReferenceOrPlaceholder, type VideoSource, + type TrackReference, observeParticipantEvents, observeParticipantMedia, roomEventSelector, @@ -33,7 +33,6 @@ import { type Observable, Subject, combineLatest, - distinctUntilKeyChanged, filter, fromEvent, interval, @@ -60,14 +59,11 @@ import { type ObservableScope } from "./ObservableScope"; export function observeTrackReference$( participant: Participant, source: Track.Source, -): Observable { +): Observable { return observeParticipantMedia(participant).pipe( - map(() => ({ - participant: participant, - publication: participant.getTrackPublication(source), - source, - })), - distinctUntilKeyChanged("publication"), + map(() => participant.getTrackPublication(source)), + distinctUntilChanged(), + map((publication) => publication && { participant, publication, source }), ); } @@ -226,7 +222,7 @@ abstract class BaseMediaViewModel { /** * The LiveKit video track for this media. */ - public readonly video$: Behavior; + public readonly video$: Behavior; /** * Whether there should be a warning that this media is unencrypted. */ @@ -241,10 +237,12 @@ abstract class BaseMediaViewModel { private observeTrackReference$( source: Track.Source, - ): Behavior { + ): Behavior { return this.scope.behavior( this.participant$.pipe( - switchMap((p) => (!p ? of(null) : observeTrackReference$(p, source))), + switchMap((p) => + !p ? of(undefined) : observeTrackReference$(p, source), + ), ), ); } @@ -268,7 +266,7 @@ abstract class BaseMediaViewModel { encryptionSystem: EncryptionSystem, audioSource: AudioSource, videoSource: VideoSource, - livekitRoom$: Behavior, + protected readonly livekitRoom$: Behavior, public readonly focusUrl$: Behavior, public readonly displayName$: Behavior, public readonly mxcAvatarUrl$: Behavior, @@ -281,8 +279,8 @@ abstract class BaseMediaViewModel { [audio$, this.video$], (a, v) => encryptionSystem.kind !== E2eeType.NONE && - (a?.publication?.isEncrypted === false || - v?.publication?.isEncrypted === false), + (a?.publication.isEncrypted === false || + v?.publication.isEncrypted === false), ), ); @@ -471,7 +469,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { private readonly videoTrack$: Observable = this.video$.pipe( switchMap((v) => { - const track = v?.publication?.track; + const track = v?.publication.track; if (!(track instanceof LocalVideoTrack)) return of(null); return merge( // Watch for track restarts because they indicate a camera switch. @@ -596,6 +594,21 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { * A remote participant's user media. */ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { + /** + * Whether we are waiting for this user's LiveKit participant to exist. This + * could be because either we or the remote party are still connecting. + */ + public readonly waitingForMedia$ = this.scope.behavior( + combineLatest( + [this.livekitRoom$, this.participant$], + (livekitRoom, participant) => + // If livekitRoom is undefined, the user is not attempting to publish on + // any transport and so we shouldn't expect a participant. (They might + // be a subscribe-only bot for example.) + livekitRoom !== undefined && participant === null, + ), + ); + // This private field is used to override the value from the superclass private __speaking$: Behavior; public get speaking$(): Behavior { diff --git a/src/state/MuteStates.test.ts b/src/state/MuteStates.test.ts new file mode 100644 index 00000000..f2a6e35f --- /dev/null +++ b/src/state/MuteStates.test.ts @@ -0,0 +1,212 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { BehaviorSubject } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { MuteStates, MuteState } from "./MuteStates"; +import { + type AudioOutputDeviceLabel, + type DeviceLabel, + type MediaDevice, + type SelectedAudioOutputDevice, + type SelectedDevice, +} from "./MediaDevices"; +import { constant } from "./Behavior"; +import { ObservableScope } from "./ObservableScope"; +import { flushPromises, mockMediaDevices } from "../utils/test"; + +const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); +vi.mock("../UrlParams", () => ({ getUrlParams })); + +let testScope: ObservableScope; + +beforeEach(() => { + testScope = new ObservableScope(); +}); + +afterEach(() => { + testScope.end(); +}); + +describe("MuteState", () => { + test("should automatically mute if force mute is set", async () => { + const forceMute$ = new BehaviorSubject(false); + + const deviceStub = { + available$: constant( + new Map([ + ["fbac11", { type: "name", name: "HD Camera" }], + ]), + ), + selected$: constant({ id: "fbac11" }), + select(): void {}, + } as unknown as MediaDevice; + + const muteState = new MuteState( + testScope, + deviceStub, + constant(true), + true, + forceMute$, + ); + let lastEnabled: boolean = false; + muteState.enabled$.subscribe((enabled) => { + lastEnabled = enabled; + }); + let setEnabled: ((enabled: boolean) => void) | null = null; + muteState.setEnabled$.subscribe((setter) => { + setEnabled = setter; + }); + + await flushPromises(); + + setEnabled!(true); + await flushPromises(); + expect(lastEnabled).toBe(true); + + // Now force mute + forceMute$.next(true); + await flushPromises(); + // Should automatically mute + expect(lastEnabled).toBe(false); + + // Try to unmute can not work + expect(setEnabled).toBeNull(); + + // Disable force mute + forceMute$.next(false); + await flushPromises(); + + // TODO I'd expect it to go back to previous state (enabled) + // but actually it goes back to the initial state from construction (disabled) + // Should go back to previous state (enabled) + // Skip for now + // expect(lastEnabled).toBe(true); + + // But yet it can be unmuted now + expect(setEnabled).not.toBeNull(); + + setEnabled!(true); + await flushPromises(); + expect(lastEnabled).toBe(true); + }); +}); + +describe("MuteStates", () => { + function aAudioOutputDevices(): MediaDevice< + AudioOutputDeviceLabel, + SelectedAudioOutputDevice + > { + const selected$ = new BehaviorSubject< + SelectedAudioOutputDevice | undefined + >({ + id: "default", + virtualEarpiece: false, + }); + return { + available$: constant( + new Map([ + ["default", { type: "speaker" }], + ["0000", { type: "speaker" }], + ["1111", { type: "earpiece" }], + ["222", { type: "name", name: "Bluetooth Speaker" }], + ]), + ), + selected$, + select(id: string): void { + if (!this.available$.getValue().has(id)) { + logger.warn(`Attempted to select unknown device id: ${id}`); + return; + } + selected$.next({ + id, + /** For test purposes we ignore this */ + virtualEarpiece: false, + }); + }, + }; + } + + function aVideoInput(): MediaDevice { + const selected$ = new BehaviorSubject( + undefined, + ); + return { + available$: constant( + new Map([ + ["0000", { type: "name", name: "HD Camera" }], + ["1111", { type: "name", name: "WebCam Pro" }], + ]), + ), + selected$, + select(id: string): void { + if (!this.available$.getValue().has(id)) { + logger.warn(`Attempted to select unknown device id: ${id}`); + return; + } + selected$.next({ id }); + }, + }; + } + + test("should mute camera when in earpiece mode", async () => { + const audioOutputDevice = aAudioOutputDevices(); + + const mediaDevices = mockMediaDevices({ + audioOutput: audioOutputDevice, + videoInput: aVideoInput(), + // other devices are not relevant for this test + }); + const muteStates = new MuteStates( + testScope, + mediaDevices, + // consider joined + constant(true), + ); + + let latestSyncedState: boolean | null = null; + muteStates.video.setHandler(async (enabled: boolean): Promise => { + logger.info(`Video mute state set to: ${enabled}`); + latestSyncedState = enabled; + return Promise.resolve(enabled); + }); + + let lastVideoEnabled: boolean = false; + muteStates.video.enabled$.subscribe((enabled) => { + lastVideoEnabled = enabled; + }); + + expect(muteStates.video.setEnabled$.value).toBeDefined(); + muteStates.video.setEnabled$.value?.(true); + await flushPromises(); + + expect(lastVideoEnabled).toBe(true); + + // Select earpiece audio output + audioOutputDevice.select("1111"); + await flushPromises(); + // Video should be automatically muted + expect(lastVideoEnabled).toBe(false); + expect(latestSyncedState).toBe(false); + + // Try to switch to speaker + audioOutputDevice.select("0000"); + await flushPromises(); + // TODO I'd expect it to go back to previous state (enabled)?? + // But maybe not? If you move the phone away from your ear you may not want it + // to automatically enable video? + expect(lastVideoEnabled).toBe(false); + + // But yet it can be unmuted now + expect(muteStates.video.setEnabled$.value).toBeDefined(); + muteStates.video.setEnabled$.value?.(true); + await flushPromises(); + expect(lastVideoEnabled).toBe(true); + }); +}); diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index 50be5e05..632e0426 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -27,7 +27,7 @@ import { ElementWidgetActions, widget } from "../widget"; import { Config } from "../config/Config"; import { getUrlParams } from "../UrlParams"; import { type ObservableScope } from "./ObservableScope"; -import { type Behavior } from "./Behavior"; +import { type Behavior, constant } from "./Behavior"; interface MuteStateData { enabled$: Observable; @@ -38,31 +38,58 @@ interface MuteStateData { export type Handler = (desired: boolean) => Promise; const defaultHandler: Handler = async (desired) => Promise.resolve(desired); -class MuteState { +/** + * Internal class - exported only for testing purposes. + * Do not use directly outside of tests. + */ +export class MuteState { + // TODO: rewrite this to explain behavior, it is not understandable, and cannot add logging private readonly enabledByDefault$ = this.enabledByConfig && !getUrlParams().skipLobby ? this.joined$.pipe(map((isJoined) => !isJoined)) : of(false); private readonly handler$ = new BehaviorSubject(defaultHandler); + public setHandler(handler: Handler): void { if (this.handler$.value !== defaultHandler) throw new Error("Multiple mute state handlers are not supported"); this.handler$.next(handler); } + public unsetHandler(): void { this.handler$.next(defaultHandler); } + private readonly canControlDevices$ = combineLatest([ + this.device.available$, + this.forceMute$, + ]).pipe( + map(([available, forceMute]) => { + return !forceMute && available.size > 0; + }), + ); + private readonly data$ = this.scope.behavior( - this.device.available$.pipe( - map((available) => available.size > 0), + this.canControlDevices$.pipe( distinctUntilChanged(), withLatestFrom( this.enabledByDefault$, - (devicesConnected, enabledByDefault) => { - if (!devicesConnected) + (canControlDevices, enabledByDefault) => { + logger.info( + `MuteState: canControlDevices: ${canControlDevices}, enabled by default: ${enabledByDefault}`, + ); + if (!canControlDevices) { + logger.info( + `MuteState: devices connected: ${canControlDevices}, disabling`, + ); + // We need to sync the mute state with the handler + // to ensure nothing is beeing published. + this.handler$.value(false).catch((err) => { + logger.error("MuteState-disable: handler error", err); + }); return { enabled$: of(false), set: null, toggle: null }; + } // Assume the default value only once devices are actually connected let enabled = enabledByDefault; @@ -135,21 +162,45 @@ class MuteState { private readonly device: MediaDevice, private readonly joined$: Observable, private readonly enabledByConfig: boolean, + /** + * An optional observable which, when it emits `true`, will force the mute. + * Used for video to stop camera when earpiece mode is on. + * @private + */ + private readonly forceMute$: Observable, ) {} } export class MuteStates { + /** + * True if the selected audio output device is an earpiece. + * Used to force-disable video when on earpiece. + */ + private readonly isEarpiece$ = combineLatest( + this.mediaDevices.audioOutput.available$, + this.mediaDevices.audioOutput.selected$, + ).pipe( + map(([available, selected]) => { + if (!selected?.id) return false; + const device = available.get(selected.id); + logger.info(`MuteStates: selected audio output device:`, device); + return device?.type === "earpiece"; + }), + ); + public readonly audio = new MuteState( this.scope, this.mediaDevices.audioInput, this.joined$, Config.get().media_devices.enable_audio, + constant(false), ); public readonly video = new MuteState( this.scope, this.mediaDevices.videoInput, this.joined$, Config.get().media_devices.enable_video, + this.isEarpiece$, ); public constructor( diff --git a/src/state/ObservableScope.test.ts b/src/state/ObservableScope.test.ts index 99f2b424..31728f39 100644 --- a/src/state/ObservableScope.test.ts +++ b/src/state/ObservableScope.test.ts @@ -5,9 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { BehaviorSubject, combineLatest, Subject } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; +import { sleep } from "matrix-js-sdk/lib/utils"; import { Epoch, @@ -102,3 +103,137 @@ describe("Epoch", () => { s$.complete(); }); }); + +describe("Reconcile", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("should wait clean up before processing next", async () => { + vi.useFakeTimers(); + const scope = new ObservableScope(); + const behavior$ = new BehaviorSubject(0); + + const setup = vi.fn().mockImplementation(async () => await sleep(100)); + const cleanup = vi + .fn() + .mockImplementation(async (n: number) => await sleep(100)); + scope.reconcile(behavior$, async (value) => { + await setup(); + return async (): Promise => { + await cleanup(value); + }; + }); + // Let the initial setup process + await vi.advanceTimersByTimeAsync(120); + expect(setup).toHaveBeenCalledTimes(1); + expect(cleanup).toHaveBeenCalledTimes(0); + + // Send next value + behavior$.next(1); + await vi.advanceTimersByTimeAsync(50); + // Should not have started setup for 1 yet + expect(setup).toHaveBeenCalledTimes(1); + expect(cleanup).toHaveBeenCalledTimes(1); + expect(cleanup).toHaveBeenCalledWith(0); + + // Let cleanup finish + await vi.advanceTimersByTimeAsync(50); + // Now setup for 1 should have started + expect(setup).toHaveBeenCalledTimes(2); + }); + + it("should skip intermediates values that are not setup", async () => { + vi.useFakeTimers(); + const scope = new ObservableScope(); + const behavior$ = new BehaviorSubject(0); + + const setup = vi + .fn() + .mockImplementation(async (n: number) => await sleep(100)); + + const cleanupLock = Promise.withResolvers(); + const cleanup = vi + .fn() + .mockImplementation(async (n: number) => await cleanupLock.promise); + + scope.reconcile(behavior$, async (value) => { + await setup(value); + return async (): Promise => { + await cleanup(value); + }; + }); + // Let the initial setup process (0) + await vi.advanceTimersByTimeAsync(120); + + // Send 4 next values quickly + behavior$.next(1); + behavior$.next(2); + behavior$.next(3); + behavior$.next(4); + + await vi.advanceTimersByTimeAsync(3000); + // should have only called cleanup for 0 + expect(cleanup).toHaveBeenCalledTimes(1); + expect(cleanup).toHaveBeenCalledWith(0); + // Let cleanup finish + cleanupLock.resolve(undefined); + await vi.advanceTimersByTimeAsync(120); + + // Now setup for 4 should have started, skipping 1,2,3 + expect(setup).toHaveBeenCalledTimes(2); + expect(setup).toHaveBeenCalledWith(4); + expect(setup).not.toHaveBeenCalledWith(1); + expect(setup).not.toHaveBeenCalledWith(2); + expect(setup).not.toHaveBeenCalledWith(3); + }); + + it("should wait for setup to complete before starting cleanup", async () => { + vi.useFakeTimers(); + const scope = new ObservableScope(); + const behavior$ = new BehaviorSubject(0); + + const setup = vi + .fn() + .mockImplementation(async (n: number) => await sleep(3000)); + + const cleanupLock = Promise.withResolvers(); + const cleanup = vi + .fn() + .mockImplementation(async (n: number) => await cleanupLock.promise); + + scope.reconcile(behavior$, async (value) => { + await setup(value); + return async (): Promise => { + await cleanup(value); + }; + }); + + await vi.advanceTimersByTimeAsync(500); + // Setup for 0 should be in progress + expect(setup).toHaveBeenCalledTimes(1); + + behavior$.next(1); + await vi.advanceTimersByTimeAsync(500); + + // Should not have started setup for 1 yet + expect(setup).not.toHaveBeenCalledWith(1); + // Should not have called cleanup yet, because the setup for 0 is not done + expect(cleanup).toHaveBeenCalledTimes(0); + + // Let setup for 0 finish + await vi.advanceTimersByTimeAsync(2500 + 100); + // Now cleanup for 0 should have started + expect(cleanup).toHaveBeenCalledTimes(1); + expect(cleanup).toHaveBeenCalledWith(0); + + cleanupLock.resolve(undefined); + await vi.advanceTimersByTimeAsync(100); + // Now setup for 1 should have started + expect(setup).toHaveBeenCalledWith(1); + }); +}); diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 27f501c7..e3fc644f 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -123,8 +123,22 @@ export class ObservableScope { callback: (value: T) => Promise<(() => Promise) | void>, ): void { let latestValue: T | typeof nothing = nothing; - let reconciledValue: T | typeof nothing = nothing; + let reconcilePromise: Promise | undefined = undefined; let cleanUp: (() => Promise) | void = undefined; + let prevVal: T | typeof nothing = nothing; + + // While this loop runs it will process the latest from `value$` until it caught up with the updates. + // It might skip updates from `value$` and only process the newest value after callback has resolved. + const reconcileLoop = async (): Promise => { + while (latestValue !== prevVal) { + await cleanUp?.(); // Call the previous value's clean-up handler + prevVal = latestValue; + + if (latestValue !== nothing) cleanUp = await callback(latestValue); // Sync current value... + // `latestValue` might have gotten updated during the `await callback`. That is why we loop here + } + }; + value$ .pipe( catchError(() => EMPTY), // Ignore errors @@ -132,23 +146,15 @@ export class ObservableScope { endWith(nothing), // Clean up when the scope ends ) .subscribe((value) => { - void (async (): Promise => { - if (latestValue === nothing) { - latestValue = value; - while (latestValue !== reconciledValue) { - await cleanUp?.(); // Call the previous value's clean-up handler - reconciledValue = latestValue; - if (latestValue !== nothing) - cleanUp = await callback(latestValue); // Sync current value - } - // Reset to signal that reconciliation is done for now - latestValue = nothing; - } else { - // There's already an instance of the above 'while' loop running - // concurrently. Just update the latest value and let it be handled. - latestValue = value; - } - })(); + // Always track the latest value! The `reconcileLoop` will run until it "processed" the "last" `latestValue`. + latestValue = value; + // There's already an instance of the below 'reconcileLoop' loop running + // concurrently. So lets let the loop handle it. NEVER instanciate two `reconcileLoop`s. + if (reconcilePromise) return; + + reconcilePromise = reconcileLoop().finally(() => { + reconcilePromise = undefined; + }); }); } diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index 38f22122..690870e6 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -27,6 +27,7 @@ import type { ReactionOption } from "../reactions"; import { observeSpeaker$ } from "./observeSpeaker.ts"; import { generateItems } from "../utils/observable.ts"; import { ScreenShare } from "./ScreenShare.ts"; +import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts"; /** * Sorting bins defining the order in which media tiles appear in the layout. @@ -68,40 +69,46 @@ enum SortingBin { * for inclusion in the call layout and tracks associated screen shares. */ export class UserMedia { - public readonly vm: UserMediaViewModel = this.participant$.value?.isLocal - ? new LocalUserMediaViewModel( - this.scope, - this.id, - this.userId, - this.participant$ as Behavior, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.mediaDevices, - this.displayName$, - this.mxcAvatarUrl$, - this.scope.behavior(this.handRaised$), - this.scope.behavior(this.reaction$), - ) - : new RemoteUserMediaViewModel( - this.scope, - this.id, - this.userId, - this.participant$ as Behavior, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.pretendToBeDisconnected$, - this.displayName$, - this.mxcAvatarUrl$, - this.scope.behavior(this.handRaised$), - this.scope.behavior(this.reaction$), - ); + public readonly vm: UserMediaViewModel = + this.participant.type === "local" + ? new LocalUserMediaViewModel( + this.scope, + this.id, + this.userId, + this.participant.value$, + this.encryptionSystem, + this.livekitRoom$, + this.focusUrl$, + this.mediaDevices, + this.displayName$, + this.mxcAvatarUrl$, + this.scope.behavior(this.handRaised$), + this.scope.behavior(this.reaction$), + ) + : new RemoteUserMediaViewModel( + this.scope, + this.id, + this.userId, + this.participant.value$, + this.encryptionSystem, + this.livekitRoom$, + this.focusUrl$, + this.pretendToBeDisconnected$, + this.displayName$, + this.mxcAvatarUrl$, + this.scope.behavior(this.handRaised$), + this.scope.behavior(this.reaction$), + ); private readonly speaker$ = this.scope.behavior( observeSpeaker$(this.vm.speaking$), ); + // TypeScript needs this widening of the type to happen in a separate statement + private readonly participant$: Behavior< + LocalParticipant | RemoteParticipant | null + > = this.participant.value$; + /** * All screen share media associated with this user media. */ @@ -184,9 +191,7 @@ export class UserMedia { private readonly scope: ObservableScope, public readonly id: string, private readonly userId: string, - private readonly participant$: Behavior< - LocalParticipant | RemoteParticipant | null - >, + private readonly participant: TaggedParticipant, private readonly encryptionSystem: EncryptionSystem, private readonly livekitRoom$: Behavior, private readonly focusUrl$: Behavior, diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index e3172a22..9bc0efb2 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -12,7 +12,11 @@ import { axe } from "vitest-axe"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { GridTile } from "./GridTile"; -import { mockRtcMembership, createRemoteMedia } from "../utils/test"; +import { + mockRtcMembership, + createRemoteMedia, + mockRemoteParticipant, +} from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsSenderProvider } from "../reactions/useReactionsSender"; import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; @@ -31,11 +35,11 @@ test("GridTile is accessible", async () => { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", }, - { + mockRemoteParticipant({ setVolume() {}, getTrackPublication: () => ({}) as Partial as RemoteTrackPublication, - }, + }), ); const fakeRtcSession = { diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 57409869..2f750c50 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -69,6 +69,7 @@ interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; mirror: boolean; locallyMuted: boolean; + waitingForMedia?: boolean; primaryButton?: ReactNode; menuStart?: ReactNode; menuEnd?: ReactNode; @@ -79,6 +80,7 @@ const UserMediaTile: FC = ({ vm, showSpeakingIndicators, locallyMuted, + waitingForMedia, primaryButton, menuStart, menuEnd, @@ -148,7 +150,7 @@ const UserMediaTile: FC = ({ const tile = ( = ({ raisedHandTime={handRaised ?? undefined} currentReaction={reaction ?? undefined} raisedHandOnClick={raisedHandOnClick} - localParticipant={vm.local} + waitingForMedia={waitingForMedia} focusUrl={focusUrl} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} @@ -290,6 +292,7 @@ const RemoteUserMediaTile: FC = ({ ...props }) => { const { t } = useTranslation(); + const waitingForMedia = useBehavior(vm.waitingForMedia$); const locallyMuted = useBehavior(vm.locallyMuted$); const localVolume = useBehavior(vm.localVolume$); const onSelectMute = useCallback( @@ -311,6 +314,7 @@ const RemoteUserMediaTile: FC = ({ { video: trackReference, userId: "@alice:example.com", mxcAvatarUrl: undefined, - localParticipant: false, focusable: true, }; @@ -66,24 +65,13 @@ describe("MediaView", () => { }); }); - describe("with no participant", () => { - it("shows avatar for local user", () => { - render( - , - ); + describe("with no video", () => { + it("shows avatar", () => { + render(); expect( screen.getByRole("img", { name: "@alice:example.com" }), ).toBeVisible(); - expect(screen.queryAllByText("Waiting for media...").length).toBe(0); - }); - it("shows avatar and label for remote user", () => { - render( - , - ); - expect( - screen.getByRole("img", { name: "@alice:example.com" }), - ).toBeVisible(); - expect(screen.getByText("Waiting for media...")).toBeVisible(); + expect(screen.queryByTestId("video")).toBe(null); }); }); @@ -94,6 +82,22 @@ describe("MediaView", () => { }); }); + describe("waitingForMedia", () => { + test("defaults to false", () => { + render(); + expect(screen.queryAllByText("Waiting for media...").length).toBe(0); + }); + test("shows and is accessible", async () => { + const { container } = render( + + + , + ); + expect(await axe(container)).toHaveNoViolations(); + expect(screen.getByText("Waiting for media...")).toBeVisible(); + }); + }); + describe("unencryptedWarning", () => { test("is shown and accessible", async () => { const { container } = render( diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index e8a30cd4..8bb38d94 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -43,7 +43,7 @@ interface Props extends ComponentProps { raisedHandTime?: Date; currentReaction?: ReactionOption; raisedHandOnClick?: () => void; - localParticipant: boolean; + waitingForMedia?: boolean; audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; // The focus url, mainly for debugging purposes @@ -71,7 +71,7 @@ export const MediaView: FC = ({ raisedHandTime, currentReaction, raisedHandOnClick, - localParticipant, + waitingForMedia, audioStreamStats, videoStreamStats, focusUrl, @@ -129,7 +129,7 @@ export const MediaView: FC = ({ /> )} - {!video && !localParticipant && ( + {waitingForMedia && (
{t("video_tile.waiting_for_media")}
diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index fb7008b8..981c0369 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -17,6 +17,7 @@ import { mockRtcMembership, createLocalMedia, createRemoteMedia, + mockRemoteParticipant, } from "../utils/test"; import { SpotlightTileViewModel } from "../state/TileViewModel"; import { constant } from "../state/Behavior"; @@ -33,7 +34,7 @@ test("SpotlightTile is accessible", async () => { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", }, - {}, + mockRemoteParticipant({}), ); const vm2 = createLocalMedia( diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 48dd0f8c..8109784f 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -38,6 +38,7 @@ import { type MediaViewModel, ScreenShareViewModel, type UserMediaViewModel, + type RemoteUserMediaViewModel, } from "../state/MediaViewModel"; import { useInitial } from "../useInitial"; import { useMergedRefs } from "../useMergedRefs"; @@ -84,6 +85,21 @@ const SpotlightLocalUserMediaItem: FC = ({ SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem"; +interface SpotlightRemoteUserMediaItemProps + extends SpotlightUserMediaItemBaseProps { + vm: RemoteUserMediaViewModel; +} + +const SpotlightRemoteUserMediaItem: FC = ({ + vm, + ...props +}) => { + const waitingForMedia = useBehavior(vm.waitingForMedia$); + return ( + + ); +}; + interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps { vm: UserMediaViewModel; } @@ -103,7 +119,7 @@ const SpotlightUserMediaItem: FC = ({ return vm instanceof LocalUserMediaViewModel ? ( ) : ( - + ); }; diff --git a/src/useAudioContext.tsx b/src/useAudioContext.tsx index 59334dda..4d08dde8 100644 --- a/src/useAudioContext.tsx +++ b/src/useAudioContext.tsx @@ -22,9 +22,12 @@ import * as controls from "./controls"; * Play a sound though a given AudioContext. Will take * care of connecting the correct buffer and gating * through gain. - * @param volume The volume to play at. * @param ctx The context to play through. * @param buffer The buffer to play. + * @param volume The volume to play at. + * @param stereoPan The stereo pan to apply. + * @param delayS Delay in seconds before starting playing. + * @param abort Optional AbortController that can be used to stop playback. * @returns A promise that resolves when the sound has finished playing. */ async function playSound( @@ -55,9 +58,11 @@ async function playSound( * Play a sound though a given AudioContext, looping until stopped. Will take * care of connecting the correct buffer and gating * through gain. - * @param volume The volume to play at. * @param ctx The context to play through. * @param buffer The buffer to play. + * @param volume The volume to play at. + * @param stereoPan The stereo pan to apply. + * @param delayS Delay in seconds between each loop. * @returns A function used to end the sound. This function will return a promise when the sound has stopped. */ function playSoundLooping( @@ -120,7 +125,7 @@ interface UseAudioContext { /** * Add an audio context which can be used to play * a set of preloaded sounds. - * @param props + * @param props The properties for the audio context. * @returns Either an instance that can be used to play sounds, or null if not ready. */ export function useAudioContext( diff --git a/src/utils/displayname.ts b/src/utils/displayname.ts index 5ab5de9b..bc49b29e 100644 --- a/src/utils/displayname.ts +++ b/src/utils/displayname.ts @@ -77,6 +77,13 @@ export function shouldDisambiguate( ); } +/** + * Calculates a display name for a member, optionally disambiguating it. + * @param member - The member to calculate the display name for. + * @param member.rawDisplayName - The raw display name of the member + * @param member.userId - The user ID of the member + * @param disambiguate - Whether to disambiguate the display name. + */ export function calculateDisplayName( member: { rawDisplayName?: string; userId: string }, disambiguate: boolean, diff --git a/src/utils/errors.ts b/src/utils/errors.ts index b77c0ff0..cddf90de 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -13,6 +13,8 @@ export enum ErrorCode { */ MISSING_MATRIX_RTC_TRANSPORT = "MISSING_MATRIX_RTC_TRANSPORT", CONNECTION_LOST_ERROR = "CONNECTION_LOST_ERROR", + INTERNAL_MEMBERSHIP_MANAGER = "INTERNAL_MEMBERSHIP_MANAGER", + FAILED_TO_START_LIVEKIT = "FAILED_TO_START_LIVEKIT", /** LiveKit indicates that the server has hit its track limits */ INSUFFICIENT_CAPACITY_ERROR = "INSUFFICIENT_CAPACITY_ERROR", E2EE_NOT_SUPPORTED = "E2EE_NOT_SUPPORTED", @@ -27,6 +29,7 @@ export enum ErrorCategory { NETWORK_CONNECTIVITY = "NETWORK_CONNECTIVITY", CLIENT_CONFIGURATION = "CLIENT_CONFIGURATION", UNKNOWN = "UNKNOWN", + SYSTEM_FAILURE = "SYSTEM_FAILURE", // SYSTEM_FAILURE / FEDERATION_FAILURE .. } @@ -54,9 +57,16 @@ export class ElementCallError extends Error { } } +/** + * Configuration problem due to no MatrixRTC backend/SFU is exposed via .well-known and no fallback configured. + */ export class MatrixRTCTransportMissingError extends ElementCallError { public domain: string; + /** + * Creates an instance of MatrixRTCTransportMissingError. + * @param domain - The domain where the MatrixRTC transport is missing. + */ public constructor(domain: string) { super( t("error.call_is_not_supported"), @@ -72,6 +82,9 @@ export class MatrixRTCTransportMissingError extends ElementCallError { } } +/** + * Error indicating that the connection to the call was lost and could not be re-established. + */ export class ConnectionLostError extends ElementCallError { public constructor() { super( @@ -83,6 +96,30 @@ export class ConnectionLostError extends ElementCallError { } } +/** + * Error indicating a failure in the membership manager causing the join call + * operation to fail. + */ +export class MembershipManagerError extends ElementCallError { + /** + * Creates an instance of MembershipManagerError. + * + * @param error - The underlying error that caused the membership manager failure. + */ + public constructor(error: Error) { + super( + t("error.membership_manager"), + ErrorCode.INTERNAL_MEMBERSHIP_MANAGER, + ErrorCategory.SYSTEM_FAILURE, + t("error.membership_manager_description"), + error, + ); + } +} + +/** + * Error indicating that end-to-end encryption is not supported in the current environment. + */ export class E2EENotSupportedError extends ElementCallError { public constructor() { super( @@ -94,7 +131,14 @@ export class E2EENotSupportedError extends ElementCallError { } } +/** + * Error indicating an unknown issue occurred during a call operation. + */ export class UnknownCallError extends ElementCallError { + /** + * Creates an instance of UnknownCallError. + * @param error - The underlying error that caused the unknown issue. + */ public constructor(error: Error) { super( t("error.generic"), @@ -107,7 +151,14 @@ export class UnknownCallError extends ElementCallError { } } +/** + * Error indicating a failure to obtain an OpenID token. + */ export class FailToGetOpenIdToken extends ElementCallError { + /** + * Creates an instance of FailToGetOpenIdToken. + * @param error - The underlying error that caused the failure. + */ public constructor(error: Error) { super( t("error.generic"), @@ -120,6 +171,27 @@ export class FailToGetOpenIdToken extends ElementCallError { } } +/** + * Error indicating a failure to start publishing on a LiveKit connection. + */ +export class FailToStartLivekitConnection extends ElementCallError { + /** + * Creates an instance of FailToStartLivekitConnection. + * @param e - An optional error message providing additional context. + */ + public constructor(e?: string) { + super( + t("error.failed_to_start_livekit"), + ErrorCode.FAILED_TO_START_LIVEKIT, + ErrorCategory.NETWORK_CONNECTIVITY, + e, + ); + } +} + +/** + * Error indicating that a LiveKit's server has hit its track limits. + */ export class InsufficientCapacityError extends ElementCallError { public constructor() { super( @@ -131,6 +203,10 @@ export class InsufficientCapacityError extends ElementCallError { } } +/** + * Error indicating that room creation is restricted by the SFU. + * Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service) + */ export class SFURoomCreationRestrictedError extends ElementCallError { public constructor() { super( diff --git a/src/utils/matrix.ts b/src/utils/matrix.ts index 0a2b5c1a..4e3ae3c3 100644 --- a/src/utils/matrix.ts +++ b/src/utils/matrix.ts @@ -188,7 +188,6 @@ function fullAliasFromRoomName(roomName: string, client: MatrixClient): string { * Applies some basic sanitisation to a room name that the user * has given us * @param input The room name from the user - * @param client A matrix client object */ export function sanitiseRoomNameInput(input: string): string { // check to see if the user has entered a fully qualified room @@ -304,8 +303,9 @@ export async function createRoom( /** * Returns an absolute URL to that will load Element Call with the given room * @param roomId ID of the room - * @param roomName Name of the room * @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses + * @param roomName Name of the room + * @param viaServers Optional list of servers to include as 'via' parameters in the URL */ export function getAbsoluteRoomUrl( roomId: string, @@ -321,8 +321,9 @@ export function getAbsoluteRoomUrl( /** * Returns a relative URL to that will load Element Call with the given room * @param roomId ID of the room - * @param roomName Name of the room * @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses + * @param roomName Name of the room + * @param viaServers Optional list of servers to include as 'via' parameters in the URL */ export function getRelativeRoomUrl( roomId: string, diff --git a/src/utils/media.ts b/src/utils/media.ts index cdd81aa7..3750aa4e 100644 --- a/src/utils/media.ts +++ b/src/utils/media.ts @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. /** * Finds a media device with label matching 'deviceName' * @param deviceName The label of the device to look for + * @param kind The kind of media device to look for * @param devices The list of devices to search * @returns A matching media device or undefined if no matching device was found */ diff --git a/src/utils/observable.test.ts b/src/utils/observable.test.ts index d1034e7b..be677367 100644 --- a/src/utils/observable.test.ts +++ b/src/utils/observable.test.ts @@ -5,11 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { test } from "vitest"; -import { Subject } from "rxjs"; +import { expect, test } from "vitest"; +import { type Observable, of, Subject, switchMap } from "rxjs"; import { withTestScheduler } from "./test"; -import { generateItems, pauseWhen } from "./observable"; +import { filterBehavior, generateItems, pauseWhen } from "./observable"; +import { type Behavior } from "../state/Behavior"; test("pauseWhen", () => { withTestScheduler(({ behavior, expectObservable }) => { @@ -72,3 +73,31 @@ test("generateItems", () => { expectObservable(scope4$).toBe(scope4Marbles); }); }); + +test("filterBehavior", () => { + withTestScheduler(({ behavior, expectObservable }) => { + // Filtering the input should segment it into 2 modes of non-null behavior. + const inputMarbles = " abcxabx"; + const filteredMarbles = "a--xa-x"; + + const input$ = behavior(inputMarbles, { + a: "a", + b: "b", + c: "c", + x: null, + }); + const filtered$: Observable | null> = input$.pipe( + filterBehavior((value) => typeof value === "string"), + ); + + expectObservable(filtered$).toBe(filteredMarbles, { + a: expect.any(Object), + x: null, + }); + expectObservable( + filtered$.pipe( + switchMap((value$) => (value$ === null ? of(null) : value$)), + ), + ).toBe(inputMarbles, { a: "a", b: "b", c: "c", x: null }); + }); +}); diff --git a/src/utils/observable.ts b/src/utils/observable.ts index 053921cd..9739353f 100644 --- a/src/utils/observable.ts +++ b/src/utils/observable.ts @@ -22,6 +22,7 @@ import { withLatestFrom, BehaviorSubject, type OperatorFunction, + distinctUntilChanged, } from "rxjs"; import { type Behavior } from "../state/Behavior"; @@ -134,7 +135,6 @@ interface ItemHandle { * requested at a later time, and destroyed (have their scope ended) when the * key is no longer requested. * - * @param input$ The input value to be mapped. * @param generator A generator function yielding a tuple of keys and the * currently associated data for each item that it wants to exist. * @param factory A function constructing an individual item, given the item's key, @@ -185,6 +185,28 @@ export function generateItemsWithEpoch< ); } +/** + * Segments a behavior into periods during which its value matches the filter + * (outputting a behavior with a narrowed type) and periods during which it does + * not match (outputting null). + */ +export function filterBehavior( + predicate: (value: T) => value is S, +): OperatorFunction | null> { + return (input$) => + input$.pipe( + scan | null>((acc$, input) => { + if (predicate(input)) { + const output$ = acc$ ?? new BehaviorSubject(input); + output$.next(input); + return output$; + } + return null; + }, null), + distinctUntilChanged(), + ); +} + function generateItemsInternal< Input, Keys extends [unknown, ...unknown[]], diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index 4cf330b7..f915bb19 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -59,3 +59,17 @@ export const daveRTLRtcMember = mockRtcMembership("@dave2:example.org", "DDDD"); export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "\u202eevaD", }); + +export const testJWTToken = [ + {}, // header + { + // payload + sub: "@me:example.org:ABCDEF", + video: { + room: "!example_room_id", + }, + }, + {}, // signature +] + .map((d) => global.btoa(JSON.stringify(d))) + .join("."); diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 98c45d86..0745be72 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -37,6 +37,7 @@ import { import { aliceRtcMember, localRtcMember } from "./test-fixtures"; import { type RaisedHandInfo, type ReactionInfo } from "../reactions"; import { constant } from "../state/Behavior"; +import { MatrixRTCMode } from "../settings/settings"; mockConfig({ livekit: { livekit_service_url: "https://example.com" } }); @@ -162,6 +163,7 @@ export function getBasicCallViewModelEnvironment( setE2EEEnabled: async () => Promise.resolve(), }), connectionState$: constant(ConnectionState.Connected), + matrixRTCMode$: constant(MatrixRTCMode.Legacy), ...callViewModelOptions, }, handRaisedSubject$, diff --git a/src/utils/test.ts b/src/utils/test.ts index 471d35d8..9a845908 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -44,12 +44,12 @@ import { Track, } from "livekit-client"; import { randomUUID } from "crypto"; -import { - type RoomAndToDeviceEvents, - type RoomAndToDeviceEventsHandlerMap, -} from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport"; import { type TrackReference } from "@livekit/components-core"; import EventEmitter from "events"; +import { + type KeyTransportEvents, + type KeyTransportEventsHandlerMap, +} from "matrix-js-sdk/lib/matrixrtc/IKeyTransport"; import { LocalUserMediaViewModel, @@ -284,6 +284,8 @@ export function mockLivekitRoom( ): LivekitRoom { const livekitRoom = { options: {}, + setE2EEEnabled: vi.fn(), + ...mockEmitter(), ...room, } as Partial as LivekitRoom; @@ -306,7 +308,11 @@ export function mockLocalParticipant( return { isLocal: true, trackPublications: new Map(), - unpublishTracks: async () => Promise.resolve(), + publishTrack: vi.fn(), + unpublishTracks: vi.fn().mockResolvedValue([]), + createTracks: vi.fn(), + setMicrophoneEnabled: vi.fn(), + setCameraEnabled: vi.fn(), getTrackPublication: () => ({}) as Partial as LocalTrackPublication, ...mockEmitter(), @@ -315,12 +321,12 @@ export function mockLocalParticipant( } export function createLocalMedia( - localRtcMember: CallMembership, + rtcMember: CallMembership, roomMember: Partial, localParticipant: LocalParticipant, mediaDevices: MediaDevices, ): LocalUserMediaViewModel { - const member = mockMatrixRoomMember(localRtcMember, roomMember); + const member = mockMatrixRoomMember(rtcMember, roomMember); return new LocalUserMediaViewModel( testScope(), "local", @@ -355,23 +361,26 @@ export function mockRemoteParticipant( } export function createRemoteMedia( - localRtcMember: CallMembership, + rtcMember: CallMembership, roomMember: Partial, - participant: Partial, + participant: RemoteParticipant | null, + livekitRoom: LivekitRoom | undefined = mockLivekitRoom( + {}, + { + remoteParticipants$: of(participant ? [participant] : []), + }, + ), ): RemoteUserMediaViewModel { - const member = mockMatrixRoomMember(localRtcMember, roomMember); - const remoteParticipant = mockRemoteParticipant(participant); + const member = mockMatrixRoomMember(rtcMember, roomMember); return new RemoteUserMediaViewModel( testScope(), "remote", member.userId, - of(remoteParticipant), + constant(participant), { kind: E2eeType.PER_PARTICIPANT, }, - constant( - mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }), - ), + constant(livekitRoom), constant("https://rtc-example.org"), constant(false), constant(member.rawDisplayName ?? "nodisplayname"), @@ -394,9 +403,9 @@ export function mockConfig( } export class MockRTCSession extends TypedEventEmitter< - MatrixRTCSessionEvent | RoomAndToDeviceEvents | MembershipManagerEvent, - MatrixRTCSessionEventHandlerMap & - RoomAndToDeviceEventsHandlerMap & + MatrixRTCSessionEvent | MembershipManagerEvent | KeyTransportEvents, + KeyTransportEventsHandlerMap & + MatrixRTCSessionEventHandlerMap & MembershipManagerEventHandlerMap > { public asMockedSession(): MockedObject { diff --git a/src/widget.ts b/src/widget.ts index 60163c7c..7862df33 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -64,6 +64,12 @@ export const widget = ((): WidgetHelpers | null => { try { const { widgetId, parentUrl } = getUrlParams(); + const { roomId, userId, deviceId, baseUrl, e2eEnabled, allowIceFallback } = + getUrlParams(); + if (!roomId) throw new Error("Room ID must be supplied"); + if (!userId) throw new Error("User ID must be supplied"); + if (!deviceId) throw new Error("Device ID must be supplied"); + if (!baseUrl) throw new Error("Base URL must be supplied"); if (widgetId && parentUrl) { const parentOrigin = new URL(parentUrl).origin; logger.info("Widget API is available"); @@ -92,19 +98,6 @@ export const widget = ((): WidgetHelpers | null => { // We need to do this now rather than later because it has capabilities to // request, and is responsible for starting the transport (should it be?) - const { - roomId, - userId, - deviceId, - baseUrl, - e2eEnabled, - allowIceFallback, - } = getUrlParams(); - if (!roomId) throw new Error("Room ID must be supplied"); - if (!userId) throw new Error("User ID must be supplied"); - if (!deviceId) throw new Error("Device ID must be supplied"); - if (!baseUrl) throw new Error("Base URL must be supplied"); - // These are all the event types the app uses const sendEvent = [ EventType.CallNotify, // Sent as a deprecated fallback diff --git a/tsconfig.json b/tsconfig.json index e864ecfc..0f9e7c66 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -50,6 +50,11 @@ "plugins": [{ "name": "typescript-eslint-language-service" }] }, - "include": ["./src/**/*.ts", "./src/**/*.tsx", "./playwright/**/*.ts"], + "include": [ + "./src/**/*.ts", + "./src/**/*.tsx", + "./playwright/**/*.ts", + "./sdk/**/*.ts" + ], "exclude": ["**.test.ts"] } diff --git a/vite-sdk.config.ts b/vite-sdk.config.ts new file mode 100644 index 00000000..48fb6f22 --- /dev/null +++ b/vite-sdk.config.ts @@ -0,0 +1,28 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { defineConfig, mergeConfig } from "vite"; +import nodePolyfills from "vite-plugin-node-stdlib-browser"; + +const base = "./"; + +// Config for embedded deployments (possibly hosted under a non-root path) +export default defineConfig(() => ({ + worker: { format: "es" as const }, + base, // Use relative URLs to allow the app to be hosted under any path + build: { + sourcemap: true, + manifest: true, + lib: { + formats: ["es" as const], + entry: "./sdk/main.ts", + name: "MatrixrtcSdk", + fileName: "matrixrtc-sdk", + }, + }, + plugins: [nodePolyfills()], +})); diff --git a/vite.config.ts b/vite.config.ts index a0bb9de5..97d643ec 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -7,14 +7,17 @@ Please see LICENSE in the repository root for full details. import { loadEnv, + PluginOption, searchForWorkspaceRoot, type ConfigEnv, type UserConfig, } from "vite"; import svgrPlugin from "vite-plugin-svgr"; import { createHtmlPlugin } from "vite-plugin-html"; + import { codecovVitePlugin } from "@codecov/vite-plugin"; import { sentryVitePlugin } from "@sentry/vite-plugin"; + import react from "@vitejs/plugin-react"; import { realpathSync } from "fs"; import * as fs from "node:fs"; @@ -31,7 +34,7 @@ export default ({ // In future we might be able to do what is needed via code splitting at // build time. process.env.VITE_PACKAGE = packageType ?? "full"; - const plugins = [ + const plugins: PluginOption[] = [ react(), svgrPlugin({ svgrOptions: { @@ -41,16 +44,6 @@ export default ({ }, }), - createHtmlPlugin({ - entry: "src/main.tsx", - inject: { - data: { - brand: env.VITE_PRODUCT_NAME || "Element Call", - packageType: process.env.VITE_PACKAGE, - }, - }, - }), - codecovVitePlugin({ enableBundleAnalysis: process.env.CODECOV_TOKEN !== undefined, bundleName: "element-call", @@ -73,6 +66,18 @@ export default ({ ); } + plugins.push( + createHtmlPlugin({ + entry: "src/main.tsx", + inject: { + data: { + brand: env.VITE_PRODUCT_NAME || "Element Call", + packageType: process.env.VITE_PACKAGE, + }, + }, + }), + ); + // The crypto WASM module is imported dynamically. Since it's common // for developers to use a linked copy of matrix-js-sdk or Rust // crypto (which could reside anywhere on their file system), Vite diff --git a/yarn.lock b/yarn.lock index f7b4242e..505a046d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2224,6 +2224,26 @@ __metadata: languageName: node linkType: hard +"@es-joy/jsdoccomment@npm:~0.76.0": + version: 0.76.0 + resolution: "@es-joy/jsdoccomment@npm:0.76.0" + dependencies: + "@types/estree": "npm:^1.0.8" + "@typescript-eslint/types": "npm:^8.46.0" + comment-parser: "npm:1.4.1" + esquery: "npm:^1.6.0" + jsdoc-type-pratt-parser: "npm:~6.10.0" + checksum: 10c0/8fe4edec7d60562787ea8c77193ebe8737a9e28ec3143d383506b63890d0ffd45a2813e913ad1f00f227cb10e3a1fb913e5a696b33d499dc564272ff1a6f3fdb + languageName: node + linkType: hard + +"@es-joy/resolve.exports@npm:1.2.0": + version: 1.2.0 + resolution: "@es-joy/resolve.exports@npm:1.2.0" + checksum: 10c0/7e4713471f5eccb17a925a12415a2d9e372a42376813a19f6abd9c35e8d01ab1403777265817da67c6150cffd4f558d9ad51e26a8de6911dad89d9cb7eedacd8 + languageName: node + linkType: hard + "@esbuild/aix-ppc64@npm:0.25.1": version: 0.25.1 resolution: "@esbuild/aix-ppc64@npm:0.25.1" @@ -2486,7 +2506,26 @@ __metadata: languageName: node linkType: hard -"@floating-ui/dom@npm:1.6.13, @floating-ui/dom@npm:^1.0.0": +"@floating-ui/core@npm:^1.7.3": + version: 1.7.3 + resolution: "@floating-ui/core@npm:1.7.3" + dependencies: + "@floating-ui/utils": "npm:^0.2.10" + checksum: 10c0/edfc23800122d81df0df0fb780b7328ae6c5f00efbb55bd48ea340f4af8c5b3b121ceb4bb81220966ab0f87b443204d37105abdd93d94846468be3243984144c + languageName: node + linkType: hard + +"@floating-ui/dom@npm:1.7.4": + version: 1.7.4 + resolution: "@floating-ui/dom@npm:1.7.4" + dependencies: + "@floating-ui/core": "npm:^1.7.3" + "@floating-ui/utils": "npm:^0.2.10" + checksum: 10c0/da6166c25f9b0729caa9f498685a73a0e28251613b35d27db8de8014bc9d045158a23c092b405321a3d67c2064909b6e2a7e6c1c9cc0f62967dca5779f5aef30 + languageName: node + linkType: hard + +"@floating-ui/dom@npm:^1.0.0": version: 1.6.13 resolution: "@floating-ui/dom@npm:1.6.13" dependencies: @@ -2522,6 +2561,13 @@ __metadata: languageName: node linkType: hard +"@floating-ui/utils@npm:^0.2.10": + version: 0.2.10 + resolution: "@floating-ui/utils@npm:0.2.10" + checksum: 10c0/e9bc2a1730ede1ee25843937e911ab6e846a733a4488623cd353f94721b05ec2c9ec6437613a2ac9379a94c2fd40c797a2ba6fa1df2716f5ce4aa6ddb1cf9ea4 + languageName: node + linkType: hard + "@floating-ui/utils@npm:^0.2.9": version: 0.2.9 resolution: "@floating-ui/utils@npm:0.2.9" @@ -2711,6 +2757,13 @@ __metadata: languageName: node linkType: hard +"@jridgewell/sourcemap-codec@npm:^1.5.5": + version: 1.5.5 + resolution: "@jridgewell/sourcemap-codec@npm:1.5.5" + checksum: 10c0/f9e538f302b63c0ebc06eecb1dd9918dd4289ed36147a0ddce35d6ea4d7ebbda243cda7b2213b6a5e1d8087a298d5cf630fb2bd39329cdecb82017023f6081a0 + languageName: node + linkType: hard + "@jridgewell/trace-mapping@npm:^0.3.23, @jridgewell/trace-mapping@npm:^0.3.24, @jridgewell/trace-mapping@npm:^0.3.25": version: 0.3.25 resolution: "@jridgewell/trace-mapping@npm:0.3.25" @@ -2731,25 +2784,25 @@ __metadata: languageName: node linkType: hard -"@livekit/components-core@npm:0.12.11, @livekit/components-core@npm:^0.12.0": - version: 0.12.11 - resolution: "@livekit/components-core@npm:0.12.11" +"@livekit/components-core@npm:0.12.12, @livekit/components-core@npm:^0.12.0": + version: 0.12.12 + resolution: "@livekit/components-core@npm:0.12.12" dependencies: - "@floating-ui/dom": "npm:1.6.13" + "@floating-ui/dom": "npm:1.7.4" loglevel: "npm:1.9.1" rxjs: "npm:7.8.2" peerDependencies: livekit-client: ^2.15.14 tslib: ^2.6.2 - checksum: 10c0/9c2ac3d30bb8cc9067ae0b2049784f81e90e57df9eabf7edbaf3c8ceb65a63f644a4e6abeb6cc38d3ebe52663d8dbb88535e01a965011f365d5ae1f3daf86052 + checksum: 10c0/788ae01fa6c58a0edbd629f4195f2f3a7bc94660d2fb729af8b27cab2b151abe36cd0a666989811c6187e51d32c847119853010a82be55844750ab3978079c38 languageName: node linkType: hard "@livekit/components-react@npm:^2.0.0": - version: 2.9.16 - resolution: "@livekit/components-react@npm:2.9.16" + version: 2.9.17 + resolution: "@livekit/components-react@npm:2.9.17" dependencies: - "@livekit/components-core": "npm:0.12.11" + "@livekit/components-core": "npm:0.12.12" clsx: "npm:2.1.1" events: "npm:^3.3.0" jose: "npm:^6.0.12" @@ -2763,7 +2816,7 @@ __metadata: peerDependenciesMeta: "@livekit/krisp-noise-filter": optional: true - checksum: 10c0/4ba4ff473c5a29d3107412733a6676a3b708d70684ed463e9b34cda26abb3d2f317c2828a52e730837b756de9df3fc248260d6f390aedebfb6ec96ef63c7b151 + checksum: 10c0/ba64ada37d4b3ce4d5ee7c5b2a6bddbffc17c2e641e95881aac9f02b4ff7428105e0a372d364e50ff124e988b7426d322d94caabdb55b634aebf0144d7e37f99 languageName: node linkType: hard @@ -2795,10 +2848,10 @@ __metadata: languageName: node linkType: hard -"@matrix-org/matrix-sdk-crypto-wasm@npm:^15.3.0": - version: 15.3.0 - resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:15.3.0" - checksum: 10c0/45628f36b7b0e54a8777ae67a7233dbdf3e3cf14e0d95d21f62f89a7ea7e3f907232f1eb7b1262193b1e227759fad47af829dcccc103ded89011f13c66f01d76 +"@matrix-org/matrix-sdk-crypto-wasm@npm:^16.0.0": + version: 16.0.0 + resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:16.0.0" + checksum: 10c0/13b4ede3e618da819957abff778afefcf3baf9a2faac04a36bb5a07a44fae2ea05fbfa072eb3408d48b2b7b9aaf27242ce52c594c8ce9bf1fb8b3aade2832be1 languageName: node linkType: hard @@ -3233,14 +3286,14 @@ __metadata: languageName: node linkType: hard -"@playwright/test@npm:^1.56.1": - version: 1.56.1 - resolution: "@playwright/test@npm:1.56.1" +"@playwright/test@npm:^1.57.0": + version: 1.57.0 + resolution: "@playwright/test@npm:1.57.0" dependencies: - playwright: "npm:1.56.1" + playwright: "npm:1.57.0" bin: playwright: cli.js - checksum: 10c0/2b5b0e1f2e6a18f6e5ce6897c7440ca78f64e0b004834e9808e93ad2b78b96366b562ae4366602669cf8ad793a43d85481b58541e74be71e905e732d833dd691 + checksum: 10c0/35ba4b28be72bf0a53e33dbb11c6cff848fb9a37f49e893ce63a90675b5291ec29a1ba82c8a3b043abaead129400f0589623e9ace2e6a1c8eaa409721ecc3774 languageName: node linkType: hard @@ -4266,6 +4319,22 @@ __metadata: languageName: node linkType: hard +"@rollup/plugin-inject@npm:^5.0.3": + version: 5.0.5 + resolution: "@rollup/plugin-inject@npm:5.0.5" + dependencies: + "@rollup/pluginutils": "npm:^5.0.1" + estree-walker: "npm:^2.0.2" + magic-string: "npm:^0.30.3" + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + checksum: 10c0/22d10cf44fa56a6683d5ac4df24a9003379b3dcaae9897f5c30c844afc2ebca83cfaa5557f13a1399b1c8a0d312c3217bcacd508b7ebc4b2cbee401bd1ec8be2 + languageName: node + linkType: hard + "@rollup/pluginutils@npm:^4.2.0": version: 4.2.1 resolution: "@rollup/pluginutils@npm:4.2.1" @@ -4276,6 +4345,22 @@ __metadata: languageName: node linkType: hard +"@rollup/pluginutils@npm:^5.0.1": + version: 5.3.0 + resolution: "@rollup/pluginutils@npm:5.3.0" + dependencies: + "@types/estree": "npm:^1.0.0" + estree-walker: "npm:^2.0.2" + picomatch: "npm:^4.0.2" + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + checksum: 10c0/001834bf62d7cf5bac424d2617c113f7f7d3b2bf3c1778cbcccb72cdc957b68989f8e7747c782c2b911f1dde8257f56f8ac1e779e29e74e638e3f1e2cac2bcd0 + languageName: node + linkType: hard + "@rollup/pluginutils@npm:^5.1.3": version: 5.1.3 resolution: "@rollup/pluginutils@npm:5.1.3" @@ -4646,6 +4731,13 @@ __metadata: languageName: node linkType: hard +"@sindresorhus/base62@npm:^1.0.0": + version: 1.0.0 + resolution: "@sindresorhus/base62@npm:1.0.0" + checksum: 10c0/9a14df0f058fdf4731c30f0f05728a4822144ee42236030039d7fa5a1a1072c2879feba8091fd4a17c8922d1056bc07bada77c31fddc3e15836fc05a266fd918 + languageName: node + linkType: hard + "@stylistic/eslint-plugin@npm:^3.0.0": version: 3.1.0 resolution: "@stylistic/eslint-plugin@npm:3.1.0" @@ -4944,7 +5036,7 @@ __metadata: languageName: node linkType: hard -"@types/estree@npm:1.0.8": +"@types/estree@npm:1.0.8, @types/estree@npm:^1.0.8": version: 1.0.8 resolution: "@types/estree@npm:1.0.8" checksum: 10c0/39d34d1afaa338ab9763f37ad6066e3f349444f9052b9676a7cc0252ef9485a41c6d81c9c4e0d26e9077993354edf25efc853f3224dd4b447175ef62bdcc86a5 @@ -5274,6 +5366,13 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/types@npm:^8.46.0": + version: 8.51.0 + resolution: "@typescript-eslint/types@npm:8.51.0" + checksum: 10c0/eb3473d0bb71eb886438f35887b620ffadae7853b281752a40c73158aee644d136adeb82549be7d7c30f346fe888b2e979dff7e30e67b35377e8281018034529 + languageName: node + linkType: hard + "@typescript-eslint/typescript-estree@npm:5.62.0": version: 5.62.0 resolution: "@typescript-eslint/typescript-estree@npm:5.62.0" @@ -5634,6 +5733,15 @@ __metadata: languageName: node linkType: hard +"acorn@npm:^8.15.0": + version: 8.15.0 + resolution: "acorn@npm:8.15.0" + bin: + acorn: bin/acorn + checksum: 10c0/dec73ff59b7d6628a01eebaece7f2bdb8bb62b9b5926dcad0f8931f2b8b79c2be21f6c68ac095592adb5adb15831a3635d9343e6a91d028bbe85d564875ec3ec + languageName: node + linkType: hard + "acorn@npm:^8.9.0": version: 8.12.1 resolution: "acorn@npm:8.12.1" @@ -5743,6 +5851,13 @@ __metadata: languageName: node linkType: hard +"are-docs-informative@npm:^0.0.2": + version: 0.0.2 + resolution: "are-docs-informative@npm:0.0.2" + checksum: 10c0/f0326981bd699c372d268b526b170a28f2e1aec2cf99d7de0686083528427ecdf6ae41fef5d9988e224a5616298af747ad8a76e7306b0a7c97cc085a99636d60 + languageName: node + linkType: hard + "argparse@npm:^2.0.1": version: 2.0.1 resolution: "argparse@npm:2.0.1" @@ -5915,6 +6030,30 @@ __metadata: languageName: node linkType: hard +"asn1.js@npm:^4.10.1": + version: 4.10.1 + resolution: "asn1.js@npm:4.10.1" + dependencies: + bn.js: "npm:^4.0.0" + inherits: "npm:^2.0.1" + minimalistic-assert: "npm:^1.0.0" + checksum: 10c0/afa7f3ab9e31566c80175a75b182e5dba50589dcc738aa485be42bdd787e2a07246a4b034d481861123cbe646a7656f318f4f1cad2e9e5e808a210d5d6feaa88 + languageName: node + linkType: hard + +"assert@npm:^2.0.0": + version: 2.1.0 + resolution: "assert@npm:2.1.0" + dependencies: + call-bind: "npm:^1.0.2" + is-nan: "npm:^1.3.2" + object-is: "npm:^1.1.5" + object.assign: "npm:^4.1.4" + util: "npm:^0.12.5" + checksum: 10c0/7271a5da883c256a1fa690677bf1dd9d6aa882139f2bed1cd15da4f9e7459683e1da8e32a203d6cc6767e5e0f730c77a9532a87b896b4b0af0dd535f668775f0 + languageName: node + linkType: hard + "assertion-error@npm:^2.0.1": version: 2.0.1 resolution: "assertion-error@npm:2.0.1" @@ -6105,6 +6244,20 @@ __metadata: languageName: node linkType: hard +"bn.js@npm:^4.0.0, bn.js@npm:^4.1.0, bn.js@npm:^4.11.9": + version: 4.12.2 + resolution: "bn.js@npm:4.12.2" + checksum: 10c0/09a249faa416a9a1ce68b5f5ec8bbca87fe54e5dd4ef8b1cc8a4969147b80035592bddcb1e9cc814c3ba79e573503d5c5178664b722b509fb36d93620dba9b57 + languageName: node + linkType: hard + +"bn.js@npm:^5.2.1, bn.js@npm:^5.2.2": + version: 5.2.2 + resolution: "bn.js@npm:5.2.2" + checksum: 10c0/cb97827d476aab1a0194df33cd84624952480d92da46e6b4a19c32964aa01553a4a613502396712704da2ec8f831cf98d02e74ca03398404bd78a037ba93f2ab + languageName: node + linkType: hard + "boolbase@npm:^1.0.0": version: 1.0.0 resolution: "boolbase@npm:1.0.0" @@ -6180,6 +6333,96 @@ __metadata: languageName: node linkType: hard +"brorand@npm:^1.0.1, brorand@npm:^1.1.0": + version: 1.1.0 + resolution: "brorand@npm:1.1.0" + checksum: 10c0/6f366d7c4990f82c366e3878492ba9a372a73163c09871e80d82fb4ae0d23f9f8924cb8a662330308206e6b3b76ba1d528b4601c9ef73c2166b440b2ea3b7571 + languageName: node + linkType: hard + +"browser-resolve@npm:^2.0.0": + version: 2.0.0 + resolution: "browser-resolve@npm:2.0.0" + dependencies: + resolve: "npm:^1.17.0" + checksum: 10c0/06c43adf3cb1939825ab9a4ac355b23272820ee421a20d04f62e0dabd9ea305e497b97f3ac027f87d53c366483aafe8673bbe1aaa5e41cd69eeafa65ac5fda6e + languageName: node + linkType: hard + +"browserify-aes@npm:^1.0.4, browserify-aes@npm:^1.2.0": + version: 1.2.0 + resolution: "browserify-aes@npm:1.2.0" + dependencies: + buffer-xor: "npm:^1.0.3" + cipher-base: "npm:^1.0.0" + create-hash: "npm:^1.1.0" + evp_bytestokey: "npm:^1.0.3" + inherits: "npm:^2.0.1" + safe-buffer: "npm:^5.0.1" + checksum: 10c0/967f2ae60d610b7b252a4cbb55a7a3331c78293c94b4dd9c264d384ca93354c089b3af9c0dd023534efdc74ffbc82510f7ad4399cf82bc37bc07052eea485f18 + languageName: node + linkType: hard + +"browserify-cipher@npm:^1.0.1": + version: 1.0.1 + resolution: "browserify-cipher@npm:1.0.1" + dependencies: + browserify-aes: "npm:^1.0.4" + browserify-des: "npm:^1.0.0" + evp_bytestokey: "npm:^1.0.0" + checksum: 10c0/aa256dcb42bc53a67168bbc94ab85d243b0a3b56109dee3b51230b7d010d9b78985ffc1fb36e145c6e4db151f888076c1cfc207baf1525d3e375cbe8187fe27d + languageName: node + linkType: hard + +"browserify-des@npm:^1.0.0": + version: 1.0.2 + resolution: "browserify-des@npm:1.0.2" + dependencies: + cipher-base: "npm:^1.0.1" + des.js: "npm:^1.0.0" + inherits: "npm:^2.0.1" + safe-buffer: "npm:^5.1.2" + checksum: 10c0/943eb5d4045eff80a6cde5be4e5fbb1f2d5002126b5a4789c3c1aae3cdddb1eb92b00fb92277f512288e5c6af330730b1dbabcf7ce0923e749e151fcee5a074d + languageName: node + linkType: hard + +"browserify-rsa@npm:^4.0.0, browserify-rsa@npm:^4.1.1": + version: 4.1.1 + resolution: "browserify-rsa@npm:4.1.1" + dependencies: + bn.js: "npm:^5.2.1" + randombytes: "npm:^2.1.0" + safe-buffer: "npm:^5.2.1" + checksum: 10c0/b650ee1192e3d7f3d779edc06dd96ed8720362e72ac310c367b9d7fe35f7e8dbb983c1829142b2b3215458be8bf17c38adc7224920843024ed8cf39e19c513c0 + languageName: node + linkType: hard + +"browserify-sign@npm:^4.2.3": + version: 4.2.5 + resolution: "browserify-sign@npm:4.2.5" + dependencies: + bn.js: "npm:^5.2.2" + browserify-rsa: "npm:^4.1.1" + create-hash: "npm:^1.2.0" + create-hmac: "npm:^1.1.7" + elliptic: "npm:^6.6.1" + inherits: "npm:^2.0.4" + parse-asn1: "npm:^5.1.9" + readable-stream: "npm:^2.3.8" + safe-buffer: "npm:^5.2.1" + checksum: 10c0/6192f9696934bbba58932d098face34c2ab9cac09feed826618b86b8c00a897dab7324cd9aa7d6cb1597064f197264ad72fa5418d4d52bf3c8f9b9e0e124655e + languageName: node + linkType: hard + +"browserify-zlib@npm:^0.2.0": + version: 0.2.0 + resolution: "browserify-zlib@npm:0.2.0" + dependencies: + pako: "npm:~1.0.5" + checksum: 10c0/9ab10b6dc732c6c5ec8ebcbe5cb7fe1467f97402c9b2140113f47b5f187b9438f93a8e065d8baf8b929323c18324fbf1105af479ee86d9d36cab7d7ef3424ad9 + languageName: node + linkType: hard + "browserslist@npm:^4.24.0, browserslist@npm:^4.24.3, browserslist@npm:^4.24.4": version: 4.24.4 resolution: "browserslist@npm:4.24.4" @@ -6224,6 +6467,23 @@ __metadata: languageName: node linkType: hard +"buffer-xor@npm:^1.0.3": + version: 1.0.3 + resolution: "buffer-xor@npm:1.0.3" + checksum: 10c0/fd269d0e0bf71ecac3146187cfc79edc9dbb054e2ee69b4d97dfb857c6d997c33de391696d04bdd669272751fa48e7872a22f3a6c7b07d6c0bc31dbe02a4075c + languageName: node + linkType: hard + +"buffer@npm:^5.7.1": + version: 5.7.1 + resolution: "buffer@npm:5.7.1" + dependencies: + base64-js: "npm:^1.3.1" + ieee754: "npm:^1.1.13" + checksum: 10c0/27cac81cff434ed2876058d72e7c4789d11ff1120ef32c9de48f59eab58179b66710c488987d295ae89a228f835fc66d088652dffeb8e3ba8659f80eb091d55e + languageName: node + linkType: hard + "buffer@npm:^6.0.3": version: 6.0.3 resolution: "buffer@npm:6.0.3" @@ -6241,6 +6501,13 @@ __metadata: languageName: node linkType: hard +"builtin-status-codes@npm:^3.0.0": + version: 3.0.0 + resolution: "builtin-status-codes@npm:3.0.0" + checksum: 10c0/c37bbba11a34c4431e56bd681b175512e99147defbe2358318d8152b3a01df7bf25e0305873947e5b350073d5ef41a364a22b37e48f1fb6d2fe6d5286a0f348c + languageName: node + linkType: hard + "bytesish@npm:^0.4.1": version: 0.4.4 resolution: "bytesish@npm:0.4.4" @@ -6295,7 +6562,7 @@ __metadata: languageName: node linkType: hard -"call-bind@npm:^1.0.2, call-bind@npm:^1.0.7, call-bind@npm:^1.0.8": +"call-bind@npm:^1.0.0, call-bind@npm:^1.0.2, call-bind@npm:^1.0.7, call-bind@npm:^1.0.8": version: 1.0.8 resolution: "call-bind@npm:1.0.8" dependencies: @@ -6358,24 +6625,10 @@ __metadata: languageName: node linkType: hard -"caniuse-lite@npm:^1.0.30001688": - version: 1.0.30001701 - resolution: "caniuse-lite@npm:1.0.30001701" - checksum: 10c0/a814bd4dd8b49645ca51bc6ee42120660a36394bb54eb6084801d3f2bbb9471e5e1a9a8a25f44f83086a032d46e66b33031e2aa345f699b90a7e84a9836b819c - languageName: node - linkType: hard - -"caniuse-lite@npm:^1.0.30001702": - version: 1.0.30001720 - resolution: "caniuse-lite@npm:1.0.30001720" - checksum: 10c0/ba9f963364ec4bfc8359d15d7e2cf365185fa1fddc90b4f534c71befedae9b3dd0cd2583a25ffc168a02d7b61b6c18b59bda0a1828ea2a5250fd3e35c2c049e9 - languageName: node - linkType: hard - -"caniuse-lite@npm:^1.0.30001726": - version: 1.0.30001726 - resolution: "caniuse-lite@npm:1.0.30001726" - checksum: 10c0/2c5f91da7fd9ebf8c6b432818b1498ea28aca8de22b30dafabe2a2a6da1e014f10e67e14f8e68e872a0867b6b4cd6001558dde04e3ab9770c9252ca5c8849d0e +"caniuse-lite@npm:^1.0.30001688, caniuse-lite@npm:^1.0.30001702, caniuse-lite@npm:^1.0.30001726": + version: 1.0.30001757 + resolution: "caniuse-lite@npm:1.0.30001757" + checksum: 10c0/3ccb71fa2bf1f8c96ff1bf9b918b08806fed33307e20a3ce3259155fda131eaf96cfcd88d3d309c8fd7f8285cc71d89a3b93648a1c04814da31c301f98508d42 languageName: node linkType: hard @@ -6509,6 +6762,17 @@ __metadata: languageName: node linkType: hard +"cipher-base@npm:^1.0.0, cipher-base@npm:^1.0.1, cipher-base@npm:^1.0.3": + version: 1.0.7 + resolution: "cipher-base@npm:1.0.7" + dependencies: + inherits: "npm:^2.0.4" + safe-buffer: "npm:^5.2.1" + to-buffer: "npm:^1.2.2" + checksum: 10c0/53c5046a9d9b60c586479b8f13fde263c3f905e13f11e8e04c7a311ce399c91d9c3ec96642332e0de077d356e1014ee12bba96f74fbaad0de750f49122258836 + languageName: node + linkType: hard + "classnames@npm:^2.3.1, classnames@npm:^2.5.1": version: 2.5.1 resolution: "classnames@npm:2.5.1" @@ -6644,6 +6908,13 @@ __metadata: languageName: node linkType: hard +"comment-parser@npm:1.4.1": + version: 1.4.1 + resolution: "comment-parser@npm:1.4.1" + checksum: 10c0/d6c4be3f5be058f98b24f2d557f745d8fe1cc9eb75bebbdccabd404a0e1ed41563171b16285f593011f8b6a5ec81f564fb1f2121418ac5cbf0f49255bf0840dd + languageName: node + linkType: hard + "common-tags@npm:^1.8.0": version: 1.8.2 resolution: "common-tags@npm:1.8.2" @@ -6672,6 +6943,20 @@ __metadata: languageName: node linkType: hard +"console-browserify@npm:^1.1.0": + version: 1.2.0 + resolution: "console-browserify@npm:1.2.0" + checksum: 10c0/89b99a53b7d6cee54e1e64fa6b1f7ac24b844b4019c5d39db298637e55c1f4ffa5c165457ad984864de1379df2c8e1886cbbdac85d9dbb6876a9f26c3106f226 + languageName: node + linkType: hard + +"constants-browserify@npm:^1.0.0": + version: 1.0.0 + resolution: "constants-browserify@npm:1.0.0" + checksum: 10c0/ab49b1d59a433ed77c964d90d19e08b2f77213fb823da4729c0baead55e3c597f8f97ebccfdfc47bd896d43854a117d114c849a6f659d9986420e97da0f83ac5 + languageName: node + linkType: hard + "content-type@npm:^1.0.4": version: 1.0.5 resolution: "content-type@npm:1.0.5" @@ -6744,6 +7029,50 @@ __metadata: languageName: node linkType: hard +"create-ecdh@npm:^4.0.4": + version: 4.0.4 + resolution: "create-ecdh@npm:4.0.4" + dependencies: + bn.js: "npm:^4.1.0" + elliptic: "npm:^6.5.3" + checksum: 10c0/77b11a51360fec9c3bce7a76288fc0deba4b9c838d5fb354b3e40c59194d23d66efe6355fd4b81df7580da0661e1334a235a2a5c040b7569ba97db428d466e7f + languageName: node + linkType: hard + +"create-hash@npm:^1.1.0, create-hash@npm:^1.2.0": + version: 1.2.0 + resolution: "create-hash@npm:1.2.0" + dependencies: + cipher-base: "npm:^1.0.1" + inherits: "npm:^2.0.1" + md5.js: "npm:^1.3.4" + ripemd160: "npm:^2.0.1" + sha.js: "npm:^2.4.0" + checksum: 10c0/d402e60e65e70e5083cb57af96d89567954d0669e90550d7cec58b56d49c4b193d35c43cec8338bc72358198b8cbf2f0cac14775b651e99238e1cf411490f915 + languageName: node + linkType: hard + +"create-hmac@npm:^1.1.7": + version: 1.1.7 + resolution: "create-hmac@npm:1.1.7" + dependencies: + cipher-base: "npm:^1.0.3" + create-hash: "npm:^1.1.0" + inherits: "npm:^2.0.1" + ripemd160: "npm:^2.0.0" + safe-buffer: "npm:^5.0.1" + sha.js: "npm:^2.4.8" + checksum: 10c0/24332bab51011652a9a0a6d160eed1e8caa091b802335324ae056b0dcb5acbc9fcf173cf10d128eba8548c3ce98dfa4eadaa01bd02f44a34414baee26b651835 + languageName: node + linkType: hard + +"create-require@npm:^1.1.1": + version: 1.1.1 + resolution: "create-require@npm:1.1.1" + checksum: 10c0/157cbc59b2430ae9a90034a5f3a1b398b6738bf510f713edc4d4e45e169bc514d3d99dd34d8d01ca7ae7830b5b8b537e46ae8f3c8f932371b0875c0151d7ec91 + languageName: node + linkType: hard + "cross-spawn@npm:^7.0.2": version: 7.0.3 resolution: "cross-spawn@npm:7.0.3" @@ -6766,6 +7095,26 @@ __metadata: languageName: node linkType: hard +"crypto-browserify@npm:^3.12.1": + version: 3.12.1 + resolution: "crypto-browserify@npm:3.12.1" + dependencies: + browserify-cipher: "npm:^1.0.1" + browserify-sign: "npm:^4.2.3" + create-ecdh: "npm:^4.0.4" + create-hash: "npm:^1.2.0" + create-hmac: "npm:^1.1.7" + diffie-hellman: "npm:^5.0.3" + hash-base: "npm:~3.0.4" + inherits: "npm:^2.0.4" + pbkdf2: "npm:^3.1.2" + public-encrypt: "npm:^4.0.3" + randombytes: "npm:^2.1.0" + randomfill: "npm:^1.0.4" + checksum: 10c0/184a2def7b16628e79841243232ab5497f18d8e158ac21b7ce90ab172427d0a892a561280adc08f9d4d517bce8db2a5b335dc21abb970f787f8e874bd7b9db7d + languageName: node + linkType: hard + "css-blank-pseudo@npm:^7.0.1": version: 7.0.1 resolution: "css-blank-pseudo@npm:7.0.1" @@ -6976,6 +7325,18 @@ __metadata: languageName: node linkType: hard +"debug@npm:^4.4.3": + version: 4.4.3 + resolution: "debug@npm:4.4.3" + dependencies: + ms: "npm:^2.1.3" + peerDependenciesMeta: + supports-color: + optional: true + checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 + languageName: node + linkType: hard + "decamelize@npm:^1.2.0": version: 1.2.0 resolution: "decamelize@npm:1.2.0" @@ -7054,6 +7415,16 @@ __metadata: languageName: node linkType: hard +"des.js@npm:^1.0.0": + version: 1.1.0 + resolution: "des.js@npm:1.1.0" + dependencies: + inherits: "npm:^2.0.1" + minimalistic-assert: "npm:^1.0.0" + checksum: 10c0/671354943ad67493e49eb4c555480ab153edd7cee3a51c658082fcde539d2690ed2a4a0b5d1f401f9cde822edf3939a6afb2585f32c091f2d3a1b1665cd45236 + languageName: node + linkType: hard + "detect-libc@npm:^1.0.3": version: 1.0.3 resolution: "detect-libc@npm:1.0.3" @@ -7070,6 +7441,17 @@ __metadata: languageName: node linkType: hard +"diffie-hellman@npm:^5.0.3": + version: 5.0.3 + resolution: "diffie-hellman@npm:5.0.3" + dependencies: + bn.js: "npm:^4.1.0" + miller-rabin: "npm:^4.0.0" + randombytes: "npm:^2.0.0" + checksum: 10c0/ce53ccafa9ca544b7fc29b08a626e23a9b6562efc2a98559a0c97b4718937cebaa9b5d7d0a05032cc9c1435e9b3c1532b9e9bf2e0ede868525922807ad6e1ecf + languageName: node + linkType: hard + "dijkstrajs@npm:^1.0.1": version: 1.0.3 resolution: "dijkstrajs@npm:1.0.3" @@ -7140,6 +7522,13 @@ __metadata: languageName: node linkType: hard +"domain-browser@npm:4.22.0": + version: 4.22.0 + resolution: "domain-browser@npm:4.22.0" + checksum: 10c0/2ef7eda6d2161038fda0c9aa4c9e18cc7a0baa89ea6be975d449527c2eefd4b608425db88508e2859acc472f46f402079274b24bd75e3fb506f28c5dba203129 + languageName: node + linkType: hard + "domelementtype@npm:^2.0.1, domelementtype@npm:^2.2.0, domelementtype@npm:^2.3.0": version: 2.3.0 resolution: "domelementtype@npm:2.3.0" @@ -7272,7 +7661,14 @@ __metadata: "@livekit/protocol": "npm:^1.42.2" "@livekit/track-processors": "npm:^0.5.5" "@mediapipe/tasks-vision": "npm:^0.10.18" - "@playwright/test": "npm:^1.56.1" + "@opentelemetry/api": "npm:^1.4.0" + "@opentelemetry/core": "npm:^2.0.0" + "@opentelemetry/exporter-trace-otlp-http": "npm:^0.203.0" + "@opentelemetry/resources": "npm:^2.0.0" + "@opentelemetry/sdk-trace-base": "npm:^2.0.0" + "@opentelemetry/sdk-trace-web": "npm:^2.0.0" + "@opentelemetry/semantic-conventions": "npm:^1.25.1" + "@playwright/test": "npm:^1.57.0" "@radix-ui/react-dialog": "npm:^1.0.4" "@radix-ui/react-slider": "npm:^1.1.2" "@radix-ui/react-visually-hidden": "npm:^1.0.3" @@ -7311,6 +7707,7 @@ __metadata: eslint-config-prettier: "npm:^10.0.0" eslint-plugin-deprecate: "npm:^0.8.2" eslint-plugin-import: "npm:^2.26.0" + eslint-plugin-jsdoc: "npm:^61.5.0" eslint-plugin-jsx-a11y: "npm:^6.5.1" eslint-plugin-matrix-org: "npm:2.1.0" eslint-plugin-react: "npm:^7.29.4" @@ -7327,8 +7724,9 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21" - matrix-widget-api: "npm:^1.13.0" + matrix-js-sdk: "matrix-org/matrix-js-sdk#2218ec4e3102e841ba3e794e1c492c0a5aa6c1c3" + matrix-widget-api: "npm:^1.14.0" + node-stdlib-browser: "npm:^1.3.1" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" pako: "npm:^2.0.4" @@ -7351,12 +7749,28 @@ __metadata: vite: "npm:^7.0.0" vite-plugin-generate-file: "npm:^0.3.0" vite-plugin-html: "npm:^3.2.2" + vite-plugin-node-stdlib-browser: "npm:^0.2.1" vite-plugin-svgr: "npm:^4.0.0" vitest: "npm:^3.0.0" vitest-axe: "npm:^1.0.0-pre.3" languageName: unknown linkType: soft +"elliptic@npm:^6.5.3, elliptic@npm:^6.6.1": + version: 6.6.1 + resolution: "elliptic@npm:6.6.1" + dependencies: + bn.js: "npm:^4.11.9" + brorand: "npm:^1.1.0" + hash.js: "npm:^1.0.0" + hmac-drbg: "npm:^1.0.1" + inherits: "npm:^2.0.4" + minimalistic-assert: "npm:^1.0.1" + minimalistic-crypto-utils: "npm:^1.0.1" + checksum: 10c0/8b24ef782eec8b472053793ea1e91ae6bee41afffdfcb78a81c0a53b191e715cbe1292aa07165958a9bbe675bd0955142560b1a007ffce7d6c765bcaf951a867 + languageName: node + linkType: hard + "emoji-regex@npm:^8.0.0": version: 8.0.0 resolution: "emoji-regex@npm:8.0.0" @@ -7868,6 +8282,30 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-jsdoc@npm:^61.5.0": + version: 61.5.0 + resolution: "eslint-plugin-jsdoc@npm:61.5.0" + dependencies: + "@es-joy/jsdoccomment": "npm:~0.76.0" + "@es-joy/resolve.exports": "npm:1.2.0" + are-docs-informative: "npm:^0.0.2" + comment-parser: "npm:1.4.1" + debug: "npm:^4.4.3" + escape-string-regexp: "npm:^4.0.0" + espree: "npm:^10.4.0" + esquery: "npm:^1.6.0" + html-entities: "npm:^2.6.0" + object-deep-merge: "npm:^2.0.0" + parse-imports-exports: "npm:^0.2.4" + semver: "npm:^7.7.3" + spdx-expression-parse: "npm:^4.0.0" + to-valid-identifier: "npm:^1.0.0" + peerDependencies: + eslint: ^7.0.0 || ^8.0.0 || ^9.0.0 + checksum: 10c0/fabb04f6efe58a167a0839d3c05676a76080c6e91d98a269fa768c1bfd835aa0ded5822d400da2874216177044d2d227ebe241d73e923f3fe1c08bafd19cfd3d + languageName: node + linkType: hard + "eslint-plugin-jsx-a11y@npm:^6.5.1": version: 6.10.2 resolution: "eslint-plugin-jsx-a11y@npm:6.10.2" @@ -8102,6 +8540,17 @@ __metadata: languageName: node linkType: hard +"espree@npm:^10.4.0": + version: 10.4.0 + resolution: "espree@npm:10.4.0" + dependencies: + acorn: "npm:^8.15.0" + acorn-jsx: "npm:^5.3.2" + eslint-visitor-keys: "npm:^4.2.1" + checksum: 10c0/c63fe06131c26c8157b4083313cb02a9a54720a08e21543300e55288c40e06c3fc284bdecf108d3a1372c5934a0a88644c98714f38b6ae8ed272b40d9ea08d6b + languageName: node + linkType: hard + "espree@npm:^9.6.0, espree@npm:^9.6.1": version: 9.6.1 resolution: "espree@npm:9.6.1" @@ -8168,13 +8617,24 @@ __metadata: languageName: node linkType: hard -"events@npm:^3.2.0, events@npm:^3.3.0": +"events@npm:^3.0.0, events@npm:^3.2.0, events@npm:^3.3.0": version: 3.3.0 resolution: "events@npm:3.3.0" checksum: 10c0/d6b6f2adbccbcda74ddbab52ed07db727ef52e31a61ed26db9feb7dc62af7fc8e060defa65e5f8af9449b86b52cc1a1f6a79f2eafcf4e62add2b7a1fa4a432f6 languageName: node linkType: hard +"evp_bytestokey@npm:^1.0.0, evp_bytestokey@npm:^1.0.3": + version: 1.0.3 + resolution: "evp_bytestokey@npm:1.0.3" + dependencies: + md5.js: "npm:^1.3.4" + node-gyp: "npm:latest" + safe-buffer: "npm:^5.1.1" + checksum: 10c0/77fbe2d94a902a80e9b8f5a73dcd695d9c14899c5e82967a61b1fc6cbbb28c46552d9b127cff47c45fcf684748bdbcfa0a50410349109de87ceb4b199ef6ee99 + languageName: node + linkType: hard + "expect-type@npm:^1.2.1": version: 1.2.1 resolution: "expect-type@npm:1.2.1" @@ -8570,6 +9030,13 @@ __metadata: languageName: node linkType: hard +"generator-function@npm:^2.0.0": + version: 2.0.1 + resolution: "generator-function@npm:2.0.1" + checksum: 10c0/8a9f59df0f01cfefafdb3b451b80555e5cf6d76487095db91ac461a0e682e4ff7a9dbce15f4ecec191e53586d59eece01949e05a4b4492879600bbbe8e28d6b8 + languageName: node + linkType: hard + "gensync@npm:^1.0.0-beta.2": version: 1.0.0-beta.2 resolution: "gensync@npm:1.0.0-beta.2" @@ -8872,6 +9339,38 @@ __metadata: languageName: node linkType: hard +"hash-base@npm:^3.0.0, hash-base@npm:^3.1.2": + version: 3.1.2 + resolution: "hash-base@npm:3.1.2" + dependencies: + inherits: "npm:^2.0.4" + readable-stream: "npm:^2.3.8" + safe-buffer: "npm:^5.2.1" + to-buffer: "npm:^1.2.1" + checksum: 10c0/f3b7fae1853b31340048dd659f40f5260ca6f3ff53b932f807f4ab701ee09039f6e9dbe1841723ff61e20f3f69d6387a352e4ccc5f997dedb0d375c7d88bc15e + languageName: node + linkType: hard + +"hash-base@npm:~3.0.4": + version: 3.0.5 + resolution: "hash-base@npm:3.0.5" + dependencies: + inherits: "npm:^2.0.4" + safe-buffer: "npm:^5.2.1" + checksum: 10c0/6dc185b79bad9b6d525cd132a588e4215380fdc36fec6f7a8a58c5db8e3b642557d02ad9c367f5e476c7c3ad3ccffa3607f308b124e1ed80e3b80a1b254db61e + languageName: node + linkType: hard + +"hash.js@npm:^1.0.0, hash.js@npm:^1.0.3": + version: 1.1.7 + resolution: "hash.js@npm:1.1.7" + dependencies: + inherits: "npm:^2.0.3" + minimalistic-assert: "npm:^1.0.1" + checksum: 10c0/41ada59494eac5332cfc1ce6b7ebdd7b88a3864a6d6b08a3ea8ef261332ed60f37f10877e0c825aaa4bddebf164fbffa618286aeeec5296675e2671cbfa746c4 + languageName: node + linkType: hard + "hasown@npm:^2.0.0, hasown@npm:^2.0.2": version: 2.0.2 resolution: "hasown@npm:2.0.2" @@ -8909,6 +9408,17 @@ __metadata: languageName: node linkType: hard +"hmac-drbg@npm:^1.0.1": + version: 1.0.1 + resolution: "hmac-drbg@npm:1.0.1" + dependencies: + hash.js: "npm:^1.0.3" + minimalistic-assert: "npm:^1.0.0" + minimalistic-crypto-utils: "npm:^1.0.1" + checksum: 10c0/f3d9ba31b40257a573f162176ac5930109816036c59a09f901eb2ffd7e5e705c6832bedfff507957125f2086a0ab8f853c0df225642a88bf1fcaea945f20600d + languageName: node + linkType: hard + "hoist-non-react-statics@npm:^3.3.2": version: 3.3.2 resolution: "hoist-non-react-statics@npm:3.3.2" @@ -8934,6 +9444,13 @@ __metadata: languageName: node linkType: hard +"html-entities@npm:^2.6.0": + version: 2.6.0 + resolution: "html-entities@npm:2.6.0" + checksum: 10c0/7c8b15d9ea0cd00dc9279f61bab002ba6ca8a7a0f3c36ed2db3530a67a9621c017830d1d2c1c65beb9b8e3436ea663e9cf8b230472e0e413359399413b27c8b7 + languageName: node + linkType: hard + "html-escaper@npm:^2.0.0": version: 2.0.2 resolution: "html-escaper@npm:2.0.2" @@ -8996,6 +9513,13 @@ __metadata: languageName: node linkType: hard +"https-browserify@npm:^1.0.0": + version: 1.0.0 + resolution: "https-browserify@npm:1.0.0" + checksum: 10c0/e17b6943bc24ea9b9a7da5714645d808670af75a425f29baffc3284962626efdc1eb3aa9bbffaa6e64028a6ad98af5b09fabcb454a8f918fb686abfdc9e9b8ae + languageName: node + linkType: hard + "https-proxy-agent@npm:^5.0.0": version: 5.0.1 resolution: "https-proxy-agent@npm:5.0.1" @@ -9075,7 +9599,7 @@ __metadata: languageName: node linkType: hard -"ieee754@npm:^1.2.1": +"ieee754@npm:^1.1.13, ieee754@npm:^1.2.1": version: 1.2.1 resolution: "ieee754@npm:1.2.1" checksum: 10c0/b0782ef5e0935b9f12883a2e2aa37baa75da6e66ce6515c168697b42160807d9330de9a32ec1ed73149aea02e0d822e572bca6f1e22bdcbd2149e13b050b17bb @@ -9137,7 +9661,7 @@ __metadata: languageName: node linkType: hard -"inherits@npm:2, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.3": +"inherits@npm:2, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.3, inherits@npm:~2.0.4": version: 2.0.4 resolution: "inherits@npm:2.0.4" checksum: 10c0/4e531f648b29039fb7426fb94075e6545faa1eb9fe83c29f0b6d9e7263aceb4289d2d4557db0d428188eeb449cc7c5e77b0a0b2c4e248ff2a65933a0dee49ef2 @@ -9165,6 +9689,16 @@ __metadata: languageName: node linkType: hard +"is-arguments@npm:^1.0.4": + version: 1.2.0 + resolution: "is-arguments@npm:1.2.0" + dependencies: + call-bound: "npm:^1.0.2" + has-tostringtag: "npm:^1.0.2" + checksum: 10c0/6377344b31e9fcb707c6751ee89b11f132f32338e6a782ec2eac9393b0cbd32235dad93052998cda778ee058754860738341d8114910d50ada5615912bb929fc + languageName: node + linkType: hard + "is-array-buffer@npm:^3.0.4, is-array-buffer@npm:^3.0.5": version: 3.0.5 resolution: "is-array-buffer@npm:3.0.5" @@ -9304,6 +9838,19 @@ __metadata: languageName: node linkType: hard +"is-generator-function@npm:^1.0.7": + version: 1.1.2 + resolution: "is-generator-function@npm:1.1.2" + dependencies: + call-bound: "npm:^1.0.4" + generator-function: "npm:^2.0.0" + get-proto: "npm:^1.0.1" + has-tostringtag: "npm:^1.0.2" + safe-regex-test: "npm:^1.1.0" + checksum: 10c0/83da102e89c3e3b71d67b51d47c9f9bc862bceb58f87201727e27f7fa19d1d90b0ab223644ecaee6fc6e3d2d622bb25c966fbdaf87c59158b01ce7c0fe2fa372 + languageName: node + linkType: hard + "is-glob@npm:^4.0.0, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3, is-glob@npm:~4.0.1": version: 4.0.3 resolution: "is-glob@npm:4.0.3" @@ -9320,6 +9867,16 @@ __metadata: languageName: node linkType: hard +"is-nan@npm:^1.3.2": + version: 1.3.2 + resolution: "is-nan@npm:1.3.2" + dependencies: + call-bind: "npm:^1.0.0" + define-properties: "npm:^1.1.3" + checksum: 10c0/8bfb286f85763f9c2e28ea32e9127702fe980ffd15fa5d63ade3be7786559e6e21355d3625dd364c769c033c5aedf0a2ed3d4025d336abf1b9241e3d9eddc5b0 + languageName: node + linkType: hard + "is-negated-glob@npm:^1.0.0": version: 1.0.0 resolution: "is-negated-glob@npm:1.0.0" @@ -9442,7 +9999,7 @@ __metadata: languageName: node linkType: hard -"is-typed-array@npm:^1.1.13, is-typed-array@npm:^1.1.14, is-typed-array@npm:^1.1.15": +"is-typed-array@npm:^1.1.13, is-typed-array@npm:^1.1.14, is-typed-array@npm:^1.1.15, is-typed-array@npm:^1.1.3": version: 1.1.15 resolution: "is-typed-array@npm:1.1.15" dependencies: @@ -9521,6 +10078,13 @@ __metadata: languageName: node linkType: hard +"isomorphic-timers-promises@npm:^1.0.1": + version: 1.0.1 + resolution: "isomorphic-timers-promises@npm:1.0.1" + checksum: 10c0/3b4761d0012ebe6b6382246079fc667f3513f36fe4042638f2bfb7db1557e4f1acd33a9c9907706c04270890ec6434120f132f3f300161a42a7dd8628926c8a4 + languageName: node + linkType: hard + "istanbul-lib-coverage@npm:^3.0.0, istanbul-lib-coverage@npm:^3.2.2": version: 3.2.2 resolution: "istanbul-lib-coverage@npm:3.2.2" @@ -9656,6 +10220,13 @@ __metadata: languageName: node linkType: hard +"jsdoc-type-pratt-parser@npm:~6.10.0": + version: 6.10.0 + resolution: "jsdoc-type-pratt-parser@npm:6.10.0" + checksum: 10c0/8ea395df0cae0e41d4bdba5f8d81b8d3e467fe53d1e4182a5d4e653235a5f17d60ed137343d68dbc74fa10e767f1c58fb85b1f6d5489c2cf16fc7216cc6d3e1a + languageName: node + linkType: hard + "jsdom@npm:^26.0.0": version: 26.1.0 resolution: "jsdom@npm:26.1.0" @@ -9899,8 +10470,8 @@ __metadata: linkType: hard "livekit-client@npm:^2.13.0": - version: 2.16.0 - resolution: "livekit-client@npm:2.16.0" + version: 2.16.1 + resolution: "livekit-client@npm:2.16.1" dependencies: "@livekit/mutex": "npm:1.1.1" "@livekit/protocol": "npm:1.42.2" @@ -9914,7 +10485,7 @@ __metadata: webrtc-adapter: "npm:^9.0.1" peerDependencies: "@types/dom-mediacapture-record": ^1 - checksum: 10c0/5d03adc5d09efde343ab894db397529dff26117598e773b23a5df90a4fb166bde12c6bb1f2cfd1d28dbaf93fe9f275026d7abb75f2ffd2ba816393a2d58e6c7e + checksum: 10c0/a16f7e603730410b640991cdb1a9d5ad3b0b06b23b8fe2e76674e6a9ba54ed4c8e7e4f6ad15fe67c8de6557da161e1265b40802c4e3701db3fe0d7ea9250b1e2 languageName: node linkType: hard @@ -10062,6 +10633,15 @@ __metadata: languageName: node linkType: hard +"magic-string@npm:^0.30.3": + version: 0.30.21 + resolution: "magic-string@npm:0.30.21" + dependencies: + "@jridgewell/sourcemap-codec": "npm:^1.5.5" + checksum: 10c0/299378e38f9a270069fc62358522ddfb44e94244baa0d6a8980ab2a9b2490a1d03b236b447eee309e17eb3bddfa482c61259d47960eb018a904f0ded52780c4a + languageName: node + linkType: hard + "magicast@npm:^0.3.5": version: 0.3.5 resolution: "magicast@npm:0.3.5" @@ -10125,35 +10705,46 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21": - version: 38.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=e7f5bec51b6f70501a025b79fe5021c933385b21" +"matrix-js-sdk@matrix-org/matrix-js-sdk#2218ec4e3102e841ba3e794e1c492c0a5aa6c1c3": + version: 39.3.0 + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=2218ec4e3102e841ba3e794e1c492c0a5aa6c1c3" dependencies: "@babel/runtime": "npm:^7.12.5" - "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" + "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" another-json: "npm:^0.2.0" bs58: "npm:^6.0.0" content-type: "npm:^1.0.4" jwt-decode: "npm:^4.0.0" loglevel: "npm:^1.9.2" matrix-events-sdk: "npm:0.0.1" - matrix-widget-api: "npm:^1.10.0" + matrix-widget-api: "npm:^1.14.0" oidc-client-ts: "npm:^3.0.1" p-retry: "npm:7" - sdp-transform: "npm:^2.14.1" + sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/7adffdc183affd2d3ee1e8497cad6ca7904a37f98328ff7bc15aa6c1829dc9f9a92f8e1bd6260432a33626ff2a839644de938270163e73438b7294675cd954e4 + checksum: 10c0/feca51c7ada5a56aa6cfb74f29bd1640a20804e9de689d23f10c5227e07ba4f66ebbb9606e1384390dca277a6942886706198394717694a9cfb1f20cd36ca377 languageName: node linkType: hard -"matrix-widget-api@npm:^1.10.0, matrix-widget-api@npm:^1.13.0": - version: 1.13.1 - resolution: "matrix-widget-api@npm:1.13.1" +"matrix-widget-api@npm:^1.14.0": + version: 1.15.0 + resolution: "matrix-widget-api@npm:1.15.0" dependencies: "@types/events": "npm:^3.0.0" events: "npm:^3.2.0" - checksum: 10c0/25ded744922755b3eb65f4e171cf6cff1a2e0fe43fc3fecbb13e565e41d8af066daa817dd2c3c7d921b996af399eec3b23df70ab7b682cf422d9cee7ca202512 + checksum: 10c0/1c08b5284cd98aed312d95594335e1391d937dfad70ef862a1f90fdbaaa27709e1c44dcda37f8045e4814779d8d5816d240aee396d52cfd9b37fbf243a6baf6a + languageName: node + linkType: hard + +"md5.js@npm:^1.3.4": + version: 1.3.5 + resolution: "md5.js@npm:1.3.5" + dependencies: + hash-base: "npm:^3.0.0" + inherits: "npm:^2.0.1" + safe-buffer: "npm:^5.1.2" + checksum: 10c0/b7bd75077f419c8e013fc4d4dada48be71882e37d69a44af65a2f2804b91e253441eb43a0614423a1c91bb830b8140b0dc906bc797245e2e275759584f4efcc5 languageName: node linkType: hard @@ -10174,6 +10765,18 @@ __metadata: languageName: node linkType: hard +"miller-rabin@npm:^4.0.0": + version: 4.0.1 + resolution: "miller-rabin@npm:4.0.1" + dependencies: + bn.js: "npm:^4.0.0" + brorand: "npm:^1.0.1" + bin: + miller-rabin: bin/miller-rabin + checksum: 10c0/26b2b96f6e49dbcff7faebb78708ed2f5f9ae27ac8cbbf1d7c08f83cf39bed3d418c0c11034dce997da70d135cc0ff6f3a4c15dc452f8e114c11986388a64346 + languageName: node + linkType: hard + "mime-db@npm:1.52.0": version: 1.52.0 resolution: "mime-db@npm:1.52.0" @@ -10197,6 +10800,20 @@ __metadata: languageName: node linkType: hard +"minimalistic-assert@npm:^1.0.0, minimalistic-assert@npm:^1.0.1": + version: 1.0.1 + resolution: "minimalistic-assert@npm:1.0.1" + checksum: 10c0/96730e5601cd31457f81a296f521eb56036e6f69133c0b18c13fe941109d53ad23a4204d946a0d638d7f3099482a0cec8c9bb6d642604612ce43ee536be3dddd + languageName: node + linkType: hard + +"minimalistic-crypto-utils@npm:^1.0.1": + version: 1.0.1 + resolution: "minimalistic-crypto-utils@npm:1.0.1" + checksum: 10c0/790ecec8c5c73973a4fbf2c663d911033e8494d5fb0960a4500634766ab05d6107d20af896ca2132e7031741f19888154d44b2408ada0852446705441383e9f8 + languageName: node + linkType: hard + "minimatch@npm:^3.0.2, minimatch@npm:^3.0.4, minimatch@npm:^3.0.5, minimatch@npm:^3.1.1, minimatch@npm:^3.1.2": version: 3.1.2 resolution: "minimatch@npm:3.1.2" @@ -10447,6 +11064,41 @@ __metadata: languageName: node linkType: hard +"node-stdlib-browser@npm:^1.3.1": + version: 1.3.1 + resolution: "node-stdlib-browser@npm:1.3.1" + dependencies: + assert: "npm:^2.0.0" + browser-resolve: "npm:^2.0.0" + browserify-zlib: "npm:^0.2.0" + buffer: "npm:^5.7.1" + console-browserify: "npm:^1.1.0" + constants-browserify: "npm:^1.0.0" + create-require: "npm:^1.1.1" + crypto-browserify: "npm:^3.12.1" + domain-browser: "npm:4.22.0" + events: "npm:^3.0.0" + https-browserify: "npm:^1.0.0" + isomorphic-timers-promises: "npm:^1.0.1" + os-browserify: "npm:^0.3.0" + path-browserify: "npm:^1.0.1" + pkg-dir: "npm:^5.0.0" + process: "npm:^0.11.10" + punycode: "npm:^1.4.1" + querystring-es3: "npm:^0.2.1" + readable-stream: "npm:^3.6.0" + stream-browserify: "npm:^3.0.0" + stream-http: "npm:^3.2.0" + string_decoder: "npm:^1.0.0" + timers-browserify: "npm:^2.0.4" + tty-browserify: "npm:0.0.1" + url: "npm:^0.11.4" + util: "npm:^0.12.4" + vm-browserify: "npm:^1.0.1" + checksum: 10c0/5b0cb5d4499b1b1c73f54db3e9e69b2a3a8aebe2ead2e356b0a03c1dfca6b5c5d2f6516e24301e76dc7b68999b9d0ae3da6c3f1dec421eed80ad6cb9eec0f356 + languageName: node + linkType: hard + "nopt@npm:^8.0.0": version: 8.1.0 resolution: "nopt@npm:8.1.0" @@ -10523,6 +11175,13 @@ __metadata: languageName: node linkType: hard +"object-deep-merge@npm:^2.0.0": + version: 2.0.0 + resolution: "object-deep-merge@npm:2.0.0" + checksum: 10c0/69e8741131ad49fa8720fb96007a3c82dca1119b5d874151d2ecbcc3b44ccd46e8553c7a30b0abcba752c099ba361bbba97f33a68c9ae54c57eed7be116ffc97 + languageName: node + linkType: hard + "object-inspect@npm:^1.13.3": version: 1.13.3 resolution: "object-inspect@npm:1.13.3" @@ -10537,6 +11196,16 @@ __metadata: languageName: node linkType: hard +"object-is@npm:^1.1.5": + version: 1.1.6 + resolution: "object-is@npm:1.1.6" + dependencies: + call-bind: "npm:^1.0.7" + define-properties: "npm:^1.2.1" + checksum: 10c0/506af444c4dce7f8e31f34fc549e2fb8152d6b9c4a30c6e62852badd7f520b579c679af433e7a072f9d78eb7808d230dc12e1cf58da9154dfbf8813099ea0fe0 + languageName: node + linkType: hard + "object-keys@npm:^1.1.1": version: 1.1.1 resolution: "object-keys@npm:1.1.1" @@ -10648,6 +11317,13 @@ __metadata: languageName: node linkType: hard +"os-browserify@npm:^0.3.0": + version: 0.3.0 + resolution: "os-browserify@npm:0.3.0" + checksum: 10c0/6ff32cb1efe2bc6930ad0fd4c50e30c38010aee909eba8d65be60af55efd6cbb48f0287e3649b4e3f3a63dce5a667b23c187c4293a75e557f0d5489d735bcf52 + languageName: node + linkType: hard + "own-keys@npm:^1.0.1": version: 1.0.1 resolution: "own-keys@npm:1.0.1" @@ -10780,6 +11456,13 @@ __metadata: languageName: node linkType: hard +"pako@npm:~1.0.5": + version: 1.0.11 + resolution: "pako@npm:1.0.11" + checksum: 10c0/86dd99d8b34c3930345b8bbeb5e1cd8a05f608eeb40967b293f72fe469d0e9c88b783a8777e4cc7dc7c91ce54c5e93d88ff4b4f060e6ff18408fd21030d9ffbe + languageName: node + linkType: hard + "param-case@npm:^3.0.4": version: 3.0.4 resolution: "param-case@npm:3.0.4" @@ -10799,6 +11482,28 @@ __metadata: languageName: node linkType: hard +"parse-asn1@npm:^5.0.0, parse-asn1@npm:^5.1.9": + version: 5.1.9 + resolution: "parse-asn1@npm:5.1.9" + dependencies: + asn1.js: "npm:^4.10.1" + browserify-aes: "npm:^1.2.0" + evp_bytestokey: "npm:^1.0.3" + pbkdf2: "npm:^3.1.5" + safe-buffer: "npm:^5.2.1" + checksum: 10c0/6dfe27c121be3d63ebbf95f03d2ae0a07dd716d44b70b0bd3458790a822a80de05361c62147271fd7b845dcc2d37755d9c9c393064a3438fe633779df0bc07e7 + languageName: node + linkType: hard + +"parse-imports-exports@npm:^0.2.4": + version: 0.2.4 + resolution: "parse-imports-exports@npm:0.2.4" + dependencies: + parse-statements: "npm:1.0.11" + checksum: 10c0/51b729037208abdf65c4a1f8e9ed06f4e7ccd907c17c668a64db54b37d95bb9e92081f8b16e4133e14102af3cb4e89870975b6ad661b4d654e9ec8f4fb5c77d6 + languageName: node + linkType: hard + "parse-json@npm:^5.0.0, parse-json@npm:^5.2.0": version: 5.2.0 resolution: "parse-json@npm:5.2.0" @@ -10811,6 +11516,13 @@ __metadata: languageName: node linkType: hard +"parse-statements@npm:1.0.11": + version: 1.0.11 + resolution: "parse-statements@npm:1.0.11" + checksum: 10c0/48960e085019068a5f5242e875fd9d21ec87df2e291acf5ad4e4887b40eab6929a8c8d59542acb85a6497e870c5c6a24f5ab7f980ef5f907c14cc5f7984a93f3 + languageName: node + linkType: hard + "parse5-htmlparser2-tree-adapter@npm:^7.0.0": version: 7.1.0 resolution: "parse5-htmlparser2-tree-adapter@npm:7.1.0" @@ -10849,6 +11561,13 @@ __metadata: languageName: node linkType: hard +"path-browserify@npm:^1.0.1": + version: 1.0.1 + resolution: "path-browserify@npm:1.0.1" + checksum: 10c0/8b8c3fd5c66bd340272180590ae4ff139769e9ab79522e2eb82e3d571a89b8117c04147f65ad066dccfb42fcad902e5b7d794b3d35e0fd840491a8ddbedf8c66 + languageName: node + linkType: hard + "path-exists@npm:^4.0.0": version: 4.0.0 resolution: "path-exists@npm:4.0.0" @@ -10922,6 +11641,20 @@ __metadata: languageName: node linkType: hard +"pbkdf2@npm:^3.1.2, pbkdf2@npm:^3.1.5": + version: 3.1.5 + resolution: "pbkdf2@npm:3.1.5" + dependencies: + create-hash: "npm:^1.2.0" + create-hmac: "npm:^1.1.7" + ripemd160: "npm:^2.0.3" + safe-buffer: "npm:^5.2.1" + sha.js: "npm:^2.4.12" + to-buffer: "npm:^1.2.1" + checksum: 10c0/ea42e8695e49417eefabb19a08ab19a602cc6cc72d2df3f109c39309600230dee3083a6f678d5d42fe035d6ae780038b80ace0e68f9792ee2839bf081fe386f3 + languageName: node + linkType: hard + "picocolors@npm:^1.0.0, picocolors@npm:^1.1.1": version: 1.1.1 resolution: "picocolors@npm:1.1.1" @@ -10950,27 +11683,36 @@ __metadata: languageName: node linkType: hard -"playwright-core@npm:1.56.1": - version: 1.56.1 - resolution: "playwright-core@npm:1.56.1" - bin: - playwright-core: cli.js - checksum: 10c0/ffd40142b99c68678b387445d5b42f1fee4ab0b65d983058c37f342e5629f9cdbdac0506ea80a0dfd41a8f9f13345bad54e9a8c35826ef66dc765f4eb3db8da7 +"pkg-dir@npm:^5.0.0": + version: 5.0.0 + resolution: "pkg-dir@npm:5.0.0" + dependencies: + find-up: "npm:^5.0.0" + checksum: 10c0/793a496d685dc55bbbdbbb22d884535c3b29241e48e3e8d37e448113a71b9e42f5481a61fdc672d7322de12fbb2c584dd3a68bf89b18fffce5c48a390f911bc5 languageName: node linkType: hard -"playwright@npm:1.56.1": - version: 1.56.1 - resolution: "playwright@npm:1.56.1" +"playwright-core@npm:1.57.0": + version: 1.57.0 + resolution: "playwright-core@npm:1.57.0" + bin: + playwright-core: cli.js + checksum: 10c0/798e35d83bf48419a8c73de20bb94d68be5dde68de23f95d80a0ebe401e3b83e29e3e84aea7894d67fa6c79d2d3d40cc5bcde3e166f657ce50987aaa2421b6a9 + languageName: node + linkType: hard + +"playwright@npm:1.57.0": + version: 1.57.0 + resolution: "playwright@npm:1.57.0" dependencies: fsevents: "npm:2.3.2" - playwright-core: "npm:1.56.1" + playwright-core: "npm:1.57.0" dependenciesMeta: fsevents: optional: true bin: playwright: cli.js - checksum: 10c0/8e9965aede86df0f4722063385748498977b219630a40a10d1b82b8bd8d4d4e9b6b65ecbfa024331a30800163161aca292fb6dd7446c531a1ad25f4155625ab4 + checksum: 10c0/ab03c99a67b835bdea9059f516ad3b6e42c21025f9adaa161a4ef6bc7ca716dcba476d287140bb240d06126eb23f889a8933b8f5f1f1a56b80659d92d1358899 languageName: node linkType: hard @@ -11446,6 +12188,13 @@ __metadata: languageName: node linkType: hard +"process@npm:^0.11.10": + version: 0.11.10 + resolution: "process@npm:0.11.10" + checksum: 10c0/40c3ce4b7e6d4b8c3355479df77aeed46f81b279818ccdc500124e6a5ab882c0cc81ff7ea16384873a95a74c4570b01b120f287abbdd4c877931460eca6084b3 + languageName: node + linkType: hard + "progress@npm:^2.0.3": version: 2.0.3 resolution: "progress@npm:2.0.3" @@ -11498,6 +12247,27 @@ __metadata: languageName: node linkType: hard +"public-encrypt@npm:^4.0.3": + version: 4.0.3 + resolution: "public-encrypt@npm:4.0.3" + dependencies: + bn.js: "npm:^4.1.0" + browserify-rsa: "npm:^4.0.0" + create-hash: "npm:^1.1.0" + parse-asn1: "npm:^5.0.0" + randombytes: "npm:^2.0.1" + safe-buffer: "npm:^5.1.2" + checksum: 10c0/6c2cc19fbb554449e47f2175065d6b32f828f9b3badbee4c76585ac28ae8641aafb9bb107afc430c33c5edd6b05dbe318df4f7d6d7712b1093407b11c4280700 + languageName: node + linkType: hard + +"punycode@npm:^1.4.1": + version: 1.4.1 + resolution: "punycode@npm:1.4.1" + checksum: 10c0/354b743320518aef36f77013be6e15da4db24c2b4f62c5f1eb0529a6ed02fbaf1cb52925785f6ab85a962f2b590d9cd5ad730b70da72b5f180e2556b8bd3ca08 + languageName: node + linkType: hard + "punycode@npm:^2.1.0, punycode@npm:^2.3.1": version: 2.3.1 resolution: "punycode@npm:2.3.1" @@ -11518,6 +12288,22 @@ __metadata: languageName: node linkType: hard +"qs@npm:^6.12.3": + version: 6.14.0 + resolution: "qs@npm:6.14.0" + dependencies: + side-channel: "npm:^1.1.0" + checksum: 10c0/8ea5d91bf34f440598ee389d4a7d95820e3b837d3fd9f433871f7924801becaa0cd3b3b4628d49a7784d06a8aea9bc4554d2b6d8d584e2d221dc06238a42909c + languageName: node + linkType: hard + +"querystring-es3@npm:^0.2.1": + version: 0.2.1 + resolution: "querystring-es3@npm:0.2.1" + checksum: 10c0/476938c1adb45c141f024fccd2ffd919a3746e79ed444d00e670aad68532977b793889648980e7ca7ff5ffc7bfece623118d0fbadcaf217495eeb7059ae51580 + languageName: node + linkType: hard + "queue-microtask@npm:^1.2.2": version: 1.2.3 resolution: "queue-microtask@npm:1.2.3" @@ -11536,6 +12322,25 @@ __metadata: languageName: node linkType: hard +"randombytes@npm:^2.0.0, randombytes@npm:^2.0.1, randombytes@npm:^2.0.5, randombytes@npm:^2.1.0": + version: 2.1.0 + resolution: "randombytes@npm:2.1.0" + dependencies: + safe-buffer: "npm:^5.1.0" + checksum: 10c0/50395efda7a8c94f5dffab564f9ff89736064d32addf0cc7e8bf5e4166f09f8ded7a0849ca6c2d2a59478f7d90f78f20d8048bca3cdf8be09d8e8a10790388f3 + languageName: node + linkType: hard + +"randomfill@npm:^1.0.4": + version: 1.0.4 + resolution: "randomfill@npm:1.0.4" + dependencies: + randombytes: "npm:^2.0.5" + safe-buffer: "npm:^5.1.0" + checksum: 10c0/11aeed35515872e8f8a2edec306734e6b74c39c46653607f03c68385ab8030e2adcc4215f76b5e4598e028c4750d820afd5c65202527d831d2a5f207fe2bc87c + languageName: node + linkType: hard + "react-dom@npm:19": version: 19.1.0 resolution: "react-dom@npm:19.1.0" @@ -11730,18 +12535,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^3.4.0": - version: 3.6.2 - resolution: "readable-stream@npm:3.6.2" - dependencies: - inherits: "npm:^2.0.3" - string_decoder: "npm:^1.1.1" - util-deprecate: "npm:^1.0.1" - checksum: 10c0/e37be5c79c376fdd088a45fa31ea2e423e5d48854be7a22a58869b4e84d25047b193f6acb54f1012331e1bcd667ffb569c01b99d36b0bd59658fb33f513511b7 - languageName: node - linkType: hard - -"readable-stream@npm:~2.3.6": +"readable-stream@npm:^2.3.8, readable-stream@npm:~2.3.6": version: 2.3.8 resolution: "readable-stream@npm:2.3.8" dependencies: @@ -11756,6 +12550,17 @@ __metadata: languageName: node linkType: hard +"readable-stream@npm:^3.4.0, readable-stream@npm:^3.5.0, readable-stream@npm:^3.6.0": + version: 3.6.2 + resolution: "readable-stream@npm:3.6.2" + dependencies: + inherits: "npm:^2.0.3" + string_decoder: "npm:^1.1.1" + util-deprecate: "npm:^1.0.1" + checksum: 10c0/e37be5c79c376fdd088a45fa31ea2e423e5d48854be7a22a58869b4e84d25047b193f6acb54f1012331e1bcd667ffb569c01b99d36b0bd59658fb33f513511b7 + languageName: node + linkType: hard + "readdirp@npm:^4.0.1": version: 4.1.2 resolution: "readdirp@npm:4.1.2" @@ -11943,6 +12748,13 @@ __metadata: languageName: node linkType: hard +"reserved-identifiers@npm:^1.0.0": + version: 1.2.0 + resolution: "reserved-identifiers@npm:1.2.0" + checksum: 10c0/b82651b12e6c608e80463c3753d275bc20fd89294d0415f04e670aeec3611ae3582ddc19e8fedd497e7d0bcbfaddab6a12823ec86e855b1e6a245e0a734eb43d + languageName: node + linkType: hard + "resolve-from@npm:^4.0.0": version: 4.0.0 resolution: "resolve-from@npm:4.0.0" @@ -11972,6 +12784,19 @@ __metadata: languageName: node linkType: hard +"resolve@npm:^1.17.0": + version: 1.22.11 + resolution: "resolve@npm:1.22.11" + dependencies: + is-core-module: "npm:^2.16.1" + path-parse: "npm:^1.0.7" + supports-preserve-symlinks-flag: "npm:^1.0.0" + bin: + resolve: bin/resolve + checksum: 10c0/f657191507530f2cbecb5815b1ee99b20741ea6ee02a59c57028e9ec4c2c8d7681afcc35febbd554ac0ded459db6f2d8153382c53a2f266cee2575e512674409 + languageName: node + linkType: hard + "resolve@npm:^1.22.10": version: 1.22.10 resolution: "resolve@npm:1.22.10" @@ -12011,6 +12836,19 @@ __metadata: languageName: node linkType: hard +"resolve@patch:resolve@npm%3A^1.17.0#optional!builtin": + version: 1.22.11 + resolution: "resolve@patch:resolve@npm%3A1.22.11#optional!builtin::version=1.22.11&hash=c3c19d" + dependencies: + is-core-module: "npm:^2.16.1" + path-parse: "npm:^1.0.7" + supports-preserve-symlinks-flag: "npm:^1.0.0" + bin: + resolve: bin/resolve + checksum: 10c0/ee5b182f2e37cb1165465e58c6abc797fec0a80b5ba3231607beb4677db0c9291ac010c47cf092b6daa2b7f518d69a0e21888e7e2b633f68d501a874212a8c63 + languageName: node + linkType: hard + "resolve@patch:resolve@npm%3A^1.22.10#optional!builtin": version: 1.22.10 resolution: "resolve@patch:resolve@npm%3A1.22.10#optional!builtin::version=1.22.10&hash=c3c19d" @@ -12084,6 +12922,16 @@ __metadata: languageName: node linkType: hard +"ripemd160@npm:^2.0.0, ripemd160@npm:^2.0.1, ripemd160@npm:^2.0.3": + version: 2.0.3 + resolution: "ripemd160@npm:2.0.3" + dependencies: + hash-base: "npm:^3.1.2" + inherits: "npm:^2.0.4" + checksum: 10c0/3f472fb453241cfe692a77349accafca38dbcdc9d96d5848c088b2932ba41eb968630ecff7b175d291c7487a4945aee5a81e30c064d1f94e36070f7e0c37ed6c + languageName: node + linkType: hard + "rollup@npm:^4.43.0": version: 4.50.1 resolution: "rollup@npm:4.50.1" @@ -12231,6 +13079,13 @@ __metadata: languageName: node linkType: hard +"safe-buffer@npm:^5.0.1, safe-buffer@npm:^5.1.0, safe-buffer@npm:^5.1.1, safe-buffer@npm:^5.1.2, safe-buffer@npm:^5.2.1, safe-buffer@npm:~5.2.0": + version: 5.2.1 + resolution: "safe-buffer@npm:5.2.1" + checksum: 10c0/6501914237c0a86e9675d4e51d89ca3c21ffd6a31642efeba25ad65720bce6921c9e7e974e5be91a786b25aa058b5303285d3c15dbabf983a919f5f630d349f3 + languageName: node + linkType: hard + "safe-buffer@npm:~5.1.0, safe-buffer@npm:~5.1.1": version: 5.1.2 resolution: "safe-buffer@npm:5.1.2" @@ -12238,13 +13093,6 @@ __metadata: languageName: node linkType: hard -"safe-buffer@npm:~5.2.0": - version: 5.2.1 - resolution: "safe-buffer@npm:5.2.1" - checksum: 10c0/6501914237c0a86e9675d4e51d89ca3c21ffd6a31642efeba25ad65720bce6921c9e7e974e5be91a786b25aa058b5303285d3c15dbabf983a919f5f630d349f3 - languageName: node - linkType: hard - "safe-push-apply@npm:^1.0.0": version: 1.0.0 resolution: "safe-push-apply@npm:1.0.0" @@ -12306,7 +13154,7 @@ __metadata: languageName: node linkType: hard -"sdp-transform@npm:^2.14.1, sdp-transform@npm:^2.15.0": +"sdp-transform@npm:^2.15.0": version: 2.15.0 resolution: "sdp-transform@npm:2.15.0" bin: @@ -12315,6 +13163,15 @@ __metadata: languageName: node linkType: hard +"sdp-transform@npm:^3.0.0": + version: 3.0.0 + resolution: "sdp-transform@npm:3.0.0" + bin: + sdp-verify: checker.js + checksum: 10c0/828a4595041ba64c86b29075aa4007ab384519b1fa29882db59ccb83b54b2b2a33b60848293f8da537fe151c52f5844fc17c8325396cac309fb19e2e81ec5bf4 + languageName: node + linkType: hard + "sdp@npm:^3.2.0": version: 3.2.0 resolution: "sdp@npm:3.2.0" @@ -12358,6 +13215,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:^7.7.3": + version: 7.7.3 + resolution: "semver@npm:7.7.3" + bin: + semver: bin/semver.js + checksum: 10c0/4afe5c986567db82f44c8c6faef8fe9df2a9b1d98098fc1721f57c696c4c21cebd572f297fc21002f81889492345b8470473bc6f4aff5fb032a6ea59ea2bc45e + languageName: node + linkType: hard + "set-blocking@npm:^2.0.0": version: 2.0.0 resolution: "set-blocking@npm:2.0.0" @@ -12409,6 +13275,26 @@ __metadata: languageName: node linkType: hard +"setimmediate@npm:^1.0.4": + version: 1.0.5 + resolution: "setimmediate@npm:1.0.5" + checksum: 10c0/5bae81bfdbfbd0ce992893286d49c9693c82b1bcc00dcaaf3a09c8f428fdeacf4190c013598b81875dfac2b08a572422db7df779a99332d0fce186d15a3e4d49 + languageName: node + linkType: hard + +"sha.js@npm:^2.4.0, sha.js@npm:^2.4.12, sha.js@npm:^2.4.8": + version: 2.4.12 + resolution: "sha.js@npm:2.4.12" + dependencies: + inherits: "npm:^2.0.4" + safe-buffer: "npm:^5.2.1" + to-buffer: "npm:^1.2.0" + bin: + sha.js: bin.js + checksum: 10c0/9d36bdd76202c8116abbe152a00055ccd8a0099cb28fc17c01fa7bb2c8cffb9ca60e2ab0fe5f274ed6c45dc2633d8c39cf7ab050306c231904512ba9da4d8ab1 + languageName: node + linkType: hard + "shebang-command@npm:^2.0.0": version: 2.0.0 resolution: "shebang-command@npm:2.0.0" @@ -12606,6 +13492,16 @@ __metadata: languageName: node linkType: hard +"spdx-expression-parse@npm:^4.0.0": + version: 4.0.0 + resolution: "spdx-expression-parse@npm:4.0.0" + dependencies: + spdx-exceptions: "npm:^2.1.0" + spdx-license-ids: "npm:^3.0.0" + checksum: 10c0/965c487e77f4fb173f1c471f3eef4eb44b9f0321adc7f93d95e7620da31faa67d29356eb02523cd7df8a7fc1ec8238773cdbf9e45bd050329d2b26492771b736 + languageName: node + linkType: hard + "spdx-license-ids@npm:^3.0.0": version: 3.0.20 resolution: "spdx-license-ids@npm:3.0.20" @@ -12653,6 +13549,16 @@ __metadata: languageName: node linkType: hard +"stream-browserify@npm:^3.0.0": + version: 3.0.0 + resolution: "stream-browserify@npm:3.0.0" + dependencies: + inherits: "npm:~2.0.4" + readable-stream: "npm:^3.5.0" + checksum: 10c0/ec3b975a4e0aa4b3dc5e70ffae3fc8fd29ac725353a14e72f213dff477b00330140ad014b163a8cbb9922dfe90803f81a5ea2b269e1bbfd8bd71511b88f889ad + languageName: node + linkType: hard + "stream-composer@npm:^1.0.2": version: 1.0.2 resolution: "stream-composer@npm:1.0.2" @@ -12662,6 +13568,18 @@ __metadata: languageName: node linkType: hard +"stream-http@npm:^3.2.0": + version: 3.2.0 + resolution: "stream-http@npm:3.2.0" + dependencies: + builtin-status-codes: "npm:^3.0.0" + inherits: "npm:^2.0.4" + readable-stream: "npm:^3.6.0" + xtend: "npm:^4.0.2" + checksum: 10c0/f128fb8076d60cd548f229554b6a1a70c08a04b7b2afd4dbe7811d20f27f7d4112562eb8bce86d72a8691df3b50573228afcf1271e55e81f981536c67498bc41 + languageName: node + linkType: hard + "streamx@npm:^2.12.0, streamx@npm:^2.12.5, streamx@npm:^2.13.2, streamx@npm:^2.14.0": version: 2.22.0 resolution: "streamx@npm:2.22.0" @@ -12778,7 +13696,7 @@ __metadata: languageName: node linkType: hard -"string_decoder@npm:^1.1.1": +"string_decoder@npm:^1.0.0, string_decoder@npm:^1.1.1": version: 1.3.0 resolution: "string_decoder@npm:1.3.0" dependencies: @@ -12980,6 +13898,15 @@ __metadata: languageName: node linkType: hard +"timers-browserify@npm:^2.0.4": + version: 2.0.12 + resolution: "timers-browserify@npm:2.0.12" + dependencies: + setimmediate: "npm:^1.0.4" + checksum: 10c0/98e84db1a685bc8827c117a8bc62aac811ad56a995d07938fc7ed8cdc5bf3777bfe2d4e5da868847194e771aac3749a20f6cdd22091300fe889a76fe214a4641 + languageName: node + linkType: hard + "tinybench@npm:^2.9.0": version: 2.9.0 resolution: "tinybench@npm:2.9.0" @@ -13053,6 +13980,17 @@ __metadata: languageName: node linkType: hard +"to-buffer@npm:^1.2.0, to-buffer@npm:^1.2.1, to-buffer@npm:^1.2.2": + version: 1.2.2 + resolution: "to-buffer@npm:1.2.2" + dependencies: + isarray: "npm:^2.0.5" + safe-buffer: "npm:^5.2.1" + typed-array-buffer: "npm:^1.0.3" + checksum: 10c0/56bc56352f14a2c4a0ab6277c5fc19b51e9534882b98eb068b39e14146591e62fa5b06bf70f7fed1626230463d7e60dca81e815096656e5e01c195c593873d12 + languageName: node + linkType: hard + "to-regex-range@npm:^5.0.1": version: 5.0.1 resolution: "to-regex-range@npm:5.0.1" @@ -13071,6 +14009,16 @@ __metadata: languageName: node linkType: hard +"to-valid-identifier@npm:^1.0.0": + version: 1.0.0 + resolution: "to-valid-identifier@npm:1.0.0" + dependencies: + "@sindresorhus/base62": "npm:^1.0.0" + reserved-identifiers: "npm:^1.0.0" + checksum: 10c0/569b49f43b5aaaa20677e67f0f1cdcff344855149934cfb80c793c7ac7c30e191b224bc81cab40fb57641af9ca73795c78053c164a2addc617671e2d22c13a4a + languageName: node + linkType: hard + "toggle-selection@npm:^1.0.6": version: 1.0.6 resolution: "toggle-selection@npm:1.0.6" @@ -13190,6 +14138,13 @@ __metadata: languageName: node linkType: hard +"tty-browserify@npm:0.0.1": + version: 0.0.1 + resolution: "tty-browserify@npm:0.0.1" + checksum: 10c0/5e34883388eb5f556234dae75b08e069b9e62de12bd6d87687f7817f5569430a6dfef550b51dbc961715ae0cd0eb5a059e6e3fc34dc127ea164aa0f9b5bb033d + languageName: node + linkType: hard + "tunnel@npm:^0.0.6": version: 0.0.6 resolution: "tunnel@npm:0.0.6" @@ -13524,6 +14479,16 @@ __metadata: languageName: node linkType: hard +"url@npm:^0.11.4": + version: 0.11.4 + resolution: "url@npm:0.11.4" + dependencies: + punycode: "npm:^1.4.1" + qs: "npm:^6.12.3" + checksum: 10c0/cc93405ae4a9b97a2aa60ca67f1cb1481c0221cb4725a7341d149be5e2f9cfda26fd432d64dbbec693d16593b68b8a46aad8e5eab21f814932134c9d8620c662 + languageName: node + linkType: hard + "use-callback-ref@npm:^1.3.3": version: 1.3.3 resolution: "use-callback-ref@npm:1.3.3" @@ -13573,6 +14538,19 @@ __metadata: languageName: node linkType: hard +"util@npm:^0.12.4, util@npm:^0.12.5": + version: 0.12.5 + resolution: "util@npm:0.12.5" + dependencies: + inherits: "npm:^2.0.3" + is-arguments: "npm:^1.0.4" + is-generator-function: "npm:^1.0.7" + is-typed-array: "npm:^1.1.3" + which-typed-array: "npm:^1.1.2" + checksum: 10c0/c27054de2cea2229a66c09522d0fa1415fb12d861d08523a8846bf2e4cbf0079d4c3f725f09dcb87493549bcbf05f5798dce1688b53c6c17201a45759e7253f3 + languageName: node + linkType: hard + "uuid@npm:13": version: 13.0.0 resolution: "uuid@npm:13.0.0" @@ -13719,6 +14697,18 @@ __metadata: languageName: node linkType: hard +"vite-plugin-node-stdlib-browser@npm:^0.2.1": + version: 0.2.1 + resolution: "vite-plugin-node-stdlib-browser@npm:0.2.1" + dependencies: + "@rollup/plugin-inject": "npm:^5.0.3" + peerDependencies: + node-stdlib-browser: ^1.2.0 + vite: ^2.0.0 || ^3.0.0 || ^4.0.0 + checksum: 10c0/4686bde59d0396d8684433e1a14ddce868dc422f80e306a0c1cb5e86564d9f7c38a67865ca339e4ff57784ec4bada149034038cad6911a2dfcac8debfc9bd20a + languageName: node + linkType: hard + "vite-plugin-svgr@npm:^4.0.0": version: 4.3.0 resolution: "vite-plugin-svgr@npm:4.3.0" @@ -13857,6 +14847,13 @@ __metadata: languageName: node linkType: hard +"vm-browserify@npm:^1.0.1": + version: 1.1.2 + resolution: "vm-browserify@npm:1.1.2" + checksum: 10c0/0cc1af6e0d880deb58bc974921320c187f9e0a94f25570fca6b1bd64e798ce454ab87dfd797551b1b0cc1849307421aae0193cedf5f06bdb5680476780ee344b + languageName: node + linkType: hard + "void-elements@npm:3.1.0": version: 3.1.0 resolution: "void-elements@npm:3.1.0" @@ -14056,7 +15053,7 @@ __metadata: languageName: node linkType: hard -"which-typed-array@npm:^1.1.19": +"which-typed-array@npm:^1.1.19, which-typed-array@npm:^1.1.2": version: 1.1.19 resolution: "which-typed-array@npm:1.1.19" dependencies: @@ -14181,7 +15178,7 @@ __metadata: languageName: node linkType: hard -"xtend@npm:~4.0.1": +"xtend@npm:^4.0.2, xtend@npm:~4.0.1": version: 4.0.2 resolution: "xtend@npm:4.0.2" checksum: 10c0/366ae4783eec6100f8a02dff02ac907bf29f9a00b82ac0264b4d8b832ead18306797e283cf19de776538babfdcb2101375ec5646b59f08c52128ac4ab812ed0e