diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 067c5246..11116ed2 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. `; module.exports = { - plugins: ["matrix-org", "rxjs"], + plugins: ["matrix-org", "rxjs", "jsdoc"], extends: [ "plugin:matrix-org/react", "plugin:matrix-org/a11y", @@ -26,6 +26,13 @@ module.exports = { node: true, }, rules: { + "jsdoc/no-types": "error", + "jsdoc/empty-tags": "error", + "jsdoc/check-property-names": "error", + "jsdoc/check-values": "error", + "jsdoc/check-param-names": "warn", + // "jsdoc/require-param": "warn", + "jsdoc/require-param-description": "warn", "matrix-org/require-copyright-header": ["error", COPYRIGHT_HEADER], "jsx-a11y/media-has-caption": "off", "react/display-name": "error", @@ -75,6 +82,23 @@ module.exports = { "no-console": ["error"], }, }, + { + files: [ + "**/*.test.ts", + "**/*.test.tsx", + "**/test.ts", + "**/test.tsx", + "**/test-**", + ], + rules: { + "jsdoc/no-types": "off", + "jsdoc/empty-tags": "off", + "jsdoc/check-property-names": "off", + "jsdoc/check-values": "off", + "jsdoc/check-param-names": "off", + "jsdoc/require-param-description": "off", + }, + }, ], settings: { react: { diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 6a5c090e..7f2c58fe 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -61,6 +61,7 @@ jobs: docker_tags: | type=sha,format=short,event=branch type=raw,value=${{ github.event.release.tag_name }} + type=raw,value=latest # Like before, using ${{ env.VERSION }} above doesn't work add_docker_release_note: needs: publish_docker diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index 50498c7a..c7591847 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -47,7 +47,7 @@ services: - ecbackend livekit: - image: livekit/livekit-server:latest + image: livekit/livekit-server:v1.9.4 pull_policy: always hostname: livekit-sfu command: --dev --config /etc/livekit.yaml @@ -67,7 +67,7 @@ services: - ecbackend livekit-1: - image: livekit/livekit-server:latest + image: livekit/livekit-server:v1.9.4 pull_policy: always hostname: livekit-sfu-1 command: --dev --config /etc/livekit.yaml diff --git a/knip.ts b/knip.ts index 6b378e29..7edfaf65 100644 --- a/knip.ts +++ b/knip.ts @@ -9,7 +9,7 @@ import { type KnipConfig } from "knip"; export default { vite: { - config: ["vite.config.ts", "vite-embedded.config.ts"], + config: ["vite.config.ts", "vite-embedded.config.ts", "vite-sdk.config.ts"], }, entry: ["src/main.tsx", "i18next-parser.config.ts"], ignoreBinaries: [ diff --git a/package.json b/package.json index 62ea9f4f..336dd5d7 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "build:embedded": "yarn build:full --config vite-embedded.config.js", "build:embedded:production": "yarn build:embedded", "build:embedded:development": "yarn build:embedded --mode development", + "build:sdk": "yarn build:full --config vite-sdk.config.js", + "build:sdk:development": "yarn build:sdk --mode development", "serve": "vite preview", "prettier:check": "prettier -c .", "prettier:format": "prettier -w .", @@ -54,7 +56,7 @@ "@opentelemetry/sdk-trace-base": "^2.0.0", "@opentelemetry/sdk-trace-web": "^2.0.0", "@opentelemetry/semantic-conventions": "^1.25.1", - "@playwright/test": "^1.56.1", + "@playwright/test": "^1.57.0", "@radix-ui/react-dialog": "^1.0.4", "@radix-ui/react-slider": "^1.1.2", "@radix-ui/react-visually-hidden": "^1.0.3", @@ -93,6 +95,7 @@ "eslint-config-prettier": "^10.0.0", "eslint-plugin-deprecate": "^0.8.2", "eslint-plugin-import": "^2.26.0", + "eslint-plugin-jsdoc": "^61.5.0", "eslint-plugin-jsx-a11y": "^6.5.1", "eslint-plugin-matrix-org": "2.1.0", "eslint-plugin-react": "^7.29.4", @@ -109,8 +112,9 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21", - "matrix-widget-api": "^1.13.0", + "matrix-js-sdk": "matrix-org/matrix-js-sdk#2218ec4e3102e841ba3e794e1c492c0a5aa6c1c3", + "matrix-widget-api": "^1.14.0", + "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", "pako": "^2.0.4", @@ -133,6 +137,7 @@ "vite": "^7.0.0", "vite-plugin-generate-file": "^0.3.0", "vite-plugin-html": "^3.2.2", + "vite-plugin-node-stdlib-browser": "^0.2.1", "vite-plugin-svgr": "^4.0.0", "vitest": "^3.0.0", "vitest-axe": "^1.0.0-pre.3" diff --git a/playwright.config.ts b/playwright.config.ts index 7a8ee530..391e746f 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -38,6 +38,7 @@ export default defineConfig({ projects: [ { name: "chromium", + testIgnore: "**/mobile/**", use: { ...devices["Desktop Chrome"], permissions: [ @@ -56,9 +57,9 @@ export default defineConfig({ }, }, }, - { name: "firefox", + testIgnore: "**/mobile/**", use: { ...devices["Desktop Firefox"], ignoreHTTPSErrors: true, @@ -70,6 +71,27 @@ export default defineConfig({ }, }, }, + { + name: "mobile", + testMatch: "**/mobile/**", + use: { + ...devices["Pixel 7"], + ignoreHTTPSErrors: true, + permissions: [ + "clipboard-write", + "clipboard-read", + "microphone", + "camera", + ], + launchOptions: { + args: [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + "--mute-audio", + ], + }, + }, + }, // No safari for now, until I find a solution to fix `Not allowed to request resource` due to calling // clear http to the homeserver diff --git a/playwright/errors.spec.ts b/playwright/errors.spec.ts index 0d36f7ab..085fb0b4 100644 --- a/playwright/errors.spec.ts +++ b/playwright/errors.spec.ts @@ -7,6 +7,8 @@ Please see LICENSE in the repository root for full details. import { expect, test } from "@playwright/test"; +import { createJTWToken } from "./fixtures/jwt-token"; + test("Should show error screen if fails to get JWT token", async ({ page }) => { await page.goto("/"); @@ -93,7 +95,7 @@ test("Should show error screen if call creation is restricted", async ({ contentType: "application/json", body: JSON.stringify({ url: "wss://badurltotricktest/livekit/sfu", - jwt: "FAKE", + jwt: createJTWToken("@fake:user", "!fake:room"), }), }), ); diff --git a/playwright/fixtures/fixture-mobile-create.ts b/playwright/fixtures/fixture-mobile-create.ts new file mode 100644 index 00000000..3920c978 --- /dev/null +++ b/playwright/fixtures/fixture-mobile-create.ts @@ -0,0 +1,73 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type Browser, type Page, test, expect } from "@playwright/test"; + +export interface MobileCreateFixtures { + asMobile: { + creatorPage: Page; + inviteLink: string; + }; +} + +export const mobileTest = test.extend({ + asMobile: async ({ browser }, pUse) => { + const fixtures = await createCallAndInvite(browser); + await pUse({ + creatorPage: fixtures.page, + inviteLink: fixtures.inviteLink, + }); + }, +}); + +/** + * Create a call and generate an invite link + */ +async function createCallAndInvite( + browser: Browser, +): Promise<{ page: Page; inviteLink: string }> { + const creatorContext = await browser.newContext({ reducedMotion: "reduce" }); + const creatorPage = await creatorContext.newPage(); + + await creatorPage.goto("/"); + + // ======== + // ARRANGE: The first user creates a call as guest, join it, then click the invite button to copy the invite link + // ======== + await creatorPage.getByTestId("home_callName").click(); + await creatorPage.getByTestId("home_callName").fill("Welcome"); + await creatorPage.getByTestId("home_displayName").click(); + await creatorPage.getByTestId("home_displayName").fill("Inviter"); + await creatorPage.getByTestId("home_go").click(); + await expect(creatorPage.locator("video")).toBeVisible(); + + await creatorPage + .getByRole("button", { name: "Continue in browser" }) + .click(); + // join + await creatorPage.getByTestId("lobby_joinCall").click(); + + // Get the invite link + await creatorPage.getByRole("button", { name: "Invite" }).click(); + await expect( + creatorPage.getByRole("heading", { name: "Invite to this call" }), + ).toBeVisible(); + await expect(creatorPage.getByRole("img", { name: "QR Code" })).toBeVisible(); + await expect(creatorPage.getByTestId("modal_inviteLink")).toBeVisible(); + await expect(creatorPage.getByTestId("modal_inviteLink")).toBeVisible(); + await creatorPage.getByTestId("modal_inviteLink").click(); + + const inviteLink = (await creatorPage.evaluate( + "navigator.clipboard.readText()", + )) as string; + expect(inviteLink).toContain("room/#/"); + + return { + page: creatorPage, + inviteLink, + }; +} diff --git a/playwright/fixtures/jwt-token.ts b/playwright/fixtures/jwt-token.ts new file mode 100644 index 00000000..18119c7e --- /dev/null +++ b/playwright/fixtures/jwt-token.ts @@ -0,0 +1,22 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +export function createJTWToken(sub: string, room: string): string { + return [ + {}, // header + { + // payload + sub, + video: { + room, + }, + }, + {}, // signature + ] + .map((d) => global.btoa(JSON.stringify(d))) + .join("."); +} diff --git a/playwright/fixtures/widget-user.ts b/playwright/fixtures/widget-user.ts index 433c960b..f1f738b7 100644 --- a/playwright/fixtures/widget-user.ts +++ b/playwright/fixtures/widget-user.ts @@ -67,7 +67,6 @@ const CONFIG_JSON = { /** * Set the Element Call URL in the dev tool settings using `window.mxSettingsStore` via `page.evaluate`. - * @param page */ const setDevToolElementCallDevUrl = process.env.USE_DOCKER ? async (page: Page): Promise => { @@ -111,19 +110,27 @@ async function registerUser( await page.getByRole("textbox", { name: "Confirm password" }).click(); await page.getByRole("textbox", { name: "Confirm password" }).fill(PASSWORD); await page.getByRole("button", { name: "Register" }).click(); - const continueButton = page.getByRole("button", { name: "Continue" }); - try { - await expect(continueButton).toBeVisible({ timeout: 5000 }); - await page - .getByRole("textbox", { name: "Password", exact: true }) - .fill(PASSWORD); - await continueButton.click(); - } catch { - // continueButton not visible, continue as normal - } + await expect( page.getByRole("heading", { name: `Welcome ${username}` }), ).toBeVisible(); + + const browserUnsupportedToast = page + .getByText("Element does not support this browser") + .locator("..") + .locator(".."); + + // Dismiss incompatible browser toast + const dismissButton = browserUnsupportedToast.getByRole("button", { + name: "Dismiss", + }); + try { + await expect(dismissButton).toBeVisible({ timeout: 700 }); + await dismissButton.click(); + } catch { + // dismissButton not visible, continue as normal + } + await setDevToolElementCallDevUrl(page); const clientHandle = await page.evaluateHandle(() => diff --git a/playwright/mobile/create-call-mobile.spec.ts b/playwright/mobile/create-call-mobile.spec.ts new file mode 100644 index 00000000..141ffaae --- /dev/null +++ b/playwright/mobile/create-call-mobile.spec.ts @@ -0,0 +1,115 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "@playwright/test"; + +import { mobileTest } from "../fixtures/fixture-mobile-create.ts"; + +test("@mobile Start a new call then leave and show the feedback screen", async ({ + page, +}) => { + await page.goto("/"); + + await page.getByTestId("home_callName").click(); + await page.getByTestId("home_callName").fill("HelloCall"); + await page.getByTestId("home_displayName").click(); + await page.getByTestId("home_displayName").fill("John Doe"); + await page.getByTestId("home_go").click(); + + // await page.pause(); + await expect(page.locator("video")).toBeVisible(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + + await page.getByRole("button", { name: "Continue in browser" }).click(); + // Join the call + await page.getByTestId("lobby_joinCall").click(); + + // Ensure that the call is connected + await page + .locator("div") + .filter({ hasText: /^HelloCall$/ }) + .click(); + // Check the number of participants + await expect(page.locator("div").filter({ hasText: /^1$/ })).toBeVisible(); + // The tooltip with the name should be visible + await expect(page.getByTestId("name_tag")).toContainText("John Doe"); + + // leave the call + await page.getByTestId("incall_leave").click(); + await expect(page.getByRole("heading")).toContainText( + "John Doe, your call has ended. How did it go?", + ); + await expect(page.getByRole("main")).toContainText( + "Why not finish by setting up a password to keep your account?", + ); + + await expect( + page.getByRole("link", { name: "Not now, return to home screen" }), + ).toBeVisible(); +}); + +mobileTest( + "Test earpiece overlay in controlledAudioDevices mode", + async ({ asMobile, browser }) => { + test.slow(); // Triples the timeout + const { creatorPage, inviteLink } = asMobile; + + // ======== + // ACT: The other user use the invite link to join the call as a guest + // ======== + const guestInviteeContext = await browser.newContext({ + reducedMotion: "reduce", + }); + const guestPage = await guestInviteeContext.newPage(); + await guestPage.goto(inviteLink + "&controlledAudioDevices=true"); + + await guestPage + .getByRole("button", { name: "Continue in browser" }) + .click(); + + await guestPage.getByTestId("joincall_displayName").fill("Invitee"); + await expect(guestPage.getByTestId("joincall_joincall")).toBeVisible(); + await guestPage.getByTestId("joincall_joincall").click(); + await guestPage.getByTestId("lobby_joinCall").click(); + + // ======== + // ASSERT: check that there are two members in the call + // ======== + + // There should be two participants now + await expect( + guestPage.getByTestId("roomHeader_participants_count"), + ).toContainText("2"); + expect(await guestPage.getByTestId("videoTile").count()).toBe(2); + + // Same in creator page + await expect( + creatorPage.getByTestId("roomHeader_participants_count"), + ).toContainText("2"); + expect(await creatorPage.getByTestId("videoTile").count()).toBe(2); + + // TEST: control audio devices from the invitee page + + await guestPage.evaluate(() => { + window.controls.setAvailableAudioDevices([ + { id: "speaker", name: "Speaker", isSpeaker: true }, + { id: "earpiece", name: "Handset", isEarpiece: true }, + { id: "headphones", name: "Headphones" }, + ]); + window.controls.setAudioDevice("earpiece"); + }); + await expect( + guestPage.getByRole("heading", { name: "Handset Mode" }), + ).toBeVisible(); + await expect( + guestPage.getByRole("button", { name: "Back to Speaker Mode" }), + ).toBeVisible(); + + // Should auto-mute the video when earpiece is selected + await expect(guestPage.getByTestId("incall_videomute")).toBeDisabled(); + }, +); diff --git a/playwright/sfu-reconnect-bug.spec.ts b/playwright/sfu-reconnect-bug.spec.ts index 6138eb78..9f666f0f 100644 --- a/playwright/sfu-reconnect-bug.spec.ts +++ b/playwright/sfu-reconnect-bug.spec.ts @@ -68,11 +68,6 @@ test("When creator left, avoid reconnect to the same SFU", async ({ reducedMotion: "reduce", }); const guestCPage = await guestC.newPage(); - let sfuGetCallCount = 0; - await guestCPage.route("**/livekit/jwt/sfu/get", async (route) => { - sfuGetCallCount++; - await route.continue(); - }); // Track WebSocket connections let wsConnectionCount = 0; await guestCPage.routeWebSocket("**", (ws) => { @@ -100,5 +95,4 @@ test("When creator left, avoid reconnect to the same SFU", async ({ // https://github.com/element-hq/element-call/issues/3344 // The app used to request a new jwt token then to reconnect to the SFU expect(wsConnectionCount).toBe(1); - expect(sfuGetCallCount).toBe(2 /* the first one is for the warmup */); }); diff --git a/sdk/README.md b/sdk/README.md new file mode 100644 index 00000000..03801b83 --- /dev/null +++ b/sdk/README.md @@ -0,0 +1,35 @@ +# SDK mode + +EC can be build in sdk mode. This will result in a compiled js file that can be imported in very simple webapps. + +It allows to use matrixRTC in combination with livekit without relying on element call. + +This is done by instantiating the call view model and exposing some useful behaviors (observables) and methods. + +This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver ellowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template. + +## Widgets + +The sdk mode is particularly interesting to be used in widgets where you do not need to pay attention to matrix login/cs api ... +To create a widget see the example index.html file in this folder. And add it to EW via: +`/addwidget ` (see **url parameters** for more details on ``) + +### url parameters + +``` +widgetId = $matrix_widget_id +perParticipantE2EE = true +userId = $matrix_user_id +deviceId = $org.matrix.msc3819.matrix_device_id +baseUrl = $org.matrix.msc4039.matrix_base_url +``` + +`parentUrl = // will be inserted automatically` + +Full template use as ``: + +``` +http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id +``` + +the `$` prefixed variables will be replaced by EW on widget instantiation. (e.g. `$matrix_user_id` -> `@user:example.com` (url encoding will also be applied automatically by EW) -> `%40user%3Aexample.com`) diff --git a/sdk/helper.ts b/sdk/helper.ts new file mode 100644 index 00000000..a3d597be --- /dev/null +++ b/sdk/helper.ts @@ -0,0 +1,55 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * This file contains helper functions and types for the MatrixRTC SDK. + */ + +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { scan } from "rxjs"; + +import { widget as _widget } from "../src/widget"; +import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel"; + +export const logger = rootLogger.getChild("[MatrixRTCSdk]"); + +if (!_widget) throw Error("No widget. This webapp can only start as a widget"); +export const widget = _widget; + +export const tryMakeSticky = (): void => { + logger.info("try making sticky MatrixRTCSdk"); + void widget.api + .setAlwaysOnScreen(true) + .then(() => { + logger.info("sticky MatrixRTCSdk"); + }) + .catch((error) => { + logger.error("failed to make sticky MatrixRTCSdk", error); + }); +}; +export const TEXT_LK_TOPIC = "matrixRTC"; +/** + * simple helper operator to combine the last emitted and the current emitted value of a rxjs observable + * + * I think there should be a builtin for this but i did not find it... + */ +export const currentAndPrev = scan< + LivekitRoomItem[], + { + prev: LivekitRoomItem[]; + current: LivekitRoomItem[]; + } +>( + ({ current: lastCurrentVal }, items) => ({ + prev: lastCurrentVal, + current: items, + }), + { + prev: [], + current: [], + }, +); diff --git a/sdk/index.html b/sdk/index.html new file mode 100644 index 00000000..51110ebd --- /dev/null +++ b/sdk/index.html @@ -0,0 +1,87 @@ + + + + Godot MatrixRTC Widget + + + + + + + +
+
+ + +
+
+
+ + diff --git a/sdk/main.ts b/sdk/main.ts new file mode 100644 index 00000000..376674a4 --- /dev/null +++ b/sdk/main.ts @@ -0,0 +1,308 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * This file is the entrypoint for the sdk build of element call: `yarn build:sdk` + * use in widgets. + * It exposes the `createMatrixRTCSdk` which creates the `MatrixRTCSdk` interface (see below) that + * can be used to join a rtc session and exchange realtime data. + * It takes care of all the tricky bits: + * - sending delayed events + * - finding the right sfu + * - handling the media stream + * - sending join/leave state or sticky events + * - setting up encryption and scharing keys + */ + +import { + combineLatest, + map, + type Observable, + of, + shareReplay, + Subject, + switchMap, + tap, +} from "rxjs"; +import { + type CallMembership, + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/lib/matrixrtc"; +import { + type Room as LivekitRoom, + type TextStreamReader, + type LocalParticipant, + type RemoteParticipant, +} from "livekit-client"; + +// TODO how can this get fixed? to just be part of `livekit-client` +// Can this be done in the tsconfig.json +import { type TextStreamInfo } from "../node_modules/livekit-client/dist/src/room/types"; +import { type Behavior, constant } from "../src/state/Behavior"; +import { createCallViewModel$ } from "../src/state/CallViewModel/CallViewModel"; +import { ObservableScope } from "../src/state/ObservableScope"; +import { getUrlParams } from "../src/UrlParams"; +import { MuteStates } from "../src/state/MuteStates"; +import { MediaDevices } from "../src/state/MediaDevices"; +import { E2eeType } from "../src/e2ee/e2eeType"; +import { + currentAndPrev, + logger, + TEXT_LK_TOPIC, + tryMakeSticky, + widget, +} from "./helper"; +import { ElementWidgetActions } from "../src/widget"; +import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection"; + +interface MatrixRTCSdk { + /** + * observe connected$ to track the state. + * @returns + */ + join: () => void; + /** @throws on leave errors */ + leave: () => void; + data$: Observable<{ sender: string; data: string }>; + /** + * flattened list of members + */ + members$: Behavior< + { + connection: Connection | null; + membership: CallMembership; + participant: LocalParticipant | RemoteParticipant | null; + }[] + >; + /** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state */ + connected$: Behavior; + sendData?: (data: unknown) => Promise; +} + +export async function createMatrixRTCSdk( + application: string = "m.call", + id: string = "", +): Promise { + logger.info("Hello"); + const client = await widget.client; + logger.info("client created"); + const scope = new ObservableScope(); + const { roomId } = getUrlParams(); + if (roomId === null) throw Error("could not get roomId from url params"); + + const room = client.getRoom(roomId); + if (room === null) throw Error("could not get room from client"); + + const mediaDevices = new MediaDevices(scope); + const muteStates = new MuteStates(scope, mediaDevices, constant(true)); + const slot = { application, id }; + const rtcSession = new MatrixRTCSession( + client, + room, + MatrixRTCSession.sessionMembershipsForSlot(room, slot), + slot, + ); + const callViewModel = createCallViewModel$( + scope, + rtcSession, + room, + mediaDevices, + muteStates, + { encryptionSystem: { kind: E2eeType.PER_PARTICIPANT } }, + of({}), + of({}), + constant({ supported: false, processor: undefined }), + ); + logger.info("CallViewModelCreated"); + // create data listener + const data$ = new Subject<{ sender: string; data: string }>(); + + const lkTextStreamHandlerFunction = async ( + reader: TextStreamReader, + participantInfo: { identity: string }, + livekitRoom: LivekitRoom, + ): Promise => { + const info = reader.info; + logger.info( + `Received text stream from ${participantInfo.identity}\n` + + ` Topic: ${info.topic}\n` + + ` Timestamp: ${info.timestamp}\n` + + ` ID: ${info.id}\n` + + ` Size: ${info.size}`, // Optional, only available if the stream was sent with `sendText` + ); + + const participants = callViewModel.livekitRoomItems$.value.find( + (i) => i.livekitRoom === livekitRoom, + )?.participants; + if (participants && participants.includes(participantInfo.identity)) { + const text = await reader.readAll(); + logger.info(`Received text: ${text}`); + data$.next({ sender: participantInfo.identity, data: text }); + } else { + logger.warn( + "Received text from unknown participant", + participantInfo.identity, + ); + } + }; + + const livekitRoomItemsSub = callViewModel.livekitRoomItems$ + .pipe( + tap((beforecurrentAndPrev) => { + logger.info( + `LiveKit room items updated: ${beforecurrentAndPrev.length}`, + beforecurrentAndPrev, + ); + }), + currentAndPrev, + tap((aftercurrentAndPrev) => { + logger.info( + `LiveKit room items updated: ${aftercurrentAndPrev.current.length}, ${aftercurrentAndPrev.prev.length}`, + aftercurrentAndPrev, + ); + }), + ) + .subscribe({ + next: ({ prev, current }) => { + const prevRooms = prev.map((i) => i.livekitRoom); + const currentRooms = current.map((i) => i.livekitRoom); + const addedRooms = currentRooms.filter((r) => !prevRooms.includes(r)); + const removedRooms = prevRooms.filter((r) => !currentRooms.includes(r)); + addedRooms.forEach((r) => { + logger.info(`Registering text stream handler for room `); + r.registerTextStreamHandler( + TEXT_LK_TOPIC, + (reader, participantInfo) => + void lkTextStreamHandlerFunction(reader, participantInfo, r), + ); + }); + removedRooms.forEach((r) => { + logger.info(`Unregistering text stream handler for room `); + r.unregisterTextStreamHandler(TEXT_LK_TOPIC); + }); + }, + complete: () => { + logger.info("Livekit room items subscription completed"); + for (const item of callViewModel.livekitRoomItems$.value) { + logger.info("unregistering room item from room", item.url); + item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC); + } + }, + }); + + // create sendData function + const sendFn: Behavior<(data: string) => Promise> = + scope.behavior( + callViewModel.localMatrixLivekitMember$.pipe( + switchMap((m) => { + if (!m) + return of((data: string): never => { + throw Error("local membership not yet ready."); + }); + return m.participant.value$.pipe( + map((p) => { + if (p === null) { + return (data: string): never => { + throw Error("local participant not yet ready to send data."); + }; + } else { + return async (data: string): Promise => + p.sendText(data, { topic: TEXT_LK_TOPIC }); + } + }), + ); + }), + ), + ); + + const sendData = async (data: unknown): Promise => { + const dataString = JSON.stringify(data); + logger.info("try sending: ", dataString); + try { + await Promise.resolve(); + const info = await sendFn.value(dataString); + logger.info(`Sent text with stream ID: ${info.id}`); + } catch (e) { + logger.error("failed sending: ", dataString, e); + } + }; + + // after hangup gets called + const leaveSubs = callViewModel.leave$.subscribe(() => { + const scheduleWidgetCloseOnLeave = async (): Promise => { + const leaveResolver = Promise.withResolvers(); + logger.info("waiting for RTC leave"); + rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, (isJoined) => { + logger.info("received RTC join update: ", isJoined); + if (!isJoined) leaveResolver.resolve(); + }); + await leaveResolver.promise; + logger.info("send Unstick"); + await widget.api + .setAlwaysOnScreen(false) + .catch((e) => + logger.error( + "Failed to set call widget `alwaysOnScreen` to false", + e, + ), + ); + logger.info("send Close"); + await widget.api.transport + .send(ElementWidgetActions.Close, {}) + .catch((e) => logger.error("Failed to send close action", e)); + }; + + // schedule close first and then leave (scope.end) + void scheduleWidgetCloseOnLeave(); + + // actual hangup (ending scope will send the leave event.. its kinda odd. since you might end up closing the widget too fast) + scope.end(); + }); + + logger.info("createMatrixRTCSdk done"); + + return { + join: (): void => { + // first lets try making the widget sticky + tryMakeSticky(); + callViewModel.join(); + }, + leave: (): void => { + callViewModel.hangup(); + leaveSubs.unsubscribe(); + livekitRoomItemsSub.unsubscribe(); + }, + data$, + connected$: callViewModel.connected$, + members$: scope.behavior( + callViewModel.matrixLivekitMembers$.pipe( + switchMap((members) => { + const listOfMemberObservables = members.map((member) => + combineLatest([ + member.connection$, + member.membership$, + member.participant.value$, + ]).pipe( + map(([connection, membership, participant]) => ({ + connection, + membership, + participant, + })), + // using shareReplay instead of a Behavior here because the behavior would need + // a tricky scope.end() setup. + shareReplay({ bufferSize: 1, refCount: true }), + ), + ); + return combineLatest(listOfMemberObservables); + }), + ), + [], + ), + sendData, + }; +} diff --git a/src/UrlParams.test.ts b/src/UrlParams.test.ts index cd8fc6d5..faba394f 100644 --- a/src/UrlParams.test.ts +++ b/src/UrlParams.test.ts @@ -332,6 +332,42 @@ describe("UrlParams", () => { expect(computeUrlParams("?intent=join_existing").skipLobby).toBe(false); }); }); + + describe("noiseSuppression", () => { + it("defaults to true", () => { + expect(computeUrlParams().noiseSuppression).toBe(true); + }); + + it("is parsed", () => { + expect( + computeUrlParams("?intent=start_call&noiseSuppression=true") + .noiseSuppression, + ).toBe(true); + expect( + computeUrlParams("?intent=start_call&noiseSuppression&bar=foo") + .noiseSuppression, + ).toBe(true); + expect(computeUrlParams("?noiseSuppression=false").noiseSuppression).toBe( + false, + ); + }); + }); + + describe("echoCancellation", () => { + it("defaults to true", () => { + expect(computeUrlParams().echoCancellation).toBe(true); + }); + + it("is parsed", () => { + expect(computeUrlParams("?echoCancellation=true").echoCancellation).toBe( + true, + ); + expect(computeUrlParams("?echoCancellation=false").echoCancellation).toBe( + false, + ); + }); + }); + describe("header", () => { it("uses header if provided", () => { expect(computeUrlParams("?header=app_bar&hideHeader=true").header).toBe( diff --git a/src/UrlParams.ts b/src/UrlParams.ts index 4eb69298..f78841fb 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -233,6 +233,17 @@ export interface UrlConfiguration { */ waitForCallPickup: boolean; + /** + * Whether to enable echo cancellation for audio capture. + * Defaults to true. + */ + echoCancellation?: boolean; + /** + * Whether to enable noise suppression for audio capture. + * Defaults to true. + */ + noiseSuppression?: boolean; + callIntent?: RTCCallIntent; } interface IntentAndPlatformDerivedConfiguration { @@ -525,6 +536,8 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => { ]), waitForCallPickup: parser.getFlag("waitForCallPickup"), autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"), + noiseSuppression: parser.getFlagParam("noiseSuppression", true), + echoCancellation: parser.getFlagParam("echoCancellation", true), }; // Log the final configuration for debugging purposes. diff --git a/src/analytics/PosthogAnalytics.ts b/src/analytics/PosthogAnalytics.ts index 8b2aa91d..46223afe 100644 --- a/src/analytics/PosthogAnalytics.ts +++ b/src/analytics/PosthogAnalytics.ts @@ -247,9 +247,8 @@ export class PosthogAnalytics { // wins, and the first writer will send tracking with an ID that doesn't match the one on the server // until the next time account data is refreshed and this function is called (most likely on next // page load). This will happen pretty infrequently, so we can tolerate the possibility. - const accountDataAnalyticsId = analyticsIdGenerator(); - await this.setAccountAnalyticsId(accountDataAnalyticsId); - analyticsID = await this.hashedEcAnalyticsId(accountDataAnalyticsId); + analyticsID = analyticsIdGenerator(); + await this.setAccountAnalyticsId(analyticsID); } } catch (e) { // The above could fail due to network requests, but not essential to starting the application, @@ -270,37 +269,14 @@ export class PosthogAnalytics { private async getAnalyticsId(): Promise { const client: MatrixClient = window.matrixclient; - let accountAnalyticsId: string | null; if (widget) { - accountAnalyticsId = getUrlParams().posthogUserId; + return getUrlParams().posthogUserId; } else { const accountData = await client.getAccountDataFromServer( PosthogAnalytics.ANALYTICS_EVENT_TYPE, ); - accountAnalyticsId = accountData?.id ?? null; + return accountData?.id ?? null; } - if (accountAnalyticsId) { - // we dont just use the element web analytics ID because that would allow to associate - // users between the two posthog instances. By using a hash from the username and the element web analytics id - // it is not possible to conclude the element web posthog user id from the element call user id and vice versa. - return await this.hashedEcAnalyticsId(accountAnalyticsId); - } - return null; - } - - private async hashedEcAnalyticsId( - accountAnalyticsId: string, - ): Promise { - const client: MatrixClient = window.matrixclient; - const posthogIdMaterial = "ec" + accountAnalyticsId + client.getUserId(); - const bufferForPosthogId = await crypto.subtle.digest( - "sha-256", - new TextEncoder().encode(posthogIdMaterial), - ); - const view = new Int32Array(bufferForPosthogId); - return Array.from(view) - .map((b) => Math.abs(b).toString(16).padStart(2, "0")) - .join(""); } private async setAccountAnalyticsId(analyticsID: string): Promise { diff --git a/src/e2ee/sharedKeyManagement.ts b/src/e2ee/sharedKeyManagement.ts index c68ba453..18d007e2 100644 --- a/src/e2ee/sharedKeyManagement.ts +++ b/src/e2ee/sharedKeyManagement.ts @@ -34,8 +34,8 @@ const getRoomSharedKeyLocalStorageKey = (roomId: string): string => `room-shared-key-${roomId}`; /** - * An upto-date shared key for the room. Either from local storage or the value from `setInitialValue`. - * @param roomId + * An up-to-date shared key for the room. Either from local storage or the value from `setInitialValue`. + * @param roomId The room ID we want the shared key for. * @param setInitialValue The value we get from the URL. The hook will overwrite the local storage value with this. * @returns [roomSharedKey, setRoomSharedKey] like a react useState hook. */ diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 5b1149e9..741529b8 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -166,7 +166,11 @@ interface StereoPanAudioTrackProps { * It main purpose is to remount the AudioTrack component when switching from * audioContext to normal audio playback. * As of now the AudioTrack component does not support adding audio nodes while being mounted. - * @param param0 + * @param props The component props + * @param props.trackRef The track reference + * @param props.muted If the track should be muted + * @param props.audioContext The audio context to use + * @param props.audioNodes The audio nodes to use * @returns */ function AudioTrackWithAudioNodes({ diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts new file mode 100644 index 00000000..2a260b01 --- /dev/null +++ b/src/livekit/openIDSFU.test.ts @@ -0,0 +1,112 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + beforeEach, + afterEach, + describe, + expect, + it, + type MockedObject, + vitest, +} from "vitest"; +import fetchMock from "fetch-mock"; + +import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU"; +import { testJWTToken } from "../utils/test-fixtures"; + +const sfuUrl = "https://sfu.example.org"; + +describe("getSFUConfigWithOpenID", () => { + let matrixClient: MockedObject; + beforeEach(() => { + matrixClient = { + getOpenIdToken: vitest.fn(), + getDeviceId: vitest.fn(), + }; + }); + afterEach(() => { + vitest.clearAllMocks(); + fetchMock.reset(); + }); + it("should handle fetching a token", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: sfuUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfigWithOpenID( + matrixClient, + "https://sfu.example.org", + "!example_room_id", + ); + expect(config).toEqual({ + jwt: testJWTToken, + url: sfuUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + void (await fetchMock.flush()); + }); + it("should fail if the SFU errors", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { error: "Test failure" }, + }; + }); + try { + await getSFUConfigWithOpenID( + matrixClient, + "https://sfu.example.org", + "!example_room_id", + ); + } catch (ex) { + expect(((ex as Error).cause as Error).message).toEqual( + "SFU Config fetch failed with status code 500", + ); + void (await fetchMock.flush()); + return; + } + expect.fail("Expected test to throw;"); + }); + + it("should retry fetching the openid token", async () => { + let count = 0; + matrixClient.getOpenIdToken.mockImplementation(async () => { + count++; + if (count < 2) { + throw Error("Test failure"); + } + return Promise.resolve({ + token_type: "Bearer", + access_token: "foobar", + matrix_server_name: "example.org", + expires_in: 30, + }); + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: sfuUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfigWithOpenID( + matrixClient, + "https://sfu.example.org", + "!example_room_id", + ); + expect(config).toEqual({ + jwt: testJWTToken, + url: sfuUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + void (await fetchMock.flush()); + }); +}); diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 3ae003fb..34c98a88 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -11,9 +11,47 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { FailToGetOpenIdToken } from "../utils/errors"; import { doNetworkOperationWithRetry } from "../utils/matrix"; +/** + * Configuration and access tokens provided by the SFU on successful authentication. + */ export interface SFUConfig { url: string; jwt: string; + livekitAlias: string; + livekitIdentity: string; +} + +/** + * Decoded details from the JWT. + */ +interface SFUJWTPayload { + /** + * Expiration time for the JWT. + * Note: This value is in seconds since Unix epoch. + */ + exp: number; + /** + * Name of the instance which authored the JWT + */ + iss: string; + /** + * Time at which the JWT can start to be used. + * Note: This value is in seconds since Unix epoch. + */ + nbf: number; + /** + * Subject. The Livekit alias in this context. + */ + sub: string; + /** + * The set of permissions for the user. + */ + video: { + canPublish: boolean; + canSubscribe: boolean; + room: string; + roomJoin: boolean; + }; } // The bits we need from MatrixClient @@ -25,9 +63,9 @@ export type OpenIDClientParts = Pick< * Gets a bearer token from the homeserver and then use it to authenticate * to the matrix RTC backend in order to get acces to the SFU. * It has built-in retry for calls to the homeserver with a backoff policy. - * @param client - * @param serviceUrl - * @param matrixRoomId + * @param client The Matrix client + * @param serviceUrl The URL of the livekit SFU service + * @param matrixRoomId The Matrix room ID for which to get the SFU config * @returns Object containing the token information * @throws FailToGetOpenIdToken */ @@ -57,7 +95,17 @@ export async function getSFUConfigWithOpenID( ); logger.info(`Got JWT from call's active focus URL.`); - return sfuConfig; + // Pull the details from the JWT + const [, payloadStr] = sfuConfig.jwt.split("."); + // TODO: Prefer Uint8Array.fromBase64 when widely available + const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; + return { + jwt: sfuConfig.jwt, + url: sfuConfig.url, + livekitAlias: payload.video.room, + // NOTE: Currently unused. + livekitIdentity: payload.sub, + }; } async function getLiveKitJWT( @@ -65,7 +113,7 @@ async function getLiveKitJWT( livekitServiceURL: string, roomName: string, openIDToken: IOpenIDToken, -): Promise { +): Promise<{ url: string; jwt: string }> { try { const res = await fetch(livekitServiceURL + "/sfu/get", { method: "POST", @@ -83,6 +131,6 @@ async function getLiveKitJWT( } return await res.json(); } catch (e) { - throw new Error("SFU Config fetch failed with exception " + e); + throw new Error("SFU Config fetch failed with exception", { cause: e }); } } diff --git a/src/reactions/ReactionsReader.ts b/src/reactions/ReactionsReader.ts index 74b47c77..7ce59812 100644 --- a/src/reactions/ReactionsReader.ts +++ b/src/reactions/ReactionsReader.ts @@ -135,10 +135,10 @@ export class ReactionsReader { } /** - * Fetchest any hand wave reactions by the given sender on the given + * Fetches any hand wave reactions by the given sender on the given * membership event. - * @param membershipEventId - * @param expectedSender + * @param membershipEventId - The user membership event id. + * @param expectedSender - The expected sender of the reaction. * @returns A MatrixEvent if one was found. */ private getLastReactionEvent( diff --git a/src/room/EarpieceOverlay.module.css b/src/room/EarpieceOverlay.module.css index d0757cdb..e53a1974 100644 --- a/src/room/EarpieceOverlay.module.css +++ b/src/room/EarpieceOverlay.module.css @@ -7,34 +7,17 @@ align-items: center; justify-content: center; gap: var(--cpd-space-2x); -} - -@keyframes fade-in { - from { - opacity: 0; - } - to { - opacity: 1; - } + transition: opacity 200ms; } .overlay[data-show="true"] { - animation: fade-in 200ms; -} - -@keyframes fade-out { - from { - opacity: 1; - } - to { - opacity: 0; - display: none; - } + opacity: 1; } .overlay[data-show="false"] { - animation: fade-out 130ms forwards; + opacity: 0; pointer-events: none; + transition-duration: 130ms; } .overlay::before { diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 75438f7f..dfd11ff3 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -160,6 +160,7 @@ export const GroupCallView: FC = ({ }, [rtcSession]); // TODO move this into the callViewModel LocalMembership.ts + // We might actually not need this at all. Since we get into fatalError on those errors already? useTypedEventEmitter( rtcSession, MatrixRTCSessionEvent.MembershipManagerError, @@ -313,6 +314,7 @@ export const GroupCallView: FC = ({ const navigate = useNavigate(); + // TODO split this into leave and onDisconnect const onLeft = useCallback( ( reason: "timeout" | "user" | "allOthersLeft" | "decline" | "error", diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 6ae004d8..07adacbc 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -24,7 +24,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import classNames from "classnames"; import { BehaviorSubject, map } from "rxjs"; import { useObservable } from "observable-hooks"; -import { logger } from "matrix-js-sdk/lib/logger"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { VoiceCallSolidIcon, VolumeOnSolidIcon, @@ -88,6 +88,7 @@ import { ReactionsOverlay } from "./ReactionsOverlay"; import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { debugTileLayout as debugTileLayoutSetting, + matrixRTCMode as matrixRTCModeSetting, useSetting, } from "../settings/settings"; import { ReactionsReader } from "../reactions/ReactionsReader"; @@ -109,6 +110,8 @@ import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.t import { type Layout } from "../state/layout-types.ts"; import { ObservableScope } from "../state/ObservableScope.ts"; +const logger = rootLogger.getChild("[InCallView]"); + const maxTapDurationMs = 400; export interface ActiveCallProps @@ -127,6 +130,7 @@ export const ActiveCall: FC = (props) => { const mediaDevices = useMediaDevices(); const trackProcessorState$ = useTrackProcessorObservable$(); useEffect(() => { + logger.info("START CALL VIEW SCOPE"); const scope = new ObservableScope(); const reactionsReader = new ReactionsReader(scope, props.rtcSession); const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = @@ -141,6 +145,7 @@ export const ActiveCall: FC = (props) => { encryptionSystem: props.e2eeSystem, autoLeaveWhenOthersLeft, waitForCallPickup: waitForCallPickup && sendNotificationType === "ring", + matrixRTCMode$: matrixRTCModeSetting.value$, }, reactionsReader.raisedHands$, reactionsReader.reactions$, @@ -151,7 +156,9 @@ export const ActiveCall: FC = (props) => { setVm(vm); vm.leave$.pipe(scope.bind()).subscribe(props.onLeft); + return (): void => { + logger.info("END CALL VIEW SCOPE"); scope.end(); }; }, [ @@ -253,7 +260,7 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); - const audioParticipants = useBehavior(vm.audioParticipants$); + const audioParticipants = useBehavior(vm.livekitRoomItems$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); const windowMode = useBehavior(vm.windowMode$); @@ -270,7 +277,10 @@ export const InCallView: FC = ({ const ringOverlay = useBehavior(vm.ringOverlay$); const fatalCallError = useBehavior(vm.fatalError$); // Stop the rendering and throw for the error boundary - if (fatalCallError) throw fatalCallError; + if (fatalCallError) { + logger.debug("fatalCallError stop rendering", fatalCallError); + throw fatalCallError; + } // We need to set the proper timings on the animation based upon the sound length. const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1; diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index ad4f30b3..10e098f1 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -79,9 +79,9 @@ export const LobbyView: FC = ({ waitingForInvite, }) => { useEffect(() => { - logger.info("[Lifecycle] GroupCallView Component mounted"); + logger.info("[Lifecycle] LobbyView Component mounted"); return (): void => { - logger.info("[Lifecycle] GroupCallView Component unmounted"); + logger.info("[Lifecycle] LobbyView Component unmounted"); }; }, []); diff --git a/src/room/useLoadGroupCall.ts b/src/room/useLoadGroupCall.ts index ab6ccf64..2cd0d40b 100644 --- a/src/room/useLoadGroupCall.ts +++ b/src/room/useLoadGroupCall.ts @@ -106,22 +106,18 @@ async function joinRoomAfterInvite( export class CallTerminatedMessage extends Error { /** + * Creates a new CallTerminatedMessage. + * + * @param icon The icon to display with the message * @param messageTitle The title of the call ended screen message (translated) + * @param messageBody The message explaining the kind of termination + * (kick, ban, knock reject, etc.) (translated) + * @param reason The user-provided reason for the termination (kick/ban) */ public constructor( - /** - * The icon to display with the message. - */ public readonly icon: ComponentType>, messageTitle: string, - /** - * The message explaining the kind of termination (kick, ban, knock reject, - * etc.) (translated) - */ public readonly messageBody: string, - /** - * The user-provided reason for the termination (kick/ban) - */ public readonly reason?: string, ) { super(messageTitle); diff --git a/src/settings/rageshake.ts b/src/settings/rageshake.ts index 6c1a0f61..26d0839b 100644 --- a/src/settings/rageshake.ts +++ b/src/settings/rageshake.ts @@ -99,7 +99,7 @@ class ConsoleLogger extends EventEmitter { /** * Returns the log lines to flush to disk and empties the internal log buffer - * @return {string} \n delimited log lines + * @return \n delimited log lines */ public popLogs(): string { const logsToFlush = this.logs; @@ -109,7 +109,7 @@ class ConsoleLogger extends EventEmitter { /** * Returns lines currently in the log buffer without removing them - * @return {string} \n delimited log lines + * @return \n delimited log lines */ public peekLogs(): string { return this.logs; @@ -139,7 +139,7 @@ class IndexedDBLogStore { } /** - * @return {Promise} Resolves when the store is ready. + * @return Resolves when the store is ready. */ public async connect(): Promise { const req = this.indexedDB.open("logs"); @@ -219,7 +219,7 @@ class IndexedDBLogStore { * This guarantees that we will always eventually do a flush when flush() is * called. * - * @return {Promise} Resolved when the logs have been flushed. + * @return Resolved when the logs have been flushed. */ public flush = async (): Promise => { // check if a flush() operation is ongoing @@ -270,7 +270,7 @@ class IndexedDBLogStore { * returned are deleted at the same time, so this can be called at startup * to do house-keeping to keep the logs from growing too large. * - * @return {Promise} Resolves to an array of objects. The array is + * @return Resolves to an array of objects. The array is * sorted in time (oldest first) based on when the log file was created (the * log ID). The objects have said log ID in an "id" field and "lines" which * is a big string with all the new-line delimited logs. @@ -421,12 +421,12 @@ class IndexedDBLogStore { /** * Helper method to collect results from a Cursor and promiseify it. - * @param {ObjectStore|Index} store The store to perform openCursor on. - * @param {IDBKeyRange=} keyRange Optional key range to apply on the cursor. - * @param {Function} resultMapper A function which is repeatedly called with a + * @param store - The store to perform openCursor on. + * @param keyRange - Optional key range to apply on the cursor. + * @param resultMapper - A function which is repeatedly called with a * Cursor. * Return the data you want to keep. - * @return {Promise} Resolves to an array of whatever you returned from + * @return Resolves to an array of whatever you returned from * resultMapper. */ async function selectQuery( @@ -464,9 +464,7 @@ declare global { /** * Configure rage shaking support for sending bug reports. * Modifies globals. - * @param {boolean} setUpPersistence When true (default), the persistence will - * be set up immediately for the logs. - * @return {Promise} Resolves when set up. + * @return Resolves when set up. */ export async function init(): Promise { global.mx_rage_logger = new ConsoleLogger(); @@ -503,7 +501,7 @@ export async function init(): Promise { /** * Try to start up the rageshake storage for logs. If not possible (client unsupported) * then this no-ops. - * @return {Promise} Resolves when complete. + * @return Resolves when complete. */ async function tryInitStorage(): Promise { if (global.mx_rage_initStoragePromise) { @@ -536,7 +534,7 @@ async function tryInitStorage(): Promise { /** * Get a recent snapshot of the logs, ready for attaching to a bug report * - * @return {LogEntry[]} list of log data + * @return list of log data */ export async function getLogsForReport(): Promise { if (!global.mx_rage_logger) { diff --git a/src/state/CallViewModel/CallNotificationLifecycle.ts b/src/state/CallViewModel/CallNotificationLifecycle.ts index 2a0bf2f1..d90f35ba 100644 --- a/src/state/CallViewModel/CallNotificationLifecycle.ts +++ b/src/state/CallViewModel/CallNotificationLifecycle.ts @@ -81,7 +81,7 @@ export interface Props { localUser: { deviceId: string; userId: string }; } /** - * @returns {callPickupState$, autoLeave$} + * @returns two observables: * `callPickupState$` The current call pickup state of the call. * - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership. * Then we can conclude if we were the first one to join or not. diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index ef59270f..3205c07f 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -60,7 +60,8 @@ import { import { MediaDevices } from "../MediaDevices.ts"; import { getValue } from "../../utils/observable.ts"; import { type Behavior, constant } from "../Behavior.ts"; -import { withCallViewModel } from "./CallViewModelTestUtils.ts"; +import { withCallViewModel as withCallViewModelInMode } from "./CallViewModelTestUtils.ts"; +import { MatrixRTCMode } from "../../settings/settings.ts"; vi.mock("rxjs", async (importOriginal) => ({ ...(await importOriginal()), @@ -229,7 +230,13 @@ function mockRingEvent( // need a value to fill in for them when emitting notifications const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; -describe("CallViewModel", () => { +describe.each([ + [MatrixRTCMode.Legacy], + [MatrixRTCMode.Compatibil], + [MatrixRTCMode.Matrix_2_0], +])("CallViewModel (%s mode)", (mode) => { + const withCallViewModel = withCallViewModelInMode(mode); + test("participants are retained during a focus switch", () => { withTestScheduler(({ behavior, expectObservable }) => { // Participants disappear on frame 2 and come back on frame 3 @@ -502,6 +509,48 @@ describe("CallViewModel", () => { }); }); + test("layout reacts to window size", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const windowSizeInputMarbles = "abc"; + const expectedLayoutMarbles = " abc"; + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember]), + windowSize$: behavior(windowSizeInputMarbles, { + a: { width: 300, height: 600 }, // Start very narrow, like a phone + b: { width: 1000, height: 800 }, // Go to normal desktop window size + c: { width: 200, height: 180 }, // Go to PiP size + }), + }, + (vm) => { + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + // This is the expected one-on-one layout for a narrow window + type: "spotlight-expanded", + spotlight: [`${aliceId}:0`], + pip: `${localId}:0`, + }, + b: { + // In a larger window, expect the normal one-on-one layout + type: "one-on-one", + local: `${localId}:0`, + remote: `${aliceId}:0`, + }, + c: { + // In a PiP-sized window, we of course expect a PiP layout + type: "pip", + spotlight: [`${aliceId}:0`], + }, + }, + ); + }, + ); + }); + }); + test("spotlight speakers swap places", () => { withTestScheduler(({ behavior, schedule, expectObservable }) => { // Go immediately into spotlight mode for the test @@ -1207,7 +1256,9 @@ describe("CallViewModel", () => { rtcSession.membershipStatus = Status.Connected; }, n: () => { - rtcSession.membershipStatus = Status.Reconnecting; + // NOTE: This was removed in https://github.com/matrix-org/matrix-js-sdk/pull/5103 accidentally. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + rtcSession.membershipStatus = "Reconnecting" as any; }, }); schedule(probablyLeftMarbles, { diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 3c15958a..5324c65d 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -15,7 +15,7 @@ import { } from "livekit-client"; import { type Room as MatrixRoom } from "matrix-js-sdk"; import { - BehaviorSubject, + catchError, combineLatest, distinctUntilChanged, filter, @@ -53,11 +53,15 @@ import { ScreenShareViewModel, type UserMediaViewModel, } from "../MediaViewModel"; -import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; +import { + accumulate, + filterBehavior, + generateItems, + pauseWhen, +} from "../../utils/observable"; import { duplicateTiles, MatrixRTCMode, - matrixRTCMode, playReactionsSound, showReactions, } from "../../settings/settings"; @@ -76,7 +80,7 @@ import { } from "../../reactions"; import { shallowEquals } from "../../utils/array"; import { type MediaDevices } from "../MediaDevices"; -import { type Behavior } from "../Behavior"; +import { constant, type Behavior } from "../Behavior"; import { E2eeType } from "../../e2ee/e2eeType"; import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; import { type MuteStates } from "../MuteStates"; @@ -94,14 +98,14 @@ import { type SpotlightLandscapeLayoutMedia, type SpotlightPortraitLayoutMedia, } from "../layout-types.ts"; -import { type ElementCallError } from "../../utils/errors.ts"; +import { ElementCallError } from "../../utils/errors.ts"; import { type ObservableScope } from "../ObservableScope.ts"; import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts"; import { createLocalMembership$, enterRTCSession, - RTCBackendState, -} from "./localMember/LocalMembership.ts"; + TransportState, +} from "./localMember/LocalMember.ts"; import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; import { createMemberships$, @@ -111,7 +115,9 @@ import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; import { createMatrixLivekitMembers$, - type MatrixLivekitMember, + type TaggedParticipant, + type LocalMatrixLivekitMember, + type RemoteMatrixLivekitMember, } from "./remoteMembers/MatrixLivekitMembers.ts"; import { type AutoLeaveReason, @@ -126,6 +132,7 @@ import { } from "./remoteMembers/MatrixMemberMetadata.ts"; import { Publisher } from "./localMember/Publisher.ts"; import { type Connection } from "./remoteMembers/Connection.ts"; +import { createLayoutModeSwitch } from "./LayoutSwitch.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO @@ -147,6 +154,10 @@ export interface CallViewModelOptions { livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; /** Optional behavior overriding the local connection state, mainly for testing purposes. */ connectionState$?: Behavior; + /** Optional behavior overriding the computed window size, mainly for testing purposes. */ + windowSize$?: Behavior<{ width: number; height: number }>; + /** The version & compatibility mode of MatrixRTC that we should use. */ + matrixRTCMode$?: Behavior; } // Do not play any sounds if the participant count has exceeded this @@ -172,7 +183,7 @@ interface LayoutScanState { } type MediaItem = UserMedia | ScreenShare; -type AudioLivekitItem = { +export type LivekitRoomItem = { livekitRoom: LivekitRoom; participants: string[]; url: string; @@ -195,8 +206,11 @@ export interface CallViewModel { callPickupState$: Behavior< "unknown" | "ringing" | "timeout" | "decline" | "success" | null >; + /** Observable that emits when the user should leave the call (hangup pressed, widget action, error). + * THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is by ending the scope. + */ leave$: Observable<"user" | AutoLeaveReason>; - /** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */ + /** Call to initiate hangup. Use in conbination with reconnectino state track the async hangup process. */ hangup: () => void; // joining @@ -248,7 +262,11 @@ export interface CallViewModel { */ participantCount$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ - audioParticipants$: Behavior; + livekitRoomItems$: Behavior; + userMedia$: Behavior; + /** use the layout instead, this is just for the sdk export. */ + matrixLivekitMembers$: Behavior; + localMatrixLivekitMember$: Behavior; /** List of participants raising their hand */ handsRaised$: Behavior>; /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ @@ -331,18 +349,17 @@ export interface CallViewModel { switch: () => void; } | null>; - // connection state /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. + * Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state. */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis reconnecting$: Behavior; + + /** + * Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit + */ + connected$: Behavior; } + /** * A view model providing all the application logic needed to show the in-call * UI (may eventually be expanded to cover the lobby and feedback screens in the @@ -370,6 +387,8 @@ export function createCallViewModel$( options.encryptionSystem, matrixRTCSession, ); + const matrixRTCMode$ = + options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy); // Each hbar seperates a block of input variables required for the CallViewModel to function. // The outputs of this block is written under the hbar. @@ -402,7 +421,7 @@ export function createCallViewModel$( client, roomId: matrixRoom.roomId, useOldestMember$: scope.behavior( - matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), }); @@ -413,6 +432,8 @@ export function createCallViewModel$( livekitKeyProvider, getUrlParams().controlledAudioDevices, options.livekitRoomFactory, + getUrlParams().echoCancellation, + getUrlParams().noiseSuppression, ); const connectionManager = createConnectionManager$({ @@ -420,7 +441,18 @@ export function createCallViewModel$( connectionFactory: connectionFactory, inputTransports$: scope.behavior( combineLatest( - [localTransport$, membershipsAndTransports.transports$], + [ + localTransport$.pipe( + catchError((e: unknown) => { + logger.info( + "dont pass local transport to createConnectionManager$. localTransport$ threw an error", + e, + ); + return of(null); + }), + ), + membershipsAndTransports.transports$, + ], (localTransport, transports) => { const localTransportAsArray = localTransport ? [localTransport] : []; return transports.mapInner((transports) => [ @@ -430,7 +462,7 @@ export function createCallViewModel$( }, ), ), - logger: logger, + logger, }); const matrixLivekitMembers$ = createMatrixLivekitMembers$({ @@ -441,7 +473,7 @@ export function createCallViewModel$( }); const connectOptions$ = scope.behavior( - matrixRTCMode.value$.pipe( + matrixRTCMode$.pipe( map((mode) => ({ encryptMedia: livekitKeyProvider !== undefined, // TODO. This might need to get called again on each change of matrixRTCMode... @@ -452,13 +484,13 @@ export function createCallViewModel$( const localMembership = createLocalMembership$({ scope: scope, - homeserverConnected$: createHomeserverConnected$( + homeserverConnected: createHomeserverConnected$( scope, client, matrixRTCSession, ), muteStates: muteStates, - joinMatrixRTC: async (transport: LivekitTransport) => { + joinMatrixRTC: (transport: LivekitTransport) => { return enterRTCSession( matrixRTCSession, transport, @@ -473,7 +505,7 @@ export function createCallViewModel$( muteStates, trackProcessorState$, logger.getChild( - "[Publisher" + connection.transport.livekit_service_url + "]", + "[Publisher " + connection.transport.livekit_service_url + "]", ), ); }, @@ -495,22 +527,21 @@ export function createCallViewModel$( ), ); - const localMatrixLivekitMemberUninitialized = { - membership$: localRtcMembership$, - participant$: localMembership.participant$, - connection$: localMembership.connection$, - userId: userId, - }; - - const localMatrixLivekitMember$: Behavior = + const localMatrixLivekitMember$: Behavior = scope.behavior( localRtcMembership$.pipe( - switchMap((membership) => { - if (!membership) return of(null); - return of( - // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. - localMatrixLivekitMemberUninitialized as MatrixLivekitMember, - ); + filterBehavior((membership) => membership !== null), + map((membership$) => { + if (membership$ === null) return null; + return { + membership$, + participant: { + type: "local" as const, + value$: localMembership.participant$, + }, + connection$: localMembership.connection$, + userId, + }; }), ), ); @@ -573,24 +604,16 @@ export function createCallViewModel$( ), ); - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis - const reconnecting$ = localMembership.reconnecting$; - - const audioParticipants$ = scope.behavior( + const livekitRoomItems$ = scope.behavior( matrixLivekitMembers$.pipe( + tap((val) => { + logger.debug("matrixLivekitMembers$ updated", val.value); + }), switchMap((membersWithEpoch) => { const members = membersWithEpoch.value; const a$ = combineLatest( members.map((member) => - combineLatest([member.connection$, member.participant$]).pipe( + combineLatest([member.connection$, member.participant.value$]).pipe( map(([connection, participant]) => { // do not render audio for local participant if (!connection || !participant || participant.isLocal) @@ -610,7 +633,7 @@ export function createCallViewModel$( return a$; }), map((members) => - members.reduce((acc, curr) => { + members.reduce((acc, curr) => { if (!curr) return acc; const existing = acc.find((item) => item.url === curr.url); @@ -631,7 +654,7 @@ export function createCallViewModel$( ); const handsRaised$ = scope.behavior( - handsRaisedSubject$.pipe(pauseWhen(reconnecting$)), + handsRaisedSubject$.pipe(pauseWhen(localMembership.reconnecting$)), ); const reactions$ = scope.behavior( @@ -644,7 +667,7 @@ export function createCallViewModel$( ]), ), ), - pauseWhen(reconnecting$), + pauseWhen(localMembership.reconnecting$), ), ); @@ -668,7 +691,7 @@ export function createCallViewModel$( let localParticipantId: string | undefined = undefined; // add local member if available if (localMatrixLivekitMember) { - const { userId, participant$, connection$, membership$ } = + const { userId, participant, connection$, membership$ } = localMatrixLivekitMember; localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional // const participantId = membership$.value.membershipID; @@ -679,7 +702,7 @@ export function createCallViewModel$( dup, localParticipantId, userId, - participant$, + participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely connection$, ], data: undefined, @@ -690,7 +713,7 @@ export function createCallViewModel$( // add remote members that are available for (const { userId, - participant$, + participant, connection$, membership$, } of matrixLivekitMembers) { @@ -699,7 +722,7 @@ export function createCallViewModel$( // const participantId = membership$.value?.identity; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [dup, participantId, userId, participant$, connection$], + keys: [dup, participantId, userId, participant, connection$], data: undefined, }; } @@ -711,7 +734,7 @@ export function createCallViewModel$( dup, participantId, userId, - participant$, + participant, connection$, ) => { const livekitRoom$ = scope.behavior( @@ -730,12 +753,12 @@ export function createCallViewModel$( scope, `${participantId}:${dup}`, userId, - participant$, + participant, options.encryptionSystem, livekitRoom$, focusUrl$, mediaDevices, - reconnecting$, + localMembership.reconnecting$, displayName$, matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), @@ -940,20 +963,29 @@ export function createCallViewModel$( ), ); - const hasRemoteScreenShares$: Observable = spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + const hasRemoteScreenShares$ = scope.behavior( + spotlight$.pipe( + map((spotlight) => + spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + ), ), - distinctUntilChanged(), ); const pipEnabled$ = scope.behavior(setPipEnabled$, false); + const windowSize$ = + options.windowSize$ ?? + scope.behavior<{ width: number; height: number }>( + fromEvent(window, "resize").pipe( + startWith(null), + map(() => ({ width: window.innerWidth, height: window.innerHeight })), + ), + ); + + // A guess at what the window's mode should be based on its size and shape. const naturalWindowMode$ = scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; + windowSize$.pipe( + map(({ width, height }) => { if (height <= 400 && width <= 340) return "pip"; // Our layouts for flat windows are better at adapting to a small width // than our layouts for narrow windows are at adapting to a small height, @@ -963,7 +995,6 @@ export function createCallViewModel$( return "normal"; }), ), - "normal", ); /** @@ -980,49 +1011,11 @@ export function createCallViewModel$( spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), ); - const gridModeUserSelection$ = new BehaviorSubject("grid"); - - // Callback to set the grid mode desired by the user. - // Notice that this is only a preference, the actual grid mode can be overridden - // if there is a remote screen share active. - const setGridMode = (value: GridMode): void => { - gridModeUserSelection$.next(value); - }; - /** - * The layout mode of the media tile grid. - */ - const gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - scope.behavior( - gridModeUserSelection$.pipe( - switchMap((userSelection): Observable => { - if (userSelection === "spotlight") { - // If already in spotlight mode, stay there - return of("spotlight"); - } else { - // Otherwise, check if there is a remote screen share active - // as this could force us into spotlight mode. - return combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( - map(([hasScreenShares, windowMode]): GridMode => { - const isFlatMode = windowMode === "flat"; - if (hasScreenShares || isFlatMode) { - logger.debug( - `Forcing spotlight mode, hasScreenShares=${hasScreenShares} windowMode=${windowMode}`, - ); - // override to spotlight mode - return "spotlight"; - } else { - // respect user choice - return "grid"; - } - }), - ); - } - }), - ), - "grid", - ); + const { setGridMode, gridMode$ } = createLayoutModeSwitch( + scope, + windowMode$, + hasRemoteScreenShares$, + ); const gridLayoutMedia$: Observable = combineLatest( [grid$, spotlight$], @@ -1449,13 +1442,44 @@ export function createCallViewModel$( // reassigned here to make it publicly accessible const toggleScreenSharing = localMembership.toggleScreenSharing; + const errors$ = scope.behavior<{ + transportError?: ElementCallError; + matrixError?: ElementCallError; + connectionError?: ElementCallError; + publishError?: ElementCallError; + } | null>( + localMembership.localMemberState$.pipe( + map((value) => { + const returnObject: { + transportError?: ElementCallError; + matrixError?: ElementCallError; + connectionError?: ElementCallError; + publishError?: ElementCallError; + } = {}; + if (value instanceof ElementCallError) return { transportError: value }; + if (value === TransportState.Waiting) return null; + if (value.matrix instanceof ElementCallError) + returnObject.matrixError = value.matrix; + if (value.media instanceof ElementCallError) + returnObject.publishError = value.media; + else if ( + typeof value.media === "object" && + value.media.connection instanceof ElementCallError + ) + returnObject.connectionError = value.media.connection; + return returnObject; + }), + ), + null, + ); + return { autoLeave$: autoLeave$, callPickupState$: callPickupState$, ringOverlay$: ringOverlay$, leave$: leave$, hangup: (): void => userHangup$.next(), - join: localMembership.requestConnect, + join: localMembership.requestJoinAndPublish, toggleScreenSharing: toggleScreenSharing, sharingScreen$: sharingScreen$, @@ -1465,16 +1489,21 @@ export function createCallViewModel$( unhoverScreen: (): void => screenUnhover$.next(), fatalError$: scope.behavior( - localMembership.connectionState.livekit$.pipe( - filter((v) => v.state === RTCBackendState.Error), - map((s) => s.error), + errors$.pipe( + map((errors) => { + logger.debug("errors$ to compute any fatal errors:", errors); + return ( + errors?.transportError ?? + errors?.matrixError ?? + errors?.connectionError ?? + null + ); + }), + filter((error) => error !== null), ), null, ), - participantCount$: participantCount$, - audioParticipants$: audioParticipants$, - handsRaised$: handsRaised$, reactions$: reactions$, joinSoundEffect$: joinSoundEffect$, @@ -1493,6 +1522,16 @@ export function createCallViewModel$( spotlight$: spotlight$, pip$: pip$, layout$: layout$, + userMedia$, + localMatrixLivekitMember$, + matrixLivekitMembers$: scope.behavior( + matrixLivekitMembers$.pipe( + map((members) => members.value), + tap((v) => { + logger.debug("matrixLivekitMembers$ updated (exported)", v); + }), + ), + ), tileStoreGeneration$: tileStoreGeneration$, showSpotlightIndicators$: showSpotlightIndicators$, showSpeakingIndicators$: showSpeakingIndicators$, @@ -1500,7 +1539,9 @@ export function createCallViewModel$( showFooter$: showFooter$, earpieceMode$: earpieceMode$, audioOutputSwitcher$: audioOutputSwitcher$, - reconnecting$: reconnecting$, + reconnecting$: localMembership.reconnecting$, + livekitRoomItems$, + connected$: localMembership.connected$, }; } diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index f86921c5..b6f53275 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -53,6 +53,7 @@ import { import { type Behavior, constant } from "../Behavior"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; import { type MediaDevices } from "../MediaDevices"; +import { type MatrixRTCMode } from "../../settings/settings"; mockConfig({ livekit: { livekit_service_url: "http://my-default-service-url.com" }, @@ -75,119 +76,130 @@ export interface CallViewModelInputs { speaking: Map>; mediaDevices: MediaDevices; initialSyncState: SyncState; + windowSize$: Behavior<{ width: number; height: number }>; } const localParticipant = mockLocalParticipant({ identity: "" }); -export function withCallViewModel( - { - remoteParticipants$ = constant([]), - rtcMembers$ = constant([localRtcMember]), - livekitConnectionState$: connectionState$ = constant( - ConnectionState.Connected, - ), - speaking = new Map(), - mediaDevices = mockMediaDevices({}), - initialSyncState = SyncState.Syncing, - }: Partial = {}, - continuation: ( - vm: CallViewModel, - rtcSession: MockRTCSession, - subjects: { raisedHands$: BehaviorSubject> }, - setSyncState: (value: SyncState) => void, - ) => void, - options: CallViewModelOptions = { - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - autoLeaveWhenOthersLeft: false, - }, -): void { - let syncState = initialSyncState; - const setSyncState = (value: SyncState): void => { - const prev = syncState; - syncState = value; - room.client.emit(ClientEvent.Sync, value, prev); - }; - const room = mockMatrixRoom({ - client: new (class extends EventEmitter { - public getUserId(): string | undefined { - return localRtcMember.userId; - } +export function withCallViewModel(mode: MatrixRTCMode) { + return ( + { + remoteParticipants$ = constant([]), + rtcMembers$ = constant([localRtcMember]), + livekitConnectionState$: connectionState$ = constant( + ConnectionState.Connected, + ), + speaking = new Map(), + mediaDevices = mockMediaDevices({}), + initialSyncState = SyncState.Syncing, + windowSize$ = constant({ width: 1000, height: 800 }), + }: Partial = {}, + continuation: ( + vm: CallViewModel, + rtcSession: MockRTCSession, + subjects: { + raisedHands$: BehaviorSubject>; + }, + setSyncState: (value: SyncState) => void, + ) => void, + options: Partial = {}, + ): void => { + let syncState = initialSyncState; + const setSyncState = (value: SyncState): void => { + const prev = syncState; + syncState = value; + room.client.emit(ClientEvent.Sync, value, prev); + }; + const room = mockMatrixRoom({ + client: new (class extends EventEmitter { + public getUserId(): string | undefined { + return localRtcMember.userId; + } - public getDeviceId(): string { - return localRtcMember.deviceId; - } + public getDeviceId(): string { + return localRtcMember.deviceId; + } - public getDomain(): string { - return "example.com"; - } + public getDomain(): string { + return "example.com"; + } - public getSyncState(): SyncState { - return syncState; - } - })() as Partial as MatrixClient, - getMembers: () => Array.from(roomMembers.values()), - getMembersWithMembership: () => Array.from(roomMembers.values()), - }); - const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$); - const participantsSpy = vi - .spyOn(ComponentsCore, "connectedParticipantsObserver") - .mockReturnValue(remoteParticipants$); - const mediaSpy = vi - .spyOn(ComponentsCore, "observeParticipantMedia") - .mockImplementation((p) => - of({ participant: p } as Partial< - ComponentsCore.ParticipantMedia - > as ComponentsCore.ParticipantMedia), + public getSyncState(): SyncState { + return syncState; + } + })() as Partial as MatrixClient, + getMembers: () => Array.from(roomMembers.values()), + getMembersWithMembership: () => Array.from(roomMembers.values()), + }); + const rtcSession = new MockRTCSession(room, []).withMemberships( + rtcMembers$, ); - const eventsSpy = vi - .spyOn(ComponentsCore, "observeParticipantEvents") - .mockImplementation((p, ...eventTypes) => { - if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) { - return (speaking.get(p) ?? of(false)).pipe( - map((s): Participant => ({ ...p, isSpeaking: s }) as Participant), - ); - } else { - return of(p); - } + const participantsSpy = vi + .spyOn(ComponentsCore, "connectedParticipantsObserver") + .mockReturnValue(remoteParticipants$); + const mediaSpy = vi + .spyOn(ComponentsCore, "observeParticipantMedia") + .mockImplementation((p) => + of({ participant: p } as Partial< + ComponentsCore.ParticipantMedia + > as ComponentsCore.ParticipantMedia), + ); + const eventsSpy = vi + .spyOn(ComponentsCore, "observeParticipantEvents") + .mockImplementation((p, ...eventTypes) => { + if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) { + return (speaking.get(p) ?? of(false)).pipe( + map((s): Participant => ({ ...p, isSpeaking: s }) as Participant), + ); + } else { + return of(p); + } + }); + + const roomEventSelectorSpy = vi + .spyOn(ComponentsCore, "roomEventSelector") + .mockImplementation((_room, _eventType) => of()); + const muteStates = mockMuteStates(); + const raisedHands$ = new BehaviorSubject>( + {}, + ); + const reactions$ = new BehaviorSubject>({}); + + const vm = createCallViewModel$( + testScope(), + rtcSession.asMockedSession(), + room, + mediaDevices, + muteStates, + { + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + autoLeaveWhenOthersLeft: false, + livekitRoomFactory: (): LivekitRoom => + mockLivekitRoom({ + localParticipant, + disconnect: async () => Promise.resolve(), + setE2EEEnabled: async () => Promise.resolve(), + }), + connectionState$, + windowSize$, + matrixRTCMode$: constant(mode), + ...options, + }, + raisedHands$, + reactions$, + new BehaviorSubject({ + processor: undefined, + supported: undefined, + }), + ); + + onTestFinished(() => { + participantsSpy.mockRestore(); + mediaSpy.mockRestore(); + eventsSpy.mockRestore(); + roomEventSelectorSpy.mockRestore(); }); - const roomEventSelectorSpy = vi - .spyOn(ComponentsCore, "roomEventSelector") - .mockImplementation((_room, _eventType) => of()); - const muteStates = mockMuteStates(); - const raisedHands$ = new BehaviorSubject>({}); - const reactions$ = new BehaviorSubject>({}); - - const vm = createCallViewModel$( - testScope(), - rtcSession.asMockedSession(), - room, - mediaDevices, - muteStates, - { - ...options, - livekitRoomFactory: (): LivekitRoom => - mockLivekitRoom({ - localParticipant, - disconnect: async () => Promise.resolve(), - setE2EEEnabled: async () => Promise.resolve(), - }), - connectionState$, - }, - raisedHands$, - reactions$, - new BehaviorSubject({ - processor: undefined, - supported: undefined, - }), - ); - - onTestFinished(() => { - participantsSpy.mockRestore(); - mediaSpy.mockRestore(); - eventsSpy.mockRestore(); - roomEventSelectorSpy.mockRestore(); - }); - - continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); + continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); + }; } diff --git a/src/state/CallViewModel/LayoutSwitch.test.ts b/src/state/CallViewModel/LayoutSwitch.test.ts new file mode 100644 index 00000000..0d184017 --- /dev/null +++ b/src/state/CallViewModel/LayoutSwitch.test.ts @@ -0,0 +1,132 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, test } from "vitest"; + +import { createLayoutModeSwitch } from "./LayoutSwitch"; +import { testScope, withTestScheduler } from "../../utils/test"; + +function testLayoutSwitch({ + windowMode = "n", + hasScreenShares = "n", + userSelection = "", + expectedGridMode, +}: { + windowMode?: string; + hasScreenShares?: string; + userSelection?: string; + expectedGridMode: string; +}): void { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const { gridMode$, setGridMode } = createLayoutModeSwitch( + testScope(), + behavior(windowMode, { n: "normal", N: "narrow", f: "flat" }), + behavior(hasScreenShares, { y: true, n: false }), + ); + schedule(userSelection, { + g: () => setGridMode("grid"), + s: () => setGridMode("spotlight"), + }); + expectObservable(gridMode$).toBe(expectedGridMode, { + g: "grid", + s: "spotlight", + }); + }); +} + +describe("default mode", () => { + test("uses grid layout by default", () => + testLayoutSwitch({ + expectedGridMode: "g", + })); + + test("uses spotlight mode when window mode is flat", () => + testLayoutSwitch({ + windowMode: " f", + expectedGridMode: "s", + })); +}); + +test("allows switching modes manually", () => + testLayoutSwitch({ + userSelection: " --sgs", + expectedGridMode: "g-sgs", + })); + +test("switches to spotlight mode when there is a remote screen share", () => + testLayoutSwitch({ + hasScreenShares: " n--y", + expectedGridMode: "g--s", + })); + +test("can manually switch to grid when there is a screenshare", () => + testLayoutSwitch({ + hasScreenShares: " n-y", + userSelection: " ---g", + expectedGridMode: "g-sg", + })); + +test("auto-switches after manually selecting grid", () => + testLayoutSwitch({ + // Two screenshares will happen in sequence. There is a screen share that + // forces spotlight, then the user manually switches back to grid. + hasScreenShares: " n-y-ny", + userSelection: " ---g", + expectedGridMode: "g-sg-s", + // If we did want to respect manual selection, the expectation would be: g-sg + })); + +test("switches back to grid mode when the remote screen share ends", () => + testLayoutSwitch({ + hasScreenShares: " n--y--n", + expectedGridMode: "g--s--g", + })); + +test("auto-switches to spotlight again after first screen share ends", () => + testLayoutSwitch({ + hasScreenShares: " nyny", + expectedGridMode: "gsgs", + })); + +test("switches manually to grid after screen share while manually in spotlight", () => + testLayoutSwitch({ + // Initially, no one is sharing. Then the user manually switches to spotlight. + // After a screen share starts, the user manually switches to grid. + hasScreenShares: " n-y", + userSelection: " -s-g", + expectedGridMode: "gs-g", + })); + +test("auto-switches to spotlight when in flat window mode", () => + testLayoutSwitch({ + // First normal, then narrow, then flat. + windowMode: " nNf", + expectedGridMode: "g-s", + })); + +test("allows switching modes manually when in flat window mode", () => + testLayoutSwitch({ + // Window becomes flat, then user switches to grid and back. + // Finally the window returns to a normal shape. + windowMode: " nf--n", + userSelection: " --gs", + expectedGridMode: "gsgsg", + })); + +test("stays in spotlight while there are screen shares even when window mode changes", () => + testLayoutSwitch({ + windowMode: " nfn", + hasScreenShares: " y", + expectedGridMode: "s", + })); + +test("ignores end of screen share until window mode returns to normal", () => + testLayoutSwitch({ + windowMode: " nf-n", + hasScreenShares: " y-n", + expectedGridMode: "s--g", + })); diff --git a/src/state/CallViewModel/LayoutSwitch.ts b/src/state/CallViewModel/LayoutSwitch.ts new file mode 100644 index 00000000..97a4ee6f --- /dev/null +++ b/src/state/CallViewModel/LayoutSwitch.ts @@ -0,0 +1,93 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + combineLatest, + map, + Subject, + startWith, + skipWhile, + switchMap, +} from "rxjs"; + +import { type GridMode, type WindowMode } from "./CallViewModel.ts"; +import { constant, type Behavior } from "../Behavior.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; + +/** + * Creates a layout mode switch that allows switching between grid and spotlight modes. + * The actual layout mode might switch automatically to spotlight if there is a + * remote screen share active or if the window mode is flat. + * + * @param scope - The observable scope to manage subscriptions. + * @param windowMode$ - The current window mode. + * @param hasRemoteScreenShares$ - A behavior indicating if there are remote screen shares active. + */ +export function createLayoutModeSwitch( + scope: ObservableScope, + windowMode$: Behavior, + hasRemoteScreenShares$: Behavior, +): { + gridMode$: Behavior; + setGridMode: (value: GridMode) => void; +} { + const userSelection$ = new Subject(); + // Callback to set the grid mode desired by the user. + // Notice that this is only a preference, the actual grid mode can be overridden + // if there is a remote screen share active. + const setGridMode = (value: GridMode): void => userSelection$.next(value); + + /** + * The natural grid mode - the mode that the grid would prefer to be in, + * not accounting for the user's manual selections. + */ + const naturalGridMode$ = scope.behavior( + combineLatest( + [hasRemoteScreenShares$, windowMode$], + (hasRemoteScreenShares, windowMode) => + // When there are screen shares or the window is flat (as with a phone + // in landscape orientation), spotlight is a better experience. + // We want screen shares to be big and readable, and we want flipping + // your phone into landscape to be a quick way of maximising the + // spotlight tile. + hasRemoteScreenShares || windowMode === "flat" ? "spotlight" : "grid", + ), + ); + + /** + * The layout mode of the media tile grid. + */ + const gridMode$ = scope.behavior( + // Whenever the user makes a selection, we enter a new mode of behavior: + userSelection$.pipe( + map((selection) => { + if (selection === "grid") + // The user has selected grid mode. Start by respecting their choice, + // but then follow the natural mode again as soon as it matches. + return naturalGridMode$.pipe( + skipWhile((naturalMode) => naturalMode !== selection), + startWith(selection), + ); + + // The user has selected spotlight mode. If this matches the natural + // mode, then follow the natural mode going forward. + return selection === naturalGridMode$.value + ? naturalGridMode$ + : constant(selection); + }), + // Initially the mode of behavior is to just follow the natural grid mode. + startWith(naturalGridMode$), + // Switch between each mode of behavior. + switchMap((mode$) => mode$), + ), + ); + + return { + gridMode$, + setGridMode, + }; +} diff --git a/src/state/CallViewModel/localMember/HomeserverConnected.test.ts b/src/state/CallViewModel/localMember/HomeserverConnected.test.ts index 1f61e533..87ca35d0 100644 --- a/src/state/CallViewModel/localMember/HomeserverConnected.test.ts +++ b/src/state/CallViewModel/localMember/HomeserverConnected.test.ts @@ -97,106 +97,106 @@ describe("createHomeserverConnected$", () => { // LLM generated test cases. They are a bit overkill but I improved the mocking so it is // easy enough to read them so I think they can stay. it("is false when sync state is not Syncing", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); - expect(hsConnected$.value).toBe(false); + const hsConnected = createHomeserverConnected$(scope, client, session); + expect(hsConnected.combined$.value).toBe(false); }); it("remains false while membership status is not Connected even if sync is Syncing", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); client.setSyncState(SyncState.Syncing); - expect(hsConnected$.value).toBe(false); // membership still disconnected + expect(hsConnected.combined$.value).toBe(false); // membership still disconnected }); it("is false when membership status transitions to Connected but ProbablyLeft is true", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); // Make sync loop OK client.setSyncState(SyncState.Syncing); // Indicate probable leave before connection session.setProbablyLeft(true); session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); }); it("becomes true only when all three conditions are satisfied", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); // 1. Sync loop connected client.setSyncState(SyncState.Syncing); - expect(hsConnected$.value).toBe(false); // not yet membership connected + expect(hsConnected.combined$.value).toBe(false); // not yet membership connected // 2. Membership connected session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); // probablyLeft is false + expect(hsConnected.combined$.value).toBe(true); // probablyLeft is false }); it("drops back to false when sync loop leaves Syncing", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); // Reach connected state client.setSyncState(SyncState.Syncing); session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); // Sync loop error => should flip false client.setSyncState(SyncState.Error); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); }); it("drops back to false when membership status becomes disconnected", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); client.setSyncState(SyncState.Syncing); session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); session.setMembershipStatus(Status.Disconnected); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); }); it("drops to false when ProbablyLeft is emitted after being true", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); client.setSyncState(SyncState.Syncing); session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); session.setProbablyLeft(true); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); }); it("recovers to true if ProbablyLeft becomes false again while other conditions remain true", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); client.setSyncState(SyncState.Syncing); session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); session.setProbablyLeft(true); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); // Simulate clearing the flag (in realistic scenario membership manager would update) session.setProbablyLeft(false); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); }); it("composite sequence reflects each individual failure reason", () => { - const hsConnected$ = createHomeserverConnected$(scope, client, session); + const hsConnected = createHomeserverConnected$(scope, client, session); // Initially false (sync error + disconnected + not probably left) - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); // Fix sync only client.setSyncState(SyncState.Syncing); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); // Fix membership session.setMembershipStatus(Status.Connected); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); // Introduce probablyLeft -> false session.setProbablyLeft(true); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); // Restore notProbablyLeft -> true again session.setProbablyLeft(false); - expect(hsConnected$.value).toBe(true); + expect(hsConnected.combined$.value).toBe(true); // Drop sync -> false client.setSyncState(SyncState.Error); - expect(hsConnected$.value).toBe(false); + expect(hsConnected.combined$.value).toBe(false); }); }); diff --git a/src/state/CallViewModel/localMember/HomeserverConnected.ts b/src/state/CallViewModel/localMember/HomeserverConnected.ts index e1c28078..c8bcd021 100644 --- a/src/state/CallViewModel/localMember/HomeserverConnected.ts +++ b/src/state/CallViewModel/localMember/HomeserverConnected.ts @@ -25,6 +25,11 @@ import { type NodeStyleEventEmitter } from "../../../utils/test"; */ const logger = rootLogger.getChild("[HomeserverConnected]"); +export interface HomeserverConnected { + combined$: Behavior; + rtsSession$: Behavior; +} + /** * Behavior representing whether we consider ourselves connected to the Matrix homeserver * for the purposes of a MatrixRTC session. @@ -39,7 +44,7 @@ export function createHomeserverConnected$( client: NodeStyleEventEmitter & Pick, matrixRTCSession: NodeStyleEventEmitter & Pick, -): Behavior { +): HomeserverConnected { const syncing$ = ( fromEvent(client, ClientEvent.Sync) as Observable<[SyncState]> ).pipe( @@ -47,12 +52,15 @@ export function createHomeserverConnected$( map(([state]) => state === SyncState.Syncing), ); - const membershipConnected$ = fromEvent( - matrixRTCSession, - MembershipManagerEvent.StatusChanged, - ).pipe( - startWith(null), - map(() => matrixRTCSession.membershipStatus === Status.Connected), + const rtsSession$ = scope.behavior( + fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe( + map(() => matrixRTCSession.membershipStatus ?? Status.Unknown), + ), + Status.Unknown, + ); + + const membershipConnected$ = rtsSession$.pipe( + map((status) => status === Status.Connected), ); // This is basically notProbablyLeft$ @@ -71,15 +79,13 @@ export function createHomeserverConnected$( map(() => matrixRTCSession.probablyLeft !== true), ); - const connectedCombined$ = and$( - syncing$, - membershipConnected$, - certainlyConnected$, - ).pipe( - tap((connected) => { - logger.info(`Homeserver connected update: ${connected}`); - }), + const combined$ = scope.behavior( + and$(syncing$, membershipConnected$, certainlyConnected$).pipe( + tap((connected) => { + logger.info(`Homeserver connected update: ${connected}`); + }), + ), ); - return scope.behavior(connectedCombined$); + return { combined$, rtsSession$ }; } diff --git a/src/state/CallViewModel/localMember/LocalMembership.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts similarity index 69% rename from src/state/CallViewModel/localMember/LocalMembership.test.ts rename to src/state/CallViewModel/localMember/LocalMember.test.ts index cff5c06d..0d77611b 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. */ import { + Status as RTCMemberStatus, type LivekitTransport, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; @@ -14,11 +15,7 @@ import { describe, expect, it, vi } from "vitest"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { BehaviorSubject, map, of } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import { - ConnectionState as LivekitConnectionState, - type LocalParticipant, - type LocalTrack, -} from "livekit-client"; +import { type LocalParticipant, type LocalTrack } from "livekit-client"; import { MatrixRTCMode } from "../../../settings/settings"; import { @@ -29,15 +26,17 @@ import { withTestScheduler, } from "../../../utils/test"; import { + TransportState, createLocalMembership$, enterRTCSession, - RTCBackendState, -} from "./LocalMembership"; + PublishState, + TrackState, +} from "./LocalMember"; import { MatrixRTCTransportMissingError } from "../../../utils/errors"; import { Epoch, ObservableScope } from "../../ObservableScope"; import { constant } from "../../Behavior"; import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; -import { type Connection } from "../remoteMembers/Connection"; +import { ConnectionState, type Connection } from "../remoteMembers/Connection"; import { type Publisher } from "./Publisher"; const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; @@ -51,7 +50,7 @@ vi.mock("@livekit/components-core", () => ({ describe("LocalMembership", () => { describe("enterRTCSession", () => { - it("It joins the correct Session", async () => { + it("It joins the correct Session", () => { const focusFromOlderMembership = { type: "livekit", livekit_service_url: "http://my-oldest-member-service-url.com", @@ -107,7 +106,7 @@ describe("LocalMembership", () => { joinRoomSession: vi.fn(), }) as unknown as MatrixRTCSession; - await enterRTCSession( + enterRTCSession( mockedSession, { livekit_alias: "roomId", @@ -136,7 +135,7 @@ describe("LocalMembership", () => { ); }); - it("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => { + it("It should not fail with configuration error if homeserver config has livekit url but not fallback", () => { mockConfig({}); vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({ "org.matrix.msc4143.rtc_foci": [ @@ -165,7 +164,7 @@ describe("LocalMembership", () => { joinRoomSession: vi.fn(), }) as unknown as MatrixRTCSession; - await enterRTCSession( + enterRTCSession( mockedSession, { livekit_alias: "roomId", @@ -190,7 +189,6 @@ describe("LocalMembership", () => { leaveRoomSession: () => {}, } as unknown as MatrixRTCSession, muteStates: mockMuteStates(), - isHomeserverConnected: constant(true), trackProcessorState$: constant({ supported: false, processor: undefined, @@ -198,20 +196,20 @@ describe("LocalMembership", () => { logger: logger, createPublisherFactory: vi.fn(), joinMatrixRTC: async (): Promise => {}, - homeserverConnected$: constant(true), + homeserverConnected: { + combined$: constant(true), + rtsSession$: constant(RTCMemberStatus.Connected), + }, }; it("throws error on missing RTC config error", () => { withTestScheduler(({ scope, hot, expectObservable }) => { - const goodTransport = { - livekit_service_url: "other", - } as LivekitTransport; - - const localTransport$ = scope.behavior( + const localTransport$ = scope.behavior( hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), - goodTransport, + null, ); + // we do not need any connection data since we want to fail before reaching that. const mockConnectionManager = { transports$: scope.behavior( localTransport$.pipe(map((t) => new Epoch([t]))), @@ -227,15 +225,11 @@ describe("LocalMembership", () => { connectionManager: mockConnectionManager, localTransport$, }); + localMembership.requestJoinAndPublish(); - expectObservable(localMembership.connectionState.livekit$).toBe("ne", { - n: { state: RTCBackendState.WaitingForConnection }, - e: { - state: RTCBackendState.Error, - error: expect.toSatisfy( - (e) => e instanceof MatrixRTCTransportMissingError, - ), - }, + expectObservable(localMembership.localMemberState$).toBe("ne", { + n: TransportState.Waiting, + e: expect.toSatisfy((e) => e instanceof MatrixRTCTransportMissingError), }); }); }); @@ -247,33 +241,26 @@ describe("LocalMembership", () => { livekit_service_url: "b", } as LivekitTransport; - const connectionManagerData = new ConnectionManagerData(); - - connectionManagerData.add( - { - livekitRoom: mockLivekitRoom({ - localParticipant: { - isScreenShareEnabled: false, - trackPublications: [], - } as unknown as LocalParticipant, - }), - state$: constant({ - state: "ConnectedToLkRoom", - livekitConnectionState$: constant(LivekitConnectionState.Connected), - }), - transport: aTransport, - } as unknown as Connection, - [], - ); - connectionManagerData.add( - { - state$: constant({ - state: "ConnectedToLkRoom", - }), - transport: bTransport, - } as unknown as Connection, - [], - ); + const connectionTransportAConnected = { + livekitRoom: mockLivekitRoom({ + localParticipant: { + isScreenShareEnabled: false, + trackPublications: [], + } as unknown as LocalParticipant, + }), + state$: constant(ConnectionState.LivekitConnected), + transport: aTransport, + } as unknown as Connection; + const connectionTransportAConnecting = { + ...connectionTransportAConnected, + state$: constant(ConnectionState.LivekitConnecting), + livekitRoom: mockLivekitRoom({}), + } as unknown as Connection; + const connectionTransportBConnected = { + state$: constant(ConnectionState.LivekitConnected), + transport: bTransport, + livekitRoom: mockLivekitRoom({}), + } as unknown as Connection; it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => { const scope = new ObservableScope(); @@ -281,13 +268,17 @@ describe("LocalMembership", () => { const localTransport$ = new BehaviorSubject(aTransport); const publishers: Publisher[] = []; - + let seed = 0; defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation( () => { + const a = seed; + seed += 1; + logger.info(`creating [${a}]`); const p = { - stopPublishing: vi.fn(), + stopPublishing: vi.fn().mockImplementation(() => { + logger.info(`stopPublishing [${a}]`); + }), stopTracks: vi.fn(), - publishing$: constant(false), }; publishers.push(p as unknown as Publisher); return p; @@ -298,6 +289,9 @@ describe("LocalMembership", () => { typeof vi.fn >; + const connectionManagerData = new ConnectionManagerData(); + connectionManagerData.add(connectionTransportAConnected, []); + connectionManagerData.add(connectionTransportBConnected, []); createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, @@ -322,7 +316,7 @@ describe("LocalMembership", () => { await flushPromises(); // stop all tracks after ending scopes expect(publishers[1].stopPublishing).toHaveBeenCalled(); - expect(publishers[1].stopTracks).toHaveBeenCalled(); + // expect(publishers[1].stopTracks).toHaveBeenCalled(); defaultCreateLocalMemberValues.createPublisherFactory.mockReset(); }); @@ -357,6 +351,9 @@ describe("LocalMembership", () => { typeof vi.fn >; + const connectionManagerData = new ConnectionManagerData(); + connectionManagerData.add(connectionTransportAConnected, []); + // connectionManagerData.add(connectionTransportB, []); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, @@ -367,15 +364,17 @@ describe("LocalMembership", () => { }); await flushPromises(); expect(publisherFactory).toHaveBeenCalledOnce(); - expect(localMembership.tracks$.value.length).toBe(0); + // expect(localMembership.tracks$.value.length).toBe(0); + expect(publishers[0].createAndSetupTracks).not.toHaveBeenCalled(); localMembership.startTracks(); await flushPromises(); - expect(localMembership.tracks$.value.length).toBe(2); + expect(publishers[0].createAndSetupTracks).toHaveBeenCalled(); + // expect(localMembership.tracks$.value.length).toBe(2); scope.end(); await flushPromises(); // stop all tracks after ending scopes expect(publishers[0].stopPublishing).toHaveBeenCalled(); - expect(publishers[0].stopTracks).toHaveBeenCalled(); + // expect(publishers[0].stopTracks).toHaveBeenCalled(); publisherFactory.mockClear(); }); // TODO add an integration test combining publisher and localMembership @@ -383,10 +382,11 @@ describe("LocalMembership", () => { it("tracks livekit state correctly", async () => { const scope = new ObservableScope(); + const connectionManagerData = new ConnectionManagerData(); const localTransport$ = new BehaviorSubject(null); - const connectionManagerData$ = new BehaviorSubject< - Epoch - >(new Epoch(new ConnectionManagerData())); + const connectionManagerData$ = new BehaviorSubject( + new Epoch(connectionManagerData), + ); const publishers: Publisher[] = []; const tracks$ = new BehaviorSubject([]); @@ -432,61 +432,96 @@ describe("LocalMembership", () => { }); await flushPromises(); - expect(localMembership.connectionState.livekit$.value).toStrictEqual({ - state: RTCBackendState.WaitingForTransport, - }); + expect(localMembership.localMemberState$.value).toStrictEqual( + TransportState.Waiting, + ); localTransport$.next(aTransport); await flushPromises(); - expect(localMembership.connectionState.livekit$.value).toStrictEqual({ - state: RTCBackendState.WaitingForConnection, + expect(localMembership.localMemberState$.value).toStrictEqual({ + matrix: RTCMemberStatus.Connected, + media: { connection: null, tracks: TrackState.WaitingForUser }, }); - connectionManagerData$.next(new Epoch(connectionManagerData)); + + const connectionManagerData2 = new ConnectionManagerData(); + connectionManagerData2.add( + // clone because we will mutate this later. + { ...connectionTransportAConnecting } as unknown as Connection, + [], + ); + + connectionManagerData$.next(new Epoch(connectionManagerData2)); await flushPromises(); - expect(localMembership.connectionState.livekit$.value).toStrictEqual({ - state: RTCBackendState.Initialized, + expect(localMembership.localMemberState$.value).toStrictEqual({ + matrix: RTCMemberStatus.Connected, + media: { + connection: ConnectionState.LivekitConnecting, + tracks: TrackState.WaitingForUser, + }, }); + + ( + connectionManagerData2.getConnectionForTransport(aTransport)! + .state$ as BehaviorSubject + ).next(ConnectionState.LivekitConnected); + expect(localMembership.localMemberState$.value).toStrictEqual({ + matrix: RTCMemberStatus.Connected, + media: { + connection: ConnectionState.LivekitConnected, + tracks: TrackState.WaitingForUser, + }, + }); + expect(publisherFactory).toHaveBeenCalledOnce(); - expect(localMembership.tracks$.value.length).toBe(0); + // expect(localMembership.tracks$.value.length).toBe(0); // ------- localMembership.startTracks(); // ------- await flushPromises(); - expect(localMembership.connectionState.livekit$.value).toStrictEqual({ - state: RTCBackendState.CreatingTracks, - }); + // expect(localMembership.localMemberState$.value).toStrictEqual({ + // matrix: RTCMemberStatus.Connected, + // media: { + // tracks: TrackState.Creating, + // connection: ConnectionState.LivekitConnected, + // }, + // }); createTrackResolver.resolve(); await flushPromises(); - expect(localMembership.connectionState.livekit$.value).toStrictEqual({ - state: RTCBackendState.ReadyToPublish, - }); + expect( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (localMembership.localMemberState$.value as any).media, + ).toStrictEqual(PublishState.WaitingForUser); // ------- - localMembership.requestConnect(); + localMembership.requestJoinAndPublish(); // ------- - expect(localMembership.connectionState.livekit$.value).toStrictEqual({ - state: RTCBackendState.WaitingToPublish, - }); + expect( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (localMembership.localMemberState$.value as any).media, + ).toStrictEqual(PublishState.Publishing); publishResolver.resolve(); await flushPromises(); - expect(localMembership.connectionState.livekit$.value).toStrictEqual({ - state: RTCBackendState.Connected, - }); + expect( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (localMembership.localMemberState$.value as any).media, + ).toStrictEqual(PublishState.Publishing); + expect(publishers[0].stopPublishing).not.toHaveBeenCalled(); - expect(localMembership.connectionState.livekit$.isStopped).toBe(false); + expect(localMembership.localMemberState$.isStopped).toBe(false); scope.end(); await flushPromises(); // stays in connected state because it is stopped before the update to tracks update the state. - expect(localMembership.connectionState.livekit$.value).toStrictEqual({ - state: RTCBackendState.Connected, - }); + expect( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (localMembership.localMemberState$.value as any).media, + ).toStrictEqual(PublishState.Publishing); // stop all tracks after ending scopes expect(publishers[0].stopPublishing).toHaveBeenCalled(); - expect(publishers[0].stopTracks).toHaveBeenCalled(); + // expect(publishers[0].stopTracks).toHaveBeenCalled(); }); // TODO add tests for matrix local matrix participation. }); diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMember.ts similarity index 56% rename from src/state/CallViewModel/localMember/LocalMembership.ts rename to src/state/CallViewModel/localMember/LocalMember.ts index 60ae79b8..dc22db23 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -6,15 +6,16 @@ Please see LICENSE in the repository root for full details. */ import { - type LocalTrack, type Participant, ParticipantEvent, type LocalParticipant, type ScreenShareCaptureOptions, - ConnectionState, + RoomEvent, + MediaDeviceFailure, } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; import { + Status as RTCSessionStatus, type LivekitTransport, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; @@ -24,10 +25,11 @@ import { combineLatest, distinctUntilChanged, from, + fromEvent, map, type Observable, of, - scan, + pairwise, startWith, switchMap, tap, @@ -35,74 +37,73 @@ import { import { type Logger } from "matrix-js-sdk/lib/logger"; import { deepCompare } from "matrix-js-sdk/lib/utils"; -import { constant, type Behavior } from "../../Behavior"; -import { type IConnectionManager } from "../remoteMembers/ConnectionManager"; -import { ObservableScope } from "../../ObservableScope"; -import { type Publisher } from "./Publisher"; -import { type MuteStates } from "../../MuteStates"; -import { and$ } from "../../../utils/observable"; +import { type Behavior } from "../../Behavior.ts"; +import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts"; +import { type ObservableScope } from "../../ObservableScope.ts"; +import { type Publisher } from "./Publisher.ts"; +import { type MuteStates } from "../../MuteStates.ts"; import { ElementCallError, + FailToStartLivekitConnection, MembershipManagerError, UnknownCallError, -} from "../../../utils/errors"; -import { ElementWidgetActions, widget } from "../../../widget"; +} from "../../../utils/errors.ts"; +import { ElementWidgetActions, widget } from "../../../widget.ts"; import { getUrlParams } from "../../../UrlParams.ts"; import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; import { MatrixRTCMode } from "../../../settings/settings.ts"; import { Config } from "../../../config/Config.ts"; -import { type Connection } from "../remoteMembers/Connection.ts"; +import { + ConnectionState, + type Connection, + type FailedToStartError, +} from "../remoteMembers/Connection.ts"; +import { type HomeserverConnected } from "./HomeserverConnected.ts"; +import { and$ } from "../../../utils/observable.ts"; -export enum RTCBackendState { - Error = "error", +export enum TransportState { /** Not even a transport is available to the LocalMembership */ - WaitingForTransport = "waiting_for_transport", - /** A connection appeared so we can initialise the publisher */ - WaitingForConnection = "waiting_for_connection", - /** Connection and transport arrived, publisher Initialized */ - Initialized = "Initialized", - CreatingTracks = "creating_tracks", - ReadyToPublish = "ready_to_publish", - WaitingToPublish = "waiting_to_publish", - Connected = "connected", - Disconnected = "disconnected", - Disconnecting = "disconnecting", + Waiting = "transport_waiting", } -type LocalMemberRtcBackendState = - | { state: RTCBackendState.Error; error: ElementCallError } - | { state: RTCBackendState.WaitingForTransport } - | { state: RTCBackendState.WaitingForConnection } - | { state: RTCBackendState.Initialized } - | { state: RTCBackendState.CreatingTracks } - | { state: RTCBackendState.ReadyToPublish } - | { state: RTCBackendState.WaitingToPublish } - | { state: RTCBackendState.Connected } - | { state: RTCBackendState.Disconnected } - | { state: RTCBackendState.Disconnecting }; - -export enum MatrixState { - WaitingForTransport = "waiting_for_transport", - Ready = "ready", - Connecting = "connecting", - Connected = "connected", - Disconnected = "disconnected", - Error = "Error", +export enum PublishState { + WaitingForUser = "publish_waiting_for_user", + // XXX: This state is removed for now since we do not have full control over + // track publication anymore with the publisher abstraction, might come back in the future? + // /** Implies lk connection is connected */ + // Starting = "publish_start_publishing", + /** Implies lk connection is connected */ + Publishing = "publish_publishing", } -type LocalMemberMatrixState = - | { state: MatrixState.Connected } - | { state: MatrixState.WaitingForTransport } - | { state: MatrixState.Ready } - | { state: MatrixState.Connecting } - | { state: MatrixState.Disconnected } - | { state: MatrixState.Error; error: Error }; - -export interface LocalMemberConnectionState { - livekit$: Behavior; - matrix$: Behavior; +// TODO not sure how to map that correctly with the +// new publisher that does not manage tracks itself anymore +export enum TrackState { + /** The track is waiting for user input to create tracks (waiting to call `startTracks()`) */ + WaitingForUser = "tracks_waiting_for_user", + // XXX: This state is removed for now since we do not have full control over + // track creation anymore with the publisher abstraction, might come back in the future? + // /** Implies lk connection is connected */ + // Creating = "tracks_creating", + /** Implies lk connection is connected */ + Ready = "tracks_ready", } +export type LocalMemberMediaState = + | { + tracks: TrackState; + connection: ConnectionState | FailedToStartError; + } + | PublishState + | ElementCallError; +export type LocalMemberState = + | ElementCallError + | TransportState.Waiting + | { + media: LocalMemberMediaState; + matrix: ElementCallError | RTCSessionStatus; + }; + /* * - get well known * - get oldest membership @@ -122,8 +123,8 @@ interface Props { muteStates: MuteStates; connectionManager: IConnectionManager; createPublisherFactory: (connection: Connection) => Publisher; - joinMatrixRTC: (transport: LivekitTransport) => Promise; - homeserverConnected$: Behavior; + joinMatrixRTC: (transport: LivekitTransport) => void; + homeserverConnected: HomeserverConnected; localTransport$: Behavior; matrixRTCSession: Pick< MatrixRTCSession, @@ -137,7 +138,16 @@ interface Props { * We want * - a publisher * - - * @param param0 + * @param props The properties required to create the local membership. + * @param props.scope The observable scope to use. + * @param props.connectionManager The connection manager to get connections from. + * @param props.createPublisherFactory Factory to create a publisher once we have a connection. + * @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport. + * @param props.homeserverConnected The homeserver connected state. + * @param props.localTransport$ The local transport to use for publishing. + * @param props.logger The logger to use. + * @param props.muteStates The mute states for video and audio. + * @param props.matrixRTCSession The matrix RTC session to join. * @returns * - publisher: The handle to create tracks and publish them to the room. * - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication) @@ -149,7 +159,7 @@ export const createLocalMembership$ = ({ scope, connectionManager, localTransport$: localTransportCanThrow$, - homeserverConnected$, + homeserverConnected, createPublisherFactory, joinMatrixRTC, logger: parentLogger, @@ -157,28 +167,37 @@ export const createLocalMembership$ = ({ matrixRTCSession, }: Props): { /** - * This starts audio and video tracks. They will be reused when calling `requestConnect`. + * This request to start audio and video tracks. + * Can be called early to pre-emptively get media permissions and start devices. */ - startTracks: () => Behavior; + startTracks: () => void; /** - * This sets a inner state (shouldConnect) to true and instructs the js-sdk and livekit to keep the user + * This sets a inner state (shouldPublish) to true and instructs the js-sdk and livekit to keep the user * connected to matrix and livekit. */ - requestConnect: () => void; + requestJoinAndPublish: () => void; requestDisconnect: () => void; - connectionState: LocalMemberConnectionState; + localMemberState$: Behavior; sharingScreen$: Behavior; /** * Callback to toggle screen sharing. If null, screen sharing is not possible. */ toggleScreenSharing: (() => void) | null; - tracks$: Behavior; + // tracks$: Behavior; participant$: Behavior; connection$: Behavior; - homeserverConnected$: Behavior; - // this needs to be discussed - /** @deprecated use state instead*/ + /** + * Tracks the homserver and livekit connected state and based on that computes reconnecting. + */ reconnecting$: Behavior; + /** Shorthand for homeserverConnected.rtcSession === Status.Disconnected + * Direct translation to the js-sdk membership manager connection `Status`. + */ + disconnected$: Behavior; + /** + * Fully connected + */ + connected$: Behavior; } => { const logger = parentLogger.getChild("[LocalMembership]"); logger.debug(`Creating local membership..`); @@ -197,7 +216,7 @@ export const createLocalMembership$ = ({ : new Error("Unknown error from localTransport"), ); } - setLivekitError(error); + setTransportError(error); return of(null); }), ), @@ -224,94 +243,59 @@ export const createLocalMembership$ = ({ ), ); - const localConnectionState$ = localConnection$.pipe( - switchMap((connection) => (connection ? connection.state$ : of(null))), - ); - - // /** - // * Whether we are "fully" connected to the call. Accounts for both the - // * connection to the MatrixRTC session and the LiveKit publish connection. - // */ - const connected$ = scope.behavior( - and$( - homeserverConnected$.pipe( - tap((v) => logger.debug("matrix: Connected state changed", v)), - ), - localConnectionState$.pipe( - switchMap((state) => { - logger.debug("livekit: Connected state changed", state); - if (!state) return of(false); - if (state.state === "ConnectedToLkRoom") { - logger.debug( - "livekit: Connected state changed (inner livekitConnectionState$)", - state.livekitConnectionState$.value, - ); - return state.livekitConnectionState$.pipe( - map((lkState) => lkState === ConnectionState.Connected), - ); - } - return of(false); - }), - ), - ).pipe(tap((v) => logger.debug("combined: Connected state changed", v))), + // Tracks error that happen when creating the local tracks. + const mediaErrors$ = localConnection$.pipe( + switchMap((connection) => { + if (!connection) { + return of(null); + } else { + return fromEvent( + connection.livekitRoom, + RoomEvent.MediaDevicesError, + (error: Error) => { + return MediaDeviceFailure.getFailure(error) ?? null; + }, + ); + } + }), ); + mediaErrors$.pipe(scope.bind()).subscribe((error) => { + if (error) { + logger.error(`Failed to create local tracks:`, error); + setMatrixError( + // TODO is it fatal? Do we need to create a new Specialized Error? + new UnknownCallError(new Error(`Media device error: ${error}`)), + ); + } + }); // MATRIX RELATED - // /** - // * Whether we should tell the user that we're reconnecting to the call. - // */ - // DISCUSSION is there a better way to do this? - // sth that is more deriectly implied from the membership manager of the js sdk. (fromEvent(matrixRTCSession, Reconnecting)) ??? or similar - const reconnecting$ = scope.behavior( - connected$.pipe( - // We are reconnecting if we previously had some successful initial - // connection but are now disconnected - scan( - ({ connectedPreviously }, connectedNow) => ({ - connectedPreviously: connectedPreviously || connectedNow, - reconnecting: connectedPreviously && !connectedNow, - }), - { connectedPreviously: false, reconnecting: false }, - ), - map(({ reconnecting }) => reconnecting), - ), - ); - // This should be used in a combineLatest with publisher$ to connect. // to make it possible to call startTracks before the preferredTransport$ has resolved. const trackStartRequested = Promise.withResolvers(); // This should be used in a combineLatest with publisher$ to connect. // to make it possible to call startTracks before the preferredTransport$ has resolved. - const connectRequested$ = new BehaviorSubject(false); + const joinAndPublishRequested$ = new BehaviorSubject(false); /** * The publisher is stored in here an abstracts creating and publishing tracks. */ const publisher$ = new BehaviorSubject(null); - /** - * Extract the tracks from the published. Also reacts to changing publishers. - */ - const tracks$ = scope.behavior( - publisher$.pipe(switchMap((p) => (p?.tracks$ ? p.tracks$ : constant([])))), - ); - const publishing$ = scope.behavior( - publisher$.pipe(switchMap((p) => p?.publishing$ ?? constant(false))), - ); - const startTracks = (): Behavior => { + const startTracks = (): void => { trackStartRequested.resolve(); - return tracks$; + // This used to return the tracks, but now they are only accessible via the publisher. }; - const requestConnect = (): void => { + const requestJoinAndPublish = (): void => { trackStartRequested.resolve(); - connectRequested$.next(true); + joinAndPublishRequested$.next(true); }; const requestDisconnect = (): void => { - connectRequested$.next(false); + joinAndPublishRequested$.next(false); }; // Take care of the publisher$ @@ -323,25 +307,30 @@ export const createLocalMembership$ = ({ // - overwrite current publisher scope.reconcile(localConnection$, async (connection) => { if (connection !== null) { - publisher$.next(createPublisherFactory(connection)); + const publisher = createPublisherFactory(connection); + publisher$.next(publisher); + // Clean-up callback + return Promise.resolve(async (): Promise => { + await publisher.stopPublishing(); + await publisher.stopTracks(); + }); } - return Promise.resolve(async (): Promise => { - await publisher$?.value?.stopPublishing(); - publisher$?.value?.stopTracks(); - }); }); // Use reconcile here to not run concurrent createAndSetupTracks calls // `tracks$` will update once they are ready. scope.reconcile( scope.behavior( - combineLatest([publisher$, tracks$, from(trackStartRequested.promise)]), + combineLatest([ + publisher$ /*, tracks$*/, + from(trackStartRequested.promise), + ]), null, ), async (valueIfReady) => { if (!valueIfReady) return; - const [publisher, tracks] = valueIfReady; - if (publisher && tracks.length === 0) { + const [publisher] = valueIfReady; + if (publisher) { await publisher.createAndSetupTracks().catch((e) => logger.error(e)); } }, @@ -349,140 +338,215 @@ export const createLocalMembership$ = ({ // Based on `connectRequested$` we start publishing tracks. (once they are there!) scope.reconcile( - scope.behavior(combineLatest([publisher$, tracks$, connectRequested$])), - async ([publisher, tracks, shouldConnect]) => { - if (shouldConnect === publisher?.publishing$.value) return; - if (tracks.length !== 0 && shouldConnect) { + scope.behavior(combineLatest([publisher$, joinAndPublishRequested$])), + async ([publisher, shouldJoinAndPublish]) => { + // Get the current publishing state to avoid redundant calls. + const isPublishing = publisher?.shouldPublish === true; + if (shouldJoinAndPublish && !isPublishing) { try { await publisher?.startPublishing(); } catch (error) { - setLivekitError(error as ElementCallError); + const message = + error instanceof Error ? error.message : String(error); + setPublishError(new FailToStartLivekitConnection(message)); } - } else if (tracks.length !== 0 && !shouldConnect) { + } else if (isPublishing) { try { await publisher?.stopPublishing(); } catch (error) { - setLivekitError(new UnknownCallError(error as Error)); + setPublishError(new UnknownCallError(error as Error)); } } }, ); - const fatalLivekitError$ = new BehaviorSubject(null); - const setLivekitError = (e: ElementCallError): void => { - if (fatalLivekitError$.value !== null) - logger.error("Multiple Livkit Errors:", e); - else fatalLivekitError$.next(e); + // STATE COMPUTATION + + // These are non fatal since we can join a room and concume media even though publishing failed. + const publishError$ = new BehaviorSubject(null); + const setPublishError = (e: ElementCallError): void => { + if (publishError$.value !== null) { + logger.error("Multiple Media Errors:", e); + } else { + publishError$.next(e); + } }; - const livekitState$: Behavior = scope.behavior( + + const fatalTransportError$ = new BehaviorSubject( + null, + ); + + const setTransportError = (e: ElementCallError): void => { + if (fatalTransportError$.value !== null) { + logger.error("Multiple Transport Errors:", e); + } else { + fatalTransportError$.next(e); + } + }; + + const localConnectionState$ = localConnection$.pipe( + switchMap((connection) => (connection ? connection.state$ : of(null))), + ); + + const mediaState$: Behavior = scope.behavior( combineLatest([ - publisher$, + localConnectionState$, localTransport$, - tracks$.pipe( - tap((t) => { - logger.info("tracks$: ", t); - }), - ), - publishing$, - connectRequested$, + joinAndPublishRequested$, from(trackStartRequested.promise).pipe( map(() => true), startWith(false), ), - fatalLivekitError$, ]).pipe( map( ([ - publisher, + localConnectionState, localTransport, - tracks, - publishing, - shouldConnect, + shouldPublish, shouldStartTracks, - error, ]) => { - // read this: - // if(!) return {state: ...} - // if(!) return {state: } - // - // as: - // We do have but not yet so we are in - if (error !== null) return { state: RTCBackendState.Error, error }; - const hasTracks = tracks.length > 0; - if (!localTransport) - return { state: RTCBackendState.WaitingForTransport }; - if (!publisher) - return { state: RTCBackendState.WaitingForConnection }; - if (!shouldStartTracks) return { state: RTCBackendState.Initialized }; - if (!hasTracks) return { state: RTCBackendState.CreatingTracks }; - if (!shouldConnect) return { state: RTCBackendState.ReadyToPublish }; - if (!publishing) return { state: RTCBackendState.WaitingToPublish }; - return { state: RTCBackendState.Connected }; + if (!localTransport) return null; + const trackState: TrackState = shouldStartTracks + ? TrackState.Ready + : TrackState.WaitingForUser; + + if ( + localConnectionState !== ConnectionState.LivekitConnected || + trackState !== TrackState.Ready + ) + return { + connection: localConnectionState, + tracks: trackState, + }; + if (!shouldPublish) return PublishState.WaitingForUser; + // if (!publishing) return PublishState.Starting; + return PublishState.Publishing; }, ), distinctUntilChanged(deepCompare), ), ); - const fatalMatrixError$ = new BehaviorSubject(null); const setMatrixError = (e: ElementCallError): void => { - if (fatalMatrixError$.value !== null) + if (fatalMatrixError$.value !== null) { logger.error("Multiple Matrix Errors:", e); - else fatalMatrixError$.next(e); + } else { + fatalMatrixError$.next(e); + } }; - const matrixState$: Behavior = scope.behavior( + + const localMemberState$ = scope.behavior( combineLatest([ - localTransport$, - connectRequested$, - homeserverConnected$, + mediaState$, + homeserverConnected.rtsSession$, + fatalMatrixError$, + fatalTransportError$, + publishError$, ]).pipe( - map(([localTransport, connectRequested, homeserverConnected]) => { - if (!localTransport) return { state: MatrixState.WaitingForTransport }; - if (!connectRequested) return { state: MatrixState.Ready }; - if (!homeserverConnected) return { state: MatrixState.Connecting }; - return { state: MatrixState.Connected }; - }), + map( + ([ + mediaState, + rtcSessionStatus, + fatalMatrixError, + fatalTransportError, + publishError, + ]) => { + if (fatalTransportError !== null) return fatalTransportError; + // `mediaState` will be 'null' until the transport/connection appears. + if (mediaState && rtcSessionStatus) + return { + matrix: fatalMatrixError ?? rtcSessionStatus, + media: publishError ?? mediaState, + }; + return TransportState.Waiting; + }, + ), ), ); - // Keep matrix rtc session in sync with localTransport$, connectRequested$ and muteStates.video.enabled$ + /** + * Whether we are "fully" connected to the call. Accounts for both the + * connection to the MatrixRTC session and the LiveKit publish connection. + */ + const matrixAndLivekitConnected$ = scope.behavior( + and$( + homeserverConnected.combined$, + localConnectionState$.pipe( + map((state) => state === ConnectionState.LivekitConnected), + ), + ).pipe( + tap((v) => logger.debug("livekit+matrix: Connected state changed", v)), + ), + ); + + /** + * Whether we should tell the user that we're reconnecting to the call. + */ + const reconnecting$ = scope.behavior( + matrixAndLivekitConnected$.pipe( + pairwise(), + map(([prev, current]) => prev === true && current === false), + ), + false, + ); + + // inform the widget about the connect and disconnect intent from the user. + scope + .behavior(joinAndPublishRequested$.pipe(pairwise(), scope.bind()), [ + undefined, + joinAndPublishRequested$.value, + ]) + .subscribe(([prev, current]) => { + if (!widget) return; + // JOIN prev=false (was left) => current-true (now joiend) + if (!prev && current) { + widget.api.transport + .send(ElementWidgetActions.JoinCall, {}) + .catch((e) => { + logger.error("Failed to send join action", e); + }); + } + // LEAVE prev=false (was joined) => current-true (now left) + if (prev && !current) { + widget.api.transport + .send(ElementWidgetActions.HangupCall, {}) + .catch((e) => { + logger.error("Failed to send hangup action", e); + }); + } + }); + + combineLatest([muteStates.video.enabled$, homeserverConnected.combined$]) + .pipe(scope.bind()) + .subscribe(([videoEnabled, connected]) => { + if (!connected) return; + void matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"); + }); + + // Keep matrix rtc session in sync with localTransport$, connectRequested$ scope.reconcile( - scope.behavior(combineLatest([localTransport$, connectRequested$])), + scope.behavior(combineLatest([localTransport$, joinAndPublishRequested$])), async ([transport, shouldConnect]) => { + if (!transport) return; + // if shouldConnect=false we will do the disconnect as the cleanup from the previous reconcile iteration. if (!shouldConnect) return; - if (!transport) return; try { - await joinMatrixRTC(transport); + joinMatrixRTC(transport); } catch (error) { logger.error("Error entering RTC session", error); if (error instanceof Error) setMatrixError(new MembershipManagerError(error)); } - // Update our member event when our mute state changes. - const callIntentScope = new ObservableScope(); - // because this uses its own scope, we can start another reconciliation for the duration of one connection. - callIntentScope.reconcile( - muteStates.video.enabled$, - async (videoEnabled) => - matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"), - ); - - return async (): Promise => { - callIntentScope.end(); + return Promise.resolve(async (): Promise => { try { - // Update matrixRTCSession to allow udpating the transport without leaving the session! - await matrixRTCSession.leaveRoomSession(); + // TODO Update matrixRTCSession to allow udpating the transport without leaving the session! + await matrixRTCSession.leaveRoomSession(1000); } catch (e) { logger.error("Error leaving RTC session", e); } - try { - await widget?.api.transport.send(ElementWidgetActions.HangupCall, {}); - } catch (e) { - logger.error("Failed to send hangup action", e); - } - }; + }); }, ); @@ -497,7 +561,7 @@ export const createLocalMembership$ = ({ // pause tracks during the initial joining sequence too until we're sure // that our own media is displayed on screen. // TODO refactor this based no livekitState$ - combineLatest([participant$, homeserverConnected$]) + combineLatest([participant$, homeserverConnected.combined$]) .pipe(scope.bind()) .subscribe(([participant, connected]) => { if (!participant) return; @@ -582,16 +646,17 @@ export const createLocalMembership$ = ({ return { startTracks, - requestConnect, + requestJoinAndPublish, requestDisconnect, - connectionState: { - livekit$: livekitState$, - matrix$: matrixState$, - }, - tracks$, + localMemberState$, participant$, - homeserverConnected$, reconnecting$, + connected$: matrixAndLivekitConnected$, + disconnected$: scope.behavior( + homeserverConnected.rtsSession$.pipe( + map((state) => state === RTCSessionStatus.Disconnected), + ), + ), sharingScreen$, toggleScreenSharing, connection$: localConnection$, @@ -620,17 +685,19 @@ interface EnterRTCSessionOptions { * - Delay events management * - Handles retries (fails only after several attempts) * - * @param rtcSession - * @param transport - * @param options + * @param rtcSession - The MatrixRTCSession to join. + * @param transport - The LivekitTransport to use for this session. + * @param options - Options for entering the RTC session. + * @param options.encryptMedia - Whether to encrypt media. + * @param options.matrixRTCMode - The Matrix RTC mode to use. * @throws If the widget could not send ElementWidgetActions.JoinCall action. */ // Exported for unit testing -export async function enterRTCSession( +export function enterRTCSession( rtcSession: MatrixRTCSession, transport: LivekitTransport, { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions, -): Promise { +): void { PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); @@ -669,7 +736,4 @@ export async function enterRTCSession( unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0, }, ); - if (widget) { - await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); - } } diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index c1c36fa5..3e69bf2c 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -5,9 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + afterEach, + beforeEach, + describe, + expect, + it, + type MockedObject, + vi, +} from "vitest"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject } from "rxjs"; +import { BehaviorSubject, lastValueFrom } from "rxjs"; +import fetchMock from "fetch-mock"; import { mockConfig, flushPromises } from "../../../utils/test"; import { createLocalTransport$ } from "./LocalTransport"; @@ -18,31 +27,22 @@ import { FailToGetOpenIdToken, } from "../../../utils/errors"; import * as openIDSFU from "../../../livekit/openIDSFU"; +import { customLivekitUrl } from "../../../settings/settings"; +import { testJWTToken } from "../../../utils/test-fixtures"; describe("LocalTransport", () => { + const openIdResponse: openIDSFU.SFUConfig = { + url: "https://lk.example.org", + jwt: testJWTToken, + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + }; + let scope: ObservableScope; - beforeEach(() => (scope = new ObservableScope())); - afterEach(() => scope.end()); - - it("throws if config is missing", async () => { - const localTransport$ = createLocalTransport$({ - scope, - roomId: "!room:example.org", - useOldestMember$: constant(false), - memberships$: constant(new Epoch([])), - client: { - getDomain: () => "", - // These won't be called in this error path but satisfy the type - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, - }); - await flushPromises(); - - expect(() => localTransport$.value).toThrow( - new MatrixRTCTransportMissingError(""), - ); + beforeEach(() => { + scope = new ObservableScope(); }); + afterEach(() => scope.end()); it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => { // Provide a valid config so makeTransportInternal resolves a transport @@ -61,12 +61,14 @@ describe("LocalTransport", () => { const errors: Error[] = []; const localTransport$ = createLocalTransport$({ scope, - roomId: "!room:example.org", + roomId: "!example_room_id", useOldestMember$: constant(false), memberships$: constant(new Epoch([])), client: { // Use empty domain to skip .well-known and use config directly getDomain: () => "", + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: async () => Promise.resolve([]), getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), }, @@ -84,41 +86,6 @@ describe("LocalTransport", () => { expect(() => localTransport$.value).toThrow(expectedError); }); - it("emits preferred transport after OpenID resolves", async () => { - // Use config so transport discovery succeeds, but delay OpenID JWT fetch - mockConfig({ - livekit: { livekit_service_url: "https://lk.example.org" }, - }); - - const openIdResolver = Promise.withResolvers(); - - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue( - openIdResolver.promise, - ); - - const localTransport$ = createLocalTransport$({ - scope, - roomId: "!room:example.org", - useOldestMember$: constant(false), - memberships$: constant(new Epoch([])), - client: { - getDomain: () => "", - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, - }); - - openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" }); - expect(localTransport$.value).toBe(null); - await flushPromises(); - // final - expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!room:example.org", - livekit_service_url: "https://lk.example.org", - type: "livekit", - }); - }); - it("updates local transport when oldest member changes", async () => { // Use config so transport discovery succeeds, but delay OpenID JWT fetch mockConfig({ @@ -133,24 +100,171 @@ describe("LocalTransport", () => { const localTransport$ = createLocalTransport$({ scope, - roomId: "!room:example.org", + roomId: "!example_room_id", useOldestMember$: constant(true), memberships$, client: { getDomain: () => "", + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: async () => Promise.resolve([]), getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), }, }); - openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" }); + openIdResolver.resolve?.(openIdResponse); expect(localTransport$.value).toBe(null); await flushPromises(); // final expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!room:example.org", + livekit_alias: "!example_room_id", livekit_service_url: "https://lk.example.org", type: "livekit", }); }); + + type LocalTransportProps = Parameters[0]; + + describe("transport configuration mechanisms", () => { + let localTransportOpts: LocalTransportProps & { + client: MockedObject; + }; + let openIdResolver: PromiseWithResolvers; + beforeEach(() => { + mockConfig({}); + customLivekitUrl.setValue(customLivekitUrl.defaultValue); + localTransportOpts = { + scope, + roomId: "!example_room_id", + useOldestMember$: constant(false), + memberships$: constant(new Epoch([])), + client: { + getDomain: vi.fn().mockReturnValue(""), + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), + getOpenIdToken: vi.fn(), + getDeviceId: vi.fn(), + }, + }; + openIdResolver = Promise.withResolvers(); + vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue( + openIdResolver.promise, + ); + }); + + afterEach(() => { + fetchMock.reset(); + }); + + it("supports getting transport via application config", async () => { + mockConfig({ + livekit: { livekit_service_url: "https://lk.example.org" }, + }); + const localTransport$ = createLocalTransport$(localTransportOpts); + openIdResolver.resolve?.(openIdResponse); + expect(localTransport$.value).toBe(null); + await flushPromises(); + expect(localTransport$.value).toStrictEqual({ + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }); + }); + it("supports getting transport via user settings", async () => { + customLivekitUrl.setValue("https://lk.example.org"); + const localTransport$ = createLocalTransport$(localTransportOpts); + openIdResolver.resolve?.(openIdResponse); + expect(localTransport$.value).toBe(null); + await flushPromises(); + expect(localTransport$.value).toStrictEqual({ + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }); + }); + it("supports getting transport via backend", async () => { + localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ + { type: "livekit", livekit_service_url: "https://lk.example.org" }, + ]); + const localTransport$ = createLocalTransport$(localTransportOpts); + openIdResolver.resolve?.(openIdResponse); + expect(localTransport$.value).toBe(null); + await flushPromises(); + expect(localTransport$.value).toStrictEqual({ + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }); + }); + it("fails fast if the openID request fails for backend config", async () => { + localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ + { type: "livekit", livekit_service_url: "https://lk.example.org" }, + ]); + openIdResolver.reject( + new FailToGetOpenIdToken(new Error("Test driven error")), + ); + try { + await lastValueFrom(createLocalTransport$(localTransportOpts)); + throw Error("Expected test to throw"); + } catch (ex) { + expect(ex).toBeInstanceOf(FailToGetOpenIdToken); + } + }); + it("supports getting transport via well-known", async () => { + localTransportOpts.client.getDomain.mockReturnValue("example.org"); + fetchMock.getOnce("https://example.org/.well-known/matrix/client", { + "org.matrix.msc4143.rtc_foci": [ + { type: "livekit", livekit_service_url: "https://lk.example.org" }, + ], + }); + const localTransport$ = createLocalTransport$(localTransportOpts); + openIdResolver.resolve?.(openIdResponse); + expect(localTransport$.value).toBe(null); + await flushPromises(); + expect(localTransport$.value).toStrictEqual({ + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }); + expect(fetchMock.done()).toEqual(true); + }); + it("fails fast if the openId request fails for the well-known config", async () => { + localTransportOpts.client.getDomain.mockReturnValue("example.org"); + fetchMock.getOnce("https://example.org/.well-known/matrix/client", { + "org.matrix.msc4143.rtc_foci": [ + { type: "livekit", livekit_service_url: "https://lk.example.org" }, + ], + }); + openIdResolver.reject( + new FailToGetOpenIdToken(new Error("Test driven error")), + ); + try { + await lastValueFrom(createLocalTransport$(localTransportOpts)); + throw Error("Expected test to throw"); + } catch (ex) { + expect(ex).toBeInstanceOf(FailToGetOpenIdToken); + } + }); + it("throws if no options are available", async () => { + const localTransport$ = createLocalTransport$({ + scope, + roomId: "!example_room_id", + useOldestMember$: constant(false), + memberships$: constant(new Epoch([])), + client: { + getDomain: () => "", + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: async () => Promise.resolve([]), + // These won't be called in this error path but satisfy the type + getOpenIdToken: vi.fn(), + getDeviceId: vi.fn(), + }, + }); + await flushPromises(); + + expect(() => localTransport$.value).toThrow( + new MatrixRTCTransportMissingError(""), + ); + }); + }); }); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 0a85bbc1..e72b076f 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -8,11 +8,11 @@ Please see LICENSE in the repository root for full details. import { type CallMembership, isLivekitTransport, - type LivekitTransportConfig, type LivekitTransport, isLivekitTransportConfig, + type Transport, } from "matrix-js-sdk/lib/matrixrtc"; -import { type MatrixClient } from "matrix-js-sdk"; +import { MatrixError, type MatrixClient } from "matrix-js-sdk"; import { combineLatest, distinctUntilChanged, @@ -27,7 +27,10 @@ import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { type Behavior } from "../../Behavior.ts"; import { type Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { Config } from "../../../config/Config.ts"; -import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; +import { + FailToGetOpenIdToken, + MatrixRTCTransportMissingError, +} from "../../../utils/errors.ts"; import { getSFUConfigWithOpenID, type OpenIDClientParts, @@ -45,7 +48,8 @@ const logger = rootLogger.getChild("[LocalTransport]"); interface Props { scope: ObservableScope; memberships$: Behavior>; - client: Pick & OpenIDClientParts; + client: Pick & + OpenIDClientParts; roomId: string; useOldestMember$: Behavior; } @@ -85,7 +89,7 @@ export const createLocalTransport$ = ({ * The transport that we would personally prefer to publish on (if not for the * transport preferences of others, perhaps). * - * @throws + * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ const preferredTransport$: Behavior = scope.behavior( customLivekitUrl.value$.pipe( @@ -116,73 +120,150 @@ export const createLocalTransport$ = ({ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; /** + * Determine the correct Transport for the current session, including + * validating auth against the service to ensure it's correct. + * Prefers in order: * - * @param client - * @param roomId - * @returns + * 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw. + * 2. The transports returned via the homeserver. + * 3. The transports returned via .well-known. + * 4. The transport configured in Element Call's config. + * + * @param client The authenticated Matrix client for the current user + * @param roomId The ID of the room to be connected to. + * @param urlFromDevSettings Override URL provided by the user's local config. + * @returns A fully validated transport config. * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ async function makeTransport( - client: Pick & OpenIDClientParts, + client: Pick & + OpenIDClientParts, roomId: string, urlFromDevSettings: string | null, ): Promise { - let transport: LivekitTransport | undefined; logger.trace("Searching for a preferred transport"); - //TODO refactor this to use the jwt service returned alias. - const livekitAlias = roomId; + + // We will call `getSFUConfigWithOpenID` once per transport here as it's our + // only mechanism of valiation. This means we will also ask the + // homeserver for a OpenID token a few times. Since OpenID tokens are single + // use we don't want to risk any issues by re-using a token. + // + // If the OpenID request were to fail then it's acceptable for us to fail + // this function early, as we assume the homeserver has got some problems. // DEVTOOL: Highest priority: Load from devtool setting if (urlFromDevSettings !== null) { - const transportFromStorage: LivekitTransport = { + logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings); + // Validate that the SFU is up. Otherwise, we want to fail on this + // as we don't permit other SFUs. + const config = await getSFUConfigWithOpenID( + client, + urlFromDevSettings, + roomId, + ); + return { type: "livekit", livekit_service_url: urlFromDevSettings, - livekit_alias: livekitAlias, + livekit_alias: config.livekitAlias, }; - logger.info( - "Using LiveKit transport from dev tools: ", - transportFromStorage, - ); - transport = transportFromStorage; } - // WELL_KNOWN: Prioritize the .well-known/matrix/client, if available, over the configured SFU + async function getFirstUsableTransport( + transports: Transport[], + ): Promise { + for (const potentialTransport of transports) { + if (isLivekitTransportConfig(potentialTransport)) { + try { + const { livekitAlias } = await getSFUConfigWithOpenID( + client, + potentialTransport.livekit_service_url, + roomId, + ); + return { + ...potentialTransport, + livekit_alias: livekitAlias, + }; + } catch (ex) { + if (ex instanceof FailToGetOpenIdToken) { + // Explictly throw these + throw ex; + } + logger.debug( + `Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`, + ex, + ); + } + } + } + return null; + } + + // MSC4143: Attempt to fetch transports from backend. + if ("_unstable_getRTCTransports" in client) { + try { + const selectedTransport = await getFirstUsableTransport( + await client._unstable_getRTCTransports(), + ); + if (selectedTransport) { + logger.info("Using backend-configured SFU", selectedTransport); + return selectedTransport; + } + } catch (ex) { + if (ex instanceof MatrixError && ex.httpStatus === 404) { + // Expected, this is an unstable endpoint and it's not required. + logger.debug("Backend does not provide any RTC transports", ex); + } else if (ex instanceof FailToGetOpenIdToken) { + throw ex; + } else { + // We got an error that wasn't just missing support for the feature, so log it loudly. + logger.error( + "Unexpected error fetching RTC transports from backend", + ex, + ); + } + } + } + + // Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available. const domain = client.getDomain(); - if (domain && transport === undefined) { + if (domain) { // we use AutoDiscovery instead of relying on the MatrixClient having already // been fully configured and started const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ FOCI_WK_KEY ]; - if (Array.isArray(wellKnownFoci)) { - const wellKnownTransport: LivekitTransportConfig | undefined = - wellKnownFoci.find((f) => f && isLivekitTransportConfig(f)); - if (wellKnownTransport !== undefined) { - logger.info("Using LiveKit transport from .well-known: ", transport); - transport = { ...wellKnownTransport, livekit_alias: livekitAlias }; - } + const selectedTransport = Array.isArray(wellKnownFoci) + ? await getFirstUsableTransport(wellKnownFoci) + : null; + if (selectedTransport) { + logger.info("Using .well-known SFU", selectedTransport); + return selectedTransport; } } // CONFIG: Least prioritized; Load from config file const urlFromConf = Config.get().livekit?.livekit_service_url; - if (urlFromConf && transport === undefined) { - const transportFromConf: LivekitTransport = { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; - logger.info("Using LiveKit transport from config: ", transportFromConf); - transport = transportFromConf; + if (urlFromConf) { + try { + const { livekitAlias } = await getSFUConfigWithOpenID( + client, + urlFromConf, + roomId, + ); + const selectedTransport: LivekitTransport = { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; + logger.info("Using config SFU", selectedTransport); + return selectedTransport; + } catch (ex) { + if (ex instanceof FailToGetOpenIdToken) { + throw ex; + } + logger.error("Failed to validate config SFU", ex); + } } - if (!transport) throw new MatrixRTCTransportMissingError(domain ?? ""); // this will call the jwt/sfu/get endpoint to pre create the livekit room. - - await getSFUConfigWithOpenID( - client, - transport.livekit_service_url, - transport.livekit_alias, - ); - - return transport; + throw new MatrixRTCTransportMissingError(domain ?? ""); } diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index 9b3e5b2a..38a80bed 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -5,66 +5,320 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +import { afterEach, beforeEach, describe, expect, it, test, vi } from "vitest"; import { - afterEach, - beforeEach, - describe, - expect, - it, - type Mock, - vi, -} from "vitest"; -import { ConnectionState as LivekitConenctionState } from "livekit-client"; -import { type BehaviorSubject } from "rxjs"; + ConnectionState as LivekitConnectionState, + LocalParticipant, + type LocalTrack, + type LocalTrackPublication, + ParticipantEvent, + Track, +} from "livekit-client"; +import { BehaviorSubject } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { ObservableScope } from "../../ObservableScope"; import { constant } from "../../Behavior"; import { + flushPromises, mockLivekitRoom, - mockLocalParticipant, mockMediaDevices, } from "../../../utils/test"; import { Publisher } from "./Publisher"; -import { - type Connection, - type ConnectionState, -} from "../remoteMembers/Connection"; +import { type Connection } from "../remoteMembers/Connection"; import { type MuteStates } from "../../MuteStates"; -import { FailToStartLivekitConnection } from "../../../utils/errors"; -describe("Publisher", () => { - let scope: ObservableScope; - let connection: Connection; - let muteStates: MuteStates; - beforeEach(() => { - muteStates = { - audio: { - enabled$: constant(false), - unsetHandler: vi.fn(), - setHandler: vi.fn(), - }, - video: { - enabled$: constant(false), - unsetHandler: vi.fn(), - setHandler: vi.fn(), - }, - } as unknown as MuteStates; - scope = new ObservableScope(); - connection = { - state$: constant({ - state: "ConnectedToLkRoom", - livekitConnectionState$: constant(LivekitConenctionState.Connected), - }), - livekitRoom: mockLivekitRoom({ - localParticipant: mockLocalParticipant({}), - }), - } as unknown as Connection; +let scope: ObservableScope; + +beforeEach(() => { + scope = new ObservableScope(); +}); + +afterEach(() => scope.end()); + +function createMockLocalTrack(source: Track.Source): LocalTrack { + const track = { + source, + isMuted: false, + isUpstreamPaused: false, + } as Partial as LocalTrack; + + vi.mocked(track).mute = vi.fn().mockImplementation(() => { + track.isMuted = true; + }); + vi.mocked(track).unmute = vi.fn().mockImplementation(() => { + track.isMuted = false; + }); + vi.mocked(track).pauseUpstream = vi.fn().mockImplementation(() => { + // @ts-expect-error - for that test we want to set isUpstreamPaused directly + track.isUpstreamPaused = true; + }); + vi.mocked(track).resumeUpstream = vi.fn().mockImplementation(() => { + // @ts-expect-error - for that test we want to set isUpstreamPaused directly + track.isUpstreamPaused = false; }); - afterEach(() => scope.end()); + return track; +} - it("throws if livekit room could not publish", async () => { +function createMockMuteState(enabled$: BehaviorSubject): { + enabled$: BehaviorSubject; + setHandler: (h: (enabled: boolean) => void) => void; + unsetHandler: () => void; +} { + let currentHandler = (enabled: boolean): void => {}; + + const ms = { + enabled$, + setHandler: vi.fn().mockImplementation((h: (enabled: boolean) => void) => { + currentHandler = h; + }), + unsetHandler: vi.fn().mockImplementation(() => { + currentHandler = (enabled: boolean): void => {}; + }), + }; + // forward enabled$ emissions to the current handler + enabled$.subscribe((enabled) => { + logger.info(`MockMuteState: enabled changed to ${enabled}`); + currentHandler(enabled); + }); + + return ms; +} + +let connection: Connection; +let muteStates: MuteStates; +let localParticipant: LocalParticipant; +let audioEnabled$: BehaviorSubject; +let videoEnabled$: BehaviorSubject; +let trackPublications: LocalTrackPublication[]; +// use it to control when track creation resolves, default to resolved +let createTrackLock: Promise; + +beforeEach(() => { + trackPublications = []; + audioEnabled$ = new BehaviorSubject(false); + videoEnabled$ = new BehaviorSubject(false); + createTrackLock = Promise.resolve(); + + muteStates = { + audio: createMockMuteState(audioEnabled$), + video: createMockMuteState(videoEnabled$), + } as unknown as MuteStates; + + const mockSendDataPacket = vi.fn(); + const mockEngine = { + client: { + sendUpdateLocalMetadata: vi.fn(), + }, + on: vi.fn().mockReturnThis(), + sendDataPacket: mockSendDataPacket, + }; + + localParticipant = new LocalParticipant( + "local-sid", + "local-identity", + // @ts-expect-error - for that test we want a real LocalParticipant to have the pending publications logic + mockEngine, + { + adaptiveStream: true, + dynacase: false, + audioCaptureDefaults: {}, + videoCaptureDefaults: {}, + stopLocalTrackOnUnpublish: true, + reconnectPolicy: "always", + disconnectOnPageLeave: true, + }, + new Map(), + {}, + ); + + vi.mocked(localParticipant).createTracks = vi + .fn() + .mockImplementation(async (opts) => { + const tracks: LocalTrack[] = []; + if (opts.audio) { + tracks.push(createMockLocalTrack(Track.Source.Microphone)); + } + if (opts.video) { + tracks.push(createMockLocalTrack(Track.Source.Camera)); + } + await createTrackLock; + return tracks; + }); + + vi.mocked(localParticipant).publishTrack = vi + .fn() + .mockImplementation(async (track: LocalTrack) => { + const pub = { + track, + source: track.source, + mute: track.mute, + unmute: track.unmute, + } as Partial as LocalTrackPublication; + trackPublications.push(pub); + localParticipant.emit(ParticipantEvent.LocalTrackPublished, pub); + return Promise.resolve(pub); + }); + + vi.mocked(localParticipant).getTrackPublication = vi + .fn() + .mockImplementation((source: Track.Source) => { + return trackPublications.find((pub) => pub.track?.source === source); + }); + + connection = { + state$: constant({ + state: "ConnectedToLkRoom", + livekitConnectionState$: constant(LivekitConnectionState.Connected), + }), + livekitRoom: mockLivekitRoom({ + localParticipant: localParticipant, + }), + } as unknown as Connection; +}); + +describe("Publisher", () => { + let publisher: Publisher; + + beforeEach(() => { + publisher = new Publisher( + scope, + connection, + mockMediaDevices({}), + muteStates, + constant({ supported: false, processor: undefined }), + logger, + ); + }); + + afterEach(() => {}); + + it("Should not create tracks if started muted to avoid unneeded permission requests", async () => { + const createTracksSpy = vi.spyOn( + connection.livekitRoom.localParticipant, + "createTracks", + ); + + audioEnabled$.next(false); + videoEnabled$.next(false); + await publisher.createAndSetupTracks(); + + expect(createTracksSpy).not.toHaveBeenCalled(); + }); + + it("Should minimize permission request by querying create at once", async () => { + const enableCameraAndMicrophoneSpy = vi.spyOn( + localParticipant, + "enableCameraAndMicrophone", + ); + const createTracksSpy = vi.spyOn(localParticipant, "createTracks"); + + audioEnabled$.next(true); + videoEnabled$.next(true); + await publisher.createAndSetupTracks(); + await flushPromises(); + + expect(enableCameraAndMicrophoneSpy).toHaveBeenCalled(); + + // It should create both at once + expect(createTracksSpy).toHaveBeenCalledWith({ + audio: true, + video: true, + }); + }); + + it("Ensure no data is streamed until publish has been called", async () => { + audioEnabled$.next(true); + await publisher.createAndSetupTracks(); + + // The track should be created and paused + expect(localParticipant.createTracks).toHaveBeenCalledWith({ + audio: true, + video: undefined, + }); + await flushPromises(); + expect(localParticipant.publishTrack).toHaveBeenCalled(); + + await flushPromises(); + const track = localParticipant.getTrackPublication( + Track.Source.Microphone, + )?.track; + expect(track).toBeDefined(); + expect(track!.pauseUpstream).toHaveBeenCalled(); + expect(track!.isUpstreamPaused).toBe(true); + }); + + it("Ensure resume upstream when published is called", async () => { + videoEnabled$.next(true); + await publisher.createAndSetupTracks(); + // await flushPromises(); + await publisher.startPublishing(); + + const track = localParticipant.getTrackPublication( + Track.Source.Camera, + )?.track; + expect(track).toBeDefined(); + // expect(track.pauseUpstream).toHaveBeenCalled(); + expect(track!.isUpstreamPaused).toBe(false); + }); + + describe("Mute states", () => { + let publisher: Publisher; + beforeEach(() => { + publisher = new Publisher( + scope, + connection, + mockMediaDevices({}), + muteStates, + constant({ supported: false, processor: undefined }), + logger, + ); + }); + + test.each([ + { mutes: { audioEnabled: true, videoEnabled: false } }, + { mutes: { audioEnabled: true, videoEnabled: false } }, + ])("only create the tracks that are unmuted $mutes", async ({ mutes }) => { + // Ensure all muted + audioEnabled$.next(mutes.audioEnabled); + videoEnabled$.next(mutes.videoEnabled); + + vi.mocked(connection.livekitRoom.localParticipant).createTracks = vi + .fn() + .mockResolvedValue([]); + + await publisher.createAndSetupTracks(); + + expect( + connection.livekitRoom.localParticipant.createTracks, + ).toHaveBeenCalledOnce(); + + expect( + connection.livekitRoom.localParticipant.createTracks, + ).toHaveBeenCalledWith({ + audio: mutes.audioEnabled ? true : undefined, + video: mutes.videoEnabled ? true : undefined, + }); + }); + }); + + it("does mute unmute audio", async () => {}); +}); + +describe("Bug fix", () => { + // There is a race condition when creating and publishing tracks while the mute state changes. + // This race condition could cause tracks to be published even though they are muted at the + // beginning of a call coming from lobby. + // This is caused by our stack using manually the low level API to create and publish tracks, + // but also using the higher level setMicrophoneEnabled and setCameraEnabled functions that also create + // and publish tracks, and managing pending publications. + // Race is as follow, on creation of the Publisher we create the tracks then publish them. + // If in the middle of that process the mute state changes: + // - the `setMicrophoneEnabled` will be no-op because it is not aware of our created track and can't see any pending publication + // - If start publication is requested it will publish the track even though there was a mute request. + it("wrongly publish tracks while muted", async () => { + // setLogLevel(`debug`); const publisher = new Publisher( scope, connection, @@ -73,68 +327,34 @@ describe("Publisher", () => { constant({ supported: false, processor: undefined }), logger, ); + audioEnabled$.next(true); - // should do nothing if no tracks have been created yet. - await publisher.startPublishing(); - expect( - connection.livekitRoom.localParticipant.publishTrack, - ).not.toHaveBeenCalled(); + const resolvers = Promise.withResolvers(); + createTrackLock = resolvers.promise; - await expect(publisher.createAndSetupTracks()).rejects.toThrow( - Error("audio and video is false"), - ); - - (muteStates.audio.enabled$ as BehaviorSubject).next(true); - - ( - connection.livekitRoom.localParticipant.createTracks as Mock - ).mockResolvedValue([{}, {}]); - - await expect(publisher.createAndSetupTracks()).resolves.not.toThrow(); - expect( - connection.livekitRoom.localParticipant.createTracks, - ).toHaveBeenCalledOnce(); - - // failiour due to localParticipant.publishTrack - ( - connection.livekitRoom.localParticipant.publishTrack as Mock - ).mockRejectedValue(Error("testError")); - - await expect(publisher.startPublishing()).rejects.toThrow( - new FailToStartLivekitConnection("testError"), - ); - - // does not try other conenction after the first one failed - expect( - connection.livekitRoom.localParticipant.publishTrack, - ).toHaveBeenCalledTimes(1); - - // failiour due to connection.state$ - const beforeState = connection.state$.value; - (connection.state$ as BehaviorSubject).next({ - state: "FailedToStart", - error: Error("testStartError"), + // Initially the audio is unmuted, so creating tracks should publish the audio track + const createTracks = publisher.createAndSetupTracks(); + void publisher.startPublishing(); + void createTracks.then(() => { + void publisher.startPublishing(); }); + // now mute the audio before allowing track creation to complete + audioEnabled$.next(false); + resolvers.resolve(undefined); + await createTracks; - await expect(publisher.startPublishing()).rejects.toThrow( - new FailToStartLivekitConnection("testStartError"), - ); - (connection.state$ as BehaviorSubject).next(beforeState); + await flushPromises(); - // does not try other conenction after the first one failed - expect( - connection.livekitRoom.localParticipant.publishTrack, - ).toHaveBeenCalledTimes(1); + const track = localParticipant.getTrackPublication( + Track.Source.Microphone, + )?.track; + expect(track).toBeDefined(); - // success case - ( - connection.livekitRoom.localParticipant.publishTrack as Mock - ).mockResolvedValue({}); - - await expect(publisher.startPublishing()).resolves.not.toThrow(); - - expect( - connection.livekitRoom.localParticipant.publishTrack, - ).toHaveBeenCalledTimes(3); + try { + expect(localParticipant.publishTrack).not.toHaveBeenCalled(); + } catch { + expect(track!.mute).toHaveBeenCalled(); + expect(track!.isMuted).toBe(true); + } }); }); diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 326dedaf..27c53726 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -6,15 +6,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { + ConnectionState as LivekitConnectionState, + type LocalTrackPublication, LocalVideoTrack, + ParticipantEvent, type Room as LivekitRoom, Track, - type LocalTrack, - type LocalTrackPublication, - ConnectionState as LivekitConnectionState, } from "livekit-client"; import { - BehaviorSubject, map, NEVER, type Observable, @@ -34,10 +33,6 @@ import { getUrlParams } from "../../../UrlParams.ts"; import { observeTrackReference$ } from "../../MediaViewModel.ts"; import { type Connection } from "../remoteMembers/Connection.ts"; import { type ObservableScope } from "../../ObservableScope.ts"; -import { - ElementCallError, - FailToStartLivekitConnection, -} from "../../../utils/errors.ts"; /** * A wrapper for a Connection object. @@ -45,14 +40,21 @@ import { * The Publisher is also responsible for creating the media tracks. */ export class Publisher { + /** + * By default, livekit will start publishing tracks as soon as they are created. + * In the matrix RTC world, we want to control when tracks are published based + * on whether the user is part of the RTC session or not. + */ + public shouldPublish = false; + /** * Creates a new Publisher. * @param scope - The observable scope to use for managing the publisher. * @param connection - The connection to use for publishing. * @param devices - The media devices to use for audio and video input. * @param muteStates - The mute states for audio and video. - * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). + * @param logger - The logger to use for logging :D. */ public constructor( private scope: ObservableScope, @@ -62,7 +64,6 @@ export class Publisher { trackerProcessorState$: Behavior, private logger: Logger, ) { - this.logger.info("Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); const room = connection.livekitRoom; @@ -80,161 +81,185 @@ export class Publisher { this.scope.onEnd(() => { this.logger.info("Scope ended -> stop publishing all tracks"); void this.stopPublishing(); + muteStates.audio.unsetHandler(); + muteStates.video.unsetHandler(); }); - // TODO move mute state handling here using reconcile (instead of inside the mute state class) - // this.scope.reconcile( - // this.scope.behavior( - // combineLatest([this.muteStates.video.enabled$, this.tracks$]), - // ), - // async ([videoEnabled, tracks]) => { - // const track = tracks.find((t) => t.kind == Track.Kind.Video); - // if (!track) return; - - // if (videoEnabled) { - // await track.unmute(); - // } else { - // await track.mute(); - // } - // }, - // ); + this.connection.livekitRoom.localParticipant.on( + ParticipantEvent.LocalTrackPublished, + this.onLocalTrackPublished.bind(this), + ); } - private _tracks$ = new BehaviorSubject[]>([]); - public tracks$ = this._tracks$ as Behavior[]>; - + // LiveKit will publish the tracks as soon as they are created + // but we want to control when tracks are published. + // We cannot just mute the tracks, even if this will effectively stop the publishing, + // it would also prevent the user from seeing their own video/audio preview. + // So for that we use pauseUpStream(): Stops sending media to the server by replacing + // the sender track with null, but keeps the local MediaStreamTrack active. + // The user can still see/hear themselves locally, but remote participants see nothing. + private onLocalTrackPublished( + localTrackPublication: LocalTrackPublication, + ): void { + this.logger.info("Local track published", localTrackPublication); + const lkRoom = this.connection.livekitRoom; + if (!this.shouldPublish) { + this.pauseUpstreams(lkRoom, [localTrackPublication.source]).catch((e) => { + this.logger.error(`Failed to pause upstreams`, e); + }); + } + // also check the mute state and apply it + if (localTrackPublication.source === Track.Source.Microphone) { + const enabled = this.muteStates.audio.enabled$.value; + lkRoom.localParticipant.setMicrophoneEnabled(enabled).catch((e) => { + this.logger.error( + `Failed to enable microphone track, enabled:${enabled}`, + e, + ); + }); + } else if (localTrackPublication.source === Track.Source.Camera) { + const enabled = this.muteStates.video.enabled$.value; + lkRoom.localParticipant.setCameraEnabled(enabled).catch((e) => { + this.logger.error( + `Failed to enable camera track, enabled:${enabled}`, + e, + ); + }); + } + } /** - * Start the connection to LiveKit and publish local tracks. + * Create and setup local audio and video tracks based on the current mute states. + * It creates the tracks only if audio and/or video is enabled, to avoid unnecessary + * permission prompts. * - * This will: - * wait for the connection to be ready. - // * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) - // * 2. Use this token to request the SFU config to the MatrixRtc authentication service. - // * 3. Connect to the configured LiveKit room. - // * 4. Create local audio and video tracks based on the current mute states and publish them to the room. + * It also observes mute state changes to update LiveKit microphone/camera states accordingly. + * If a track is not created initially because disabled, it will be created when unmuting. + * + * This call is not blocking anymore, instead callers can listen to the + * `RoomEvent.MediaDevicesError` event in the LiveKit room to be notified of any errors. * - * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. - * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. */ public async createAndSetupTracks(): Promise { this.logger.debug("createAndSetupTracks called"); const lkRoom = this.connection.livekitRoom; // Observe mute state changes and update LiveKit microphone/camera states accordingly - this.observeMuteStates(this.scope); + this.observeMuteStates(); - // TODO-MULTI-SFU: Prepublish a microphone track + // Check if audio and/or video is enabled. We only create tracks if enabled, + // because it could prompt for permission, and we don't want to do that unnecessarily. const audio = this.muteStates.audio.enabled$.value; const video = this.muteStates.video.enabled$.value; - // createTracks throws if called with audio=false and video=false - if (audio || video) { - // TODO this can still throw errors? It will also prompt for permissions if not already granted - return lkRoom.localParticipant - .createTracks({ - audio, - video, - }) - .then((tracks) => { - this.logger.info( - "created track", - tracks.map((t) => t.kind + ", " + t.id), - ); - this._tracks$.next(tracks); - }) - .catch((error) => { - this.logger.error("Failed to create tracks", error); - }); + + // We don't await the creation, because livekit could block until the tracks + // are fully published, and not only that they are created. + // We don't have control on that, localParticipant creates and publishes the tracks + // asap. + // We are using the `ParticipantEvent.LocalTrackPublished` to be notified + // when tracks are actually published, and at that point + // we can pause upstream if needed (depending on if startPublishing has been called). + if (audio && video) { + // Enable both at once in order to have a single permission prompt! + void lkRoom.localParticipant.enableCameraAndMicrophone(); + } else if (audio) { + void lkRoom.localParticipant.setMicrophoneEnabled(true); + } else if (video) { + void lkRoom.localParticipant.setCameraEnabled(true); + } + + return Promise.resolve(); + } + + private async pauseUpstreams( + lkRoom: LivekitRoom, + sources: Track.Source[], + ): Promise { + for (const source of sources) { + const track = lkRoom.localParticipant.getTrackPublication(source)?.track; + if (track) { + await track.pauseUpstream(); + } else { + this.logger.warn( + `No track found for source ${source} to pause upstream`, + ); + } + } + } + + private async resumeUpstreams( + lkRoom: LivekitRoom, + sources: Track.Source[], + ): Promise { + for (const source of sources) { + const track = lkRoom.localParticipant.getTrackPublication(source)?.track; + if (track) { + await track.resumeUpstream(); + } else { + this.logger.warn( + `No track found for source ${source} to resume upstream`, + ); + } } - throw Error("audio and video is false"); } - private _publishing$ = new BehaviorSubject(false); - public publishing$ = this.scope.behavior(this._publishing$); /** + * + * Request to publish local tracks to the LiveKit room. + * This will wait for the connection to be ready before publishing. + * Livekit also have some local retry logic for publishing tracks. + * Can be called multiple times, localparticipant manages the state of published tracks (or pending publications). * * @returns - * @throws ElementCallError */ - public async startPublishing(): Promise { + public async startPublishing(): Promise { + if (this.shouldPublish) { + this.logger.debug(`Already publishing, ignoring startPublishing call`); + return; + } + this.shouldPublish = true; this.logger.debug("startPublishing called"); + const lkRoom = this.connection.livekitRoom; - const { promise, resolve, reject } = Promise.withResolvers(); - const sub = this.connection.state$.subscribe((s) => { - switch (s.state) { - case "ConnectedToLkRoom": - resolve(); - break; - case "FailedToStart": - reject( - s.error instanceof ElementCallError - ? s.error - : new FailToStartLivekitConnection(s.error.message), - ); - break; - default: - this.logger.info("waiting for connection: ", s.state); - } - }); + + // Resume upstream for both audio and video tracks + // We need to call it explicitly because call setTrackEnabled does not always + // resume upstream. It will only if you switch the track from disabled to enabled, + // but if the track is already enabled but upstream is paused, it won't resume it. + // TODO what about screen share? try { - await promise; + await this.resumeUpstreams(lkRoom, [ + Track.Source.Microphone, + Track.Source.Camera, + ]); } catch (e) { - throw e; - } finally { - sub.unsubscribe(); + this.logger.error(`Failed to resume upstreams`, e); } - - for (const track of this.tracks$.value) { - this.logger.info("publish ", this.tracks$.value.length, "tracks"); - // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally - // with a timeout. - await lkRoom.localParticipant.publishTrack(track).catch((error) => { - this.logger.error("Failed to publish track", error); - throw new FailToStartLivekitConnection( - error instanceof Error ? error.message : error, - ); - }); - this.logger.info("published track ", track.kind, track.id); - - // TODO: check if the connection is still active? and break the loop if not? - } - this._publishing$.next(true); - return this.tracks$.value; } public async stopPublishing(): Promise { this.logger.debug("stopPublishing called"); - // TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope - // actually has the right lifetime - this.muteStates.audio.unsetHandler(); - this.muteStates.video.unsetHandler(); - - const localParticipant = this.connection.livekitRoom.localParticipant; - const tracks: LocalTrack[] = []; - const addToTracksIfDefined = (p: LocalTrackPublication): void => { - if (p.track !== undefined) tracks.push(p.track); - }; - localParticipant.trackPublications.forEach(addToTracksIfDefined); - this.logger.debug( - "list of tracks to unpublish:", - tracks.map((t) => t.kind + ", " + t.id), - "start unpublishing now", - ); - await localParticipant.unpublishTracks(tracks).catch((error) => { - this.logger.error("Failed to unpublish tracks", error); - throw error; - }); - this.logger.debug( - "unpublished tracks", - tracks.map((t) => t.kind + ", " + t.id), - ); - this._publishing$.next(false); + this.shouldPublish = false; + // Pause upstream will stop sending media to the server, while keeping + // the local MediaStreamTrack active, so the user can still see themselves. + await this.pauseUpstreams(this.connection.livekitRoom, [ + Track.Source.Microphone, + Track.Source.Camera, + Track.Source.ScreenShare, + ]); } - /** - * Stops all tracks that are currently running - */ - public stopTracks(): void { - this.tracks$.value.forEach((t) => t.stop()); - this._tracks$.next([]); + public async stopTracks(): Promise { + const lkRoom = this.connection.livekitRoom; + for (const source of [ + Track.Source.Microphone, + Track.Source.Camera, + Track.Source.ScreenShare, + ]) { + const localPub = lkRoom.localParticipant.getTrackPublication(source); + if (localPub?.track) { + // stops and unpublishes the track + await lkRoom.localParticipant.unpublishTrack(localPub!.track, true); + } + } } /// Private methods @@ -331,22 +356,35 @@ export class Publisher { /** * Observe changes in the mute states and update the LiveKit room accordingly. - * @param scope * @private */ - private observeMuteStates(scope: ObservableScope): void { + private observeMuteStates(): void { const lkRoom = this.connection.livekitRoom; - this.muteStates.audio.setHandler(async (desired) => { + this.muteStates.audio.setHandler(async (enable) => { try { - await lkRoom.localParticipant.setMicrophoneEnabled(desired); + this.logger.debug( + `handler: Setting LiveKit microphone enabled: ${enable}`, + ); + await lkRoom.localParticipant.setMicrophoneEnabled(enable); + // Unmute will restart the track if it was paused upstream, + // but until explicitly requested, we want to keep it paused. + if (!this.shouldPublish && enable) { + await this.pauseUpstreams(lkRoom, [Track.Source.Microphone]); + } } catch (e) { this.logger.error("Failed to update LiveKit audio input mute state", e); } return lkRoom.localParticipant.isMicrophoneEnabled; }); - this.muteStates.video.setHandler(async (desired) => { + this.muteStates.video.setHandler(async (enable) => { try { - await lkRoom.localParticipant.setCameraEnabled(desired); + this.logger.debug(`handler: Setting LiveKit camera enabled: ${enable}`); + await lkRoom.localParticipant.setCameraEnabled(enable); + // Unmute will restart the track if it was paused upstream, + // but until explicitly requested, we want to keep it paused. + if (!this.shouldPublish && enable) { + await this.pauseUpstreams(lkRoom, [Track.Source.Camera]); + } } catch (e) { this.logger.error("Failed to update LiveKit video input mute state", e); } @@ -362,7 +400,7 @@ export class Publisher { const track$ = scope.behavior( observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( map((trackRef) => { - const track = trackRef?.publication?.track; + const track = trackRef?.publication.track; return track instanceof LocalVideoTrack ? track : null; }), ), diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 2ead768b..c1e24eb4 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -30,13 +30,17 @@ import { logger } from "matrix-js-sdk/lib/logger"; import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { Connection, + ConnectionState, type ConnectionOpts, - type ConnectionState, - type PublishingParticipant, } from "./Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; -import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; +import { + ElementCallError, + FailToGetOpenIdToken, +} from "../../../utils/errors.ts"; +import { testJWTToken } from "../../../utils/test-fixtures.ts"; +import { mockRemoteParticipant } from "../../../utils/test.ts"; let testScope: ObservableScope; @@ -47,11 +51,6 @@ let fakeLivekitRoom: MockedObject; let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; -let fakeRoomEventEmiter: EventEmitter; -// let fakeMembershipsFocusMap$: BehaviorSubject< -// { membership: CallMembership; transport: LivekitTransport }[] -// >; - const livekitFocus: LivekitTransport = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", @@ -88,22 +87,25 @@ function setupTest(): void { localParticipantEventEmiter, ), } as unknown as LocalParticipant); - fakeRoomEventEmiter = new EventEmitter(); + const fakeRoomEventEmitter = new EventEmitter(); fakeLivekitRoom = vi.mocked({ connect: vi.fn(), disconnect: vi.fn(), remoteParticipants: new Map(), localParticipant: fakeLocalParticipant, state: LivekitConnectionState.Disconnected, - on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), - off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), - addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), + on: fakeRoomEventEmitter.on.bind(fakeRoomEventEmitter), + off: fakeRoomEventEmitter.off.bind(fakeRoomEventEmitter), + addListener: fakeRoomEventEmitter.addListener.bind(fakeRoomEventEmitter), removeListener: - fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), + fakeRoomEventEmitter.removeListener.bind(fakeRoomEventEmitter), removeAllListeners: - fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), + fakeRoomEventEmitter.removeAllListeners.bind(fakeRoomEventEmitter), setE2EEEnabled: vi.fn().mockResolvedValue(undefined), + emit: (eventName: string | symbol, ...args: unknown[]) => { + fakeRoomEventEmitter.emit(eventName, ...args); + }, } as unknown as LivekitRoom); } @@ -120,12 +122,21 @@ function setupRemoteConnection(): Connection { status: 200, body: { url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: "ATOKEN", + jwt: testJWTToken, }, }; }); - fakeLivekitRoom.connect.mockResolvedValue(undefined); + fakeLivekitRoom.connect.mockImplementation(async (): Promise => { + const changeEv = RoomEvent.ConnectionStateChanged; + + fakeLivekitRoom.state = LivekitConnectionState.Connecting; + fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state); + fakeLivekitRoom.state = LivekitConnectionState.Connected; + fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state); + + return Promise.resolve(); + }); return new Connection(opts, logger); } @@ -148,7 +159,7 @@ describe("Start connection states", () => { }; const connection = new Connection(opts, logger); - expect(connection.state$.getValue().state).toEqual("Initialized"); + expect(connection.state$.getValue()).toEqual("Initialized"); }); it("fail to getOpenId token then error state", async () => { @@ -164,7 +175,7 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: ConnectionState[] = []; + const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -184,22 +195,20 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState!.state).toEqual("FetchingConfig"); + expect(capturedState!).toEqual("FetchingConfig"); deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); await vi.runAllTimersAsync(); capturedState = capturedStates.pop(); - if (capturedState!.state === "FailedToStart") { - expect(capturedState!.error.message).toEqual("Something went wrong"); + if (capturedState instanceof Error) { + expect(capturedState.message).toEqual("Something went wrong"); expect(connection.transport.livekit_alias).toEqual( livekitFocus.livekit_alias, ); } else { - expect.fail( - "Expected FailedToStart state but got " + capturedState?.state, - ); + expect.fail("Expected FailedToStart state but got " + capturedState); } }); @@ -216,7 +225,7 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: ConnectionState[] = []; + const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -238,24 +247,25 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState?.state).toEqual("FetchingConfig"); + expect(capturedState).toEqual(ConnectionState.FetchingConfig); deferredSFU.resolve(); await vi.runAllTimersAsync(); capturedState = capturedStates.pop(); - if (capturedState?.state === "FailedToStart") { - expect(capturedState?.error.message).toContain( - "SFU Config fetch failed with exception Error", + if ( + capturedState instanceof ElementCallError && + capturedState.cause instanceof Error + ) { + expect(capturedState.cause.message).toContain( + "SFU Config fetch failed with exception", ); expect(connection.transport.livekit_alias).toEqual( livekitFocus.livekit_alias, ); } else { - expect.fail( - "Expected FailedToStart state but got " + capturedState?.state, - ); + expect.fail("Expected FailedToStart state but got " + capturedState); } }); @@ -272,7 +282,7 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: ConnectionState[] = []; + const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -285,7 +295,7 @@ describe("Start connection states", () => { status: 200, body: { url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: "ATOKEN", + jwt: testJWTToken, }, }; }); @@ -302,15 +312,18 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState?.state).toEqual("FetchingConfig"); + expect(capturedState).toEqual(ConnectionState.FetchingConfig); deferredSFU.resolve(); await vi.runAllTimersAsync(); capturedState = capturedStates.pop(); - if (capturedState && capturedState?.state === "FailedToStart") { - expect(capturedState.error.message).toContain( + if ( + capturedState instanceof ElementCallError && + capturedState.cause instanceof Error + ) { + expect(capturedState.cause.message).toContain( "Failed to connect to livekit", ); expect(connection.transport.livekit_alias).toEqual( @@ -329,7 +342,7 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); - const capturedStates: ConnectionState[] = []; + const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -339,13 +352,15 @@ describe("Start connection states", () => { await vi.runAllTimersAsync(); const initialState = capturedStates.shift(); - expect(initialState?.state).toEqual("Initialized"); + expect(initialState).toEqual(ConnectionState.Initialized); const fetchingState = capturedStates.shift(); - expect(fetchingState?.state).toEqual("FetchingConfig"); + expect(fetchingState).toEqual(ConnectionState.FetchingConfig); + const disconnectedState = capturedStates.shift(); + expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected); const connectingState = capturedStates.shift(); - expect(connectingState?.state).toEqual("ConnectingToLkRoom"); + expect(connectingState).toEqual(ConnectionState.LivekitConnecting); const connectedState = capturedStates.shift(); - expect(connectedState?.state).toEqual("ConnectedToLkRoom"); + expect(connectedState).toEqual(ConnectionState.LivekitConnected); }); it("shutting down the scope should stop the connection", async () => { @@ -363,46 +378,32 @@ describe("Start connection states", () => { }); }); -function fakeRemoteLivekitParticipant( - id: string, - publications: number = 1, -): RemoteParticipant { - return { - identity: id, - getTrackPublications: () => Array(publications), - } as unknown as RemoteParticipant; -} - -describe("Publishing participants observations", () => { - it("should emit the list of publishing participants", () => { +describe("remote participants", () => { + it("emits the list of remote participants", () => { setupTest(); const connection = setupRemoteConnection(); - const bobIsAPublisher = Promise.withResolvers(); - const danIsAPublisher = Promise.withResolvers(); - const observedPublishers: PublishingParticipant[][] = []; - const s = connection.remoteParticipantsWithTracks$.subscribe( - (publishers) => { - observedPublishers.push(publishers); - if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) { - bobIsAPublisher.resolve(); - } - if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) { - danIsAPublisher.resolve(); - } - }, - ); + const observedParticipants: RemoteParticipant[][] = []; + const s = connection.remoteParticipants$.subscribe((participants) => { + observedParticipants.push(participants); + }); onTestFinished(() => s.unsubscribe()); - // The publishingParticipants$ observable is derived from the current members of the + // The remoteParticipants$ observable is derived from the current members of the // livekitRoom and the rtc membership in order to publish the members that are publishing // on this connection. let participants: RemoteParticipant[] = [ - fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 0), - fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0), - fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 0), - fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 0), + mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }), + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), + mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }), + // Mock Dan to have no published tracks. We want him to still show show up + // in the participants list. + mockRemoteParticipant({ + identity: "@dan:example.org:DEV333", + getTrackPublication: () => undefined, + getTrackPublications: () => [], + }), ]; // Let's simulate 3 members on the livekitRoom @@ -411,24 +412,26 @@ describe("Publishing participants observations", () => { ); participants.forEach((p) => - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p), + fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p), ); - // At this point there should be no publishers - expect(observedPublishers.pop()!.length).toEqual(0); + // At this point there should be ~~no~~ publishers + // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks) + // The filtering we do is just based on the matrixRTC member events. + expect(observedParticipants.pop()!.length).toEqual(4); participants = [ - fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 1), - fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1), - fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 1), - fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 2), + mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }), + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), + mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }), + mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }), ]; participants.forEach((p) => - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p), + fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p), ); // At this point there should be no publishers - expect(observedPublishers.pop()!.length).toEqual(4); + expect(observedParticipants.pop()!.length).toEqual(4); }); it("should be scoped to parent scope", (): void => { @@ -436,16 +439,14 @@ describe("Publishing participants observations", () => { const connection = setupRemoteConnection(); - let observedPublishers: PublishingParticipant[][] = []; - const s = connection.remoteParticipantsWithTracks$.subscribe( - (publishers) => { - observedPublishers.push(publishers); - }, - ); + let observedParticipants: RemoteParticipant[][] = []; + const s = connection.remoteParticipants$.subscribe((participants) => { + observedParticipants.push(participants); + }); onTestFinished(() => s.unsubscribe()); let participants: RemoteParticipant[] = [ - fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0), + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), ]; // Let's simulate 3 members on the livekitRoom @@ -454,38 +455,29 @@ describe("Publishing participants observations", () => { ); for (const participant of participants) { - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); + fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, participant); } - // At this point there should be no publishers - expect(observedPublishers.pop()!.length).toEqual(0); - - participants = [fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1)]; - - for (const participant of participants) { - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); - } - - // We should have bob has a publisher now - const publishers = observedPublishers.pop(); - expect(publishers?.length).toEqual(1); - expect(publishers?.[0]?.identity).toEqual("@bob:example.org:DEV111"); + // We should have bob as a participant now + const ps = observedParticipants.pop(); + expect(ps?.length).toEqual(1); + expect(ps?.[0]?.identity).toEqual("@bob:example.org:DEV111"); // end the parent scope testScope.end(); - observedPublishers = []; + observedParticipants = []; - // SHOULD NOT emit any more publishers as the scope is ended + // SHOULD NOT emit any more participants as the scope is ended participants = participants.filter( (p) => p.identity !== "@bob:example.org:DEV111", ); - fakeRoomEventEmiter.emit( + fakeLivekitRoom.emit( RoomEvent.ParticipantDisconnected, - fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), ); - expect(observedPublishers.length).toEqual(0); + expect(observedParticipants.length).toEqual(0); }); }); diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 4f3bbda4..41dfe665 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -12,11 +12,8 @@ import { } from "@livekit/components-core"; import { ConnectionError, - type ConnectionState as LivekitConenctionState, type Room as LivekitRoom, - type LocalParticipant, type RemoteParticipant, - RoomEvent, } from "livekit-client"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, map } from "rxjs"; @@ -30,12 +27,12 @@ import { import { type Behavior } from "../../Behavior.ts"; import { type ObservableScope } from "../../ObservableScope.ts"; import { + ElementCallError, InsufficientCapacityError, SFURoomCreationRestrictedError, + UnknownCallError, } from "../../../utils/errors.ts"; -export type PublishingParticipant = LocalParticipant | RemoteParticipant; - export interface ConnectionOpts { /** The media transport to connect to. */ transport: LivekitTransport; @@ -47,17 +44,30 @@ export interface ConnectionOpts { /** Optional factory to create the LiveKit room, mainly for testing purposes. */ livekitRoomFactory: () => LivekitRoom; } +export class FailedToStartError extends Error { + public constructor(message: string) { + super(message); + this.name = "FailedToStartError"; + } +} -export type ConnectionState = - | { state: "Initialized" } - | { state: "FetchingConfig" } - | { state: "ConnectingToLkRoom" } - | { - state: "ConnectedToLkRoom"; - livekitConnectionState$: Behavior; - } - | { state: "FailedToStart"; error: Error } - | { state: "Stopped" }; +export enum ConnectionState { + /** The start state of a connection. It has been created but nothing has loaded yet. */ + Initialized = "Initialized", + /** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room */ + FetchingConfig = "FetchingConfig", + Stopped = "Stopped", + /** The same as ConnectionState.Disconnected from `livekit-client` */ + LivekitDisconnected = "disconnected", + /** The same as ConnectionState.Connecting from `livekit-client` */ + LivekitConnecting = "connecting", + /** The same as ConnectionState.Connected from `livekit-client` */ + LivekitConnected = "connected", + /** The same as ConnectionState.Reconnecting from `livekit-client` */ + LivekitReconnecting = "reconnecting", + /** The same as ConnectionState.SignalReconnecting from `livekit-client` */ + LivekitSignalReconnecting = "signalReconnecting", +} /** * A connection to a Matrix RTC LiveKit backend. @@ -66,14 +76,14 @@ export type ConnectionState = */ export class Connection { // Private Behavior - private readonly _state$ = new BehaviorSubject({ - state: "Initialized", - }); + private readonly _state$ = new BehaviorSubject< + ConnectionState | ElementCallError + >(ConnectionState.Initialized); /** * The current state of the connection to the media transport. */ - public readonly state$: Behavior = this._state$; + public readonly state$: Behavior = this._state$; /** * The media transport to connect to. @@ -85,13 +95,13 @@ export class Connection { private scope: ObservableScope; /** - * An observable of the participants that are publishing on this connection. (Excluding our local participant) - * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`. - * It filters the participants to only those that are associated with a membership that claims to publish on this connection. + * The remote LiveKit participants that are visible on this connection. + * + * Note that this may include participants that are connected only to + * subscribe, or publishers that are otherwise unattested in MatrixRTC state. + * It is therefore more low-level than what should be presented to the user. */ - public readonly remoteParticipantsWithTracks$: Behavior< - PublishingParticipant[] - >; + public readonly remoteParticipants$: Behavior; /** * Whether the connection has been stopped. @@ -117,16 +127,24 @@ export class Connection { this.logger.debug("Starting Connection"); this.stopped = false; try { - this._state$.next({ - state: "FetchingConfig", - }); + this._state$.next(ConnectionState.FetchingConfig); + // We should already have this information after creating the localTransport. + // It would probably be better to forward this here. const { url, jwt } = await this.getSFUConfigWithOpenID(); // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - this._state$.next({ - state: "ConnectingToLkRoom", - }); + // Setup observer once we are done with getSFUConfigWithOpenID + connectionStateObserver(this.livekitRoom) + .pipe( + this.scope.bind(), + map((s) => s as unknown as ConnectionState), + ) + .subscribe((lkState) => { + // It is save to cast lkState to ConnectionState as they are fully overlapping. + this._state$.next(lkState); + }); + try { await this.livekitRoom.connect(url, jwt); } catch (e) { @@ -141,7 +159,8 @@ export class Connection { throw new InsufficientCapacityError(); } if (e.status === 404) { - // error msg is "Could not establish signal connection: requested room does not exist" + // error msg is "Failed to create call" + // error description is "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists." // The room does not exist. There are two different modes of operation for the SFU: // - the room is created on the fly when connecting (livekit `auto_create` option) // - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service) @@ -153,19 +172,16 @@ export class Connection { } // If we were stopped while connecting, don't proceed to update state. if (this.stopped) return; - - this._state$.next({ - state: "ConnectedToLkRoom", - livekitConnectionState$: this.scope.behavior( - connectionStateObserver(this.livekitRoom), - ), - }); } catch (error) { this.logger.debug(`Failed to connect to LiveKit room: ${error}`); - this._state$.next({ - state: "FailedToStart", - error: error instanceof Error ? error : new Error(`${error}`), - }); + this._state$.next( + error instanceof ElementCallError + ? error + : error instanceof Error + ? new UnknownCallError(error) + : new UnknownCallError(new Error(`${error}`)), + ); + // Its okay to ignore the throw. The error is part of the state. throw error; } } @@ -190,9 +206,7 @@ export class Connection { ); if (this.stopped) return; await this.livekitRoom.disconnect(); - this._state$.next({ - state: "Stopped", - }); + this._state$.next(ConnectionState.Stopped); this.stopped = true; } @@ -204,12 +218,12 @@ export class Connection { * * @param opts - Connection options {@link ConnectionOpts}. * - * @param logger + * @param logger - The logger to use. */ public constructor(opts: ConnectionOpts, logger: Logger) { this.logger = logger.getChild("[Connection]"); this.logger.info( - `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, + `Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, ); const { transport, client, scope } = opts; @@ -218,23 +232,9 @@ export class Connection { this.transport = transport; this.client = client; - // REMOTE participants with track!!! - // this.remoteParticipantsWithTracks$ - this.remoteParticipantsWithTracks$ = scope.behavior( - // only tracks remote participants - connectedParticipantsObserver(this.livekitRoom, { - additionalRoomEvents: [ - RoomEvent.TrackPublished, - RoomEvent.TrackUnpublished, - ], - }).pipe( - map((participants) => { - return participants.filter( - (participant) => participant.getTrackPublications().length > 0, - ); - }), - ), - [], + this.remoteParticipants$ = scope.behavior( + // Only tracks remote participants + connectedParticipantsObserver(this.livekitRoom), ); scope.onEnd(() => { diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index f58fcb76..c3364059 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -7,13 +7,15 @@ Please see LICENSE in the repository root for full details. import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { - type E2EEOptions, Room as LivekitRoom, type RoomOptions, type BaseKeyProvider, + type E2EEManagerOptions, + type BaseE2EEManager, } from "livekit-client"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import E2EEWorker from "livekit-client/e2ee-worker?worker"; +// imported as inline to support worker when loaded from a cdn (cross domain) +import E2EEWorker from "livekit-client/e2ee-worker?worker&inline"; import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; @@ -41,9 +43,11 @@ export class ECConnectionFactory implements ConnectionFactory { * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens. * @param devices - Used for video/audio out/in capture options. * @param processorState$ - Effects like background blur (only for publishing connection?) - * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit Room. + * @param livekitKeyProvider - Optional key provider for end-to-end encryption. * @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app). * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used. + * @param echoCancellation - Whether to enable echo cancellation for audio capture. + * @param noiseSuppression - Whether to enable noise suppression for audio capture. */ public constructor( private client: OpenIDClientParts, @@ -52,20 +56,24 @@ export class ECConnectionFactory implements ConnectionFactory { livekitKeyProvider: BaseKeyProvider | undefined, private controlledAudioDevices: boolean, livekitRoomFactory?: () => LivekitRoom, + echoCancellation: boolean = true, + noiseSuppression: boolean = true, ) { const defaultFactory = (): LivekitRoom => new LivekitRoom( - generateRoomOption( - this.devices, - this.processorState$.value, - livekitKeyProvider && { + generateRoomOption({ + devices: this.devices, + processorState: this.processorState$.value, + e2eeLivekitOptions: livekitKeyProvider && { keyProvider: livekitKeyProvider, // It's important that every room use a separate E2EE worker. // They get confused if given streams from multiple rooms. worker: new E2EEWorker(), }, - this.controlledAudioDevices, - ), + controlledAudioDevices: this.controlledAudioDevices, + echoCancellation, + noiseSuppression, + }), ); this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; } @@ -90,12 +98,24 @@ export class ECConnectionFactory implements ConnectionFactory { /** * Generate the initial LiveKit RoomOptions based on the current media devices and processor state. */ -function generateRoomOption( - devices: MediaDevices, - processorState: ProcessorState, - e2eeLivekitOptions: E2EEOptions | undefined, - controlledAudioDevices: boolean, -): RoomOptions { +function generateRoomOption({ + devices, + processorState, + e2eeLivekitOptions, + controlledAudioDevices, + echoCancellation, + noiseSuppression, +}: { + devices: MediaDevices; + processorState: ProcessorState; + e2eeLivekitOptions: + | E2EEManagerOptions + | { e2eeManager: BaseE2EEManager } + | undefined; + controlledAudioDevices: boolean; + echoCancellation: boolean; + noiseSuppression: boolean; +}): RoomOptions { return { ...defaultLiveKitOptions, videoCaptureDefaults: { @@ -106,6 +126,8 @@ function generateRoomOption( audioCaptureDefaults: { ...defaultLiveKitOptions.audioCaptureDefaults, deviceId: devices.audioInput.selected$.value?.id, + echoCancellation, + noiseSuppression, }, audioOutput: { // When using controlled audio devices, we don't want to set the diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index 484a44e7..280d8ff7 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { type Participant as LivekitParticipant } from "livekit-client"; +import { type RemoteParticipant } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts"; @@ -52,7 +52,7 @@ beforeEach(() => { (transport: LivekitTransport, scope: ObservableScope) => { const mockConnection = { transport, - remoteParticipantsWithTracks$: new BehaviorSubject([]), + remoteParticipants$: new BehaviorSubject([]), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); @@ -200,24 +200,21 @@ describe("connections$ stream", () => { }); describe("connectionManagerData$ stream", () => { - // Used in test to control fake connections' remoteParticipantsWithTracks$ streams - let fakePublishingParticipantsStreams: Map< - string, - Behavior - >; + // Used in test to control fake connections' remoteParticipants$ streams + let fakeRemoteParticipantsStreams: Map>; function keyForTransport(transport: LivekitTransport): string { return `${transport.livekit_service_url}|${transport.livekit_alias}`; } beforeEach(() => { - fakePublishingParticipantsStreams = new Map(); + fakeRemoteParticipantsStreams = new Map(); - function getPublishingParticipantsFor( + function getRemoteParticipantsFor( transport: LivekitTransport, - ): Behavior { + ): Behavior { return ( - fakePublishingParticipantsStreams.get(keyForTransport(transport)) ?? + fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ?? new BehaviorSubject([]) ); } @@ -227,13 +224,12 @@ describe("connectionManagerData$ stream", () => { .fn() .mockImplementation( (transport: LivekitTransport, scope: ObservableScope) => { - const fakePublishingParticipants$ = new BehaviorSubject< - LivekitParticipant[] + const fakeRemoteParticipants$ = new BehaviorSubject< + RemoteParticipant[] >([]); const mockConnection = { transport, - remoteParticipantsWithTracks$: - getPublishingParticipantsFor(transport), + remoteParticipants$: getRemoteParticipantsFor(transport), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); @@ -242,36 +238,36 @@ describe("connectionManagerData$ stream", () => { void mockConnection.stop(); }); - fakePublishingParticipantsStreams.set( + fakeRemoteParticipantsStreams.set( keyForTransport(transport), - fakePublishingParticipants$, + fakeRemoteParticipants$, ); return mockConnection; }, ); }); - test("Should report connections with the publishing participants", () => { + test("Should report connections with the remote participants", () => { withTestScheduler(({ expectObservable, schedule, behavior }) => { // Setup the fake participants streams behavior // ============================== - fakePublishingParticipantsStreams.set( + fakeRemoteParticipantsStreams.set( keyForTransport(TRANSPORT_1), behavior("oa-b", { o: [], - a: [{ identity: "user1A" } as LivekitParticipant], + a: [{ identity: "user1A" } as RemoteParticipant], b: [ - { identity: "user1A" } as LivekitParticipant, - { identity: "user1B" } as LivekitParticipant, + { identity: "user1A" } as RemoteParticipant, + { identity: "user1B" } as RemoteParticipant, ], }), ); - fakePublishingParticipantsStreams.set( + fakeRemoteParticipantsStreams.set( keyForTransport(TRANSPORT_2), behavior("o-a", { o: [], - a: [{ identity: "user2A" } as LivekitParticipant], + a: [{ identity: "user2A" } as RemoteParticipant], }), ); // ============================== @@ -289,47 +285,47 @@ describe("connectionManagerData$ stream", () => { a: expect.toSatisfy((e) => { const data: ConnectionManagerData = e.value; expect(data.getConnections().length).toBe(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); return true; }), b: expect.toSatisfy((e) => { const data: ConnectionManagerData = e.value; expect(data.getConnections().length).toBe(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0); - expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( - "user1A", - ); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[0].identity, + ).toBe("user1A"); return true; }), c: expect.toSatisfy((e) => { const data: ConnectionManagerData = e.value; expect(data.getConnections().length).toBe(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1); - expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( - "user1A", - ); - expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe( - "user2A", - ); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[0].identity, + ).toBe("user1A"); + expect( + data.getParticipantsForTransport(TRANSPORT_2)[0].identity, + ).toBe("user2A"); return true; }), d: expect.toSatisfy((e) => { const data: ConnectionManagerData = e.value; expect(data.getConnections().length).toBe(2); - expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2); - expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1); - expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( - "user1A", - ); - expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe( - "user1B", - ); - expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe( - "user2A", - ); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[0].identity, + ).toBe("user1A"); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[1].identity, + ).toBe("user1B"); + expect( + data.getParticipantsForTransport(TRANSPORT_2)[0].identity, + ).toBe("user2A"); return true; }), }); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 0b9f939c..101e34ed 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -6,13 +6,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - type LivekitTransport, - type ParticipantId, -} from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, combineLatest, map, of, switchMap, tap } from "rxjs"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { combineLatest, map, of, switchMap, tap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; +import { type RemoteParticipant } from "livekit-client"; import { type Behavior } from "../../Behavior.ts"; import { type Connection } from "./Connection.ts"; @@ -24,21 +21,18 @@ import { type ConnectionFactory } from "./ConnectionFactory.ts"; export class ConnectionManagerData { private readonly store: Map< string, - [Connection, (LocalParticipant | RemoteParticipant)[]] + { connection: Connection; participants: RemoteParticipant[] } > = new Map(); public constructor() {} - public add( - connection: Connection, - participants: (LocalParticipant | RemoteParticipant)[], - ): void { + public add(connection: Connection, participants: RemoteParticipant[]): void { const key = this.getKey(connection.transport); const existing = this.store.get(key); if (!existing) { - this.store.set(key, [connection, participants]); + this.store.set(key, { connection, participants }); } else { - existing[1].push(...participants); + existing.participants.push(...participants); } } @@ -47,58 +41,46 @@ export class ConnectionManagerData { } public getConnections(): Connection[] { - return Array.from(this.store.values()).map(([connection]) => connection); + return Array.from(this.store.values()).map(({ connection }) => connection); } public getConnectionForTransport( transport: LivekitTransport, ): Connection | null { - return this.store.get(this.getKey(transport))?.[0] ?? null; + return this.store.get(this.getKey(transport))?.connection ?? null; } - public getParticipantForTransport( + public getParticipantsForTransport( transport: LivekitTransport, - ): (LocalParticipant | RemoteParticipant)[] { + ): RemoteParticipant[] { const key = transport.livekit_service_url + "|" + transport.livekit_alias; const existing = this.store.get(key); if (existing) { - return existing[1]; + return existing.participants; } return []; } - /** - * Get all connections where the given participant is publishing. - * In theory, there could be several connections where the same participant is publishing but with - * only well behaving clients a participant should only be publishing on a single connection. - * @param participantId - */ - public getConnectionsForParticipant( - participantId: ParticipantId, - ): Connection[] { - const connections: Connection[] = []; - for (const [connection, participants] of this.store.values()) { - if (participants.some((p) => p.identity === participantId)) { - connections.push(connection); - } - } - return connections; - } } + interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; inputTransports$: Behavior>; logger: Logger; } + // TODO - write test for scopes (do we really need to bind scope) export interface IConnectionManager { connectionManagerData$: Behavior>; } + /** * Crete a `ConnectionManager` - * @param scope the observable scope used by this object. - * @param connectionFactory used to create new connections. - * @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport. + * @param props - Configuration object + * @param props.scope - The observable scope used by this object + * @param props.connectionFactory - Used to create new connections + * @param props.inputTransports$ - A list of Behaviors each containing a LIST of LivekitTransport. + * @param props.logger - The logger to use * Each of these behaviors can be interpreted as subscribed list of transports. * * Using `registerTransports` independent external modules can control what connections @@ -115,9 +97,6 @@ export function createConnectionManager$({ logger: parentLogger, }: Props): IConnectionManager { const logger = parentLogger.getChild("[ConnectionManager]"); - - const running$ = new BehaviorSubject(true); - scope.onEnd(() => running$.next(false)); // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing /** @@ -129,10 +108,7 @@ export function createConnectionManager$({ * externally this is modified via `registerTransports()`. */ const transports$ = scope.behavior( - combineLatest([running$, inputTransports$]).pipe( - map(([running, transports]) => - transports.mapInner((transport) => (running ? transport : [])), - ), + inputTransports$.pipe( map((transports) => transports.mapInner(removeDuplicateTransports)), tap(({ value: transports }) => { logger.trace( @@ -182,23 +158,25 @@ export function createConnectionManager$({ const epoch = connections.epoch; // Map the connections to list of {connection, participants}[] - const listOfConnectionsWithPublishingParticipants = - connections.value.map((connection) => { - return connection.remoteParticipantsWithTracks$.pipe( + const listOfConnectionsWithRemoteParticipants = connections.value.map( + (connection) => { + return connection.remoteParticipants$.pipe( map((participants) => ({ connection, participants, })), ); - }); + }, + ); // probably not required - if (listOfConnectionsWithPublishingParticipants.length === 0) { + + if (listOfConnectionsWithRemoteParticipants.length === 0) { return of(new Epoch(new ConnectionManagerData(), epoch)); } // combineLatest the several streams into a single stream with the ConnectionManagerData - return combineLatest(listOfConnectionsWithPublishingParticipants).pipe( + return combineLatest(listOfConnectionsWithRemoteParticipants).pipe( map( (lists) => new Epoch( diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts new file mode 100644 index 00000000..0c439a6b --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -0,0 +1,133 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { Room as LivekitRoom } from "livekit-client"; +import { BehaviorSubject } from "rxjs"; +import fetchMock from "fetch-mock"; +import { logger } from "matrix-js-sdk/lib/logger"; +import EventEmitter from "events"; + +import { ObservableScope } from "../../ObservableScope.ts"; +import { ECConnectionFactory } from "./ConnectionFactory.ts"; +import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import { exampleTransport, mockMediaDevices } from "../../../utils/test.ts"; +import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; +import { constant } from "../../Behavior"; + +// At the top of your test file, after imports +vi.mock("livekit-client", async (importOriginal) => { + return { + ...(await importOriginal()), + Room: vi.fn().mockImplementation(function (this: LivekitRoom, options) { + const emitter = new EventEmitter(); + return { + on: emitter.on.bind(emitter), + off: emitter.off.bind(emitter), + emit: emitter.emit.bind(emitter), + disconnect: vi.fn(), + remoteParticipants: new Map(), + } as unknown as LivekitRoom; + }), + }; +}); + +let testScope: ObservableScope; +let mockClient: OpenIDClientParts; + +beforeEach(() => { + testScope = new ObservableScope(); + mockClient = { + getOpenIdToken: vi.fn().mockReturnValue(""), + getDeviceId: vi.fn().mockReturnValue("DEV000"), + }; +}); + +describe("ECConnectionFactory - Audio inputs options", () => { + test.each([ + { echo: true, noise: true }, + { echo: true, noise: false }, + { echo: false, noise: true }, + { echo: false, noise: false }, + ])( + "it sets echoCancellation=$echo and noiseSuppression=$noise based on constructor parameters", + ({ echo, noise }) => { + // test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => { + const RoomConstructor = vi.mocked(LivekitRoom); + + const ecConnectionFactory = new ECConnectionFactory( + mockClient, + mockMediaDevices({}), + new BehaviorSubject({ + supported: true, + processor: undefined, + }), + undefined, + false, + undefined, + echo, + noise, + ); + ecConnectionFactory.createConnection(exampleTransport, testScope, logger); + + // Check if Room was constructed with expected options + expect(RoomConstructor).toHaveBeenCalledWith( + expect.objectContaining({ + audioCaptureDefaults: expect.objectContaining({ + echoCancellation: echo, + noiseSuppression: noise, + }), + }), + ); + }, + ); +}); + +describe("ECConnectionFactory - ControlledAudioDevice", () => { + test.each([{ controlled: true }, { controlled: false }])( + "it sets controlledAudioDevice=$controlled then uses deviceId accordingly", + ({ controlled }) => { + // test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => { + const RoomConstructor = vi.mocked(LivekitRoom); + + const ecConnectionFactory = new ECConnectionFactory( + mockClient, + mockMediaDevices({ + audioOutput: { + available$: constant(new Map()), + selected$: constant({ id: "DEV00", virtualEarpiece: false }), + select: () => {}, + }, + }), + new BehaviorSubject({ + supported: true, + processor: undefined, + }), + undefined, + controlled, + undefined, + false, + false, + ); + ecConnectionFactory.createConnection(exampleTransport, testScope, logger); + + // Check if Room was constructed with expected options + expect(RoomConstructor).toHaveBeenCalledWith( + expect.objectContaining({ + audioOutput: expect.objectContaining({ + deviceId: controlled ? undefined : "DEV00", + }), + }), + ); + }, + ); +}); + +afterEach(() => { + testScope.end(); + fetchMock.reset(); +}); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index e675f723..d26bac37 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -15,7 +15,7 @@ import { combineLatest, map, type Observable } from "rxjs"; import { type IConnectionManager } from "./ConnectionManager.ts"; import { - type MatrixLivekitMember, + type RemoteMatrixLivekitMember, createMatrixLivekitMembers$, } from "./MatrixLivekitMembers.ts"; import { @@ -91,7 +91,7 @@ test("should signal participant not yet connected to livekit", () => { }), ); - const matrixLivekitMember$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), connectionManager: { @@ -99,21 +99,24 @@ test("should signal participant not yet connected to livekit", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant$).toBe("a", { - a: null, - }); - expectObservable(data[0].connection$).toBe("a", { - a: null, - }); - return true; - }), - }); + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant.value$).toBe("a", { + a: null, + }); + expectObservable(data[0].connection$).toBe("a", { + a: null, + }); + return true; + }), + }, + ); }); }); @@ -171,7 +174,7 @@ test("should signal participant on a connection that is publishing", () => { }), ); - const matrixLivekitMember$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), connectionManager: { @@ -179,25 +182,28 @@ test("should signal participant on a connection that is publishing", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant$).toBe("a", { - a: expect.toSatisfy((participant) => { - expect(participant).toBeDefined(); - expect(participant!.identity).toEqual(bobParticipantId); - return true; - }), - }); - expectObservable(data[0].connection$).toBe("a", { - a: connection, - }); - return true; - }), - }); + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant.value$).toBe("a", { + a: expect.toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }), + }); + expectObservable(data[0].connection$).toBe("a", { + a: connection, + }); + return true; + }), + }, + ); }); }); @@ -222,7 +228,7 @@ test("should signal participant on a connection that is not publishing", () => { }), ); - const matrixLivekitMember$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), connectionManager: { @@ -230,21 +236,24 @@ test("should signal participant on a connection that is not publishing", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant$).toBe("a", { - a: null, - }); - expectObservable(data[0].connection$).toBe("a", { - a: connection, - }); - return true; - }), - }); + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant.value$).toBe("a", { + a: null, + }); + expectObservable(data[0].connection$).toBe("a", { + a: connection, + }); + return true; + }), + }, + ); }); }); @@ -283,7 +292,7 @@ describe("Publication edge case", () => { }), ); - const matrixLivekitMember$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior( membershipsWithTransport$, @@ -293,10 +302,10 @@ describe("Publication edge case", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( "a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { expect(data.length).toEqual(2); expectObservable(data[0].membership$).toBe("a", { a: bobMembership, @@ -305,7 +314,7 @@ describe("Publication edge case", () => { // The real connection should be from transportA as per the membership a: connectionA, }); - expectObservable(data[0].participant$).toBe("a", { + expectObservable(data[0].participant.value$).toBe("a", { a: expect.toSatisfy((participant) => { expect(participant).toBeDefined(); expect(participant!.identity).toEqual(bobParticipantId); @@ -349,7 +358,7 @@ describe("Publication edge case", () => { }), ); - const matrixLivekitMember$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior( membershipsWithTransport$, @@ -359,10 +368,10 @@ describe("Publication edge case", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( "a", { - a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { expect(data.length).toEqual(2); expectObservable(data[0].membership$).toBe("a", { a: bobMembership, @@ -371,7 +380,7 @@ describe("Publication edge case", () => { // The real connection should be from transportA as per the membership a: connectionA, }); - expectObservable(data[0].participant$).toBe("a", { + expectObservable(data[0].participant.value$).toBe("a", { // No participant as Bob is not publishing on his membership transport a: null, }); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 2f152630..6501adb4 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -5,10 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - type LocalParticipant as LocalLivekitParticipant, - type RemoteParticipant as RemoteLivekitParticipant, -} from "livekit-client"; +import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; import { type LivekitTransport, type CallMembership, @@ -24,22 +21,44 @@ import { generateItemsWithEpoch } from "../../../utils/observable"; const logger = rootLogger.getChild("[MatrixLivekitMembers]"); -/** - * Represents a Matrix call member and their associated LiveKit participation. - * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room - * or if it has no livekit transport at all. - */ -export interface MatrixLivekitMember { +interface LocalTaggedParticipant { + type: "local"; + value$: Behavior; +} +interface RemoteTaggedParticipant { + type: "remote"; + value$: Behavior; +} +export type TaggedParticipant = + | LocalTaggedParticipant + | RemoteTaggedParticipant; + +interface MatrixLivekitMember { membership$: Behavior; - participant$: Behavior< - LocalLivekitParticipant | RemoteLivekitParticipant | null - >; connection$: Behavior; // participantId: string; We do not want a participantId here since it will be generated by the jwt // TODO decide if we can also drop the userId. Its in the matrix membership anyways. userId: string; } +/** + * Represents the local Matrix call member and their associated LiveKit participation. + * `livekitParticipant` can be null if the member is not yet connected to the livekit room + * or if it has no livekit transport at all. + */ +export interface LocalMatrixLivekitMember extends MatrixLivekitMember { + participant: LocalTaggedParticipant; +} + +/** + * Represents a remote Matrix call member and their associated LiveKit participation. + * `livekitParticipant` can be null if the member is not yet connected to the livekit room + * or if it has no livekit transport at all. + */ +export interface RemoteMatrixLivekitMember extends MatrixLivekitMember { + participant: RemoteTaggedParticipant; +} + interface Props { scope: ObservableScope; membershipsWithTransport$: Behavior< @@ -61,7 +80,7 @@ export function createMatrixLivekitMembers$({ scope, membershipsWithTransport$, connectionManager, -}: Props): Behavior> { +}: Props): Behavior> { /** * Stream of all the call members and their associated livekit data (if available). */ @@ -91,7 +110,7 @@ export function createMatrixLivekitMembers$({ const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; const participants = transport - ? managerData.getParticipantForTransport(transport) + ? managerData.getParticipantsForTransport(transport) : []; const participant = participants.find((p) => p.identity == participantId) ?? null; @@ -108,14 +127,16 @@ export function createMatrixLivekitMembers$({ // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory. (scope, data$, participantId, userId) => { logger.debug( - `Updating data$ for participantId: ${participantId}, userId: ${userId}`, + `Generating member for participantId: ${participantId}, userId: ${userId}`, ); + const { participant$, ...rest } = scope.splitBehavior(data$); // will only get called once per `participantId, userId` pair. // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. return { participantId, userId, - ...scope.splitBehavior(data$), + participant: { type: "remote" as const, value$: participant$ }, + ...rest, }; }, ), diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index e3aa6be8..d885ddc6 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -29,10 +29,11 @@ import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx" import { areLivekitTransportsEqual, createMatrixLivekitMembers$, - type MatrixLivekitMember, + type RemoteMatrixLivekitMember, } from "./MatrixLivekitMembers.ts"; import { createConnectionManager$ } from "./ConnectionManager.ts"; import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; +import { testJWTToken } from "../../../utils/test-fixtures.ts"; // Test the integration of ConnectionManager and MatrixLivekitMerger @@ -85,7 +86,7 @@ beforeEach(() => { status: 200, body: { url: `wss://${domain}/livekit/sfu`, - jwt: "ATOKEN", + jwt: testJWTToken, }, }; }); @@ -124,15 +125,15 @@ test("bob, carl, then bob joining no tracks yet", () => { logger: logger, }); - const matrixLivekitItems$ = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, connectionManager, }); - expectObservable(matrixLivekitItems$).toBe(vMarble, { - a: expect.toSatisfy((e: Epoch) => { + expectObservable(matrixLivekitMembers$).toBe(vMarble, { + a: expect.toSatisfy((e: Epoch) => { const items = e.value; expect(items.length).toBe(1); const item = items[0]!; @@ -147,12 +148,12 @@ test("bob, carl, then bob joining no tracks yet", () => { ), ), }); - expectObservable(item.participant$).toBe("a", { + expectObservable(item.participant.value$).toBe("a", { a: null, }); return true; }), - b: expect.toSatisfy((e: Epoch) => { + b: expect.toSatisfy((e: Epoch) => { const items = e.value; expect(items.length).toBe(2); @@ -161,7 +162,7 @@ test("bob, carl, then bob joining no tracks yet", () => { expectObservable(item.membership$).toBe("a", { a: bobMembership, }); - expectObservable(item.participant$).toBe("a", { + expectObservable(item.participant.value$).toBe("a", { a: null, }); } @@ -172,7 +173,7 @@ test("bob, carl, then bob joining no tracks yet", () => { expectObservable(item.membership$).toBe("a", { a: carlMembership, }); - expectObservable(item.participant$).toBe("a", { + expectObservable(item.participant.value$).toBe("a", { a: null, }); expectObservable(item.connection$).toBe("a", { @@ -189,7 +190,7 @@ test("bob, carl, then bob joining no tracks yet", () => { } return true; }), - c: expect.toSatisfy((e: Epoch) => { + c: expect.toSatisfy((e: Epoch) => { const items = e.value; expect(items.length).toBe(3); @@ -216,7 +217,7 @@ test("bob, carl, then bob joining no tracks yet", () => { return true; }), }); - expectObservable(item.participant$).toBe("a", { + expectObservable(item.participant.value$).toBe("a", { a: null, }); } diff --git a/src/state/CallViewModelWidget.test.ts b/src/state/CallViewModelWidget.test.ts index afcf69ba..5d6442f1 100644 --- a/src/state/CallViewModelWidget.test.ts +++ b/src/state/CallViewModelWidget.test.ts @@ -15,6 +15,7 @@ import { constant } from "./Behavior.ts"; import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts"; import { ElementWidgetActions, widget } from "../widget.ts"; import { E2eeType } from "../e2ee/e2eeType.ts"; +import { MatrixRTCMode } from "../settings/settings.ts"; vi.mock("@livekit/components-core", { spy: true }); @@ -34,36 +35,43 @@ vi.mock("../widget", () => ({ }, })); -it("expect leave when ElementWidgetActions.HangupCall is called", async () => { - const pr = Promise.withResolvers(); - withCallViewModel( - { - remoteParticipants$: constant([aliceParticipant]), - rtcMembers$: constant([localRtcMember]), - }, - (vm: CallViewModel) => { - vm.leave$.subscribe((s: string) => { - pr.resolve(s); - }); +it.each([ + [MatrixRTCMode.Legacy], + [MatrixRTCMode.Compatibil], + [MatrixRTCMode.Matrix_2_0], +])( + "expect leave when ElementWidgetActions.HangupCall is called (%s mode)", + async (mode) => { + const pr = Promise.withResolvers(); + withCallViewModel(mode)( + { + remoteParticipants$: constant([aliceParticipant]), + rtcMembers$: constant([localRtcMember]), + }, + (vm: CallViewModel) => { + vm.leave$.subscribe((s: string) => { + pr.resolve(s); + }); - widget!.lazyActions!.emit( - ElementWidgetActions.HangupCall, - new CustomEvent(ElementWidgetActions.HangupCall, { - detail: { - action: "im.vector.hangup", - api: "toWidget", - data: {}, - requestId: "widgetapi-1761237395918", - widgetId: "mrUjS9T6uKUOWHMxXvLbSv0F", - }, - }), - ); - }, - { - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); + widget!.lazyActions!.emit( + ElementWidgetActions.HangupCall, + new CustomEvent(ElementWidgetActions.HangupCall, { + detail: { + action: "im.vector.hangup", + api: "toWidget", + data: {}, + requestId: "widgetapi-1761237395918", + widgetId: "mrUjS9T6uKUOWHMxXvLbSv0F", + }, + }), + ); + }, + { + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); - const source = await pr.promise; - expect(source).toBe("user"); -}); + const source = await pr.promise; + expect(source).toBe("user"); + }, +); diff --git a/src/state/MediaViewModel.test.ts b/src/state/MediaViewModel.test.ts index 61fa2d8c..92868216 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/MediaViewModel.test.ts @@ -20,6 +20,7 @@ import { createLocalMedia, createRemoteMedia, withTestScheduler, + mockRemoteParticipant, } from "../utils/test"; import { getValue } from "../utils/observable"; import { constant } from "./Behavior"; @@ -44,7 +45,11 @@ const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); test("control a participant's volume", () => { const setVolumeSpy = vi.fn(); - const vm = createRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy }); + const vm = createRemoteMedia( + rtcMembership, + {}, + mockRemoteParticipant({ setVolume: setVolumeSpy }), + ); withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab---c---d|", { a() { @@ -88,7 +93,7 @@ test("control a participant's volume", () => { }); test("toggle fit/contain for a participant's video", () => { - const vm = createRemoteMedia(rtcMembership, {}, {}); + const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); withTestScheduler(({ expectObservable, schedule }) => { schedule("-ab|", { a: () => vm.toggleFitContain(), @@ -199,3 +204,35 @@ test("switch cameras", async () => { }); expect(deviceId).toBe("front camera"); }); + +test("remote media is in waiting state when participant has not yet connected", () => { + const vm = createRemoteMedia(rtcMembership, {}, null); // null participant + expect(vm.waitingForMedia$.value).toBe(true); +}); + +test("remote media is not in waiting state when participant is connected", () => { + const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); + expect(vm.waitingForMedia$.value).toBe(false); +}); + +test("remote media is not in waiting state when participant is connected with no publications", () => { + const vm = createRemoteMedia( + rtcMembership, + {}, + mockRemoteParticipant({ + getTrackPublication: () => undefined, + getTrackPublications: () => [], + }), + ); + expect(vm.waitingForMedia$.value).toBe(false); +}); + +test("remote media is not in waiting state when user does not intend to publish anywhere", () => { + const vm = createRemoteMedia( + rtcMembership, + {}, + mockRemoteParticipant({}), + undefined, // No room (no advertised transport) + ); + expect(vm.waitingForMedia$.value).toBe(false); +}); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 74e64b93..9888d6bf 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -7,8 +7,8 @@ Please see LICENSE in the repository root for full details. import { type AudioSource, - type TrackReferenceOrPlaceholder, type VideoSource, + type TrackReference, observeParticipantEvents, observeParticipantMedia, roomEventSelector, @@ -33,7 +33,6 @@ import { type Observable, Subject, combineLatest, - distinctUntilKeyChanged, filter, fromEvent, interval, @@ -60,14 +59,11 @@ import { type ObservableScope } from "./ObservableScope"; export function observeTrackReference$( participant: Participant, source: Track.Source, -): Observable { +): Observable { return observeParticipantMedia(participant).pipe( - map(() => ({ - participant: participant, - publication: participant.getTrackPublication(source), - source, - })), - distinctUntilKeyChanged("publication"), + map(() => participant.getTrackPublication(source)), + distinctUntilChanged(), + map((publication) => publication && { participant, publication, source }), ); } @@ -226,7 +222,7 @@ abstract class BaseMediaViewModel { /** * The LiveKit video track for this media. */ - public readonly video$: Behavior; + public readonly video$: Behavior; /** * Whether there should be a warning that this media is unencrypted. */ @@ -241,10 +237,12 @@ abstract class BaseMediaViewModel { private observeTrackReference$( source: Track.Source, - ): Behavior { + ): Behavior { return this.scope.behavior( this.participant$.pipe( - switchMap((p) => (!p ? of(null) : observeTrackReference$(p, source))), + switchMap((p) => + !p ? of(undefined) : observeTrackReference$(p, source), + ), ), ); } @@ -268,7 +266,7 @@ abstract class BaseMediaViewModel { encryptionSystem: EncryptionSystem, audioSource: AudioSource, videoSource: VideoSource, - livekitRoom$: Behavior, + protected readonly livekitRoom$: Behavior, public readonly focusUrl$: Behavior, public readonly displayName$: Behavior, public readonly mxcAvatarUrl$: Behavior, @@ -281,8 +279,8 @@ abstract class BaseMediaViewModel { [audio$, this.video$], (a, v) => encryptionSystem.kind !== E2eeType.NONE && - (a?.publication?.isEncrypted === false || - v?.publication?.isEncrypted === false), + (a?.publication.isEncrypted === false || + v?.publication.isEncrypted === false), ), ); @@ -471,7 +469,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { private readonly videoTrack$: Observable = this.video$.pipe( switchMap((v) => { - const track = v?.publication?.track; + const track = v?.publication.track; if (!(track instanceof LocalVideoTrack)) return of(null); return merge( // Watch for track restarts because they indicate a camera switch. @@ -596,6 +594,21 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { * A remote participant's user media. */ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { + /** + * Whether we are waiting for this user's LiveKit participant to exist. This + * could be because either we or the remote party are still connecting. + */ + public readonly waitingForMedia$ = this.scope.behavior( + combineLatest( + [this.livekitRoom$, this.participant$], + (livekitRoom, participant) => + // If livekitRoom is undefined, the user is not attempting to publish on + // any transport and so we shouldn't expect a participant. (They might + // be a subscribe-only bot for example.) + livekitRoom !== undefined && participant === null, + ), + ); + // This private field is used to override the value from the superclass private __speaking$: Behavior; public get speaking$(): Behavior { diff --git a/src/state/MuteStates.test.ts b/src/state/MuteStates.test.ts new file mode 100644 index 00000000..f2a6e35f --- /dev/null +++ b/src/state/MuteStates.test.ts @@ -0,0 +1,212 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { BehaviorSubject } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { MuteStates, MuteState } from "./MuteStates"; +import { + type AudioOutputDeviceLabel, + type DeviceLabel, + type MediaDevice, + type SelectedAudioOutputDevice, + type SelectedDevice, +} from "./MediaDevices"; +import { constant } from "./Behavior"; +import { ObservableScope } from "./ObservableScope"; +import { flushPromises, mockMediaDevices } from "../utils/test"; + +const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); +vi.mock("../UrlParams", () => ({ getUrlParams })); + +let testScope: ObservableScope; + +beforeEach(() => { + testScope = new ObservableScope(); +}); + +afterEach(() => { + testScope.end(); +}); + +describe("MuteState", () => { + test("should automatically mute if force mute is set", async () => { + const forceMute$ = new BehaviorSubject(false); + + const deviceStub = { + available$: constant( + new Map([ + ["fbac11", { type: "name", name: "HD Camera" }], + ]), + ), + selected$: constant({ id: "fbac11" }), + select(): void {}, + } as unknown as MediaDevice; + + const muteState = new MuteState( + testScope, + deviceStub, + constant(true), + true, + forceMute$, + ); + let lastEnabled: boolean = false; + muteState.enabled$.subscribe((enabled) => { + lastEnabled = enabled; + }); + let setEnabled: ((enabled: boolean) => void) | null = null; + muteState.setEnabled$.subscribe((setter) => { + setEnabled = setter; + }); + + await flushPromises(); + + setEnabled!(true); + await flushPromises(); + expect(lastEnabled).toBe(true); + + // Now force mute + forceMute$.next(true); + await flushPromises(); + // Should automatically mute + expect(lastEnabled).toBe(false); + + // Try to unmute can not work + expect(setEnabled).toBeNull(); + + // Disable force mute + forceMute$.next(false); + await flushPromises(); + + // TODO I'd expect it to go back to previous state (enabled) + // but actually it goes back to the initial state from construction (disabled) + // Should go back to previous state (enabled) + // Skip for now + // expect(lastEnabled).toBe(true); + + // But yet it can be unmuted now + expect(setEnabled).not.toBeNull(); + + setEnabled!(true); + await flushPromises(); + expect(lastEnabled).toBe(true); + }); +}); + +describe("MuteStates", () => { + function aAudioOutputDevices(): MediaDevice< + AudioOutputDeviceLabel, + SelectedAudioOutputDevice + > { + const selected$ = new BehaviorSubject< + SelectedAudioOutputDevice | undefined + >({ + id: "default", + virtualEarpiece: false, + }); + return { + available$: constant( + new Map([ + ["default", { type: "speaker" }], + ["0000", { type: "speaker" }], + ["1111", { type: "earpiece" }], + ["222", { type: "name", name: "Bluetooth Speaker" }], + ]), + ), + selected$, + select(id: string): void { + if (!this.available$.getValue().has(id)) { + logger.warn(`Attempted to select unknown device id: ${id}`); + return; + } + selected$.next({ + id, + /** For test purposes we ignore this */ + virtualEarpiece: false, + }); + }, + }; + } + + function aVideoInput(): MediaDevice { + const selected$ = new BehaviorSubject( + undefined, + ); + return { + available$: constant( + new Map([ + ["0000", { type: "name", name: "HD Camera" }], + ["1111", { type: "name", name: "WebCam Pro" }], + ]), + ), + selected$, + select(id: string): void { + if (!this.available$.getValue().has(id)) { + logger.warn(`Attempted to select unknown device id: ${id}`); + return; + } + selected$.next({ id }); + }, + }; + } + + test("should mute camera when in earpiece mode", async () => { + const audioOutputDevice = aAudioOutputDevices(); + + const mediaDevices = mockMediaDevices({ + audioOutput: audioOutputDevice, + videoInput: aVideoInput(), + // other devices are not relevant for this test + }); + const muteStates = new MuteStates( + testScope, + mediaDevices, + // consider joined + constant(true), + ); + + let latestSyncedState: boolean | null = null; + muteStates.video.setHandler(async (enabled: boolean): Promise => { + logger.info(`Video mute state set to: ${enabled}`); + latestSyncedState = enabled; + return Promise.resolve(enabled); + }); + + let lastVideoEnabled: boolean = false; + muteStates.video.enabled$.subscribe((enabled) => { + lastVideoEnabled = enabled; + }); + + expect(muteStates.video.setEnabled$.value).toBeDefined(); + muteStates.video.setEnabled$.value?.(true); + await flushPromises(); + + expect(lastVideoEnabled).toBe(true); + + // Select earpiece audio output + audioOutputDevice.select("1111"); + await flushPromises(); + // Video should be automatically muted + expect(lastVideoEnabled).toBe(false); + expect(latestSyncedState).toBe(false); + + // Try to switch to speaker + audioOutputDevice.select("0000"); + await flushPromises(); + // TODO I'd expect it to go back to previous state (enabled)?? + // But maybe not? If you move the phone away from your ear you may not want it + // to automatically enable video? + expect(lastVideoEnabled).toBe(false); + + // But yet it can be unmuted now + expect(muteStates.video.setEnabled$.value).toBeDefined(); + muteStates.video.setEnabled$.value?.(true); + await flushPromises(); + expect(lastVideoEnabled).toBe(true); + }); +}); diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index 50be5e05..632e0426 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -27,7 +27,7 @@ import { ElementWidgetActions, widget } from "../widget"; import { Config } from "../config/Config"; import { getUrlParams } from "../UrlParams"; import { type ObservableScope } from "./ObservableScope"; -import { type Behavior } from "./Behavior"; +import { type Behavior, constant } from "./Behavior"; interface MuteStateData { enabled$: Observable; @@ -38,31 +38,58 @@ interface MuteStateData { export type Handler = (desired: boolean) => Promise; const defaultHandler: Handler = async (desired) => Promise.resolve(desired); -class MuteState { +/** + * Internal class - exported only for testing purposes. + * Do not use directly outside of tests. + */ +export class MuteState { + // TODO: rewrite this to explain behavior, it is not understandable, and cannot add logging private readonly enabledByDefault$ = this.enabledByConfig && !getUrlParams().skipLobby ? this.joined$.pipe(map((isJoined) => !isJoined)) : of(false); private readonly handler$ = new BehaviorSubject(defaultHandler); + public setHandler(handler: Handler): void { if (this.handler$.value !== defaultHandler) throw new Error("Multiple mute state handlers are not supported"); this.handler$.next(handler); } + public unsetHandler(): void { this.handler$.next(defaultHandler); } + private readonly canControlDevices$ = combineLatest([ + this.device.available$, + this.forceMute$, + ]).pipe( + map(([available, forceMute]) => { + return !forceMute && available.size > 0; + }), + ); + private readonly data$ = this.scope.behavior( - this.device.available$.pipe( - map((available) => available.size > 0), + this.canControlDevices$.pipe( distinctUntilChanged(), withLatestFrom( this.enabledByDefault$, - (devicesConnected, enabledByDefault) => { - if (!devicesConnected) + (canControlDevices, enabledByDefault) => { + logger.info( + `MuteState: canControlDevices: ${canControlDevices}, enabled by default: ${enabledByDefault}`, + ); + if (!canControlDevices) { + logger.info( + `MuteState: devices connected: ${canControlDevices}, disabling`, + ); + // We need to sync the mute state with the handler + // to ensure nothing is beeing published. + this.handler$.value(false).catch((err) => { + logger.error("MuteState-disable: handler error", err); + }); return { enabled$: of(false), set: null, toggle: null }; + } // Assume the default value only once devices are actually connected let enabled = enabledByDefault; @@ -135,21 +162,45 @@ class MuteState { private readonly device: MediaDevice, private readonly joined$: Observable, private readonly enabledByConfig: boolean, + /** + * An optional observable which, when it emits `true`, will force the mute. + * Used for video to stop camera when earpiece mode is on. + * @private + */ + private readonly forceMute$: Observable, ) {} } export class MuteStates { + /** + * True if the selected audio output device is an earpiece. + * Used to force-disable video when on earpiece. + */ + private readonly isEarpiece$ = combineLatest( + this.mediaDevices.audioOutput.available$, + this.mediaDevices.audioOutput.selected$, + ).pipe( + map(([available, selected]) => { + if (!selected?.id) return false; + const device = available.get(selected.id); + logger.info(`MuteStates: selected audio output device:`, device); + return device?.type === "earpiece"; + }), + ); + public readonly audio = new MuteState( this.scope, this.mediaDevices.audioInput, this.joined$, Config.get().media_devices.enable_audio, + constant(false), ); public readonly video = new MuteState( this.scope, this.mediaDevices.videoInput, this.joined$, Config.get().media_devices.enable_video, + this.isEarpiece$, ); public constructor( diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index 38f22122..690870e6 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -27,6 +27,7 @@ import type { ReactionOption } from "../reactions"; import { observeSpeaker$ } from "./observeSpeaker.ts"; import { generateItems } from "../utils/observable.ts"; import { ScreenShare } from "./ScreenShare.ts"; +import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts"; /** * Sorting bins defining the order in which media tiles appear in the layout. @@ -68,40 +69,46 @@ enum SortingBin { * for inclusion in the call layout and tracks associated screen shares. */ export class UserMedia { - public readonly vm: UserMediaViewModel = this.participant$.value?.isLocal - ? new LocalUserMediaViewModel( - this.scope, - this.id, - this.userId, - this.participant$ as Behavior, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.mediaDevices, - this.displayName$, - this.mxcAvatarUrl$, - this.scope.behavior(this.handRaised$), - this.scope.behavior(this.reaction$), - ) - : new RemoteUserMediaViewModel( - this.scope, - this.id, - this.userId, - this.participant$ as Behavior, - this.encryptionSystem, - this.livekitRoom$, - this.focusUrl$, - this.pretendToBeDisconnected$, - this.displayName$, - this.mxcAvatarUrl$, - this.scope.behavior(this.handRaised$), - this.scope.behavior(this.reaction$), - ); + public readonly vm: UserMediaViewModel = + this.participant.type === "local" + ? new LocalUserMediaViewModel( + this.scope, + this.id, + this.userId, + this.participant.value$, + this.encryptionSystem, + this.livekitRoom$, + this.focusUrl$, + this.mediaDevices, + this.displayName$, + this.mxcAvatarUrl$, + this.scope.behavior(this.handRaised$), + this.scope.behavior(this.reaction$), + ) + : new RemoteUserMediaViewModel( + this.scope, + this.id, + this.userId, + this.participant.value$, + this.encryptionSystem, + this.livekitRoom$, + this.focusUrl$, + this.pretendToBeDisconnected$, + this.displayName$, + this.mxcAvatarUrl$, + this.scope.behavior(this.handRaised$), + this.scope.behavior(this.reaction$), + ); private readonly speaker$ = this.scope.behavior( observeSpeaker$(this.vm.speaking$), ); + // TypeScript needs this widening of the type to happen in a separate statement + private readonly participant$: Behavior< + LocalParticipant | RemoteParticipant | null + > = this.participant.value$; + /** * All screen share media associated with this user media. */ @@ -184,9 +191,7 @@ export class UserMedia { private readonly scope: ObservableScope, public readonly id: string, private readonly userId: string, - private readonly participant$: Behavior< - LocalParticipant | RemoteParticipant | null - >, + private readonly participant: TaggedParticipant, private readonly encryptionSystem: EncryptionSystem, private readonly livekitRoom$: Behavior, private readonly focusUrl$: Behavior, diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index e3172a22..9bc0efb2 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -12,7 +12,11 @@ import { axe } from "vitest-axe"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { GridTile } from "./GridTile"; -import { mockRtcMembership, createRemoteMedia } from "../utils/test"; +import { + mockRtcMembership, + createRemoteMedia, + mockRemoteParticipant, +} from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsSenderProvider } from "../reactions/useReactionsSender"; import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; @@ -31,11 +35,11 @@ test("GridTile is accessible", async () => { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", }, - { + mockRemoteParticipant({ setVolume() {}, getTrackPublication: () => ({}) as Partial as RemoteTrackPublication, - }, + }), ); const fakeRtcSession = { diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 57409869..2f750c50 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -69,6 +69,7 @@ interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; mirror: boolean; locallyMuted: boolean; + waitingForMedia?: boolean; primaryButton?: ReactNode; menuStart?: ReactNode; menuEnd?: ReactNode; @@ -79,6 +80,7 @@ const UserMediaTile: FC = ({ vm, showSpeakingIndicators, locallyMuted, + waitingForMedia, primaryButton, menuStart, menuEnd, @@ -148,7 +150,7 @@ const UserMediaTile: FC = ({ const tile = ( = ({ raisedHandTime={handRaised ?? undefined} currentReaction={reaction ?? undefined} raisedHandOnClick={raisedHandOnClick} - localParticipant={vm.local} + waitingForMedia={waitingForMedia} focusUrl={focusUrl} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} @@ -290,6 +292,7 @@ const RemoteUserMediaTile: FC = ({ ...props }) => { const { t } = useTranslation(); + const waitingForMedia = useBehavior(vm.waitingForMedia$); const locallyMuted = useBehavior(vm.locallyMuted$); const localVolume = useBehavior(vm.localVolume$); const onSelectMute = useCallback( @@ -311,6 +314,7 @@ const RemoteUserMediaTile: FC = ({ { video: trackReference, userId: "@alice:example.com", mxcAvatarUrl: undefined, - localParticipant: false, focusable: true, }; @@ -66,24 +65,13 @@ describe("MediaView", () => { }); }); - describe("with no participant", () => { - it("shows avatar for local user", () => { - render( - , - ); + describe("with no video", () => { + it("shows avatar", () => { + render(); expect( screen.getByRole("img", { name: "@alice:example.com" }), ).toBeVisible(); - expect(screen.queryAllByText("Waiting for media...").length).toBe(0); - }); - it("shows avatar and label for remote user", () => { - render( - , - ); - expect( - screen.getByRole("img", { name: "@alice:example.com" }), - ).toBeVisible(); - expect(screen.getByText("Waiting for media...")).toBeVisible(); + expect(screen.queryByTestId("video")).toBe(null); }); }); @@ -94,6 +82,22 @@ describe("MediaView", () => { }); }); + describe("waitingForMedia", () => { + test("defaults to false", () => { + render(); + expect(screen.queryAllByText("Waiting for media...").length).toBe(0); + }); + test("shows and is accessible", async () => { + const { container } = render( + + + , + ); + expect(await axe(container)).toHaveNoViolations(); + expect(screen.getByText("Waiting for media...")).toBeVisible(); + }); + }); + describe("unencryptedWarning", () => { test("is shown and accessible", async () => { const { container } = render( diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index e8a30cd4..8bb38d94 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -43,7 +43,7 @@ interface Props extends ComponentProps { raisedHandTime?: Date; currentReaction?: ReactionOption; raisedHandOnClick?: () => void; - localParticipant: boolean; + waitingForMedia?: boolean; audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; // The focus url, mainly for debugging purposes @@ -71,7 +71,7 @@ export const MediaView: FC = ({ raisedHandTime, currentReaction, raisedHandOnClick, - localParticipant, + waitingForMedia, audioStreamStats, videoStreamStats, focusUrl, @@ -129,7 +129,7 @@ export const MediaView: FC = ({ /> )} - {!video && !localParticipant && ( + {waitingForMedia && (
{t("video_tile.waiting_for_media")}
diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index fb7008b8..981c0369 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -17,6 +17,7 @@ import { mockRtcMembership, createLocalMedia, createRemoteMedia, + mockRemoteParticipant, } from "../utils/test"; import { SpotlightTileViewModel } from "../state/TileViewModel"; import { constant } from "../state/Behavior"; @@ -33,7 +34,7 @@ test("SpotlightTile is accessible", async () => { rawDisplayName: "Alice", getMxcAvatarUrl: () => "mxc://adfsg", }, - {}, + mockRemoteParticipant({}), ); const vm2 = createLocalMedia( diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 48dd0f8c..8109784f 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -38,6 +38,7 @@ import { type MediaViewModel, ScreenShareViewModel, type UserMediaViewModel, + type RemoteUserMediaViewModel, } from "../state/MediaViewModel"; import { useInitial } from "../useInitial"; import { useMergedRefs } from "../useMergedRefs"; @@ -84,6 +85,21 @@ const SpotlightLocalUserMediaItem: FC = ({ SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem"; +interface SpotlightRemoteUserMediaItemProps + extends SpotlightUserMediaItemBaseProps { + vm: RemoteUserMediaViewModel; +} + +const SpotlightRemoteUserMediaItem: FC = ({ + vm, + ...props +}) => { + const waitingForMedia = useBehavior(vm.waitingForMedia$); + return ( + + ); +}; + interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps { vm: UserMediaViewModel; } @@ -103,7 +119,7 @@ const SpotlightUserMediaItem: FC = ({ return vm instanceof LocalUserMediaViewModel ? ( ) : ( - + ); }; diff --git a/src/useAudioContext.tsx b/src/useAudioContext.tsx index 59334dda..4d08dde8 100644 --- a/src/useAudioContext.tsx +++ b/src/useAudioContext.tsx @@ -22,9 +22,12 @@ import * as controls from "./controls"; * Play a sound though a given AudioContext. Will take * care of connecting the correct buffer and gating * through gain. - * @param volume The volume to play at. * @param ctx The context to play through. * @param buffer The buffer to play. + * @param volume The volume to play at. + * @param stereoPan The stereo pan to apply. + * @param delayS Delay in seconds before starting playing. + * @param abort Optional AbortController that can be used to stop playback. * @returns A promise that resolves when the sound has finished playing. */ async function playSound( @@ -55,9 +58,11 @@ async function playSound( * Play a sound though a given AudioContext, looping until stopped. Will take * care of connecting the correct buffer and gating * through gain. - * @param volume The volume to play at. * @param ctx The context to play through. * @param buffer The buffer to play. + * @param volume The volume to play at. + * @param stereoPan The stereo pan to apply. + * @param delayS Delay in seconds between each loop. * @returns A function used to end the sound. This function will return a promise when the sound has stopped. */ function playSoundLooping( @@ -120,7 +125,7 @@ interface UseAudioContext { /** * Add an audio context which can be used to play * a set of preloaded sounds. - * @param props + * @param props The properties for the audio context. * @returns Either an instance that can be used to play sounds, or null if not ready. */ export function useAudioContext( diff --git a/src/utils/displayname.ts b/src/utils/displayname.ts index 5ab5de9b..bc49b29e 100644 --- a/src/utils/displayname.ts +++ b/src/utils/displayname.ts @@ -77,6 +77,13 @@ export function shouldDisambiguate( ); } +/** + * Calculates a display name for a member, optionally disambiguating it. + * @param member - The member to calculate the display name for. + * @param member.rawDisplayName - The raw display name of the member + * @param member.userId - The user ID of the member + * @param disambiguate - Whether to disambiguate the display name. + */ export function calculateDisplayName( member: { rawDisplayName?: string; userId: string }, disambiguate: boolean, diff --git a/src/utils/errors.ts b/src/utils/errors.ts index bb37754a..cddf90de 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -57,9 +57,16 @@ export class ElementCallError extends Error { } } +/** + * Configuration problem due to no MatrixRTC backend/SFU is exposed via .well-known and no fallback configured. + */ export class MatrixRTCTransportMissingError extends ElementCallError { public domain: string; + /** + * Creates an instance of MatrixRTCTransportMissingError. + * @param domain - The domain where the MatrixRTC transport is missing. + */ public constructor(domain: string) { super( t("error.call_is_not_supported"), @@ -75,6 +82,9 @@ export class MatrixRTCTransportMissingError extends ElementCallError { } } +/** + * Error indicating that the connection to the call was lost and could not be re-established. + */ export class ConnectionLostError extends ElementCallError { public constructor() { super( @@ -86,7 +96,16 @@ export class ConnectionLostError extends ElementCallError { } } +/** + * Error indicating a failure in the membership manager causing the join call + * operation to fail. + */ export class MembershipManagerError extends ElementCallError { + /** + * Creates an instance of MembershipManagerError. + * + * @param error - The underlying error that caused the membership manager failure. + */ public constructor(error: Error) { super( t("error.membership_manager"), @@ -98,6 +117,9 @@ export class MembershipManagerError extends ElementCallError { } } +/** + * Error indicating that end-to-end encryption is not supported in the current environment. + */ export class E2EENotSupportedError extends ElementCallError { public constructor() { super( @@ -109,7 +131,14 @@ export class E2EENotSupportedError extends ElementCallError { } } +/** + * Error indicating an unknown issue occurred during a call operation. + */ export class UnknownCallError extends ElementCallError { + /** + * Creates an instance of UnknownCallError. + * @param error - The underlying error that caused the unknown issue. + */ public constructor(error: Error) { super( t("error.generic"), @@ -122,7 +151,14 @@ export class UnknownCallError extends ElementCallError { } } +/** + * Error indicating a failure to obtain an OpenID token. + */ export class FailToGetOpenIdToken extends ElementCallError { + /** + * Creates an instance of FailToGetOpenIdToken. + * @param error - The underlying error that caused the failure. + */ public constructor(error: Error) { super( t("error.generic"), @@ -135,7 +171,14 @@ export class FailToGetOpenIdToken extends ElementCallError { } } +/** + * Error indicating a failure to start publishing on a LiveKit connection. + */ export class FailToStartLivekitConnection extends ElementCallError { + /** + * Creates an instance of FailToStartLivekitConnection. + * @param e - An optional error message providing additional context. + */ public constructor(e?: string) { super( t("error.failed_to_start_livekit"), @@ -146,6 +189,9 @@ export class FailToStartLivekitConnection extends ElementCallError { } } +/** + * Error indicating that a LiveKit's server has hit its track limits. + */ export class InsufficientCapacityError extends ElementCallError { public constructor() { super( @@ -157,6 +203,10 @@ export class InsufficientCapacityError extends ElementCallError { } } +/** + * Error indicating that room creation is restricted by the SFU. + * Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service) + */ export class SFURoomCreationRestrictedError extends ElementCallError { public constructor() { super( diff --git a/src/utils/matrix.ts b/src/utils/matrix.ts index 0a2b5c1a..4e3ae3c3 100644 --- a/src/utils/matrix.ts +++ b/src/utils/matrix.ts @@ -188,7 +188,6 @@ function fullAliasFromRoomName(roomName: string, client: MatrixClient): string { * Applies some basic sanitisation to a room name that the user * has given us * @param input The room name from the user - * @param client A matrix client object */ export function sanitiseRoomNameInput(input: string): string { // check to see if the user has entered a fully qualified room @@ -304,8 +303,9 @@ export async function createRoom( /** * Returns an absolute URL to that will load Element Call with the given room * @param roomId ID of the room - * @param roomName Name of the room * @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses + * @param roomName Name of the room + * @param viaServers Optional list of servers to include as 'via' parameters in the URL */ export function getAbsoluteRoomUrl( roomId: string, @@ -321,8 +321,9 @@ export function getAbsoluteRoomUrl( /** * Returns a relative URL to that will load Element Call with the given room * @param roomId ID of the room - * @param roomName Name of the room * @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses + * @param roomName Name of the room + * @param viaServers Optional list of servers to include as 'via' parameters in the URL */ export function getRelativeRoomUrl( roomId: string, diff --git a/src/utils/media.ts b/src/utils/media.ts index cdd81aa7..3750aa4e 100644 --- a/src/utils/media.ts +++ b/src/utils/media.ts @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. /** * Finds a media device with label matching 'deviceName' * @param deviceName The label of the device to look for + * @param kind The kind of media device to look for * @param devices The list of devices to search * @returns A matching media device or undefined if no matching device was found */ diff --git a/src/utils/observable.test.ts b/src/utils/observable.test.ts index d1034e7b..be677367 100644 --- a/src/utils/observable.test.ts +++ b/src/utils/observable.test.ts @@ -5,11 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { test } from "vitest"; -import { Subject } from "rxjs"; +import { expect, test } from "vitest"; +import { type Observable, of, Subject, switchMap } from "rxjs"; import { withTestScheduler } from "./test"; -import { generateItems, pauseWhen } from "./observable"; +import { filterBehavior, generateItems, pauseWhen } from "./observable"; +import { type Behavior } from "../state/Behavior"; test("pauseWhen", () => { withTestScheduler(({ behavior, expectObservable }) => { @@ -72,3 +73,31 @@ test("generateItems", () => { expectObservable(scope4$).toBe(scope4Marbles); }); }); + +test("filterBehavior", () => { + withTestScheduler(({ behavior, expectObservable }) => { + // Filtering the input should segment it into 2 modes of non-null behavior. + const inputMarbles = " abcxabx"; + const filteredMarbles = "a--xa-x"; + + const input$ = behavior(inputMarbles, { + a: "a", + b: "b", + c: "c", + x: null, + }); + const filtered$: Observable | null> = input$.pipe( + filterBehavior((value) => typeof value === "string"), + ); + + expectObservable(filtered$).toBe(filteredMarbles, { + a: expect.any(Object), + x: null, + }); + expectObservable( + filtered$.pipe( + switchMap((value$) => (value$ === null ? of(null) : value$)), + ), + ).toBe(inputMarbles, { a: "a", b: "b", c: "c", x: null }); + }); +}); diff --git a/src/utils/observable.ts b/src/utils/observable.ts index 053921cd..9739353f 100644 --- a/src/utils/observable.ts +++ b/src/utils/observable.ts @@ -22,6 +22,7 @@ import { withLatestFrom, BehaviorSubject, type OperatorFunction, + distinctUntilChanged, } from "rxjs"; import { type Behavior } from "../state/Behavior"; @@ -134,7 +135,6 @@ interface ItemHandle { * requested at a later time, and destroyed (have their scope ended) when the * key is no longer requested. * - * @param input$ The input value to be mapped. * @param generator A generator function yielding a tuple of keys and the * currently associated data for each item that it wants to exist. * @param factory A function constructing an individual item, given the item's key, @@ -185,6 +185,28 @@ export function generateItemsWithEpoch< ); } +/** + * Segments a behavior into periods during which its value matches the filter + * (outputting a behavior with a narrowed type) and periods during which it does + * not match (outputting null). + */ +export function filterBehavior( + predicate: (value: T) => value is S, +): OperatorFunction | null> { + return (input$) => + input$.pipe( + scan | null>((acc$, input) => { + if (predicate(input)) { + const output$ = acc$ ?? new BehaviorSubject(input); + output$.next(input); + return output$; + } + return null; + }, null), + distinctUntilChanged(), + ); +} + function generateItemsInternal< Input, Keys extends [unknown, ...unknown[]], diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index 4cf330b7..f915bb19 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -59,3 +59,17 @@ export const daveRTLRtcMember = mockRtcMembership("@dave2:example.org", "DDDD"); export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "\u202eevaD", }); + +export const testJWTToken = [ + {}, // header + { + // payload + sub: "@me:example.org:ABCDEF", + video: { + room: "!example_room_id", + }, + }, + {}, // signature +] + .map((d) => global.btoa(JSON.stringify(d))) + .join("."); diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 98c45d86..0745be72 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -37,6 +37,7 @@ import { import { aliceRtcMember, localRtcMember } from "./test-fixtures"; import { type RaisedHandInfo, type ReactionInfo } from "../reactions"; import { constant } from "../state/Behavior"; +import { MatrixRTCMode } from "../settings/settings"; mockConfig({ livekit: { livekit_service_url: "https://example.com" } }); @@ -162,6 +163,7 @@ export function getBasicCallViewModelEnvironment( setE2EEEnabled: async () => Promise.resolve(), }), connectionState$: constant(ConnectionState.Connected), + matrixRTCMode$: constant(MatrixRTCMode.Legacy), ...callViewModelOptions, }, handRaisedSubject$, diff --git a/src/utils/test.ts b/src/utils/test.ts index 50a9add0..9a845908 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -44,12 +44,12 @@ import { Track, } from "livekit-client"; import { randomUUID } from "crypto"; -import { - type RoomAndToDeviceEvents, - type RoomAndToDeviceEventsHandlerMap, -} from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport"; import { type TrackReference } from "@livekit/components-core"; import EventEmitter from "events"; +import { + type KeyTransportEvents, + type KeyTransportEventsHandlerMap, +} from "matrix-js-sdk/lib/matrixrtc/IKeyTransport"; import { LocalUserMediaViewModel, @@ -311,6 +311,8 @@ export function mockLocalParticipant( publishTrack: vi.fn(), unpublishTracks: vi.fn().mockResolvedValue([]), createTracks: vi.fn(), + setMicrophoneEnabled: vi.fn(), + setCameraEnabled: vi.fn(), getTrackPublication: () => ({}) as Partial as LocalTrackPublication, ...mockEmitter(), @@ -319,12 +321,12 @@ export function mockLocalParticipant( } export function createLocalMedia( - localRtcMember: CallMembership, + rtcMember: CallMembership, roomMember: Partial, localParticipant: LocalParticipant, mediaDevices: MediaDevices, ): LocalUserMediaViewModel { - const member = mockMatrixRoomMember(localRtcMember, roomMember); + const member = mockMatrixRoomMember(rtcMember, roomMember); return new LocalUserMediaViewModel( testScope(), "local", @@ -359,23 +361,26 @@ export function mockRemoteParticipant( } export function createRemoteMedia( - localRtcMember: CallMembership, + rtcMember: CallMembership, roomMember: Partial, - participant: Partial, + participant: RemoteParticipant | null, + livekitRoom: LivekitRoom | undefined = mockLivekitRoom( + {}, + { + remoteParticipants$: of(participant ? [participant] : []), + }, + ), ): RemoteUserMediaViewModel { - const member = mockMatrixRoomMember(localRtcMember, roomMember); - const remoteParticipant = mockRemoteParticipant(participant); + const member = mockMatrixRoomMember(rtcMember, roomMember); return new RemoteUserMediaViewModel( testScope(), "remote", member.userId, - of(remoteParticipant), + constant(participant), { kind: E2eeType.PER_PARTICIPANT, }, - constant( - mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }), - ), + constant(livekitRoom), constant("https://rtc-example.org"), constant(false), constant(member.rawDisplayName ?? "nodisplayname"), @@ -398,9 +403,9 @@ export function mockConfig( } export class MockRTCSession extends TypedEventEmitter< - MatrixRTCSessionEvent | RoomAndToDeviceEvents | MembershipManagerEvent, - MatrixRTCSessionEventHandlerMap & - RoomAndToDeviceEventsHandlerMap & + MatrixRTCSessionEvent | MembershipManagerEvent | KeyTransportEvents, + KeyTransportEventsHandlerMap & + MatrixRTCSessionEventHandlerMap & MembershipManagerEventHandlerMap > { public asMockedSession(): MockedObject { diff --git a/src/widget.ts b/src/widget.ts index 60163c7c..7862df33 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -64,6 +64,12 @@ export const widget = ((): WidgetHelpers | null => { try { const { widgetId, parentUrl } = getUrlParams(); + const { roomId, userId, deviceId, baseUrl, e2eEnabled, allowIceFallback } = + getUrlParams(); + if (!roomId) throw new Error("Room ID must be supplied"); + if (!userId) throw new Error("User ID must be supplied"); + if (!deviceId) throw new Error("Device ID must be supplied"); + if (!baseUrl) throw new Error("Base URL must be supplied"); if (widgetId && parentUrl) { const parentOrigin = new URL(parentUrl).origin; logger.info("Widget API is available"); @@ -92,19 +98,6 @@ export const widget = ((): WidgetHelpers | null => { // We need to do this now rather than later because it has capabilities to // request, and is responsible for starting the transport (should it be?) - const { - roomId, - userId, - deviceId, - baseUrl, - e2eEnabled, - allowIceFallback, - } = getUrlParams(); - if (!roomId) throw new Error("Room ID must be supplied"); - if (!userId) throw new Error("User ID must be supplied"); - if (!deviceId) throw new Error("Device ID must be supplied"); - if (!baseUrl) throw new Error("Base URL must be supplied"); - // These are all the event types the app uses const sendEvent = [ EventType.CallNotify, // Sent as a deprecated fallback diff --git a/tsconfig.json b/tsconfig.json index e864ecfc..0f9e7c66 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -50,6 +50,11 @@ "plugins": [{ "name": "typescript-eslint-language-service" }] }, - "include": ["./src/**/*.ts", "./src/**/*.tsx", "./playwright/**/*.ts"], + "include": [ + "./src/**/*.ts", + "./src/**/*.tsx", + "./playwright/**/*.ts", + "./sdk/**/*.ts" + ], "exclude": ["**.test.ts"] } diff --git a/vite-sdk.config.ts b/vite-sdk.config.ts new file mode 100644 index 00000000..48fb6f22 --- /dev/null +++ b/vite-sdk.config.ts @@ -0,0 +1,28 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { defineConfig, mergeConfig } from "vite"; +import nodePolyfills from "vite-plugin-node-stdlib-browser"; + +const base = "./"; + +// Config for embedded deployments (possibly hosted under a non-root path) +export default defineConfig(() => ({ + worker: { format: "es" as const }, + base, // Use relative URLs to allow the app to be hosted under any path + build: { + sourcemap: true, + manifest: true, + lib: { + formats: ["es" as const], + entry: "./sdk/main.ts", + name: "MatrixrtcSdk", + fileName: "matrixrtc-sdk", + }, + }, + plugins: [nodePolyfills()], +})); diff --git a/vite.config.ts b/vite.config.ts index a0bb9de5..97d643ec 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -7,14 +7,17 @@ Please see LICENSE in the repository root for full details. import { loadEnv, + PluginOption, searchForWorkspaceRoot, type ConfigEnv, type UserConfig, } from "vite"; import svgrPlugin from "vite-plugin-svgr"; import { createHtmlPlugin } from "vite-plugin-html"; + import { codecovVitePlugin } from "@codecov/vite-plugin"; import { sentryVitePlugin } from "@sentry/vite-plugin"; + import react from "@vitejs/plugin-react"; import { realpathSync } from "fs"; import * as fs from "node:fs"; @@ -31,7 +34,7 @@ export default ({ // In future we might be able to do what is needed via code splitting at // build time. process.env.VITE_PACKAGE = packageType ?? "full"; - const plugins = [ + const plugins: PluginOption[] = [ react(), svgrPlugin({ svgrOptions: { @@ -41,16 +44,6 @@ export default ({ }, }), - createHtmlPlugin({ - entry: "src/main.tsx", - inject: { - data: { - brand: env.VITE_PRODUCT_NAME || "Element Call", - packageType: process.env.VITE_PACKAGE, - }, - }, - }), - codecovVitePlugin({ enableBundleAnalysis: process.env.CODECOV_TOKEN !== undefined, bundleName: "element-call", @@ -73,6 +66,18 @@ export default ({ ); } + plugins.push( + createHtmlPlugin({ + entry: "src/main.tsx", + inject: { + data: { + brand: env.VITE_PRODUCT_NAME || "Element Call", + packageType: process.env.VITE_PACKAGE, + }, + }, + }), + ); + // The crypto WASM module is imported dynamically. Since it's common // for developers to use a linked copy of matrix-js-sdk or Rust // crypto (which could reside anywhere on their file system), Vite diff --git a/yarn.lock b/yarn.lock index 97ca1985..782c9345 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2224,6 +2224,26 @@ __metadata: languageName: node linkType: hard +"@es-joy/jsdoccomment@npm:~0.76.0": + version: 0.76.0 + resolution: "@es-joy/jsdoccomment@npm:0.76.0" + dependencies: + "@types/estree": "npm:^1.0.8" + "@typescript-eslint/types": "npm:^8.46.0" + comment-parser: "npm:1.4.1" + esquery: "npm:^1.6.0" + jsdoc-type-pratt-parser: "npm:~6.10.0" + checksum: 10c0/8fe4edec7d60562787ea8c77193ebe8737a9e28ec3143d383506b63890d0ffd45a2813e913ad1f00f227cb10e3a1fb913e5a696b33d499dc564272ff1a6f3fdb + languageName: node + linkType: hard + +"@es-joy/resolve.exports@npm:1.2.0": + version: 1.2.0 + resolution: "@es-joy/resolve.exports@npm:1.2.0" + checksum: 10c0/7e4713471f5eccb17a925a12415a2d9e372a42376813a19f6abd9c35e8d01ab1403777265817da67c6150cffd4f558d9ad51e26a8de6911dad89d9cb7eedacd8 + languageName: node + linkType: hard + "@esbuild/aix-ppc64@npm:0.25.1": version: 0.25.1 resolution: "@esbuild/aix-ppc64@npm:0.25.1" @@ -2486,7 +2506,26 @@ __metadata: languageName: node linkType: hard -"@floating-ui/dom@npm:1.6.13, @floating-ui/dom@npm:^1.0.0": +"@floating-ui/core@npm:^1.7.3": + version: 1.7.3 + resolution: "@floating-ui/core@npm:1.7.3" + dependencies: + "@floating-ui/utils": "npm:^0.2.10" + checksum: 10c0/edfc23800122d81df0df0fb780b7328ae6c5f00efbb55bd48ea340f4af8c5b3b121ceb4bb81220966ab0f87b443204d37105abdd93d94846468be3243984144c + languageName: node + linkType: hard + +"@floating-ui/dom@npm:1.7.4": + version: 1.7.4 + resolution: "@floating-ui/dom@npm:1.7.4" + dependencies: + "@floating-ui/core": "npm:^1.7.3" + "@floating-ui/utils": "npm:^0.2.10" + checksum: 10c0/da6166c25f9b0729caa9f498685a73a0e28251613b35d27db8de8014bc9d045158a23c092b405321a3d67c2064909b6e2a7e6c1c9cc0f62967dca5779f5aef30 + languageName: node + linkType: hard + +"@floating-ui/dom@npm:^1.0.0": version: 1.6.13 resolution: "@floating-ui/dom@npm:1.6.13" dependencies: @@ -2522,6 +2561,13 @@ __metadata: languageName: node linkType: hard +"@floating-ui/utils@npm:^0.2.10": + version: 0.2.10 + resolution: "@floating-ui/utils@npm:0.2.10" + checksum: 10c0/e9bc2a1730ede1ee25843937e911ab6e846a733a4488623cd353f94721b05ec2c9ec6437613a2ac9379a94c2fd40c797a2ba6fa1df2716f5ce4aa6ddb1cf9ea4 + languageName: node + linkType: hard + "@floating-ui/utils@npm:^0.2.9": version: 0.2.9 resolution: "@floating-ui/utils@npm:0.2.9" @@ -2711,6 +2757,13 @@ __metadata: languageName: node linkType: hard +"@jridgewell/sourcemap-codec@npm:^1.5.5": + version: 1.5.5 + resolution: "@jridgewell/sourcemap-codec@npm:1.5.5" + checksum: 10c0/f9e538f302b63c0ebc06eecb1dd9918dd4289ed36147a0ddce35d6ea4d7ebbda243cda7b2213b6a5e1d8087a298d5cf630fb2bd39329cdecb82017023f6081a0 + languageName: node + linkType: hard + "@jridgewell/trace-mapping@npm:^0.3.23, @jridgewell/trace-mapping@npm:^0.3.24, @jridgewell/trace-mapping@npm:^0.3.25": version: 0.3.25 resolution: "@jridgewell/trace-mapping@npm:0.3.25" @@ -2731,25 +2784,25 @@ __metadata: languageName: node linkType: hard -"@livekit/components-core@npm:0.12.11, @livekit/components-core@npm:^0.12.0": - version: 0.12.11 - resolution: "@livekit/components-core@npm:0.12.11" +"@livekit/components-core@npm:0.12.12, @livekit/components-core@npm:^0.12.0": + version: 0.12.12 + resolution: "@livekit/components-core@npm:0.12.12" dependencies: - "@floating-ui/dom": "npm:1.6.13" + "@floating-ui/dom": "npm:1.7.4" loglevel: "npm:1.9.1" rxjs: "npm:7.8.2" peerDependencies: livekit-client: ^2.15.14 tslib: ^2.6.2 - checksum: 10c0/9c2ac3d30bb8cc9067ae0b2049784f81e90e57df9eabf7edbaf3c8ceb65a63f644a4e6abeb6cc38d3ebe52663d8dbb88535e01a965011f365d5ae1f3daf86052 + checksum: 10c0/788ae01fa6c58a0edbd629f4195f2f3a7bc94660d2fb729af8b27cab2b151abe36cd0a666989811c6187e51d32c847119853010a82be55844750ab3978079c38 languageName: node linkType: hard "@livekit/components-react@npm:^2.0.0": - version: 2.9.16 - resolution: "@livekit/components-react@npm:2.9.16" + version: 2.9.17 + resolution: "@livekit/components-react@npm:2.9.17" dependencies: - "@livekit/components-core": "npm:0.12.11" + "@livekit/components-core": "npm:0.12.12" clsx: "npm:2.1.1" events: "npm:^3.3.0" jose: "npm:^6.0.12" @@ -2763,7 +2816,7 @@ __metadata: peerDependenciesMeta: "@livekit/krisp-noise-filter": optional: true - checksum: 10c0/4ba4ff473c5a29d3107412733a6676a3b708d70684ed463e9b34cda26abb3d2f317c2828a52e730837b756de9df3fc248260d6f390aedebfb6ec96ef63c7b151 + checksum: 10c0/ba64ada37d4b3ce4d5ee7c5b2a6bddbffc17c2e641e95881aac9f02b4ff7428105e0a372d364e50ff124e988b7426d322d94caabdb55b634aebf0144d7e37f99 languageName: node linkType: hard @@ -2795,10 +2848,10 @@ __metadata: languageName: node linkType: hard -"@matrix-org/matrix-sdk-crypto-wasm@npm:^15.3.0": - version: 15.3.0 - resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:15.3.0" - checksum: 10c0/45628f36b7b0e54a8777ae67a7233dbdf3e3cf14e0d95d21f62f89a7ea7e3f907232f1eb7b1262193b1e227759fad47af829dcccc103ded89011f13c66f01d76 +"@matrix-org/matrix-sdk-crypto-wasm@npm:^16.0.0": + version: 16.0.0 + resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:16.0.0" + checksum: 10c0/13b4ede3e618da819957abff778afefcf3baf9a2faac04a36bb5a07a44fae2ea05fbfa072eb3408d48b2b7b9aaf27242ce52c594c8ce9bf1fb8b3aade2832be1 languageName: node linkType: hard @@ -3373,14 +3426,14 @@ __metadata: languageName: node linkType: hard -"@playwright/test@npm:^1.56.1": - version: 1.56.1 - resolution: "@playwright/test@npm:1.56.1" +"@playwright/test@npm:^1.57.0": + version: 1.57.0 + resolution: "@playwright/test@npm:1.57.0" dependencies: - playwright: "npm:1.56.1" + playwright: "npm:1.57.0" bin: playwright: cli.js - checksum: 10c0/2b5b0e1f2e6a18f6e5ce6897c7440ca78f64e0b004834e9808e93ad2b78b96366b562ae4366602669cf8ad793a43d85481b58541e74be71e905e732d833dd691 + checksum: 10c0/35ba4b28be72bf0a53e33dbb11c6cff848fb9a37f49e893ce63a90675b5291ec29a1ba82c8a3b043abaead129400f0589623e9ace2e6a1c8eaa409721ecc3774 languageName: node linkType: hard @@ -4479,6 +4532,22 @@ __metadata: languageName: node linkType: hard +"@rollup/plugin-inject@npm:^5.0.3": + version: 5.0.5 + resolution: "@rollup/plugin-inject@npm:5.0.5" + dependencies: + "@rollup/pluginutils": "npm:^5.0.1" + estree-walker: "npm:^2.0.2" + magic-string: "npm:^0.30.3" + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + checksum: 10c0/22d10cf44fa56a6683d5ac4df24a9003379b3dcaae9897f5c30c844afc2ebca83cfaa5557f13a1399b1c8a0d312c3217bcacd508b7ebc4b2cbee401bd1ec8be2 + languageName: node + linkType: hard + "@rollup/pluginutils@npm:^4.2.0": version: 4.2.1 resolution: "@rollup/pluginutils@npm:4.2.1" @@ -4489,6 +4558,22 @@ __metadata: languageName: node linkType: hard +"@rollup/pluginutils@npm:^5.0.1": + version: 5.3.0 + resolution: "@rollup/pluginutils@npm:5.3.0" + dependencies: + "@types/estree": "npm:^1.0.0" + estree-walker: "npm:^2.0.2" + picomatch: "npm:^4.0.2" + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + checksum: 10c0/001834bf62d7cf5bac424d2617c113f7f7d3b2bf3c1778cbcccb72cdc957b68989f8e7747c782c2b911f1dde8257f56f8ac1e779e29e74e638e3f1e2cac2bcd0 + languageName: node + linkType: hard + "@rollup/pluginutils@npm:^5.1.3": version: 5.1.3 resolution: "@rollup/pluginutils@npm:5.1.3" @@ -4859,6 +4944,13 @@ __metadata: languageName: node linkType: hard +"@sindresorhus/base62@npm:^1.0.0": + version: 1.0.0 + resolution: "@sindresorhus/base62@npm:1.0.0" + checksum: 10c0/9a14df0f058fdf4731c30f0f05728a4822144ee42236030039d7fa5a1a1072c2879feba8091fd4a17c8922d1056bc07bada77c31fddc3e15836fc05a266fd918 + languageName: node + linkType: hard + "@stylistic/eslint-plugin@npm:^3.0.0": version: 3.1.0 resolution: "@stylistic/eslint-plugin@npm:3.1.0" @@ -5157,7 +5249,7 @@ __metadata: languageName: node linkType: hard -"@types/estree@npm:1.0.8": +"@types/estree@npm:1.0.8, @types/estree@npm:^1.0.8": version: 1.0.8 resolution: "@types/estree@npm:1.0.8" checksum: 10c0/39d34d1afaa338ab9763f37ad6066e3f349444f9052b9676a7cc0252ef9485a41c6d81c9c4e0d26e9077993354edf25efc853f3224dd4b447175ef62bdcc86a5 @@ -5487,6 +5579,13 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/types@npm:^8.46.0": + version: 8.51.0 + resolution: "@typescript-eslint/types@npm:8.51.0" + checksum: 10c0/eb3473d0bb71eb886438f35887b620ffadae7853b281752a40c73158aee644d136adeb82549be7d7c30f346fe888b2e979dff7e30e67b35377e8281018034529 + languageName: node + linkType: hard + "@typescript-eslint/typescript-estree@npm:5.62.0": version: 5.62.0 resolution: "@typescript-eslint/typescript-estree@npm:5.62.0" @@ -5847,6 +5946,15 @@ __metadata: languageName: node linkType: hard +"acorn@npm:^8.15.0": + version: 8.15.0 + resolution: "acorn@npm:8.15.0" + bin: + acorn: bin/acorn + checksum: 10c0/dec73ff59b7d6628a01eebaece7f2bdb8bb62b9b5926dcad0f8931f2b8b79c2be21f6c68ac095592adb5adb15831a3635d9343e6a91d028bbe85d564875ec3ec + languageName: node + linkType: hard + "acorn@npm:^8.9.0": version: 8.12.1 resolution: "acorn@npm:8.12.1" @@ -5956,6 +6064,13 @@ __metadata: languageName: node linkType: hard +"are-docs-informative@npm:^0.0.2": + version: 0.0.2 + resolution: "are-docs-informative@npm:0.0.2" + checksum: 10c0/f0326981bd699c372d268b526b170a28f2e1aec2cf99d7de0686083528427ecdf6ae41fef5d9988e224a5616298af747ad8a76e7306b0a7c97cc085a99636d60 + languageName: node + linkType: hard + "argparse@npm:^2.0.1": version: 2.0.1 resolution: "argparse@npm:2.0.1" @@ -6128,6 +6243,30 @@ __metadata: languageName: node linkType: hard +"asn1.js@npm:^4.10.1": + version: 4.10.1 + resolution: "asn1.js@npm:4.10.1" + dependencies: + bn.js: "npm:^4.0.0" + inherits: "npm:^2.0.1" + minimalistic-assert: "npm:^1.0.0" + checksum: 10c0/afa7f3ab9e31566c80175a75b182e5dba50589dcc738aa485be42bdd787e2a07246a4b034d481861123cbe646a7656f318f4f1cad2e9e5e808a210d5d6feaa88 + languageName: node + linkType: hard + +"assert@npm:^2.0.0": + version: 2.1.0 + resolution: "assert@npm:2.1.0" + dependencies: + call-bind: "npm:^1.0.2" + is-nan: "npm:^1.3.2" + object-is: "npm:^1.1.5" + object.assign: "npm:^4.1.4" + util: "npm:^0.12.5" + checksum: 10c0/7271a5da883c256a1fa690677bf1dd9d6aa882139f2bed1cd15da4f9e7459683e1da8e32a203d6cc6767e5e0f730c77a9532a87b896b4b0af0dd535f668775f0 + languageName: node + linkType: hard + "assertion-error@npm:^2.0.1": version: 2.0.1 resolution: "assertion-error@npm:2.0.1" @@ -6318,6 +6457,20 @@ __metadata: languageName: node linkType: hard +"bn.js@npm:^4.0.0, bn.js@npm:^4.1.0, bn.js@npm:^4.11.9": + version: 4.12.2 + resolution: "bn.js@npm:4.12.2" + checksum: 10c0/09a249faa416a9a1ce68b5f5ec8bbca87fe54e5dd4ef8b1cc8a4969147b80035592bddcb1e9cc814c3ba79e573503d5c5178664b722b509fb36d93620dba9b57 + languageName: node + linkType: hard + +"bn.js@npm:^5.2.1, bn.js@npm:^5.2.2": + version: 5.2.2 + resolution: "bn.js@npm:5.2.2" + checksum: 10c0/cb97827d476aab1a0194df33cd84624952480d92da46e6b4a19c32964aa01553a4a613502396712704da2ec8f831cf98d02e74ca03398404bd78a037ba93f2ab + languageName: node + linkType: hard + "boolbase@npm:^1.0.0": version: 1.0.0 resolution: "boolbase@npm:1.0.0" @@ -6393,6 +6546,96 @@ __metadata: languageName: node linkType: hard +"brorand@npm:^1.0.1, brorand@npm:^1.1.0": + version: 1.1.0 + resolution: "brorand@npm:1.1.0" + checksum: 10c0/6f366d7c4990f82c366e3878492ba9a372a73163c09871e80d82fb4ae0d23f9f8924cb8a662330308206e6b3b76ba1d528b4601c9ef73c2166b440b2ea3b7571 + languageName: node + linkType: hard + +"browser-resolve@npm:^2.0.0": + version: 2.0.0 + resolution: "browser-resolve@npm:2.0.0" + dependencies: + resolve: "npm:^1.17.0" + checksum: 10c0/06c43adf3cb1939825ab9a4ac355b23272820ee421a20d04f62e0dabd9ea305e497b97f3ac027f87d53c366483aafe8673bbe1aaa5e41cd69eeafa65ac5fda6e + languageName: node + linkType: hard + +"browserify-aes@npm:^1.0.4, browserify-aes@npm:^1.2.0": + version: 1.2.0 + resolution: "browserify-aes@npm:1.2.0" + dependencies: + buffer-xor: "npm:^1.0.3" + cipher-base: "npm:^1.0.0" + create-hash: "npm:^1.1.0" + evp_bytestokey: "npm:^1.0.3" + inherits: "npm:^2.0.1" + safe-buffer: "npm:^5.0.1" + checksum: 10c0/967f2ae60d610b7b252a4cbb55a7a3331c78293c94b4dd9c264d384ca93354c089b3af9c0dd023534efdc74ffbc82510f7ad4399cf82bc37bc07052eea485f18 + languageName: node + linkType: hard + +"browserify-cipher@npm:^1.0.1": + version: 1.0.1 + resolution: "browserify-cipher@npm:1.0.1" + dependencies: + browserify-aes: "npm:^1.0.4" + browserify-des: "npm:^1.0.0" + evp_bytestokey: "npm:^1.0.0" + checksum: 10c0/aa256dcb42bc53a67168bbc94ab85d243b0a3b56109dee3b51230b7d010d9b78985ffc1fb36e145c6e4db151f888076c1cfc207baf1525d3e375cbe8187fe27d + languageName: node + linkType: hard + +"browserify-des@npm:^1.0.0": + version: 1.0.2 + resolution: "browserify-des@npm:1.0.2" + dependencies: + cipher-base: "npm:^1.0.1" + des.js: "npm:^1.0.0" + inherits: "npm:^2.0.1" + safe-buffer: "npm:^5.1.2" + checksum: 10c0/943eb5d4045eff80a6cde5be4e5fbb1f2d5002126b5a4789c3c1aae3cdddb1eb92b00fb92277f512288e5c6af330730b1dbabcf7ce0923e749e151fcee5a074d + languageName: node + linkType: hard + +"browserify-rsa@npm:^4.0.0, browserify-rsa@npm:^4.1.1": + version: 4.1.1 + resolution: "browserify-rsa@npm:4.1.1" + dependencies: + bn.js: "npm:^5.2.1" + randombytes: "npm:^2.1.0" + safe-buffer: "npm:^5.2.1" + checksum: 10c0/b650ee1192e3d7f3d779edc06dd96ed8720362e72ac310c367b9d7fe35f7e8dbb983c1829142b2b3215458be8bf17c38adc7224920843024ed8cf39e19c513c0 + languageName: node + linkType: hard + +"browserify-sign@npm:^4.2.3": + version: 4.2.5 + resolution: "browserify-sign@npm:4.2.5" + dependencies: + bn.js: "npm:^5.2.2" + browserify-rsa: "npm:^4.1.1" + create-hash: "npm:^1.2.0" + create-hmac: "npm:^1.1.7" + elliptic: "npm:^6.6.1" + inherits: "npm:^2.0.4" + parse-asn1: "npm:^5.1.9" + readable-stream: "npm:^2.3.8" + safe-buffer: "npm:^5.2.1" + checksum: 10c0/6192f9696934bbba58932d098face34c2ab9cac09feed826618b86b8c00a897dab7324cd9aa7d6cb1597064f197264ad72fa5418d4d52bf3c8f9b9e0e124655e + languageName: node + linkType: hard + +"browserify-zlib@npm:^0.2.0": + version: 0.2.0 + resolution: "browserify-zlib@npm:0.2.0" + dependencies: + pako: "npm:~1.0.5" + checksum: 10c0/9ab10b6dc732c6c5ec8ebcbe5cb7fe1467f97402c9b2140113f47b5f187b9438f93a8e065d8baf8b929323c18324fbf1105af479ee86d9d36cab7d7ef3424ad9 + languageName: node + linkType: hard + "browserslist@npm:^4.24.0, browserslist@npm:^4.24.3, browserslist@npm:^4.24.4": version: 4.24.4 resolution: "browserslist@npm:4.24.4" @@ -6437,6 +6680,23 @@ __metadata: languageName: node linkType: hard +"buffer-xor@npm:^1.0.3": + version: 1.0.3 + resolution: "buffer-xor@npm:1.0.3" + checksum: 10c0/fd269d0e0bf71ecac3146187cfc79edc9dbb054e2ee69b4d97dfb857c6d997c33de391696d04bdd669272751fa48e7872a22f3a6c7b07d6c0bc31dbe02a4075c + languageName: node + linkType: hard + +"buffer@npm:^5.7.1": + version: 5.7.1 + resolution: "buffer@npm:5.7.1" + dependencies: + base64-js: "npm:^1.3.1" + ieee754: "npm:^1.1.13" + checksum: 10c0/27cac81cff434ed2876058d72e7c4789d11ff1120ef32c9de48f59eab58179b66710c488987d295ae89a228f835fc66d088652dffeb8e3ba8659f80eb091d55e + languageName: node + linkType: hard + "buffer@npm:^6.0.3": version: 6.0.3 resolution: "buffer@npm:6.0.3" @@ -6454,6 +6714,13 @@ __metadata: languageName: node linkType: hard +"builtin-status-codes@npm:^3.0.0": + version: 3.0.0 + resolution: "builtin-status-codes@npm:3.0.0" + checksum: 10c0/c37bbba11a34c4431e56bd681b175512e99147defbe2358318d8152b3a01df7bf25e0305873947e5b350073d5ef41a364a22b37e48f1fb6d2fe6d5286a0f348c + languageName: node + linkType: hard + "bytesish@npm:^0.4.1": version: 0.4.4 resolution: "bytesish@npm:0.4.4" @@ -6508,7 +6775,7 @@ __metadata: languageName: node linkType: hard -"call-bind@npm:^1.0.2, call-bind@npm:^1.0.7, call-bind@npm:^1.0.8": +"call-bind@npm:^1.0.0, call-bind@npm:^1.0.2, call-bind@npm:^1.0.7, call-bind@npm:^1.0.8": version: 1.0.8 resolution: "call-bind@npm:1.0.8" dependencies: @@ -6571,24 +6838,10 @@ __metadata: languageName: node linkType: hard -"caniuse-lite@npm:^1.0.30001688": - version: 1.0.30001701 - resolution: "caniuse-lite@npm:1.0.30001701" - checksum: 10c0/a814bd4dd8b49645ca51bc6ee42120660a36394bb54eb6084801d3f2bbb9471e5e1a9a8a25f44f83086a032d46e66b33031e2aa345f699b90a7e84a9836b819c - languageName: node - linkType: hard - -"caniuse-lite@npm:^1.0.30001702": - version: 1.0.30001720 - resolution: "caniuse-lite@npm:1.0.30001720" - checksum: 10c0/ba9f963364ec4bfc8359d15d7e2cf365185fa1fddc90b4f534c71befedae9b3dd0cd2583a25ffc168a02d7b61b6c18b59bda0a1828ea2a5250fd3e35c2c049e9 - languageName: node - linkType: hard - -"caniuse-lite@npm:^1.0.30001726": - version: 1.0.30001726 - resolution: "caniuse-lite@npm:1.0.30001726" - checksum: 10c0/2c5f91da7fd9ebf8c6b432818b1498ea28aca8de22b30dafabe2a2a6da1e014f10e67e14f8e68e872a0867b6b4cd6001558dde04e3ab9770c9252ca5c8849d0e +"caniuse-lite@npm:^1.0.30001688, caniuse-lite@npm:^1.0.30001702, caniuse-lite@npm:^1.0.30001726": + version: 1.0.30001757 + resolution: "caniuse-lite@npm:1.0.30001757" + checksum: 10c0/3ccb71fa2bf1f8c96ff1bf9b918b08806fed33307e20a3ce3259155fda131eaf96cfcd88d3d309c8fd7f8285cc71d89a3b93648a1c04814da31c301f98508d42 languageName: node linkType: hard @@ -6722,6 +6975,17 @@ __metadata: languageName: node linkType: hard +"cipher-base@npm:^1.0.0, cipher-base@npm:^1.0.1, cipher-base@npm:^1.0.3": + version: 1.0.7 + resolution: "cipher-base@npm:1.0.7" + dependencies: + inherits: "npm:^2.0.4" + safe-buffer: "npm:^5.2.1" + to-buffer: "npm:^1.2.2" + checksum: 10c0/53c5046a9d9b60c586479b8f13fde263c3f905e13f11e8e04c7a311ce399c91d9c3ec96642332e0de077d356e1014ee12bba96f74fbaad0de750f49122258836 + languageName: node + linkType: hard + "classnames@npm:^2.3.1, classnames@npm:^2.5.1": version: 2.5.1 resolution: "classnames@npm:2.5.1" @@ -6857,6 +7121,13 @@ __metadata: languageName: node linkType: hard +"comment-parser@npm:1.4.1": + version: 1.4.1 + resolution: "comment-parser@npm:1.4.1" + checksum: 10c0/d6c4be3f5be058f98b24f2d557f745d8fe1cc9eb75bebbdccabd404a0e1ed41563171b16285f593011f8b6a5ec81f564fb1f2121418ac5cbf0f49255bf0840dd + languageName: node + linkType: hard + "common-tags@npm:^1.8.0": version: 1.8.2 resolution: "common-tags@npm:1.8.2" @@ -6885,6 +7156,20 @@ __metadata: languageName: node linkType: hard +"console-browserify@npm:^1.1.0": + version: 1.2.0 + resolution: "console-browserify@npm:1.2.0" + checksum: 10c0/89b99a53b7d6cee54e1e64fa6b1f7ac24b844b4019c5d39db298637e55c1f4ffa5c165457ad984864de1379df2c8e1886cbbdac85d9dbb6876a9f26c3106f226 + languageName: node + linkType: hard + +"constants-browserify@npm:^1.0.0": + version: 1.0.0 + resolution: "constants-browserify@npm:1.0.0" + checksum: 10c0/ab49b1d59a433ed77c964d90d19e08b2f77213fb823da4729c0baead55e3c597f8f97ebccfdfc47bd896d43854a117d114c849a6f659d9986420e97da0f83ac5 + languageName: node + linkType: hard + "content-type@npm:^1.0.4": version: 1.0.5 resolution: "content-type@npm:1.0.5" @@ -6957,6 +7242,50 @@ __metadata: languageName: node linkType: hard +"create-ecdh@npm:^4.0.4": + version: 4.0.4 + resolution: "create-ecdh@npm:4.0.4" + dependencies: + bn.js: "npm:^4.1.0" + elliptic: "npm:^6.5.3" + checksum: 10c0/77b11a51360fec9c3bce7a76288fc0deba4b9c838d5fb354b3e40c59194d23d66efe6355fd4b81df7580da0661e1334a235a2a5c040b7569ba97db428d466e7f + languageName: node + linkType: hard + +"create-hash@npm:^1.1.0, create-hash@npm:^1.2.0": + version: 1.2.0 + resolution: "create-hash@npm:1.2.0" + dependencies: + cipher-base: "npm:^1.0.1" + inherits: "npm:^2.0.1" + md5.js: "npm:^1.3.4" + ripemd160: "npm:^2.0.1" + sha.js: "npm:^2.4.0" + checksum: 10c0/d402e60e65e70e5083cb57af96d89567954d0669e90550d7cec58b56d49c4b193d35c43cec8338bc72358198b8cbf2f0cac14775b651e99238e1cf411490f915 + languageName: node + linkType: hard + +"create-hmac@npm:^1.1.7": + version: 1.1.7 + resolution: "create-hmac@npm:1.1.7" + dependencies: + cipher-base: "npm:^1.0.3" + create-hash: "npm:^1.1.0" + inherits: "npm:^2.0.1" + ripemd160: "npm:^2.0.0" + safe-buffer: "npm:^5.0.1" + sha.js: "npm:^2.4.8" + checksum: 10c0/24332bab51011652a9a0a6d160eed1e8caa091b802335324ae056b0dcb5acbc9fcf173cf10d128eba8548c3ce98dfa4eadaa01bd02f44a34414baee26b651835 + languageName: node + linkType: hard + +"create-require@npm:^1.1.1": + version: 1.1.1 + resolution: "create-require@npm:1.1.1" + checksum: 10c0/157cbc59b2430ae9a90034a5f3a1b398b6738bf510f713edc4d4e45e169bc514d3d99dd34d8d01ca7ae7830b5b8b537e46ae8f3c8f932371b0875c0151d7ec91 + languageName: node + linkType: hard + "cross-spawn@npm:^7.0.2": version: 7.0.3 resolution: "cross-spawn@npm:7.0.3" @@ -6979,6 +7308,26 @@ __metadata: languageName: node linkType: hard +"crypto-browserify@npm:^3.12.1": + version: 3.12.1 + resolution: "crypto-browserify@npm:3.12.1" + dependencies: + browserify-cipher: "npm:^1.0.1" + browserify-sign: "npm:^4.2.3" + create-ecdh: "npm:^4.0.4" + create-hash: "npm:^1.2.0" + create-hmac: "npm:^1.1.7" + diffie-hellman: "npm:^5.0.3" + hash-base: "npm:~3.0.4" + inherits: "npm:^2.0.4" + pbkdf2: "npm:^3.1.2" + public-encrypt: "npm:^4.0.3" + randombytes: "npm:^2.1.0" + randomfill: "npm:^1.0.4" + checksum: 10c0/184a2def7b16628e79841243232ab5497f18d8e158ac21b7ce90ab172427d0a892a561280adc08f9d4d517bce8db2a5b335dc21abb970f787f8e874bd7b9db7d + languageName: node + linkType: hard + "css-blank-pseudo@npm:^7.0.1": version: 7.0.1 resolution: "css-blank-pseudo@npm:7.0.1" @@ -7189,6 +7538,18 @@ __metadata: languageName: node linkType: hard +"debug@npm:^4.4.3": + version: 4.4.3 + resolution: "debug@npm:4.4.3" + dependencies: + ms: "npm:^2.1.3" + peerDependenciesMeta: + supports-color: + optional: true + checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 + languageName: node + linkType: hard + "decamelize@npm:^1.2.0": version: 1.2.0 resolution: "decamelize@npm:1.2.0" @@ -7267,6 +7628,16 @@ __metadata: languageName: node linkType: hard +"des.js@npm:^1.0.0": + version: 1.1.0 + resolution: "des.js@npm:1.1.0" + dependencies: + inherits: "npm:^2.0.1" + minimalistic-assert: "npm:^1.0.0" + checksum: 10c0/671354943ad67493e49eb4c555480ab153edd7cee3a51c658082fcde539d2690ed2a4a0b5d1f401f9cde822edf3939a6afb2585f32c091f2d3a1b1665cd45236 + languageName: node + linkType: hard + "detect-libc@npm:^1.0.3": version: 1.0.3 resolution: "detect-libc@npm:1.0.3" @@ -7283,6 +7654,17 @@ __metadata: languageName: node linkType: hard +"diffie-hellman@npm:^5.0.3": + version: 5.0.3 + resolution: "diffie-hellman@npm:5.0.3" + dependencies: + bn.js: "npm:^4.1.0" + miller-rabin: "npm:^4.0.0" + randombytes: "npm:^2.0.0" + checksum: 10c0/ce53ccafa9ca544b7fc29b08a626e23a9b6562efc2a98559a0c97b4718937cebaa9b5d7d0a05032cc9c1435e9b3c1532b9e9bf2e0ede868525922807ad6e1ecf + languageName: node + linkType: hard + "dijkstrajs@npm:^1.0.1": version: 1.0.3 resolution: "dijkstrajs@npm:1.0.3" @@ -7353,6 +7735,13 @@ __metadata: languageName: node linkType: hard +"domain-browser@npm:4.22.0": + version: 4.22.0 + resolution: "domain-browser@npm:4.22.0" + checksum: 10c0/2ef7eda6d2161038fda0c9aa4c9e18cc7a0baa89ea6be975d449527c2eefd4b608425db88508e2859acc472f46f402079274b24bd75e3fb506f28c5dba203129 + languageName: node + linkType: hard + "domelementtype@npm:^2.0.1, domelementtype@npm:^2.2.0, domelementtype@npm:^2.3.0": version: 2.3.0 resolution: "domelementtype@npm:2.3.0" @@ -7492,7 +7881,7 @@ __metadata: "@opentelemetry/sdk-trace-base": "npm:^2.0.0" "@opentelemetry/sdk-trace-web": "npm:^2.0.0" "@opentelemetry/semantic-conventions": "npm:^1.25.1" - "@playwright/test": "npm:^1.56.1" + "@playwright/test": "npm:^1.57.0" "@radix-ui/react-dialog": "npm:^1.0.4" "@radix-ui/react-slider": "npm:^1.1.2" "@radix-ui/react-visually-hidden": "npm:^1.0.3" @@ -7531,6 +7920,7 @@ __metadata: eslint-config-prettier: "npm:^10.0.0" eslint-plugin-deprecate: "npm:^0.8.2" eslint-plugin-import: "npm:^2.26.0" + eslint-plugin-jsdoc: "npm:^61.5.0" eslint-plugin-jsx-a11y: "npm:^6.5.1" eslint-plugin-matrix-org: "npm:2.1.0" eslint-plugin-react: "npm:^7.29.4" @@ -7547,8 +7937,9 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21" - matrix-widget-api: "npm:^1.13.0" + matrix-js-sdk: "matrix-org/matrix-js-sdk#2218ec4e3102e841ba3e794e1c492c0a5aa6c1c3" + matrix-widget-api: "npm:^1.14.0" + node-stdlib-browser: "npm:^1.3.1" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" pako: "npm:^2.0.4" @@ -7571,12 +7962,28 @@ __metadata: vite: "npm:^7.0.0" vite-plugin-generate-file: "npm:^0.3.0" vite-plugin-html: "npm:^3.2.2" + vite-plugin-node-stdlib-browser: "npm:^0.2.1" vite-plugin-svgr: "npm:^4.0.0" vitest: "npm:^3.0.0" vitest-axe: "npm:^1.0.0-pre.3" languageName: unknown linkType: soft +"elliptic@npm:^6.5.3, elliptic@npm:^6.6.1": + version: 6.6.1 + resolution: "elliptic@npm:6.6.1" + dependencies: + bn.js: "npm:^4.11.9" + brorand: "npm:^1.1.0" + hash.js: "npm:^1.0.0" + hmac-drbg: "npm:^1.0.1" + inherits: "npm:^2.0.4" + minimalistic-assert: "npm:^1.0.1" + minimalistic-crypto-utils: "npm:^1.0.1" + checksum: 10c0/8b24ef782eec8b472053793ea1e91ae6bee41afffdfcb78a81c0a53b191e715cbe1292aa07165958a9bbe675bd0955142560b1a007ffce7d6c765bcaf951a867 + languageName: node + linkType: hard + "emoji-regex@npm:^8.0.0": version: 8.0.0 resolution: "emoji-regex@npm:8.0.0" @@ -8088,6 +8495,30 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-jsdoc@npm:^61.5.0": + version: 61.5.0 + resolution: "eslint-plugin-jsdoc@npm:61.5.0" + dependencies: + "@es-joy/jsdoccomment": "npm:~0.76.0" + "@es-joy/resolve.exports": "npm:1.2.0" + are-docs-informative: "npm:^0.0.2" + comment-parser: "npm:1.4.1" + debug: "npm:^4.4.3" + escape-string-regexp: "npm:^4.0.0" + espree: "npm:^10.4.0" + esquery: "npm:^1.6.0" + html-entities: "npm:^2.6.0" + object-deep-merge: "npm:^2.0.0" + parse-imports-exports: "npm:^0.2.4" + semver: "npm:^7.7.3" + spdx-expression-parse: "npm:^4.0.0" + to-valid-identifier: "npm:^1.0.0" + peerDependencies: + eslint: ^7.0.0 || ^8.0.0 || ^9.0.0 + checksum: 10c0/fabb04f6efe58a167a0839d3c05676a76080c6e91d98a269fa768c1bfd835aa0ded5822d400da2874216177044d2d227ebe241d73e923f3fe1c08bafd19cfd3d + languageName: node + linkType: hard + "eslint-plugin-jsx-a11y@npm:^6.5.1": version: 6.10.2 resolution: "eslint-plugin-jsx-a11y@npm:6.10.2" @@ -8322,6 +8753,17 @@ __metadata: languageName: node linkType: hard +"espree@npm:^10.4.0": + version: 10.4.0 + resolution: "espree@npm:10.4.0" + dependencies: + acorn: "npm:^8.15.0" + acorn-jsx: "npm:^5.3.2" + eslint-visitor-keys: "npm:^4.2.1" + checksum: 10c0/c63fe06131c26c8157b4083313cb02a9a54720a08e21543300e55288c40e06c3fc284bdecf108d3a1372c5934a0a88644c98714f38b6ae8ed272b40d9ea08d6b + languageName: node + linkType: hard + "espree@npm:^9.6.0, espree@npm:^9.6.1": version: 9.6.1 resolution: "espree@npm:9.6.1" @@ -8388,13 +8830,24 @@ __metadata: languageName: node linkType: hard -"events@npm:^3.2.0, events@npm:^3.3.0": +"events@npm:^3.0.0, events@npm:^3.2.0, events@npm:^3.3.0": version: 3.3.0 resolution: "events@npm:3.3.0" checksum: 10c0/d6b6f2adbccbcda74ddbab52ed07db727ef52e31a61ed26db9feb7dc62af7fc8e060defa65e5f8af9449b86b52cc1a1f6a79f2eafcf4e62add2b7a1fa4a432f6 languageName: node linkType: hard +"evp_bytestokey@npm:^1.0.0, evp_bytestokey@npm:^1.0.3": + version: 1.0.3 + resolution: "evp_bytestokey@npm:1.0.3" + dependencies: + md5.js: "npm:^1.3.4" + node-gyp: "npm:latest" + safe-buffer: "npm:^5.1.1" + checksum: 10c0/77fbe2d94a902a80e9b8f5a73dcd695d9c14899c5e82967a61b1fc6cbbb28c46552d9b127cff47c45fcf684748bdbcfa0a50410349109de87ceb4b199ef6ee99 + languageName: node + linkType: hard + "expect-type@npm:^1.2.1": version: 1.2.1 resolution: "expect-type@npm:1.2.1" @@ -8790,6 +9243,13 @@ __metadata: languageName: node linkType: hard +"generator-function@npm:^2.0.0": + version: 2.0.1 + resolution: "generator-function@npm:2.0.1" + checksum: 10c0/8a9f59df0f01cfefafdb3b451b80555e5cf6d76487095db91ac461a0e682e4ff7a9dbce15f4ecec191e53586d59eece01949e05a4b4492879600bbbe8e28d6b8 + languageName: node + linkType: hard + "gensync@npm:^1.0.0-beta.2": version: 1.0.0-beta.2 resolution: "gensync@npm:1.0.0-beta.2" @@ -9092,6 +9552,38 @@ __metadata: languageName: node linkType: hard +"hash-base@npm:^3.0.0, hash-base@npm:^3.1.2": + version: 3.1.2 + resolution: "hash-base@npm:3.1.2" + dependencies: + inherits: "npm:^2.0.4" + readable-stream: "npm:^2.3.8" + safe-buffer: "npm:^5.2.1" + to-buffer: "npm:^1.2.1" + checksum: 10c0/f3b7fae1853b31340048dd659f40f5260ca6f3ff53b932f807f4ab701ee09039f6e9dbe1841723ff61e20f3f69d6387a352e4ccc5f997dedb0d375c7d88bc15e + languageName: node + linkType: hard + +"hash-base@npm:~3.0.4": + version: 3.0.5 + resolution: "hash-base@npm:3.0.5" + dependencies: + inherits: "npm:^2.0.4" + safe-buffer: "npm:^5.2.1" + checksum: 10c0/6dc185b79bad9b6d525cd132a588e4215380fdc36fec6f7a8a58c5db8e3b642557d02ad9c367f5e476c7c3ad3ccffa3607f308b124e1ed80e3b80a1b254db61e + languageName: node + linkType: hard + +"hash.js@npm:^1.0.0, hash.js@npm:^1.0.3": + version: 1.1.7 + resolution: "hash.js@npm:1.1.7" + dependencies: + inherits: "npm:^2.0.3" + minimalistic-assert: "npm:^1.0.1" + checksum: 10c0/41ada59494eac5332cfc1ce6b7ebdd7b88a3864a6d6b08a3ea8ef261332ed60f37f10877e0c825aaa4bddebf164fbffa618286aeeec5296675e2671cbfa746c4 + languageName: node + linkType: hard + "hasown@npm:^2.0.0, hasown@npm:^2.0.2": version: 2.0.2 resolution: "hasown@npm:2.0.2" @@ -9129,6 +9621,17 @@ __metadata: languageName: node linkType: hard +"hmac-drbg@npm:^1.0.1": + version: 1.0.1 + resolution: "hmac-drbg@npm:1.0.1" + dependencies: + hash.js: "npm:^1.0.3" + minimalistic-assert: "npm:^1.0.0" + minimalistic-crypto-utils: "npm:^1.0.1" + checksum: 10c0/f3d9ba31b40257a573f162176ac5930109816036c59a09f901eb2ffd7e5e705c6832bedfff507957125f2086a0ab8f853c0df225642a88bf1fcaea945f20600d + languageName: node + linkType: hard + "hoist-non-react-statics@npm:^3.3.2": version: 3.3.2 resolution: "hoist-non-react-statics@npm:3.3.2" @@ -9154,6 +9657,13 @@ __metadata: languageName: node linkType: hard +"html-entities@npm:^2.6.0": + version: 2.6.0 + resolution: "html-entities@npm:2.6.0" + checksum: 10c0/7c8b15d9ea0cd00dc9279f61bab002ba6ca8a7a0f3c36ed2db3530a67a9621c017830d1d2c1c65beb9b8e3436ea663e9cf8b230472e0e413359399413b27c8b7 + languageName: node + linkType: hard + "html-escaper@npm:^2.0.0": version: 2.0.2 resolution: "html-escaper@npm:2.0.2" @@ -9216,6 +9726,13 @@ __metadata: languageName: node linkType: hard +"https-browserify@npm:^1.0.0": + version: 1.0.0 + resolution: "https-browserify@npm:1.0.0" + checksum: 10c0/e17b6943bc24ea9b9a7da5714645d808670af75a425f29baffc3284962626efdc1eb3aa9bbffaa6e64028a6ad98af5b09fabcb454a8f918fb686abfdc9e9b8ae + languageName: node + linkType: hard + "https-proxy-agent@npm:^5.0.0": version: 5.0.1 resolution: "https-proxy-agent@npm:5.0.1" @@ -9295,7 +9812,7 @@ __metadata: languageName: node linkType: hard -"ieee754@npm:^1.2.1": +"ieee754@npm:^1.1.13, ieee754@npm:^1.2.1": version: 1.2.1 resolution: "ieee754@npm:1.2.1" checksum: 10c0/b0782ef5e0935b9f12883a2e2aa37baa75da6e66ce6515c168697b42160807d9330de9a32ec1ed73149aea02e0d822e572bca6f1e22bdcbd2149e13b050b17bb @@ -9357,7 +9874,7 @@ __metadata: languageName: node linkType: hard -"inherits@npm:2, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.3": +"inherits@npm:2, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.3, inherits@npm:~2.0.4": version: 2.0.4 resolution: "inherits@npm:2.0.4" checksum: 10c0/4e531f648b29039fb7426fb94075e6545faa1eb9fe83c29f0b6d9e7263aceb4289d2d4557db0d428188eeb449cc7c5e77b0a0b2c4e248ff2a65933a0dee49ef2 @@ -9385,6 +9902,16 @@ __metadata: languageName: node linkType: hard +"is-arguments@npm:^1.0.4": + version: 1.2.0 + resolution: "is-arguments@npm:1.2.0" + dependencies: + call-bound: "npm:^1.0.2" + has-tostringtag: "npm:^1.0.2" + checksum: 10c0/6377344b31e9fcb707c6751ee89b11f132f32338e6a782ec2eac9393b0cbd32235dad93052998cda778ee058754860738341d8114910d50ada5615912bb929fc + languageName: node + linkType: hard + "is-array-buffer@npm:^3.0.4, is-array-buffer@npm:^3.0.5": version: 3.0.5 resolution: "is-array-buffer@npm:3.0.5" @@ -9524,6 +10051,19 @@ __metadata: languageName: node linkType: hard +"is-generator-function@npm:^1.0.7": + version: 1.1.2 + resolution: "is-generator-function@npm:1.1.2" + dependencies: + call-bound: "npm:^1.0.4" + generator-function: "npm:^2.0.0" + get-proto: "npm:^1.0.1" + has-tostringtag: "npm:^1.0.2" + safe-regex-test: "npm:^1.1.0" + checksum: 10c0/83da102e89c3e3b71d67b51d47c9f9bc862bceb58f87201727e27f7fa19d1d90b0ab223644ecaee6fc6e3d2d622bb25c966fbdaf87c59158b01ce7c0fe2fa372 + languageName: node + linkType: hard + "is-glob@npm:^4.0.0, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3, is-glob@npm:~4.0.1": version: 4.0.3 resolution: "is-glob@npm:4.0.3" @@ -9540,6 +10080,16 @@ __metadata: languageName: node linkType: hard +"is-nan@npm:^1.3.2": + version: 1.3.2 + resolution: "is-nan@npm:1.3.2" + dependencies: + call-bind: "npm:^1.0.0" + define-properties: "npm:^1.1.3" + checksum: 10c0/8bfb286f85763f9c2e28ea32e9127702fe980ffd15fa5d63ade3be7786559e6e21355d3625dd364c769c033c5aedf0a2ed3d4025d336abf1b9241e3d9eddc5b0 + languageName: node + linkType: hard + "is-negated-glob@npm:^1.0.0": version: 1.0.0 resolution: "is-negated-glob@npm:1.0.0" @@ -9662,7 +10212,7 @@ __metadata: languageName: node linkType: hard -"is-typed-array@npm:^1.1.13, is-typed-array@npm:^1.1.14, is-typed-array@npm:^1.1.15": +"is-typed-array@npm:^1.1.13, is-typed-array@npm:^1.1.14, is-typed-array@npm:^1.1.15, is-typed-array@npm:^1.1.3": version: 1.1.15 resolution: "is-typed-array@npm:1.1.15" dependencies: @@ -9741,6 +10291,13 @@ __metadata: languageName: node linkType: hard +"isomorphic-timers-promises@npm:^1.0.1": + version: 1.0.1 + resolution: "isomorphic-timers-promises@npm:1.0.1" + checksum: 10c0/3b4761d0012ebe6b6382246079fc667f3513f36fe4042638f2bfb7db1557e4f1acd33a9c9907706c04270890ec6434120f132f3f300161a42a7dd8628926c8a4 + languageName: node + linkType: hard + "istanbul-lib-coverage@npm:^3.0.0, istanbul-lib-coverage@npm:^3.2.2": version: 3.2.2 resolution: "istanbul-lib-coverage@npm:3.2.2" @@ -9876,6 +10433,13 @@ __metadata: languageName: node linkType: hard +"jsdoc-type-pratt-parser@npm:~6.10.0": + version: 6.10.0 + resolution: "jsdoc-type-pratt-parser@npm:6.10.0" + checksum: 10c0/8ea395df0cae0e41d4bdba5f8d81b8d3e467fe53d1e4182a5d4e653235a5f17d60ed137343d68dbc74fa10e767f1c58fb85b1f6d5489c2cf16fc7216cc6d3e1a + languageName: node + linkType: hard + "jsdom@npm:^26.0.0": version: 26.1.0 resolution: "jsdom@npm:26.1.0" @@ -10119,8 +10683,8 @@ __metadata: linkType: hard "livekit-client@npm:^2.13.0": - version: 2.16.0 - resolution: "livekit-client@npm:2.16.0" + version: 2.16.1 + resolution: "livekit-client@npm:2.16.1" dependencies: "@livekit/mutex": "npm:1.1.1" "@livekit/protocol": "npm:1.42.2" @@ -10134,7 +10698,7 @@ __metadata: webrtc-adapter: "npm:^9.0.1" peerDependencies: "@types/dom-mediacapture-record": ^1 - checksum: 10c0/5d03adc5d09efde343ab894db397529dff26117598e773b23a5df90a4fb166bde12c6bb1f2cfd1d28dbaf93fe9f275026d7abb75f2ffd2ba816393a2d58e6c7e + checksum: 10c0/a16f7e603730410b640991cdb1a9d5ad3b0b06b23b8fe2e76674e6a9ba54ed4c8e7e4f6ad15fe67c8de6557da161e1265b40802c4e3701db3fe0d7ea9250b1e2 languageName: node linkType: hard @@ -10289,6 +10853,15 @@ __metadata: languageName: node linkType: hard +"magic-string@npm:^0.30.3": + version: 0.30.21 + resolution: "magic-string@npm:0.30.21" + dependencies: + "@jridgewell/sourcemap-codec": "npm:^1.5.5" + checksum: 10c0/299378e38f9a270069fc62358522ddfb44e94244baa0d6a8980ab2a9b2490a1d03b236b447eee309e17eb3bddfa482c61259d47960eb018a904f0ded52780c4a + languageName: node + linkType: hard + "magicast@npm:^0.3.5": version: 0.3.5 resolution: "magicast@npm:0.3.5" @@ -10352,35 +10925,46 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21": - version: 38.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=e7f5bec51b6f70501a025b79fe5021c933385b21" +"matrix-js-sdk@matrix-org/matrix-js-sdk#2218ec4e3102e841ba3e794e1c492c0a5aa6c1c3": + version: 39.3.0 + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=2218ec4e3102e841ba3e794e1c492c0a5aa6c1c3" dependencies: "@babel/runtime": "npm:^7.12.5" - "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" + "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" another-json: "npm:^0.2.0" bs58: "npm:^6.0.0" content-type: "npm:^1.0.4" jwt-decode: "npm:^4.0.0" loglevel: "npm:^1.9.2" matrix-events-sdk: "npm:0.0.1" - matrix-widget-api: "npm:^1.10.0" + matrix-widget-api: "npm:^1.14.0" oidc-client-ts: "npm:^3.0.1" p-retry: "npm:7" - sdp-transform: "npm:^2.14.1" + sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/7adffdc183affd2d3ee1e8497cad6ca7904a37f98328ff7bc15aa6c1829dc9f9a92f8e1bd6260432a33626ff2a839644de938270163e73438b7294675cd954e4 + checksum: 10c0/feca51c7ada5a56aa6cfb74f29bd1640a20804e9de689d23f10c5227e07ba4f66ebbb9606e1384390dca277a6942886706198394717694a9cfb1f20cd36ca377 languageName: node linkType: hard -"matrix-widget-api@npm:^1.10.0, matrix-widget-api@npm:^1.13.0": - version: 1.13.1 - resolution: "matrix-widget-api@npm:1.13.1" +"matrix-widget-api@npm:^1.14.0": + version: 1.15.0 + resolution: "matrix-widget-api@npm:1.15.0" dependencies: "@types/events": "npm:^3.0.0" events: "npm:^3.2.0" - checksum: 10c0/25ded744922755b3eb65f4e171cf6cff1a2e0fe43fc3fecbb13e565e41d8af066daa817dd2c3c7d921b996af399eec3b23df70ab7b682cf422d9cee7ca202512 + checksum: 10c0/1c08b5284cd98aed312d95594335e1391d937dfad70ef862a1f90fdbaaa27709e1c44dcda37f8045e4814779d8d5816d240aee396d52cfd9b37fbf243a6baf6a + languageName: node + linkType: hard + +"md5.js@npm:^1.3.4": + version: 1.3.5 + resolution: "md5.js@npm:1.3.5" + dependencies: + hash-base: "npm:^3.0.0" + inherits: "npm:^2.0.1" + safe-buffer: "npm:^5.1.2" + checksum: 10c0/b7bd75077f419c8e013fc4d4dada48be71882e37d69a44af65a2f2804b91e253441eb43a0614423a1c91bb830b8140b0dc906bc797245e2e275759584f4efcc5 languageName: node linkType: hard @@ -10401,6 +10985,18 @@ __metadata: languageName: node linkType: hard +"miller-rabin@npm:^4.0.0": + version: 4.0.1 + resolution: "miller-rabin@npm:4.0.1" + dependencies: + bn.js: "npm:^4.0.0" + brorand: "npm:^1.0.1" + bin: + miller-rabin: bin/miller-rabin + checksum: 10c0/26b2b96f6e49dbcff7faebb78708ed2f5f9ae27ac8cbbf1d7c08f83cf39bed3d418c0c11034dce997da70d135cc0ff6f3a4c15dc452f8e114c11986388a64346 + languageName: node + linkType: hard + "mime-db@npm:1.52.0": version: 1.52.0 resolution: "mime-db@npm:1.52.0" @@ -10424,6 +11020,20 @@ __metadata: languageName: node linkType: hard +"minimalistic-assert@npm:^1.0.0, minimalistic-assert@npm:^1.0.1": + version: 1.0.1 + resolution: "minimalistic-assert@npm:1.0.1" + checksum: 10c0/96730e5601cd31457f81a296f521eb56036e6f69133c0b18c13fe941109d53ad23a4204d946a0d638d7f3099482a0cec8c9bb6d642604612ce43ee536be3dddd + languageName: node + linkType: hard + +"minimalistic-crypto-utils@npm:^1.0.1": + version: 1.0.1 + resolution: "minimalistic-crypto-utils@npm:1.0.1" + checksum: 10c0/790ecec8c5c73973a4fbf2c663d911033e8494d5fb0960a4500634766ab05d6107d20af896ca2132e7031741f19888154d44b2408ada0852446705441383e9f8 + languageName: node + linkType: hard + "minimatch@npm:^3.0.2, minimatch@npm:^3.0.4, minimatch@npm:^3.0.5, minimatch@npm:^3.1.1, minimatch@npm:^3.1.2": version: 3.1.2 resolution: "minimatch@npm:3.1.2" @@ -10674,6 +11284,41 @@ __metadata: languageName: node linkType: hard +"node-stdlib-browser@npm:^1.3.1": + version: 1.3.1 + resolution: "node-stdlib-browser@npm:1.3.1" + dependencies: + assert: "npm:^2.0.0" + browser-resolve: "npm:^2.0.0" + browserify-zlib: "npm:^0.2.0" + buffer: "npm:^5.7.1" + console-browserify: "npm:^1.1.0" + constants-browserify: "npm:^1.0.0" + create-require: "npm:^1.1.1" + crypto-browserify: "npm:^3.12.1" + domain-browser: "npm:4.22.0" + events: "npm:^3.0.0" + https-browserify: "npm:^1.0.0" + isomorphic-timers-promises: "npm:^1.0.1" + os-browserify: "npm:^0.3.0" + path-browserify: "npm:^1.0.1" + pkg-dir: "npm:^5.0.0" + process: "npm:^0.11.10" + punycode: "npm:^1.4.1" + querystring-es3: "npm:^0.2.1" + readable-stream: "npm:^3.6.0" + stream-browserify: "npm:^3.0.0" + stream-http: "npm:^3.2.0" + string_decoder: "npm:^1.0.0" + timers-browserify: "npm:^2.0.4" + tty-browserify: "npm:0.0.1" + url: "npm:^0.11.4" + util: "npm:^0.12.4" + vm-browserify: "npm:^1.0.1" + checksum: 10c0/5b0cb5d4499b1b1c73f54db3e9e69b2a3a8aebe2ead2e356b0a03c1dfca6b5c5d2f6516e24301e76dc7b68999b9d0ae3da6c3f1dec421eed80ad6cb9eec0f356 + languageName: node + linkType: hard + "nopt@npm:^8.0.0": version: 8.1.0 resolution: "nopt@npm:8.1.0" @@ -10750,6 +11395,13 @@ __metadata: languageName: node linkType: hard +"object-deep-merge@npm:^2.0.0": + version: 2.0.0 + resolution: "object-deep-merge@npm:2.0.0" + checksum: 10c0/69e8741131ad49fa8720fb96007a3c82dca1119b5d874151d2ecbcc3b44ccd46e8553c7a30b0abcba752c099ba361bbba97f33a68c9ae54c57eed7be116ffc97 + languageName: node + linkType: hard + "object-inspect@npm:^1.13.3": version: 1.13.3 resolution: "object-inspect@npm:1.13.3" @@ -10764,6 +11416,16 @@ __metadata: languageName: node linkType: hard +"object-is@npm:^1.1.5": + version: 1.1.6 + resolution: "object-is@npm:1.1.6" + dependencies: + call-bind: "npm:^1.0.7" + define-properties: "npm:^1.2.1" + checksum: 10c0/506af444c4dce7f8e31f34fc549e2fb8152d6b9c4a30c6e62852badd7f520b579c679af433e7a072f9d78eb7808d230dc12e1cf58da9154dfbf8813099ea0fe0 + languageName: node + linkType: hard + "object-keys@npm:^1.1.1": version: 1.1.1 resolution: "object-keys@npm:1.1.1" @@ -10875,6 +11537,13 @@ __metadata: languageName: node linkType: hard +"os-browserify@npm:^0.3.0": + version: 0.3.0 + resolution: "os-browserify@npm:0.3.0" + checksum: 10c0/6ff32cb1efe2bc6930ad0fd4c50e30c38010aee909eba8d65be60af55efd6cbb48f0287e3649b4e3f3a63dce5a667b23c187c4293a75e557f0d5489d735bcf52 + languageName: node + linkType: hard + "own-keys@npm:^1.0.1": version: 1.0.1 resolution: "own-keys@npm:1.0.1" @@ -11007,6 +11676,13 @@ __metadata: languageName: node linkType: hard +"pako@npm:~1.0.5": + version: 1.0.11 + resolution: "pako@npm:1.0.11" + checksum: 10c0/86dd99d8b34c3930345b8bbeb5e1cd8a05f608eeb40967b293f72fe469d0e9c88b783a8777e4cc7dc7c91ce54c5e93d88ff4b4f060e6ff18408fd21030d9ffbe + languageName: node + linkType: hard + "param-case@npm:^3.0.4": version: 3.0.4 resolution: "param-case@npm:3.0.4" @@ -11026,6 +11702,28 @@ __metadata: languageName: node linkType: hard +"parse-asn1@npm:^5.0.0, parse-asn1@npm:^5.1.9": + version: 5.1.9 + resolution: "parse-asn1@npm:5.1.9" + dependencies: + asn1.js: "npm:^4.10.1" + browserify-aes: "npm:^1.2.0" + evp_bytestokey: "npm:^1.0.3" + pbkdf2: "npm:^3.1.5" + safe-buffer: "npm:^5.2.1" + checksum: 10c0/6dfe27c121be3d63ebbf95f03d2ae0a07dd716d44b70b0bd3458790a822a80de05361c62147271fd7b845dcc2d37755d9c9c393064a3438fe633779df0bc07e7 + languageName: node + linkType: hard + +"parse-imports-exports@npm:^0.2.4": + version: 0.2.4 + resolution: "parse-imports-exports@npm:0.2.4" + dependencies: + parse-statements: "npm:1.0.11" + checksum: 10c0/51b729037208abdf65c4a1f8e9ed06f4e7ccd907c17c668a64db54b37d95bb9e92081f8b16e4133e14102af3cb4e89870975b6ad661b4d654e9ec8f4fb5c77d6 + languageName: node + linkType: hard + "parse-json@npm:^5.0.0, parse-json@npm:^5.2.0": version: 5.2.0 resolution: "parse-json@npm:5.2.0" @@ -11038,6 +11736,13 @@ __metadata: languageName: node linkType: hard +"parse-statements@npm:1.0.11": + version: 1.0.11 + resolution: "parse-statements@npm:1.0.11" + checksum: 10c0/48960e085019068a5f5242e875fd9d21ec87df2e291acf5ad4e4887b40eab6929a8c8d59542acb85a6497e870c5c6a24f5ab7f980ef5f907c14cc5f7984a93f3 + languageName: node + linkType: hard + "parse5-htmlparser2-tree-adapter@npm:^7.0.0": version: 7.1.0 resolution: "parse5-htmlparser2-tree-adapter@npm:7.1.0" @@ -11076,6 +11781,13 @@ __metadata: languageName: node linkType: hard +"path-browserify@npm:^1.0.1": + version: 1.0.1 + resolution: "path-browserify@npm:1.0.1" + checksum: 10c0/8b8c3fd5c66bd340272180590ae4ff139769e9ab79522e2eb82e3d571a89b8117c04147f65ad066dccfb42fcad902e5b7d794b3d35e0fd840491a8ddbedf8c66 + languageName: node + linkType: hard + "path-exists@npm:^4.0.0": version: 4.0.0 resolution: "path-exists@npm:4.0.0" @@ -11149,6 +11861,20 @@ __metadata: languageName: node linkType: hard +"pbkdf2@npm:^3.1.2, pbkdf2@npm:^3.1.5": + version: 3.1.5 + resolution: "pbkdf2@npm:3.1.5" + dependencies: + create-hash: "npm:^1.2.0" + create-hmac: "npm:^1.1.7" + ripemd160: "npm:^2.0.3" + safe-buffer: "npm:^5.2.1" + sha.js: "npm:^2.4.12" + to-buffer: "npm:^1.2.1" + checksum: 10c0/ea42e8695e49417eefabb19a08ab19a602cc6cc72d2df3f109c39309600230dee3083a6f678d5d42fe035d6ae780038b80ace0e68f9792ee2839bf081fe386f3 + languageName: node + linkType: hard + "picocolors@npm:^1.0.0, picocolors@npm:^1.1.1": version: 1.1.1 resolution: "picocolors@npm:1.1.1" @@ -11177,27 +11903,36 @@ __metadata: languageName: node linkType: hard -"playwright-core@npm:1.56.1": - version: 1.56.1 - resolution: "playwright-core@npm:1.56.1" - bin: - playwright-core: cli.js - checksum: 10c0/ffd40142b99c68678b387445d5b42f1fee4ab0b65d983058c37f342e5629f9cdbdac0506ea80a0dfd41a8f9f13345bad54e9a8c35826ef66dc765f4eb3db8da7 +"pkg-dir@npm:^5.0.0": + version: 5.0.0 + resolution: "pkg-dir@npm:5.0.0" + dependencies: + find-up: "npm:^5.0.0" + checksum: 10c0/793a496d685dc55bbbdbbb22d884535c3b29241e48e3e8d37e448113a71b9e42f5481a61fdc672d7322de12fbb2c584dd3a68bf89b18fffce5c48a390f911bc5 languageName: node linkType: hard -"playwright@npm:1.56.1": - version: 1.56.1 - resolution: "playwright@npm:1.56.1" +"playwright-core@npm:1.57.0": + version: 1.57.0 + resolution: "playwright-core@npm:1.57.0" + bin: + playwright-core: cli.js + checksum: 10c0/798e35d83bf48419a8c73de20bb94d68be5dde68de23f95d80a0ebe401e3b83e29e3e84aea7894d67fa6c79d2d3d40cc5bcde3e166f657ce50987aaa2421b6a9 + languageName: node + linkType: hard + +"playwright@npm:1.57.0": + version: 1.57.0 + resolution: "playwright@npm:1.57.0" dependencies: fsevents: "npm:2.3.2" - playwright-core: "npm:1.56.1" + playwright-core: "npm:1.57.0" dependenciesMeta: fsevents: optional: true bin: playwright: cli.js - checksum: 10c0/8e9965aede86df0f4722063385748498977b219630a40a10d1b82b8bd8d4d4e9b6b65ecbfa024331a30800163161aca292fb6dd7446c531a1ad25f4155625ab4 + checksum: 10c0/ab03c99a67b835bdea9059f516ad3b6e42c21025f9adaa161a4ef6bc7ca716dcba476d287140bb240d06126eb23f889a8933b8f5f1f1a56b80659d92d1358899 languageName: node linkType: hard @@ -11673,6 +12408,13 @@ __metadata: languageName: node linkType: hard +"process@npm:^0.11.10": + version: 0.11.10 + resolution: "process@npm:0.11.10" + checksum: 10c0/40c3ce4b7e6d4b8c3355479df77aeed46f81b279818ccdc500124e6a5ab882c0cc81ff7ea16384873a95a74c4570b01b120f287abbdd4c877931460eca6084b3 + languageName: node + linkType: hard + "progress@npm:^2.0.3": version: 2.0.3 resolution: "progress@npm:2.0.3" @@ -11745,6 +12487,27 @@ __metadata: languageName: node linkType: hard +"public-encrypt@npm:^4.0.3": + version: 4.0.3 + resolution: "public-encrypt@npm:4.0.3" + dependencies: + bn.js: "npm:^4.1.0" + browserify-rsa: "npm:^4.0.0" + create-hash: "npm:^1.1.0" + parse-asn1: "npm:^5.0.0" + randombytes: "npm:^2.0.1" + safe-buffer: "npm:^5.1.2" + checksum: 10c0/6c2cc19fbb554449e47f2175065d6b32f828f9b3badbee4c76585ac28ae8641aafb9bb107afc430c33c5edd6b05dbe318df4f7d6d7712b1093407b11c4280700 + languageName: node + linkType: hard + +"punycode@npm:^1.4.1": + version: 1.4.1 + resolution: "punycode@npm:1.4.1" + checksum: 10c0/354b743320518aef36f77013be6e15da4db24c2b4f62c5f1eb0529a6ed02fbaf1cb52925785f6ab85a962f2b590d9cd5ad730b70da72b5f180e2556b8bd3ca08 + languageName: node + linkType: hard + "punycode@npm:^2.1.0, punycode@npm:^2.3.1": version: 2.3.1 resolution: "punycode@npm:2.3.1" @@ -11765,6 +12528,22 @@ __metadata: languageName: node linkType: hard +"qs@npm:^6.12.3": + version: 6.14.0 + resolution: "qs@npm:6.14.0" + dependencies: + side-channel: "npm:^1.1.0" + checksum: 10c0/8ea5d91bf34f440598ee389d4a7d95820e3b837d3fd9f433871f7924801becaa0cd3b3b4628d49a7784d06a8aea9bc4554d2b6d8d584e2d221dc06238a42909c + languageName: node + linkType: hard + +"querystring-es3@npm:^0.2.1": + version: 0.2.1 + resolution: "querystring-es3@npm:0.2.1" + checksum: 10c0/476938c1adb45c141f024fccd2ffd919a3746e79ed444d00e670aad68532977b793889648980e7ca7ff5ffc7bfece623118d0fbadcaf217495eeb7059ae51580 + languageName: node + linkType: hard + "queue-microtask@npm:^1.2.2": version: 1.2.3 resolution: "queue-microtask@npm:1.2.3" @@ -11783,6 +12562,25 @@ __metadata: languageName: node linkType: hard +"randombytes@npm:^2.0.0, randombytes@npm:^2.0.1, randombytes@npm:^2.0.5, randombytes@npm:^2.1.0": + version: 2.1.0 + resolution: "randombytes@npm:2.1.0" + dependencies: + safe-buffer: "npm:^5.1.0" + checksum: 10c0/50395efda7a8c94f5dffab564f9ff89736064d32addf0cc7e8bf5e4166f09f8ded7a0849ca6c2d2a59478f7d90f78f20d8048bca3cdf8be09d8e8a10790388f3 + languageName: node + linkType: hard + +"randomfill@npm:^1.0.4": + version: 1.0.4 + resolution: "randomfill@npm:1.0.4" + dependencies: + randombytes: "npm:^2.0.5" + safe-buffer: "npm:^5.1.0" + checksum: 10c0/11aeed35515872e8f8a2edec306734e6b74c39c46653607f03c68385ab8030e2adcc4215f76b5e4598e028c4750d820afd5c65202527d831d2a5f207fe2bc87c + languageName: node + linkType: hard + "react-dom@npm:19": version: 19.1.0 resolution: "react-dom@npm:19.1.0" @@ -11977,18 +12775,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^3.4.0": - version: 3.6.2 - resolution: "readable-stream@npm:3.6.2" - dependencies: - inherits: "npm:^2.0.3" - string_decoder: "npm:^1.1.1" - util-deprecate: "npm:^1.0.1" - checksum: 10c0/e37be5c79c376fdd088a45fa31ea2e423e5d48854be7a22a58869b4e84d25047b193f6acb54f1012331e1bcd667ffb569c01b99d36b0bd59658fb33f513511b7 - languageName: node - linkType: hard - -"readable-stream@npm:~2.3.6": +"readable-stream@npm:^2.3.8, readable-stream@npm:~2.3.6": version: 2.3.8 resolution: "readable-stream@npm:2.3.8" dependencies: @@ -12003,6 +12790,17 @@ __metadata: languageName: node linkType: hard +"readable-stream@npm:^3.4.0, readable-stream@npm:^3.5.0, readable-stream@npm:^3.6.0": + version: 3.6.2 + resolution: "readable-stream@npm:3.6.2" + dependencies: + inherits: "npm:^2.0.3" + string_decoder: "npm:^1.1.1" + util-deprecate: "npm:^1.0.1" + checksum: 10c0/e37be5c79c376fdd088a45fa31ea2e423e5d48854be7a22a58869b4e84d25047b193f6acb54f1012331e1bcd667ffb569c01b99d36b0bd59658fb33f513511b7 + languageName: node + linkType: hard + "readdirp@npm:^4.0.1": version: 4.1.2 resolution: "readdirp@npm:4.1.2" @@ -12190,6 +12988,13 @@ __metadata: languageName: node linkType: hard +"reserved-identifiers@npm:^1.0.0": + version: 1.2.0 + resolution: "reserved-identifiers@npm:1.2.0" + checksum: 10c0/b82651b12e6c608e80463c3753d275bc20fd89294d0415f04e670aeec3611ae3582ddc19e8fedd497e7d0bcbfaddab6a12823ec86e855b1e6a245e0a734eb43d + languageName: node + linkType: hard + "resolve-from@npm:^4.0.0": version: 4.0.0 resolution: "resolve-from@npm:4.0.0" @@ -12219,6 +13024,19 @@ __metadata: languageName: node linkType: hard +"resolve@npm:^1.17.0": + version: 1.22.11 + resolution: "resolve@npm:1.22.11" + dependencies: + is-core-module: "npm:^2.16.1" + path-parse: "npm:^1.0.7" + supports-preserve-symlinks-flag: "npm:^1.0.0" + bin: + resolve: bin/resolve + checksum: 10c0/f657191507530f2cbecb5815b1ee99b20741ea6ee02a59c57028e9ec4c2c8d7681afcc35febbd554ac0ded459db6f2d8153382c53a2f266cee2575e512674409 + languageName: node + linkType: hard + "resolve@npm:^1.22.10": version: 1.22.10 resolution: "resolve@npm:1.22.10" @@ -12258,6 +13076,19 @@ __metadata: languageName: node linkType: hard +"resolve@patch:resolve@npm%3A^1.17.0#optional!builtin": + version: 1.22.11 + resolution: "resolve@patch:resolve@npm%3A1.22.11#optional!builtin::version=1.22.11&hash=c3c19d" + dependencies: + is-core-module: "npm:^2.16.1" + path-parse: "npm:^1.0.7" + supports-preserve-symlinks-flag: "npm:^1.0.0" + bin: + resolve: bin/resolve + checksum: 10c0/ee5b182f2e37cb1165465e58c6abc797fec0a80b5ba3231607beb4677db0c9291ac010c47cf092b6daa2b7f518d69a0e21888e7e2b633f68d501a874212a8c63 + languageName: node + linkType: hard + "resolve@patch:resolve@npm%3A^1.22.10#optional!builtin": version: 1.22.10 resolution: "resolve@patch:resolve@npm%3A1.22.10#optional!builtin::version=1.22.10&hash=c3c19d" @@ -12331,6 +13162,16 @@ __metadata: languageName: node linkType: hard +"ripemd160@npm:^2.0.0, ripemd160@npm:^2.0.1, ripemd160@npm:^2.0.3": + version: 2.0.3 + resolution: "ripemd160@npm:2.0.3" + dependencies: + hash-base: "npm:^3.1.2" + inherits: "npm:^2.0.4" + checksum: 10c0/3f472fb453241cfe692a77349accafca38dbcdc9d96d5848c088b2932ba41eb968630ecff7b175d291c7487a4945aee5a81e30c064d1f94e36070f7e0c37ed6c + languageName: node + linkType: hard + "rollup@npm:^4.43.0": version: 4.50.1 resolution: "rollup@npm:4.50.1" @@ -12478,6 +13319,13 @@ __metadata: languageName: node linkType: hard +"safe-buffer@npm:^5.0.1, safe-buffer@npm:^5.1.0, safe-buffer@npm:^5.1.1, safe-buffer@npm:^5.1.2, safe-buffer@npm:^5.2.1, safe-buffer@npm:~5.2.0": + version: 5.2.1 + resolution: "safe-buffer@npm:5.2.1" + checksum: 10c0/6501914237c0a86e9675d4e51d89ca3c21ffd6a31642efeba25ad65720bce6921c9e7e974e5be91a786b25aa058b5303285d3c15dbabf983a919f5f630d349f3 + languageName: node + linkType: hard + "safe-buffer@npm:~5.1.0, safe-buffer@npm:~5.1.1": version: 5.1.2 resolution: "safe-buffer@npm:5.1.2" @@ -12485,13 +13333,6 @@ __metadata: languageName: node linkType: hard -"safe-buffer@npm:~5.2.0": - version: 5.2.1 - resolution: "safe-buffer@npm:5.2.1" - checksum: 10c0/6501914237c0a86e9675d4e51d89ca3c21ffd6a31642efeba25ad65720bce6921c9e7e974e5be91a786b25aa058b5303285d3c15dbabf983a919f5f630d349f3 - languageName: node - linkType: hard - "safe-push-apply@npm:^1.0.0": version: 1.0.0 resolution: "safe-push-apply@npm:1.0.0" @@ -12553,7 +13394,7 @@ __metadata: languageName: node linkType: hard -"sdp-transform@npm:^2.14.1, sdp-transform@npm:^2.15.0": +"sdp-transform@npm:^2.15.0": version: 2.15.0 resolution: "sdp-transform@npm:2.15.0" bin: @@ -12562,6 +13403,15 @@ __metadata: languageName: node linkType: hard +"sdp-transform@npm:^3.0.0": + version: 3.0.0 + resolution: "sdp-transform@npm:3.0.0" + bin: + sdp-verify: checker.js + checksum: 10c0/828a4595041ba64c86b29075aa4007ab384519b1fa29882db59ccb83b54b2b2a33b60848293f8da537fe151c52f5844fc17c8325396cac309fb19e2e81ec5bf4 + languageName: node + linkType: hard + "sdp@npm:^3.2.0": version: 3.2.0 resolution: "sdp@npm:3.2.0" @@ -12605,6 +13455,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:^7.7.3": + version: 7.7.3 + resolution: "semver@npm:7.7.3" + bin: + semver: bin/semver.js + checksum: 10c0/4afe5c986567db82f44c8c6faef8fe9df2a9b1d98098fc1721f57c696c4c21cebd572f297fc21002f81889492345b8470473bc6f4aff5fb032a6ea59ea2bc45e + languageName: node + linkType: hard + "set-blocking@npm:^2.0.0": version: 2.0.0 resolution: "set-blocking@npm:2.0.0" @@ -12656,6 +13515,26 @@ __metadata: languageName: node linkType: hard +"setimmediate@npm:^1.0.4": + version: 1.0.5 + resolution: "setimmediate@npm:1.0.5" + checksum: 10c0/5bae81bfdbfbd0ce992893286d49c9693c82b1bcc00dcaaf3a09c8f428fdeacf4190c013598b81875dfac2b08a572422db7df779a99332d0fce186d15a3e4d49 + languageName: node + linkType: hard + +"sha.js@npm:^2.4.0, sha.js@npm:^2.4.12, sha.js@npm:^2.4.8": + version: 2.4.12 + resolution: "sha.js@npm:2.4.12" + dependencies: + inherits: "npm:^2.0.4" + safe-buffer: "npm:^5.2.1" + to-buffer: "npm:^1.2.0" + bin: + sha.js: bin.js + checksum: 10c0/9d36bdd76202c8116abbe152a00055ccd8a0099cb28fc17c01fa7bb2c8cffb9ca60e2ab0fe5f274ed6c45dc2633d8c39cf7ab050306c231904512ba9da4d8ab1 + languageName: node + linkType: hard + "shebang-command@npm:^2.0.0": version: 2.0.0 resolution: "shebang-command@npm:2.0.0" @@ -12853,6 +13732,16 @@ __metadata: languageName: node linkType: hard +"spdx-expression-parse@npm:^4.0.0": + version: 4.0.0 + resolution: "spdx-expression-parse@npm:4.0.0" + dependencies: + spdx-exceptions: "npm:^2.1.0" + spdx-license-ids: "npm:^3.0.0" + checksum: 10c0/965c487e77f4fb173f1c471f3eef4eb44b9f0321adc7f93d95e7620da31faa67d29356eb02523cd7df8a7fc1ec8238773cdbf9e45bd050329d2b26492771b736 + languageName: node + linkType: hard + "spdx-license-ids@npm:^3.0.0": version: 3.0.20 resolution: "spdx-license-ids@npm:3.0.20" @@ -12900,6 +13789,16 @@ __metadata: languageName: node linkType: hard +"stream-browserify@npm:^3.0.0": + version: 3.0.0 + resolution: "stream-browserify@npm:3.0.0" + dependencies: + inherits: "npm:~2.0.4" + readable-stream: "npm:^3.5.0" + checksum: 10c0/ec3b975a4e0aa4b3dc5e70ffae3fc8fd29ac725353a14e72f213dff477b00330140ad014b163a8cbb9922dfe90803f81a5ea2b269e1bbfd8bd71511b88f889ad + languageName: node + linkType: hard + "stream-composer@npm:^1.0.2": version: 1.0.2 resolution: "stream-composer@npm:1.0.2" @@ -12909,6 +13808,18 @@ __metadata: languageName: node linkType: hard +"stream-http@npm:^3.2.0": + version: 3.2.0 + resolution: "stream-http@npm:3.2.0" + dependencies: + builtin-status-codes: "npm:^3.0.0" + inherits: "npm:^2.0.4" + readable-stream: "npm:^3.6.0" + xtend: "npm:^4.0.2" + checksum: 10c0/f128fb8076d60cd548f229554b6a1a70c08a04b7b2afd4dbe7811d20f27f7d4112562eb8bce86d72a8691df3b50573228afcf1271e55e81f981536c67498bc41 + languageName: node + linkType: hard + "streamx@npm:^2.12.0, streamx@npm:^2.12.5, streamx@npm:^2.13.2, streamx@npm:^2.14.0": version: 2.22.0 resolution: "streamx@npm:2.22.0" @@ -13025,7 +13936,7 @@ __metadata: languageName: node linkType: hard -"string_decoder@npm:^1.1.1": +"string_decoder@npm:^1.0.0, string_decoder@npm:^1.1.1": version: 1.3.0 resolution: "string_decoder@npm:1.3.0" dependencies: @@ -13227,6 +14138,15 @@ __metadata: languageName: node linkType: hard +"timers-browserify@npm:^2.0.4": + version: 2.0.12 + resolution: "timers-browserify@npm:2.0.12" + dependencies: + setimmediate: "npm:^1.0.4" + checksum: 10c0/98e84db1a685bc8827c117a8bc62aac811ad56a995d07938fc7ed8cdc5bf3777bfe2d4e5da868847194e771aac3749a20f6cdd22091300fe889a76fe214a4641 + languageName: node + linkType: hard + "tinybench@npm:^2.9.0": version: 2.9.0 resolution: "tinybench@npm:2.9.0" @@ -13300,6 +14220,17 @@ __metadata: languageName: node linkType: hard +"to-buffer@npm:^1.2.0, to-buffer@npm:^1.2.1, to-buffer@npm:^1.2.2": + version: 1.2.2 + resolution: "to-buffer@npm:1.2.2" + dependencies: + isarray: "npm:^2.0.5" + safe-buffer: "npm:^5.2.1" + typed-array-buffer: "npm:^1.0.3" + checksum: 10c0/56bc56352f14a2c4a0ab6277c5fc19b51e9534882b98eb068b39e14146591e62fa5b06bf70f7fed1626230463d7e60dca81e815096656e5e01c195c593873d12 + languageName: node + linkType: hard + "to-regex-range@npm:^5.0.1": version: 5.0.1 resolution: "to-regex-range@npm:5.0.1" @@ -13318,6 +14249,16 @@ __metadata: languageName: node linkType: hard +"to-valid-identifier@npm:^1.0.0": + version: 1.0.0 + resolution: "to-valid-identifier@npm:1.0.0" + dependencies: + "@sindresorhus/base62": "npm:^1.0.0" + reserved-identifiers: "npm:^1.0.0" + checksum: 10c0/569b49f43b5aaaa20677e67f0f1cdcff344855149934cfb80c793c7ac7c30e191b224bc81cab40fb57641af9ca73795c78053c164a2addc617671e2d22c13a4a + languageName: node + linkType: hard + "toggle-selection@npm:^1.0.6": version: 1.0.6 resolution: "toggle-selection@npm:1.0.6" @@ -13437,6 +14378,13 @@ __metadata: languageName: node linkType: hard +"tty-browserify@npm:0.0.1": + version: 0.0.1 + resolution: "tty-browserify@npm:0.0.1" + checksum: 10c0/5e34883388eb5f556234dae75b08e069b9e62de12bd6d87687f7817f5569430a6dfef550b51dbc961715ae0cd0eb5a059e6e3fc34dc127ea164aa0f9b5bb033d + languageName: node + linkType: hard + "tunnel@npm:^0.0.6": version: 0.0.6 resolution: "tunnel@npm:0.0.6" @@ -13771,6 +14719,16 @@ __metadata: languageName: node linkType: hard +"url@npm:^0.11.4": + version: 0.11.4 + resolution: "url@npm:0.11.4" + dependencies: + punycode: "npm:^1.4.1" + qs: "npm:^6.12.3" + checksum: 10c0/cc93405ae4a9b97a2aa60ca67f1cb1481c0221cb4725a7341d149be5e2f9cfda26fd432d64dbbec693d16593b68b8a46aad8e5eab21f814932134c9d8620c662 + languageName: node + linkType: hard + "use-callback-ref@npm:^1.3.3": version: 1.3.3 resolution: "use-callback-ref@npm:1.3.3" @@ -13820,6 +14778,19 @@ __metadata: languageName: node linkType: hard +"util@npm:^0.12.4, util@npm:^0.12.5": + version: 0.12.5 + resolution: "util@npm:0.12.5" + dependencies: + inherits: "npm:^2.0.3" + is-arguments: "npm:^1.0.4" + is-generator-function: "npm:^1.0.7" + is-typed-array: "npm:^1.1.3" + which-typed-array: "npm:^1.1.2" + checksum: 10c0/c27054de2cea2229a66c09522d0fa1415fb12d861d08523a8846bf2e4cbf0079d4c3f725f09dcb87493549bcbf05f5798dce1688b53c6c17201a45759e7253f3 + languageName: node + linkType: hard + "uuid@npm:13": version: 13.0.0 resolution: "uuid@npm:13.0.0" @@ -13966,6 +14937,18 @@ __metadata: languageName: node linkType: hard +"vite-plugin-node-stdlib-browser@npm:^0.2.1": + version: 0.2.1 + resolution: "vite-plugin-node-stdlib-browser@npm:0.2.1" + dependencies: + "@rollup/plugin-inject": "npm:^5.0.3" + peerDependencies: + node-stdlib-browser: ^1.2.0 + vite: ^2.0.0 || ^3.0.0 || ^4.0.0 + checksum: 10c0/4686bde59d0396d8684433e1a14ddce868dc422f80e306a0c1cb5e86564d9f7c38a67865ca339e4ff57784ec4bada149034038cad6911a2dfcac8debfc9bd20a + languageName: node + linkType: hard + "vite-plugin-svgr@npm:^4.0.0": version: 4.3.0 resolution: "vite-plugin-svgr@npm:4.3.0" @@ -14104,6 +15087,13 @@ __metadata: languageName: node linkType: hard +"vm-browserify@npm:^1.0.1": + version: 1.1.2 + resolution: "vm-browserify@npm:1.1.2" + checksum: 10c0/0cc1af6e0d880deb58bc974921320c187f9e0a94f25570fca6b1bd64e798ce454ab87dfd797551b1b0cc1849307421aae0193cedf5f06bdb5680476780ee344b + languageName: node + linkType: hard + "void-elements@npm:3.1.0": version: 3.1.0 resolution: "void-elements@npm:3.1.0" @@ -14303,7 +15293,7 @@ __metadata: languageName: node linkType: hard -"which-typed-array@npm:^1.1.19": +"which-typed-array@npm:^1.1.19, which-typed-array@npm:^1.1.2": version: 1.1.19 resolution: "which-typed-array@npm:1.1.19" dependencies: @@ -14428,7 +15418,7 @@ __metadata: languageName: node linkType: hard -"xtend@npm:~4.0.1": +"xtend@npm:^4.0.2, xtend@npm:~4.0.1": version: 4.0.2 resolution: "xtend@npm:4.0.2" checksum: 10c0/366ae4783eec6100f8a02dff02ac907bf29f9a00b82ac0264b4d8b832ead18306797e283cf19de776538babfdcb2101375ec5646b59f08c52128ac4ab812ed0e