diff --git a/.github/labels.yml b/.github/labels.yml index 063787d96..0747a55b7 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -102,6 +102,9 @@ - name: "backport-candidate" description: "Something that is a candidate for backport to a particular release branch" color: "0B8D85" +- name: "Claude-Review" + description: "Request code review from Claude" + color: "e57157" - name: "customer-retainer" color: "F44A5F" - name: "dependencies" diff --git a/.github/workflows/claude.yaml b/.github/workflows/claude.yaml new file mode 100644 index 000000000..1ddbeadaa --- /dev/null +++ b/.github/workflows/claude.yaml @@ -0,0 +1,90 @@ +name: Claude +on: + pull_request: + types: [labeled] + +permissions: + # The action exchanges the GitHub OIDC token for a short-lived Anthropic + # access token. GitHub API access goes through CLAUDE_GH_TOKEN instead. + id-token: write + +jobs: + review: + name: Review + # Only run on labelling and skip forks. + if: github.event.label.name == 'Claude-Review' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + # The review looks at the individual commits, not just the combined diff + fetch-depth: 0 + persist-credentials: false + - name: Review + uses: anthropics/claude-code-action@6fed3ca145920b639991cb756090506e1bcaf515 # v1.0.245 + with: + anthropic_federation_rule_id: fdrl_01XQmWo4nka1ZTqZdsdeScyr + anthropic_organization_id: dfbc9a23-d636-4653-8970-035204ef60fa + anthropic_service_account_id: svac_018Ay5qaA7ZW8X7XsbdekqUM + anthropic_workspace_id: wrkspc_01Aw8LHjBS6fZysu9yqzYifW + # The token used here must grant read access to code, issues, and metadata as well as + # read and write access to pull requests. + github_token: ${{ secrets.CLAUDE_GH_TOKEN }} + use_sticky_comment: true + track_progress: true + prompt: | + Review this pull request under the following (and no further) angles: + + 1. Scope: + + - Does the PR do one thing? + - Is the PR free of unrelated fixes and drive-by refactorings? + - If the PR is large, could it reasonably be split into smaller PRs + that still deliver value? + + 2. Commits: + + - Is the work split into self-contained commits of reasonable size? + - Are the commits arranged in a sensible order? + - Have required refactorings been split out into separate commits? + - Is the history free of "wip", checkpoint and fixup-style commits + which only correct an earlier commit in the same PR? + + 3. Code: + + - Are the code changes functionally correct? + - Do the changes introduce new security vulnerabilities or metadata + leaks? + - Is the new code performant enough? + - Don't check anything that is already validated via other GitHub + workflows like linting, formatting or tests passing. + + Read the commits themselves with `git log` and `git diff` against + the base branch of the PR. + + Drop any findings that are outside the above review angles or that don't + require resolution. Grade the severity of each finding as either "🟡 low", + "🟠 medium" or "🔴 high". Devise at least one possible resolution for each + finding. + + Report the graded findings and resolutions from the "Code" angle as inline + comments. Add a footer "_-- Finding from automated review_" to each inline + comment. + + Finally, report your overall result as PR comment. Start your message with + "# Automated review". Then add one section per review angle with: + + - A verdict of "✅ PASS" or "❌ NEEDS WORK" + - A bullet list summarizing the remaining non-inlined graded findings and + resolutions (as nested bullets). + + Be brief and concise in all your communication. Do not mention what you have + not covered and don't bring up issues that have already been fixed. + + Use Markdown links for anything that can be navigated to on GitHub such + as files, commits, issues, pull requests, workflow runs, etc. + claude_args: | + --model claude-fable-5-1 + --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(git log:*),Bash(git diff:*),Bash(git show:*)" + --max-turns 50 diff --git a/backend/app-service-othersite.yaml b/backend/app-service-othersite.yaml index ef2a7ada7..a80322baf 100644 --- a/backend/app-service-othersite.yaml +++ b/backend/app-service-othersite.yaml @@ -5,7 +5,7 @@ sender_localpart: "_lk_jwt_service" namespaces: users: - exclusive: false - regex: "@.*:homeserver-1" # Cover all local users + regex: "@.*:synapse\\.othersite\\.m\\.localhost" # Cover all local users # Required for Synapse to call POST /_matrix/app/v1/ping on this service. url: "http://auth-server-1:16080" # Enable membership look-ups via /is_joined. diff --git a/backend/app-service.yaml b/backend/app-service.yaml index b480df64e..75716ca59 100644 --- a/backend/app-service.yaml +++ b/backend/app-service.yaml @@ -5,7 +5,7 @@ sender_localpart: "_lk_jwt_service" namespaces: users: - exclusive: false - regex: "@.*:homeserver" # Cover all local users + regex: "@.*:synapse\\.m\\.localhost" # Cover all local users # Required for Synapse to call POST /_matrix/app/v1/ping on this service. url: "http://auth-server:6080" # Enable membership look-ups via /is_joined. diff --git a/backend/dev_livekit-othersite.yaml b/backend/dev_livekit-othersite.yaml index 53fc9ce94..476ecc45c 100644 --- a/backend/dev_livekit-othersite.yaml +++ b/backend/dev_livekit-othersite.yaml @@ -6,6 +6,11 @@ rtc: port_range_start: 50300 port_range_end: 50400 use_external_ip: false + # Advertise the loopback address instead of the container IP, which is not + # routable from the host when Docker runs in a VM (colima, Docker Desktop). + # The browser then reaches the SFU through the published ports. Firefox + # ignores loopback candidates, so this only works in Chromium-based browsers. + node_ip: 127.0.0.1 turn: enabled: false domain: localhost diff --git a/backend/dev_livekit.yaml b/backend/dev_livekit.yaml index 6cef4241a..576c67589 100644 --- a/backend/dev_livekit.yaml +++ b/backend/dev_livekit.yaml @@ -6,6 +6,11 @@ rtc: port_range_start: 50100 port_range_end: 50200 use_external_ip: false + # Advertise the loopback address instead of the container IP, which is not + # routable from the host when Docker runs in a VM (colima, Docker Desktop). + # The browser then reaches the SFU through the published ports. Firefox + # ignores loopback candidates, so this only works in Chromium-based browsers. + node_ip: 127.0.0.1 turn: enabled: false domain: localhost diff --git a/component/CallPortalRoot.test.tsx b/component/CallPortalRoot.test.tsx new file mode 100644 index 000000000..bfa23971b --- /dev/null +++ b/component/CallPortalRoot.test.tsx @@ -0,0 +1,57 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { type ReactNode } from "react"; +import { Tooltip, TooltipProvider } from "@vector-im/compound-web"; + +import { CallPortalRoot } from "./CallPortalRoot"; + +// A button whose accessible name comes from its tooltip, as every icon button +// in the call does: the name is lost whenever the tooltip has nowhere to go. +const LabelledButton = (): ReactNode => ( + + + + + +); + +test("floats into the host document with no root", () => { + render( + + + , + ); + expect(screen.getByRole("button", { name: "Microphone" })).toBeTruthy(); +}); + +test("floats into the root it is given", () => { + const root = document.createElement("div"); + document.body.appendChild(root); + render( + + + , + ); + const button = screen.getByRole("button", { name: "Microphone" }); + const label = document.getElementById( + button.getAttribute("aria-labelledby")!, + ); + expect(root.contains(label)).toBe(true); + root.remove(); +}); + +test("waits for a root that is not available yet", () => { + render( + + + , + ); + expect(screen.queryByRole("button", { name: "Microphone" })).toBeNull(); +}); diff --git a/component/CallPortalRoot.tsx b/component/CallPortalRoot.tsx new file mode 100644 index 000000000..281eb28c0 --- /dev/null +++ b/component/CallPortalRoot.tsx @@ -0,0 +1,26 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type FC, type ReactNode } from "react"; +import { PortalRoot } from "@vector-im/compound-web"; + +interface Props { + root: HTMLElement | null | undefined; + children: ReactNode; +} + +/** + * Where Compound floats the call's menus and tooltips. With no `root` the call + * is not wrapped at all and they go into the host document's body; `null` is + * Compound's "not available yet", and the floating parts wait for it. + */ +export const CallPortalRoot: FC = ({ root, children }): ReactNode => + root === undefined ? ( + children + ) : ( + {children} + ); diff --git a/component/api.ts b/component/api.ts index c9e7a2fd7..fa2928eb1 100644 --- a/component/api.ts +++ b/component/api.ts @@ -121,4 +121,18 @@ export interface ElementCallProps { * the most recently set language wins for all of them. */ language?: string; + /** + * Where the parts of Element Call that float above it — its menus and + * tooltips, which Compound portals out of the call — are rendered. Left out, + * they go into the host document's body, as they do for any other component + * on the host's page. + * + * A host that moves Element Call's DOM into another document, such as a + * Document Picture-in-Picture window, passes that document's body here so + * that a menu opens in the window its button is in. Changes take effect at + * once. `null` means the root is not available yet, as from a callback ref + * that has not fired: menus and tooltips wait for it rather than falling + * back to the body. + */ + portalRoot?: HTMLElement | null; } diff --git a/component/index.tsx b/component/index.tsx index 8f901c139..b3f7b6209 100644 --- a/component/index.tsx +++ b/component/index.tsx @@ -82,6 +82,7 @@ import { useTheme } from "../src/useTheme"; import { useStableValue } from "../src/useStableValue"; import styles from "./ElementCall.module.css"; import { useComponentHostBridge } from "./host"; +import { CallPortalRoot } from "./CallPortalRoot"; import { type ElementCallProps } from "./api"; import { supportedLanguages, translationsBackend } from "./localization"; @@ -154,6 +155,7 @@ export const ElementCall: FC = ({ ref, theme, language, + portalRoot, }): ReactNode => { const hostBridge = useComponentHostBridge(suppliedHostBridge, ref, theme); @@ -240,34 +242,36 @@ export const ElementCall: FC = ({ // Stands in for the standalone page's `#root`: see `.content` // in ElementCall.module.css
- - {/* Whatever goes wrong in here is shown in here. Left to - propagate, an error would unmount the host's own tree. */} - } - // A broken call should not hold the host on screen - onError={() => void hostBridge.setAlwaysOnScreen(false)} - > - - - - - - - - - - - - - + + + {/* Whatever goes wrong in here is shown in here. Left to + propagate, an error would unmount the host's own tree. */} + } + // A broken call should not hold the host on screen + onError={() => void hostBridge.setAlwaysOnScreen(false)} + > + + + + + + + + + + + + + +
)} diff --git a/component/package.json b/component/package.json index 0d45eb4d7..9929b39bb 100644 --- a/component/package.json +++ b/component/package.json @@ -39,7 +39,7 @@ }, "peerDependencies": { "@vector-im/compound-design-tokens": "^11.0.0", - "@vector-im/compound-web": "^10.0.0", + "@vector-im/compound-web": "^10.3.1", "matrix-js-sdk": "*", "react": "^19", "react-dom": "^19" diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 3de89ec38..2db9a63a0 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -3,7 +3,7 @@ networks: services: auth-service: - image: ghcr.io/element-hq/lk-jwt-service:0.7.0 + image: ghcr.io/element-hq/lk-jwt-service:0.8.0 pull_policy: always hostname: auth-server environment: @@ -33,7 +33,7 @@ services: - ecbackend auth-service-1: - image: ghcr.io/element-hq/lk-jwt-service:0.7.0 + image: ghcr.io/element-hq/lk-jwt-service:0.8.0 pull_policy: always hostname: auth-server-1 environment: diff --git a/docker-compose-playwright.yml b/docker-compose-playwright.yml index 889b94bd2..8cf23fa32 100644 --- a/docker-compose-playwright.yml +++ b/docker-compose-playwright.yml @@ -19,7 +19,7 @@ services: - ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z element-web: # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates. - image: ghcr.io/element-hq/element-web:develop@sha256:53a4fe6d60afc0647b95d9e1365d696679241821d63da94dfbfc33171a42ed6d + image: ghcr.io/element-hq/element-web:develop@sha256:06ac258c582ba3e59899cc52c9c2f4fde2e0a2dafddc1e3541c18471162828f8 element-web-1: # Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates. - image: ghcr.io/element-hq/element-web:develop@sha256:53a4fe6d60afc0647b95d9e1365d696679241821d63da94dfbfc33171a42ed6d + image: ghcr.io/element-hq/element-web:develop@sha256:06ac258c582ba3e59899cc52c9c2f4fde2e0a2dafddc1e3541c18471162828f8 diff --git a/docs/url_params.md b/docs/url_params.md index 789e3f3ce..8263055c5 100644 --- a/docs/url_params.md +++ b/docs/url_params.md @@ -61,7 +61,6 @@ These parameters are relevant to both [widget](./embedded_standalone.md) and [st | `homeserver` | | Not applicable | No | Homeserver for registering a new (guest) user, configures non-default guest user server when creating a spa link. | | `lang` | [BCP 47](https://www.rfc-editor.org/info/bcp47) code | No | No | The language the app should use. | | `password` | | No | No | E2EE password when using a shared secret. (For individual sender keys in embedded mode this is not required.) | -| `perParticipantE2EE` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Enables per participant encryption with Keys exchanged over encrypted matrix room messages. | | `controlledAudioDevices` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Whether the [global JS controls for audio devices](./controls.md#audio-devices) should be enabled, allowing the list of audio devices to be controlled by the app hosting Element Call. | | `roomId` | [Matrix Room ID](https://spec.matrix.org/v1.12/appendices/#room-ids) | Yes | No | Anything about what room we're pointed to should be from useRoomIdentifier which parses the path and resolves alias with respect to the default server name, however roomId is an exception as we need the room ID in embedded widget mode, and not the room alias (or even the via params because we are not trying to join it). This is also not validated, where it is in `useRoomIdentifier()`. | | `showControls` | `true` or `false` | No, defaults to `true` | No, defaults to `true` | Displays controls like mute, screen-share, invite, and hangup buttons during a call. | diff --git a/locales/en/app.json b/locales/en/app.json index e31c7d50a..5be1bfff6 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -59,10 +59,10 @@ "developer_mode": { "always_show_iphone_earpiece": "Show iPhone earpiece option on all platforms", "crypto_version": "Crypto version: {{version}}", - "custom_livekit_url": { - "current_url": "Currently set to: ", + "custom_transport": { + "current": "Currently set to: ", "from_config": "Currently, no overwrite is set. Url from config is used.", - "label": "Custom Livekit-url", + "label": "Custom transport JSON", "reset": "Reset overwrite", "save": "Save", "saving": "Saving..." diff --git a/package.json b/package.json index 3127614c5..cd75a791b 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,7 @@ "@typescript-eslint/utils": "^8.61.0", "@use-gesture/react": "^10.2.11", "@vector-im/compound-design-tokens": "^11.0.0", - "@vector-im/compound-web": "^10.0.0", + "@vector-im/compound-web": "^10.4.0", "@vitejs/plugin-react": "^6.0.2", "@vitest/browser-playwright": "^4.1.5", "@vitest/coverage-v8": "^4.0.18", diff --git a/playwright.config.ts b/playwright.config.ts index 847b87c84..2de044194 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -39,8 +39,20 @@ export default defineConfig({ retries: process.env.CI ? 2 : 0, /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, - /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: "html", + /** + * Reporter to use. See https://playwright.dev/docs/test-reporters + * + * On CI, `list` prints a line per test as it finishes, which keeps the job + * log moving; `github` annotates failures on the PR; `html` is uploaded as + * an artifact and must never try to open a browser there. + */ + reporter: process.env.CI + ? [["list"], ["github"], ["html", { open: "never" }]] + : [["list"], ["html"]], + /* Stop a broken run early on CI instead of waiting out every retry. */ + maxFailures: process.env.CI ? 5 : undefined, + /* Under the job's 60 minute limit, so a hung run still ends with a report. */ + globalTimeout: process.env.CI ? 50 * 60 * 1000 : undefined, /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ diff --git a/playwright/component/audio-menu.spec.ts b/playwright/component/audio-menu.spec.ts index f4d811598..3276c28a7 100644 --- a/playwright/component/audio-menu.spec.ts +++ b/playwright/component/audio-menu.spec.ts @@ -129,7 +129,7 @@ async function openDeviceList(page: Page, pane: Locator): Promise { .getByRole("button", { name: "Microphone" }) .click({ timeout: 60_000 }); await expect(page.getByRole("menu")).toBeVisible(); - const list = page.locator("[role='menu'] div[role='none']").first(); + const list = page.getByRole("menu").getByTestId("device-list"); await expect(list).toBeVisible(); return list; } diff --git a/playwright/errors.spec.ts b/playwright/errors.spec.ts index 23f6e29f3..e7eff6592 100644 --- a/playwright/errors.spec.ts +++ b/playwright/errors.spec.ts @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { expect, test } from "@playwright/test"; -import { createJTWToken } from "./fixtures/jwt-token"; +import { createJWTToken } from "./fixtures/jwt-token"; test("Should show error screen if fails to get JWT token", async ({ page }) => { await page.goto("/"); @@ -95,7 +95,7 @@ test("Should show error screen if call creation is restricted", async ({ contentType: "application/json", body: JSON.stringify({ url: "wss://badurltotricktest/livekit/sfu", - jwt: createJTWToken("@fake:user", "!fake:room"), + jwt: createJWTToken("@fake:user", "!fake:room"), }), }), ); diff --git a/playwright/fixtures/jwt-token.ts b/playwright/fixtures/jwt-token.ts index 18119c7e0..dff715f85 100644 --- a/playwright/fixtures/jwt-token.ts +++ b/playwright/fixtures/jwt-token.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -export function createJTWToken(sub: string, room: string): string { +export function createJWTToken(sub: string, room: string): string { return [ {}, // header { diff --git a/playwright/url-transport.spec.ts b/playwright/url-transport.spec.ts new file mode 100644 index 000000000..77fe00827 --- /dev/null +++ b/playwright/url-transport.spec.ts @@ -0,0 +1,493 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Page, test } from "@playwright/test"; + +import { SpaHelpers } from "./spa-helpers"; +import { type RtcMode } from "./widget/test-helpers.ts"; + +const sfuUrl = "wss://matrix-rtc.m.localhost/livekit/sfu"; +const serviceUrl = "https://matrix-rtc.m.localhost/livekit/jwt"; + +const otherHomeserver = { + base_url: "https://synapse.othersite.m.localhost", + server_name: "synapse.othersite.m.localhost", +}; +const otherSfuUrl = "wss://matrix-rtc.othersite.m.localhost/livekit/sfu"; + +interface Transport { + type: "livekit"; + url?: string; + livekit_service_url?: string; +} + +/** + * The transport shapes a homeserver may advertise while migrating to the + * latest MSC4195. `legacy` is what the e2e Synapse advertises on its own. + */ +const shapes = { + url: { type: "livekit", url: sfuUrl }, + both: { type: "livekit", url: sfuUrl, livekit_service_url: serviceUrl }, + legacy: null, +} satisfies Record; +type Shape = keyof typeof shapes; + +/** Which endpoint a page asks for its SFU tokens. */ +type Flow = "cs-api" | "service-legacy" | "service-default"; + +const cases: { + hostShape: Shape; + guestShape: Shape; + /** The MatrixRTC mode both sides use; the default is compatibility mode. */ + mode?: RtcMode; + hostFlows: Flow[]; + guestFlows: Flow[]; +}[] = [ + { + hostShape: "url", + guestShape: "url", + hostFlows: ["cs-api"], + guestFlows: ["cs-api"], + }, + { + // Publishers keep the legacy participant identity that clients predating + // MSC4195 expect, which only the JWT service assigns. + hostShape: "both", + guestShape: "both", + hostFlows: ["service-legacy"], + guestFlows: ["service-legacy"], + }, + { + // The MatrixRTC mode has no bearing on the choice. + hostShape: "both", + guestShape: "both", + mode: "2_0", + hostFlows: ["service-legacy"], + guestFlows: ["service-legacy"], + }, + { + hostShape: "url", + guestShape: "legacy", + hostFlows: ["cs-api", "service-default"], + guestFlows: ["service-legacy", "cs-api"], + }, +]; + +for (const { hostShape, guestShape, mode, hostFlows, guestFlows } of cases) { + test(`One to one call with ${hostShape} host and ${guestShape} guest transport${mode === "2_0" ? " in Matrix 2.0 mode" : ""}`, async ({ + browser, + page, + browserName, + }) => { + skipOnFirefox(browserName); + + // The host starts the call. + await advertiseTransport(page, shapes[hostShape]); + const hostRequests = recordAuthRequests(page); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true, mode); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest joins. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await advertiseTransport(guestPage, shapes[guestShape]); + const guestRequests = recordAuthRequests(guestPage); + await SpaHelpers.joinCallFromInviteLink( + guestPage, + inviteLink, + "Pevara", + mode, + ); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // The host and guest ran through the expected flows. + expect(flowsOf(hostRequests)).toEqual(new Set(hostFlows)); + expect(flowsOf(guestRequests)).toEqual(new Set(guestFlows)); + + // All requests succeeded and the SFU URL was set on C-S requests. + for (const request of [...hostRequests, ...guestRequests]) { + expect(request.status).toBe(200); + if (request.flow === "cs-api") expect(request.sfuUrl).toBe(sfuUrl); + // Clients predating MSC4195 match a publisher by the legacy identity, + // which only the JWT service's legacy endpoint assigns. The other flows + // hash it. + expect(request.identity).toMatch( + request.flow === "service-legacy" ? legacyIdentity : hashedIdentity, + ); + } + }); +} + +test("Subscriber falls back to the JWT service when the homeserver lacks MSC4195", async ({ + browser, + page, + browserName, +}) => { + skipOnFirefox(browserName); + + // The host starts the call with both transport shapes. + await advertiseTransport(page, shapes.both); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest joins but their C-S requests fail. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await rejectCsApiTokens(guestPage); + const guestRequests = recordAuthRequests(guestPage); + await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink, "Pevara"); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // The guest ran through all flows. + expect(flowsOf(guestRequests)).toEqual( + new Set(["cs-api", "service-legacy", "service-default"]), + ); + + // The C-S token requests failed. + for (const request of guestRequests) { + expect(request.status).toBe(request.flow === "cs-api" ? 404 : 200); + } + + // The C-S token requests preceeded the fallback request. + const flows = guestRequests.map((r) => r.flow); + expect(flows.indexOf("service-default")).toBeGreaterThan( + flows.indexOf("cs-api"), + ); +}); + +test("Publisher shows an error when the homeserver lacks MSC4195", async ({ + page, + browserName, +}) => { + skipOnFirefox(browserName); + + // The host tries to start a call with only the C-S transport shape available but + // their C-S requests fail. + await advertiseTransport(page, shapes.url); + await rejectCsApiTokens(page); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + + // We land on an error page. + await expect( + page.getByRole("heading", { name: "Something went wrong" }), + ).toBeVisible(); + await expect( + page.getByText( + "The authorization service for your media server (SFU) is out of date.", + ), + ).toBeVisible(); +}); + +/** + * Delegation follows the flow that authenticated the publisher: the homeserver + * for a `url`-only transport, the JWT service wherever there is one. + */ +const delegationCases: { shape: Shape; via: Delegation["via"] }[] = [ + { shape: "url", via: "homeserver" }, + { shape: "legacy", via: "service" }, + { shape: "both", via: "service" }, +]; + +for (const { shape, via } of delegationCases) { + test(`Delayed leave event delegated through the ${via} for a ${shape} transport gets sent on disconnect`, async ({ + browser, + page, + browserName, + }) => { + skipOnFirefox(browserName); + + // The host starts the call. + await advertiseTransport(page, shapes[shape]); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest joins. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await advertiseTransport(guestPage, shapes[shape]); + const delayedLeave = recordDelayedLeave(guestPage); + const delegations = recordDelegations(guestPage); + await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink, "Pevara"); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // The guest handed its delayed leave over, after probing for delegation + // support with bodiless requests, which count as supported unless they + // 404. The homeserver is always probed; the JWT service only when there + // is one. + await expect + .poll(() => delegations.at(-1)) + .toEqual({ + via, + status: 200, + delayId: delayedLeave.delayId, + ...(via === "homeserver" && { sfuUrl }), + }); + const probes = delegations.slice(0, -1); + expect(probes.some((probe) => probe.via === via)).toBe(true); + for (const probe of probes) { + expect(probe.status).toBeGreaterThanOrEqual(400); + expect(probe.status).not.toBe(404); + } + + // The delay is long enough that a leave within the window below cannot + // come from the delayed event expiring on its own. + expect(delayedLeave.delayMs).toBeGreaterThan(60_000); + + // The guest drops without leaving. + await guestPage.reload(); + + // The host eventually sees the guest go once the delayed event fires. + await expect(page.getByTestId("videoTile")).toHaveCount(1, { + timeout: 30_000, + }); + }); +} + +test("Subscribers get tokens for a remote homeserver's SFU over federation", async ({ + browser, + page, + browserName, +}) => { + skipOnFirefox(browserName); + + // The host lives on the other homeserver and publishes on its SFU. + await useHomeserver(page, otherHomeserver); + await advertiseTransport(page, { type: "livekit", url: otherSfuUrl }); + const hostRequests = recordAuthRequests(page); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest lives on the default homeserver and joins over federation. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await advertiseTransport(guestPage, shapes.url); + const guestRequests = recordAuthRequests(guestPage); + await SpaHelpers.joinCallFromInviteLink( + guestPage, + `${inviteLink}&viaServers=${otherHomeserver.server_name}`, + "Pevara", + ); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // Each side got a token for its own SFU and for the other side's SFU, all + // through its own homeserver. + for (const requests of [hostRequests, guestRequests]) { + expect(flowsOf(requests)).toEqual(new Set(["cs-api"])); + expect(new Set(requests.map((r) => r.sfuUrl))).toEqual( + new Set([sfuUrl, otherSfuUrl]), + ); + for (const request of requests) expect(request.status).toBe(200); + } +}); + +/** Skips the calling test if we're running on Firefox. */ +function skipOnFirefox(browserName: string): void { + test.skip( + browserName === "firefox", + "The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled", + ); +} + +/** Points this page's Element Call at another homeserver. */ +async function useHomeserver( + page: Page, + homeserver: typeof otherHomeserver, +): Promise { + await page.route("**/config.json", async (route) => { + const response = await route.fetch(); + const config = (await response.json()) as Record; + await route.fulfill({ + response, + json: { + ...config, + default_server_config: { "m.homeserver": homeserver }, + }, + }); + }); +} + +/** + * Maps recorded requests to a set of flows. Publisher and subscriber connections + * start concurrently so using arrays for comparison isn't convenient. + */ +function flowsOf(requests: AuthRequest[]): Set { + return new Set(requests.map((r) => r.flow)); +} + +/** + * Replaces the transports the homeserver advertises to this page. The real + * response is fetched first so its CORS headers carry over. + */ +async function advertiseTransport( + page: Page, + transport: Transport | null, +): Promise { + if (transport === null) return; + await page.route("**/org.matrix.msc4143/rtc/transports", async (route) => { + if (route.request().method() !== "GET") return route.continue(); + const response = await route.fetch(); + await route.fulfill({ response, json: { rtc_transports: [transport] } }); + }); +} + +/** + * Makes the homeserver look like one without MSC4195: its `get_token` + * endpoint answers as Synapse does for an unknown route. + */ +async function rejectCsApiTokens(page: Page): Promise { + await page.route("**/rtc/livekit/get_token", async (route) => { + if (route.request().method() !== "POST") return route.continue(); + await route.fulfill({ + status: 404, + json: { errcode: "M_UNRECOGNIZED", error: "Unrecognized request" }, + headers: { "Access-Control-Allow-Origin": "*" }, + }); + }); +} + +/** A recorded delayed leave event. */ +interface DelayedLeave { + delayMs?: number; + delayId?: string; +} + +/** Records the delayed leave event the page schedules on joining. */ +function recordDelayedLeave(page: Page): DelayedLeave { + const delayedLeave: DelayedLeave = {}; + page.on("response", (response) => { + const delay = new URL(response.request().url()).searchParams.get( + "org.matrix.msc4140.delay", + ); + if (delay === null || response.status() !== 200) return; + delayedLeave.delayMs = Number(delay); + response + .json() + .then((body: { delay_id: string }) => { + delayedLeave.delayId = body.delay_id; + }) + .catch(() => {}); + }); + return delayedLeave; +} + +/** A recorded delegation request for a delayed leave event. */ +interface Delegation { + /** Who was asked to send the leave event on the guest's behalf. */ + via: "homeserver" | "service"; + status: number; + sfuUrl?: string; + delayId?: string; +} + +/** + * Records requests that delegate a delayed leave event, or probe for support: + * the MSC4195 `delegate_delayed_leave` endpoint of the homeserver, and the JWT + * service's `delegate_delayed_leave` probe and `/sfu/get` with delay fields. + */ +function recordDelegations(page: Page): Delegation[] { + const delegations: Delegation[] = []; + page.on("response", (response) => { + const request = response.request(); + if (request.method() !== "POST") return; + const url = request.url(); + const body = request.postDataJSON() as { + url?: string; + delay_id?: string; + } | null; + if (url.endsWith("/rtc/livekit/delegate_delayed_leave")) { + delegations.push({ + via: "homeserver", + status: response.status(), + ...(body && { sfuUrl: body.url, delayId: body.delay_id }), + }); + } else if (url.endsWith("/livekit/jwt/delegate_delayed_leave")) { + delegations.push({ via: "service", status: response.status() }); + } else if (url.endsWith("/livekit/jwt/sfu/get") && body?.delay_id) { + delegations.push({ + via: "service", + status: response.status(), + delayId: body.delay_id, + }); + } + }); + return delegations; +} + +/** A recorded token request. */ +interface AuthRequest { + flow: Flow; + status: number; + /** The SFU a request to the MSC4195 `get_token` endpoint asked for. */ + sfuUrl?: string; + /** The participant identity the issued token carries. */ + identity?: string; +} + +/** The legacy `:` participant identity. */ +const legacyIdentity = /^@[^:]+:[^:]+:[A-Z]+$/; +/** A hashed participant identity: unpadded base64 of a SHA-256. */ +const hashedIdentity = /^[A-Za-z0-9+/]{43}$/; + +/** Reads the subject of a JWT without verifying it. */ +function jwtSubject(jwt: string): string { + const [, payload] = jwt.split("."); + return ( + JSON.parse(Buffer.from(payload, "base64url").toString()) as { + sub: string; + } + ).sub; +} + +/** Records token requests, including the identity each token carries. */ +function recordAuthRequests(page: Page): AuthRequest[] { + const requests: AuthRequest[] = []; + page.on("response", (response) => { + const request = response.request(); + if (request.method() !== "POST") return; + const url = request.url(); + let entry: AuthRequest; + if (url.endsWith("/rtc/livekit/get_token")) { + entry = { + flow: "cs-api", + status: response.status(), + sfuUrl: (request.postDataJSON() as { url: string }).url, + }; + } else if (url.endsWith("/livekit/jwt/sfu/get")) { + entry = { flow: "service-legacy", status: response.status() }; + } else if (url.endsWith("/livekit/jwt/get_token")) { + entry = { flow: "service-default", status: response.status() }; + } else { + return; + } + requests.push(entry); + if (entry.status !== 200) return; + response + .json() + .then((body: { jwt: string }) => { + entry.identity = jwtSubject(body.jwt); + }) + .catch(() => {}); + }); + return requests; +} diff --git a/playwright/widget/federated-call.test.ts b/playwright/widget/federated-call.test.ts index 61f4750b3..8359dd233 100644 --- a/playwright/widget/federated-call.test.ts +++ b/playwright/widget/federated-call.test.ts @@ -37,8 +37,6 @@ modePairs.forEach(([rtcMode1, rtcMode2]) => { await TestHelpers.acceptRoomInvite(roomName, timo.page); - await florian.page.pause(); - await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget( florian.page, rtcMode1, @@ -74,8 +72,6 @@ modePairs.forEach(([rtcMode1, rtcMode2]) => { await TestHelpers.expectVisibleVideoCount(frame, 2); } - - // await florian.page.pause(); }, ); }); diff --git a/playwright/widget/url-transport.test.ts b/playwright/widget/url-transport.test.ts new file mode 100644 index 000000000..c4e39cfdd --- /dev/null +++ b/playwright/widget/url-transport.test.ts @@ -0,0 +1,108 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Page, test } from "@playwright/test"; + +import { widgetTest } from "../fixtures/widget-user.ts"; +import { HOST1, TestHelpers } from "./test-helpers.ts"; + +const sfuUrl = "wss://matrix-rtc.m.localhost/livekit/sfu"; +const serviceUrl = "https://matrix-rtc.m.localhost/livekit/jwt"; + +/** Which endpoint a page asks for its SFU tokens. */ +type Flow = "cs-api" | "service-legacy" | "service-default"; + +widgetTest( + "Call between a url + livekit_service_url transport and a legacy one", + async ({ addUser, browserName }) => { + test.skip( + browserName === "firefox", + "The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled", + ); + test.slow(); + + const [host, guest] = await Promise.all([ + addUser("Androl", HOST1), + addUser("Pevara", HOST1), + ]); + + // The host's homeserver has started advertising a `url` next to its JWT + // service; the guest's still advertises the JWT service alone. + await advertiseTransport(host.page, { + type: "livekit", + url: sfuUrl, + livekit_service_url: serviceUrl, + }); + // Element Web answers the widget from a cache it filled at login, so make + // it fetch the transports again now that they are rerouted + await host.clientHandle.evaluate(async (client) => { + await client.cachedRtcTransports.refresh(); + }); + const hostFlows = recordFlows(host.page); + const guestFlows = recordFlows(guest.page); + + const roomName = "Transport Room"; + await TestHelpers.createRoom(roomName, host.page, [guest.mxId]); + await TestHelpers.acceptRoomInvite(roomName, guest.page); + + await TestHelpers.startCallInCurrentRoom(host.page, false); + await TestHelpers.joinCallFromLobby(host.page); + await TestHelpers.joinCallInCurrentRoom(guest.page); + + for (const user of [host, guest]) { + const frame = user.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await expect(frame.getByTestId("videoTile")).toHaveCount(2, { + timeout: 10000, + }); + await expect(frame.getByText("Waiting for media...")).not.toBeVisible({ + timeout: 10000, + }); + await TestHelpers.expectVisibleVideoCount(frame, 2); + } + + // Element Web does not expose the MSC4195 endpoints to widgets yet, so + // neither side reached the CS API. The host published through the JWT + // service's legacy endpoint and subscribed to the guest through its + // default one; the guest published the same way and fell back to the + // default endpoint for the host's transport. + expect(hostFlows).toEqual(new Set(["service-legacy", "service-default"])); + expect(guestFlows).toEqual(new Set(["service-legacy", "service-default"])); + }, +); + +/** + * Replaces the transports the homeserver advertises to this page, including + * to the widget host, which fetches them on the widget's behalf. The real + * response is fetched first so its CORS headers carry over. + */ +async function advertiseTransport( + page: Page, + transport: { type: "livekit"; url?: string; livekit_service_url?: string }, +): Promise { + await page.route("**/org.matrix.msc4143/rtc/transports", async (route) => { + if (route.request().method() !== "GET") return route.continue(); + const response = await route.fetch(); + await route.fulfill({ response, json: { rtc_transports: [transport] } }); + }); +} + +/** Records which token endpoints the page's widget asks. */ +function recordFlows(page: Page): Set { + const flows = new Set(); + page.on("response", (response) => { + const request = response.request(); + if (request.method() !== "POST") return; + const url = request.url(); + if (url.endsWith("/rtc/livekit/get_token")) flows.add("cs-api"); + else if (url.endsWith("/livekit/jwt/sfu/get")) flows.add("service-legacy"); + else if (url.endsWith("/livekit/jwt/get_token")) + flows.add("service-default"); + }); + return flows; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c55527011..cc0d93beb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -132,10 +132,10 @@ importers: version: 10.3.1(react@19.2.8) '@vector-im/compound-design-tokens': specifier: ^11.0.0 - version: 11.1.0(@types/react@19.2.18)(react@19.2.8) + version: 11.2.0(@types/react@19.2.18)(react@19.2.8) '@vector-im/compound-web': - specifier: ^10.0.0 - version: 10.1.0(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@11.1.0(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8) + specifier: ^10.4.0 + version: 10.4.0(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@11.2.0(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8) '@vitejs/plugin-react': specifier: ^6.0.2 version: 6.1.1(@rolldown/plugin-babel@0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.0)(rolldown@1.2.7)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0)))(babel-plugin-react-compiler@1.0.0)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0)) @@ -3223,8 +3223,8 @@ packages: peerDependencies: react: '>= 16.8.0' - '@vector-im/compound-design-tokens@11.1.0': - resolution: {integrity: sha512-vkMcy/G5VBlVle8umQNpctKHqZITS8XOQprsgcbSGRmwW080Z+TXV/vPsEXhgq8JKdxURkLhua0zQUufCeo46A==} + '@vector-im/compound-design-tokens@11.2.0': + resolution: {integrity: sha512-XGwO63ZqFVXdkhCVLm3wa/tpx2Lw8pyaQYskObrLZIUkjs3aUmgq8CNhGcjkqRJXYGsLNJE2azov4DaM74h2PQ==} peerDependencies: '@adobe/leonardo-contrast-colors': ^1.0.0 '@types/react': '*' @@ -3240,13 +3240,13 @@ packages: react: optional: true - '@vector-im/compound-web@10.1.0': - resolution: {integrity: sha512-Lhw9Q+RZo1GtDYjF5PXWTD0zdfbrz/vnSVBYFHC8Hq+YeDule2QfGbTRo8IVHeV7jK6ZzsLc8zb6553eeIRLYg==} + '@vector-im/compound-web@10.4.0': + resolution: {integrity: sha512-Tt5hsNK7ulhGdRuy9lzSm5C+T8M/4O4kLDFLkaU+Hi1kVWj6tj/5awW+U+GvtpWRh5llbHN2HRCh79+gQhek+A==} peerDependencies: '@fontsource/inconsolata': ^5 '@fontsource/inter': ^5 '@types/react': '*' - '@vector-im/compound-design-tokens': '>=1.6.1 <11.0.0' + '@vector-im/compound-design-tokens': '>=1.6.1 <12.0.0' react: ^18 || ^19.0.0 peerDependenciesMeta: '@types/react': @@ -8667,12 +8667,12 @@ snapshots: '@use-gesture/core': 10.3.1 react: 19.2.8 - '@vector-im/compound-design-tokens@11.1.0(@types/react@19.2.18)(react@19.2.8)': + '@vector-im/compound-design-tokens@11.2.0(@types/react@19.2.18)(react@19.2.8)': optionalDependencies: '@types/react': 19.2.18 react: 19.2.8 - '@vector-im/compound-web@10.1.0(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@11.1.0(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)': + '@vector-im/compound-web@10.4.0(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@11.2.0(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)': dependencies: '@floating-ui/react': 0.27.20(react-dom@19.2.8(react@19.2.8))(react@19.2.8) '@floating-ui/react-dom': 2.1.9(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -8684,7 +8684,7 @@ snapshots: '@radix-ui/react-progress': 1.1.16(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) '@radix-ui/react-separator': 1.1.15(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) '@radix-ui/react-slot': 1.3.3(@types/react@19.2.18)(react@19.2.8) - '@vector-im/compound-design-tokens': 11.1.0(@types/react@19.2.18)(react@19.2.8) + '@vector-im/compound-design-tokens': 11.2.0(@types/react@19.2.18)(react@19.2.8) classnames: 2.5.1 react: 19.2.8 vaul: 1.1.2(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) diff --git a/sdk-target-based-on-call-view-model/README.md b/sdk-target-based-on-call-view-model/README.md index 806a6357f..4ead105ae 100644 --- a/sdk-target-based-on-call-view-model/README.md +++ b/sdk-target-based-on-call-view-model/README.md @@ -31,7 +31,7 @@ This file needs to be hosted. Locally (via `npx serve -l 81234 --cors`) or on a Now you just need to add the widget to element web via: ``` -/addwidget http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id +/addwidget http://localhost:3000?widgetId=$matrix_widget_id&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id ``` ## Widgets @@ -47,7 +47,6 @@ by the matrixRTCSdk to start the postmessage widget api (communication between t ``` widgetId = $matrix_widget_id -perParticipantE2EE = true userId = $matrix_user_id deviceId = $org.matrix.msc3819.matrix_device_id baseUrl = $org.matrix.msc4039.matrix_base_url @@ -58,7 +57,7 @@ baseUrl = $org.matrix.msc4039.matrix_base_url Full template use as ``: ``` -http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id +http://localhost:3000?widgetId=$matrix_widget_id&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id ``` the `$` prefixed variables will be replaced by EW on widget instantiation. (e.g. `$matrix_user_id` -> `@user:example.com` (url encoding will also be applied automatically by EW) -> `%40user%3Aexample.com`) diff --git a/sdk-target-based-on-call-view-model/main.ts b/sdk-target-based-on-call-view-model/main.ts index 6cd37032d..3932d1190 100644 --- a/sdk-target-based-on-call-view-model/main.ts +++ b/sdk-target-based-on-call-view-model/main.ts @@ -231,7 +231,7 @@ export async function createMatrixRTCSdk( complete: () => { logger.info("Livekit room items subscription completed"); for (const item of callViewModel.livekitRoomItems$.value) { - logger.info("unregistering room item from room", item.url); + logger.info("unregistering room item from room", item.transport); item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC); } }, diff --git a/src/Modal.test.tsx b/src/Modal.test.tsx index 74670b6db..223bbbc8d 100644 --- a/src/Modal.test.tsx +++ b/src/Modal.test.tsx @@ -11,6 +11,7 @@ import { type ReactNode, useState } from "react"; import userEvent from "@testing-library/user-event"; import { Modal } from "./Modal"; +import { RootElementProvider } from "./RootElementContext"; const originalMatchMedia = window.matchMedia; afterEach(() => { @@ -70,3 +71,31 @@ test("the modal renders as a drawer in mobile viewports", () => { ); expect(queryByRole("dialog")).toMatchSnapshot(); }); + +test("the modal can be closed by clicking the backdrop, in another window too", async () => { + // A dialog moved into another window, as a host does for a Document + // Picture-in-Picture window: its nodes are not `Node`s of this window + const frame = document.createElement("iframe"); + document.body.appendChild(frame); + const otherBody = frame.contentDocument!.body; + + function ModalFn(): ReactNode { + const [isOpen, setOpen] = useState(true); + return ( + + setOpen(false)}> +

This is the content.

+ + + ); + } + render(); + const dialog = otherBody.querySelector('[role="dialog"]'); + expect(dialog).not.toBeNull(); + const backdrop = dialog!.previousElementSibling as HTMLElement; + + const user = userEvent.setup({ document: frame.contentDocument! }); + await user.click(backdrop); + expect(otherBody.querySelector('[role="dialog"]')).toBeNull(); + frame.remove(); +}); diff --git a/src/Modal.tsx b/src/Modal.tsx index 46316d370..01b2c76bc 100644 --- a/src/Modal.tsx +++ b/src/Modal.tsx @@ -160,6 +160,12 @@ export const Modal: FC = ({ onDismiss?.()} /> { - try { - const parsedUrl = new URL(url); - return parsedUrl.hostname; // Returns "kdk.cpm" - } catch (error) { - console.error("Invalid URL:", error); - return url; - } -}; - // This is only used in developer mode for debugging purposes, so we don't need full localization export const RTCConnectionStats: FC = ({ audio, video, - focusUrl, + transport, rtcBackendIdentity, - ...rest }) => { const [showModal, setShowModal] = useState(false); const [modalContents, setModalContents] = useState< @@ -76,10 +67,10 @@ export const RTCConnectionStats: FC = ({ rtcBackendIdentity:{rtcBackendIdentity} - {focusUrl && ( + {transport && (
- -  {extractDomain(focusUrl)} + +  {JSON.stringify(transport)}
)} diff --git a/src/UrlConfiguration.ts b/src/UrlConfiguration.ts index 6de204bb9..2840bad1a 100644 --- a/src/UrlConfiguration.ts +++ b/src/UrlConfiguration.ts @@ -86,11 +86,6 @@ export interface UrlConfiguration { * user's homeserver doesn't provide any. */ allowIceFallback: boolean; - - /** - * Whether the app should use per participant keys for E2EE. - */ - perParticipantE2EE: boolean; /** * Whether the global JS controls for audio output devices should be enabled, * allowing the list of output devices to be controlled by the app hosting @@ -174,7 +169,6 @@ export function configurationForIntent(intent: UserIntent): UrlConfiguration { showControls: true, hideScreensharing: false, allowIceFallback: true, - perParticipantE2EE: true, controlledAudioDevices: platform === "desktop" ? false : true, skipLobby: true, returnToLobby: false, @@ -229,7 +223,6 @@ export function configurationForIntent(intent: UserIntent): UrlConfiguration { showControls: true, hideScreensharing: false, allowIceFallback: false, - perParticipantE2EE: false, controlledAudioDevices: false, skipLobby: false, returnToLobby: false, diff --git a/src/UrlParams.test.ts b/src/UrlParams.test.ts index 65be8ae85..c1f1aeeb0 100644 --- a/src/UrlParams.test.ts +++ b/src/UrlParams.test.ts @@ -230,7 +230,6 @@ describe("UrlParams", () => { showControls: true, hideScreensharing: false, allowIceFallback: false, - perParticipantE2EE: false, controlledAudioDevices: false, skipLobby: false, returnToLobby: false, @@ -243,7 +242,6 @@ describe("UrlParams", () => { showControls: true, hideScreensharing: false, allowIceFallback: true, - perParticipantE2EE: true, controlledAudioDevices: platform === "desktop" ? false : true, skipLobby: true, returnToLobby: false, @@ -256,7 +254,6 @@ describe("UrlParams", () => { showControls: true, hideScreensharing: false, allowIceFallback: true, - perParticipantE2EE: true, controlledAudioDevices: platform === "desktop" ? false : true, skipLobby: false, returnToLobby: false, @@ -499,7 +496,6 @@ describe("UrlParams", () => { // A host owns navigation, so Element Call must not offer a way out of // the room confineToRoom: true, - perParticipantE2EE: true, // The lobby first, so that the user picks their devices rather than // being thrown into the call by the act of being rendered skipLobby: false, @@ -512,7 +508,6 @@ describe("UrlParams", () => { expect(configurationForIntent(UserIntent.Unknown)).toMatchObject({ confineToRoom: false, header: HeaderStyle.Standard, - perParticipantE2EE: false, }); }); }); diff --git a/src/UrlParams.ts b/src/UrlParams.ts index 120d57533..c3b4229de 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -350,7 +350,6 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => { showControls: parser.getFlag("showControls"), hideScreensharing: parser.getFlag("hideScreensharing"), allowIceFallback: parser.getFlag("allowIceFallback"), - perParticipantE2EE: parser.getFlag("perParticipantE2EE"), controlledAudioDevices: parser.getFlag("controlledAudioDevices"), skipLobby: isWidget ? parser.getFlag("skipLobby") : false, // In SPA mode the user should always exit to the home screen when hanging @@ -501,20 +500,14 @@ export function generateUrlSearchParams( const params = new URLSearchParams(); // The password shouldn't need URL encoding here (we generate URL-safe ones) but encode // it in case it came from another client that generated a non url-safe one - switch (encryptionSystem?.kind) { - case E2eeType.SHARED_KEY: { - const encodedPassword = encodeURIComponent(encryptionSystem.secret); - if (encodedPassword !== encryptionSystem.secret) { - logger.info( - "Encoded call password used non URL-safe chars: buggy client?", - ); - } - params.set("password", encodedPassword); - break; + if (encryptionSystem?.kind === E2eeType.SHARED_KEY) { + const encodedPassword = encodeURIComponent(encryptionSystem.secret); + if (encodedPassword !== encryptionSystem.secret) { + logger.info( + "Encoded call password used non URL-safe chars: buggy client?", + ); } - case E2eeType.PER_PARTICIPANT: - params.set("perParticipantE2EE", "true"); - break; + params.set("password", encodedPassword); } params.set("roomId", roomId); viaServers?.forEach((s) => params.set("viaServers", s)); diff --git a/src/__snapshots__/AppBar.test.tsx.snap b/src/__snapshots__/AppBar.test.tsx.snap index 99b322ac3..93f11e96c 100644 --- a/src/__snapshots__/AppBar.test.tsx.snap +++ b/src/__snapshots__/AppBar.test.tsx.snap @@ -15,7 +15,7 @@ exports[`AppBar > renders 1`] = ` tabindex="0" >
renders with title and subtitle 1`] = ` tabindex="0" >
= ({ // Keeps the items the menu's own children for assistive tech. role="none" className={styles.deviceList} + data-testid="device-list" style={ { "--device-list-max-height": diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index 5d18695a9..5f9910662 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -97,11 +97,19 @@ export interface ConfigOptions { }; }; - // Describes the LiveKit configuration to be used. + /** + * Describes the LiveKit configuration to be used. + */ livekit?: { - // The link to the service that returns a livekit url and token to use it. - // This is a fallback link in case the homeserver in use does not advertise - // a livekit service url over the transports endpoint. + /** + * The link to the service that returns a livekit url and token to use it. + * This is a fallback link in case the homeserver in use does not advertise + * a livekit service url over the transports endpoint. + * + * @deprecated Homeservers should advertise available transports over the + * /_matrix/client/unstable/org.matrix.msc4143/rtc/transports endpoint + * instead. + */ livekit_service_url: string; }; diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index e4573cda9..ab252a0e7 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -24,6 +24,7 @@ import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; import { + exampleTransportLocator, mockLocalParticipant, mockMediaDevices, mockRemoteParticipant, @@ -118,7 +119,7 @@ function renderTestComponent( p.identity)} livekitRoom={livekitRoom} - url={""} + transport={exampleTransportLocator} /> , ); @@ -150,7 +151,7 @@ it("should not render or warn for the local participant", () => { , ); @@ -177,7 +178,7 @@ it("should warn only once per unexpected participant", () => { , ); diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 95d68c056..4ca1d3bae 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -19,12 +19,10 @@ import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; +import { type TransportLocator } from "./auth"; export interface MatrixAudioRendererProps { - /** - * The service URL of the LiveKit room. - */ - url: string; + transport: TransportLocator; livekitRoom: LivekitRoom; /** * The list of participant identities to render audio for. @@ -54,7 +52,7 @@ export interface MatrixAudioRendererProps { * @public */ export function LivekitRoomAudioRenderer({ - url, + transport, livekitRoom, validIdentities, muted, @@ -88,7 +86,7 @@ export function LivekitRoomAudioRenderer({ warnedIdentities.current.add(identity); // Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call. logger.warn( - `Audio track ${identity} from ${url} has no matching matrix call member`, + `Audio track ${identity} from ${JSON.stringify(transport)} has no matching matrix call member`, `current members: ${validIdentities.join()}`, `track will not get rendered`, ); diff --git a/src/livekit/auth/csApi.ts b/src/livekit/auth/csApi.ts new file mode 100644 index 000000000..d62111f02 --- /dev/null +++ b/src/livekit/auth/csApi.ts @@ -0,0 +1,137 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { ConnectionError, HTTPError, type MatrixClient } from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { extractFullConfigFromToken, type SFUConfig } from "./types.ts"; +import { doNetworkOperationWithRetry } from "../../utils/matrix.ts"; + +/** + * In widget mode the host may refuse an action before any request is made + * which surfaces as a plain error. Asking again cannot change its answer, + * so only errors from the network are worth retrying. + */ +function isNetworkError(e: unknown): boolean { + return e instanceof HTTPError || e instanceof ConnectionError; +} + +export type ClientGetTokenParts = Pick< + MatrixClient, + "_unstable_getLivekitToken" +>; + +export interface GetSFUConfigParams { + /** + * The Matrix client. + */ + client: ClientGetTokenParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The WebSocket URL of the SFU for which we wish to get an access token. + */ + url: string; + /** + * The name of the homeserver to which the SFU belongs. + */ + serverName: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; +} + +/** + * Gets an {@link SFUConfig} appropriate for connecting to a given SFU, using + * the MSC4195 LiveKit client-server API endpoints. + */ +export async function getSFUConfig({ + client, + membership, + url, + serverName, + roomId, + slotId, +}: GetSFUConfigParams): Promise { + const res = await doNetworkOperationWithRetry( + async () => + client._unstable_getLivekitToken({ + url, + server_name: serverName, + room_id: roomId, + slot_id: slotId, + member_id: membership.memberId, + }), + isNetworkError, + ); + return extractFullConfigFromToken({ url, jwt: res.jwt }); +} + +export type ClientDelegationParts = Pick< + MatrixClient, + "_unstable_delegateDelayedLeave" | "baseUrl" +>; + +export interface DelegateDelayedLeaveParams { + /** + * The Matrix client. + */ + client: ClientDelegationParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The WebSocket URL of the SFU to which we are publishing and wish to + * delegate the delayed leave event. + */ + url: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * The delay ID of the leave event to be delegated. + */ + delayId: string; +} + +/** + * Delegates a delayed leave event to a given SFU, so that the event will + * be sent on our behalf whenever we disconnect from the SFU. + */ +export async function delegateDelayedLeave({ + client, + membership, + url, + roomId, + slotId, + delayId, +}: DelegateDelayedLeaveParams): Promise { + await doNetworkOperationWithRetry( + async () => + client._unstable_delegateDelayedLeave({ + url, + room_id: roomId, + slot_id: slotId, + member_id: membership.memberId, + delay_id: delayId, + }), + isNetworkError, + ); +} diff --git a/src/livekit/auth/index.test.ts b/src/livekit/auth/index.test.ts new file mode 100644 index 000000000..6c48c5032 --- /dev/null +++ b/src/livekit/auth/index.test.ts @@ -0,0 +1,306 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, expect, type Mocked, vi, describe, it } from "vitest"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { + aliceDeviceId, + aliceId, + aliceUserId, + createJWTToken, +} from "../../utils/test-fixtures"; +import type * as csApi from "./csApi"; +import * as openID from "./openID"; +import { delegateDelayedLeave, getSFUConfig } from "."; +import { mocked } from "storybook/test"; +import { ConnectionError, MatrixError } from "matrix-js-sdk"; +import { NoMatrix2AuthorizationService } from "../../utils/errors"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +vi.mock("./openID", () => ({ getSFUConfig: vi.fn() })); +vi.useFakeTimers(); + +const csApiJWTToken = createJWTToken(aliceId, "via CS API"); +const openIDJWTToken = createJWTToken(aliceId, "via OpenID"); + +mocked(openID).getSFUConfig.mockResolvedValue({ + url: "mock-openID-url", + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + livekitIdentity: aliceId, +}); + +const client: Mocked< + csApi.ClientGetTokenParts & + csApi.ClientDelegationParts & + openID.ClientOpenIDParts +> = { + baseUrl: "https://matrix.example.org", + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: csApiJWTToken }), + _unstable_delegateDelayedLeave: vi.fn().mockResolvedValue({}), + getOpenIdToken: vi + .fn() + .mockResolvedValue({ access_token: "mock-openID-token" }), +}; + +const membership: CallMembershipIdentityParts = { + userId: aliceUserId, + deviceId: aliceDeviceId, + memberId: "123", +}; + +const roomId = "!room:example.org"; +const slotId = "m.call#ROOM"; + +const unsupportedError = new MatrixError({ errcode: "M_UNSUPPORTED" }, 404); + +const urlTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://livekit.example.org", +}; + +const serviceUrlTransport: UnstableLivekitTransport = { + type: "livekit", + livekit_service_url: "https://livekit-jwt.example.org", +}; + +const urlAndServiceUrlTransport: UnstableLivekitTransport = { + ...urlTransport, + ...serviceUrlTransport, +}; + +afterEach(() => vi.clearAllMocks()); + +describe("getSFUConfig", () => { + const defaultParams = { + client, + membership, + serverName: "example.org", + role: "subscriber" as const, + roomId, + slotId, + logger, + }; + + it("uses CS API with url transport", async () => { + expect( + await getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).toEqual( + expect.objectContaining({ + jwt: csApiJWTToken, + livekitAlias: "via CS API", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("does not fall back to OpenID with url transport", async () => { + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + await expect( + getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(expect.any(NoMatrix2AuthorizationService)); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("uses OpenID with livekit_service_url transport", async () => { + expect( + await getSFUConfig({ ...defaultParams, transport: serviceUrlTransport }), + ).toEqual( + expect.objectContaining({ + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + }), + ); + expect(client._unstable_getLivekitToken).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("uses CS API with url + livekit_service_url transport", async () => { + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual( + expect.objectContaining({ + jwt: csApiJWTToken, + livekitAlias: "via CS API", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("falls back to OpenID with url + livekit_service_url transport", async () => { + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual( + expect.objectContaining({ + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("falls back to OpenID when the widget host refuses the CS API", async () => { + // In widget mode a host without MSC4533 rejects the request before it + // reaches the homeserver, with a plain error rather than a Matrix error + client._unstable_getLivekitToken.mockRejectedValueOnce( + new Error( + "The rtc_livekit_get_token action is not supported by the client.", + ), + ); + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual(expect.objectContaining({ jwt: openIDJWTToken })); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("surfaces the widget host's refusal with url transport", async () => { + const error = new Error( + "The rtc_livekit_get_token action is not supported by the client.", + ); + client._unstable_getLivekitToken.mockRejectedValueOnce(error); + await expect( + getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(error); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("uses CS API when publishing with url transport", async () => { + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlTransport, + role: "publisher", + }), + ).toEqual(expect.objectContaining({ jwt: csApiJWTToken })); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("does not fall back to OpenID when publishing", async () => { + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + await expect( + getSFUConfig({ + ...defaultParams, + transport: urlTransport, + role: "publisher", + }), + ).rejects.toThrow(expect.any(NoMatrix2AuthorizationService)); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("uses OpenID when publishing with url + livekit_service_url transport", async () => { + // Subscribers predating MSC4195 expect a legacy participant identity from + // anyone advertising a state event, which only the JWT service assigns + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + role: "publisher", + }), + ).toEqual( + expect.objectContaining({ + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + }), + ); + expect(client._unstable_getLivekitToken).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalledWith( + expect.objectContaining({ role: "publisher" }), + ); + }); + + it("does not fall back to OpenID on connection error", async () => { + const client_ = { + ...client, + _unstable_getLivekitToken: vi + .fn() + .mockRejectedValue(new ConnectionError("No connection")), + }; + await Promise.all([ + expect( + getSFUConfig({ + ...defaultParams, + client: client_, + transport: urlAndServiceUrlTransport, + }), + ).rejects.toThrow(expect.any(ConnectionError)), + vi.runAllTimersAsync(), // Allow request to be retried + ]); + expect(client_._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); +}); + +describe("delegateDelayedLeave", () => { + const defaultParams = { + client, + membership, + roomId, + slotId, + delayId: "delay123", + logger, + }; + + it("uses CS API with url transport", async () => { + await delegateDelayedLeave({ ...defaultParams, transport: urlTransport }); + expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("does not fall back to OpenID with url transport", async () => { + client._unstable_delegateDelayedLeave.mockRejectedValueOnce( + unsupportedError, + ); + await expect( + delegateDelayedLeave({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(); + expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("uses OpenID with livekit_service_url transport", async () => { + await delegateDelayedLeave({ + ...defaultParams, + transport: serviceUrlTransport, + }); + expect(client._unstable_delegateDelayedLeave).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("uses OpenID with url + livekit_service_url transport", async () => { + // Must match the flow that got the token, so the identities line up + await delegateDelayedLeave({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }); + expect(client._unstable_delegateDelayedLeave).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalledWith( + expect.objectContaining({ + role: "publisher", + delayId: "delay123", + delayEndpointBaseUrl: client.baseUrl, + }), + ); + }); +}); diff --git a/src/livekit/auth/index.ts b/src/livekit/auth/index.ts new file mode 100644 index 000000000..292dac4d4 --- /dev/null +++ b/src/livekit/auth/index.ts @@ -0,0 +1,253 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { ConnectionError, MatrixError } from "matrix-js-sdk"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import * as openID from "./openID"; +import * as csApi from "./csApi"; +import { type SFUConfig } from "./types"; +import { NoMatrix2AuthorizationService } from "../../utils/errors"; + +export type { ClientOpenIDParts } from "./openID"; +export type { ClientGetTokenParts, ClientDelegationParts } from "./csApi"; +export type * from "./types"; + +/** + * Checks whether an error means "this homeserver does not implement the + * (unstable) MSC4195 endpoints". + * + * MSC4195 documents `M_NOT_FOUND` for this, but a homeserver without support + * answers with whatever it uses for unknown endpoints (Synapse: 404 with + * `M_UNRECOGNIZED`), so we accept any 404 as "unsupported". + * @param e The error to check. + */ +function isEndpointUnsupported(e: unknown): boolean { + return ( + e instanceof MatrixError && + (e.httpStatus === 404 || + e.errcode === "M_NOT_FOUND" || + e.errcode === "M_UNRECOGNIZED") + ); +} + +/** + * Checks whether an error means the MSC4195 CS API cannot be used from here: + * the homeserver does not implement it, or the request never reached the + * homeserver because in widget mode the host does not expose the endpoint + * (MSC4533), which the widget API reports as a plain error rather than a + * Matrix error. A connection error is neither, as the endpoint may well exist. + * @param e The error to check. + */ +function isCsApiUnavailable(e: unknown): boolean { + if (isEndpointUnsupported(e)) return true; + return !(e instanceof MatrixError) && !(e instanceof ConnectionError); +} + +export interface GetSFUConfigParams { + /** + * The Matrix client. + */ + client: csApi.ClientGetTokenParts & openID.ClientOpenIDParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The transport for which we wish to get an access token. + */ + transport: UnstableLivekitTransport; + /** + * The name of the homeserver to which the {@link transport} belongs. + */ + serverName: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * Whether we want to publish or only subscribe on the {@link transport}. + */ + role: "publisher" | "subscriber"; + logger: Logger; +} + +/** + * Gets an {@link SFUConfig} appropriate for connecting to a given transport. + */ +export async function getSFUConfig({ + client, + membership, + transport, + serverName, + roomId, + slotId, + role, + logger, +}: GetSFUConfigParams): Promise { + const viaOpenID = async (serviceUrl: string): Promise => + await openID.getSFUConfig({ + client, + membership, + serviceUrl, + roomId, + slotId, + role, + logger, + }); + + const flow = chooseAuthFlow(transport, role); + if (flow.kind === "openID") return await viaOpenID(flow.serviceUrl); + + try { + return await csApi.getSFUConfig({ + client, + membership, + url: flow.url, + serverName, + roomId, + slotId, + }); + } catch (e) { + // Publishers should never fall back. If a homeserver advertises a `url` + // but doesn't support MSC4195 itself, that's a misconfiguration which + // the admin should know about so they can prepare for the future + // deprecation of `livekit_service_url`. + const mayFallBack = + role === "subscriber" && "livekit_service_url" in transport; + if (mayFallBack && isCsApiUnavailable(e)) { + logger.warn( + `Cannot use the MSC4195 get_token endpoint (${e}), falling back to the OpenID flow with service ${transport.livekit_service_url}`, + ); + return await viaOpenID(transport.livekit_service_url); + } + if (isEndpointUnsupported(e)) { + logger.error( + `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${flow.url}`, + ); + throw new NoMatrix2AuthorizationService(e as Error); + } + throw e; + } +} + +export interface DelegateDelayedLeaveParams { + /** + * The Matrix client. + */ + client: csApi.ClientDelegationParts & openID.ClientOpenIDParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The transport to which we are publishing and wish to delegate the delayed + * leave event. + */ + transport: UnstableLivekitTransport; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * The delay ID of the leave event to be delegated. + */ + delayId: string; + logger: Logger; +} + +/** + * Delegates a delayed leave event to a given transport, so that the event will + * be sent on our behalf whenever we disconnect from the transport's SFU. + */ +export async function delegateDelayedLeave({ + client, + membership, + transport, + roomId, + slotId, + delayId, + logger, +}: DelegateDelayedLeaveParams): Promise { + // It's important that we use the same flow (CS API or OpenID) as whatever we + // would've used to get a JWT token for this transport, since the participant + // identity computed by the service needs to match. + const flow = chooseAuthFlow(transport, "publisher"); + if (flow.kind === "csApi") { + try { + await csApi.delegateDelayedLeave({ + client, + membership, + url: flow.url, + roomId, + slotId, + delayId, + }); + } catch (e) { + if (isEndpointUnsupported(e)) + logger.error( + "Server misconfigured: transport contains a 'url' field but server does not support the MSC4195 delegate_delayed_leave endpoint", + ); + throw e; + } + } else { + // This will technically cause the service to issue a new JWT token, but + // it's safe to discard. We're only interested in triggering delegation. + await openID.getSFUConfig({ + client, + membership, + serviceUrl: flow.serviceUrl, + roomId, + slotId, + role: "publisher", + delayEndpointBaseUrl: client.baseUrl, + delayId, + logger, + }); + } +} + +/** + * The flow with which the local member authenticates against a transport. + */ +export type AuthFlow = + | { kind: "csApi"; url: string } + | { kind: "openID"; serviceUrl: string }; + +/** + * Chooses between the MSC4195 CS API and the legacy OpenID flow against the + * transport's JWT service. + * + * Only the JWT service's `/sfu/get` endpoint assigns the legacy participant + * identity that clients predating MSC4195 expect from anyone advertising a + * membership state event. So a publisher keeps using the JWT service wherever + * the transport still offers one, even if it also has a `url`. Subscribers may + * use the CS API regardless, as nobody matches their identity. + * @param transport The transport to authenticate against. + * @param role Whether we want to publish or only subscribe on the transport. + */ +export function chooseAuthFlow( + transport: UnstableLivekitTransport, + role: "publisher" | "subscriber", +): AuthFlow { + if ("url" in transport) { + const useCsApi = + !("livekit_service_url" in transport) || role === "subscriber"; + if (useCsApi) return { kind: "csApi", url: transport.url }; + } + return { kind: "openID", serviceUrl: transport.livekit_service_url }; +} diff --git a/src/livekit/auth/openID.test.ts b/src/livekit/auth/openID.test.ts new file mode 100644 index 000000000..7498b8fe1 --- /dev/null +++ b/src/livekit/auth/openID.test.ts @@ -0,0 +1,230 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + beforeEach, + afterEach, + describe, + expect, + it, + type MockedObject, + vitest, +} from "vitest"; +import fetchMock from "fetch-mock"; +import { MatrixError } from "matrix-js-sdk"; + +import { + getSFUConfig, + type GetSFUConfigParams, + type ClientOpenIDParts, +} from "./openID"; +import { testJWTToken } from "../../utils/test-fixtures"; +import { ownMemberMock } from "../../utils/test"; +import { FailToGetOpenIdToken } from "../../utils/errors"; +import { logger } from "matrix-js-sdk/lib/logger"; + +const serviceUrl = "https://sfu.example.org"; + +const delayParams = { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", +}; + +describe("getSFUConfig", () => { + let matrixClient: MockedObject; + let params: GetSFUConfigParams; + beforeEach(() => { + fetchMock.catch(404); + matrixClient = { getOpenIdToken: vitest.fn() }; + params = { + client: matrixClient, + membership: ownMemberMock, + serviceUrl: "https://sfu.example.org", + roomId: "!example_room_id", + slotId: "m.call#ROOM", + role: "publisher", + logger, + }; + }); + afterEach(async () => { + void (await fetchMock.flush()); + vitest.clearAllMocks(); + fetchMock.reset(); + }); + + it("should handle fetching a token", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfig(params); + expect(config).toEqual({ + jwt: testJWTToken, + url: serviceUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + }); + + it("should fail if the SFU errors", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { + errcode: "M_LOOKUP_FAILED", + error: "Failed to look up user info from homeserver", + }, + }; + }); + try { + await getSFUConfig(params); + } catch (ex: unknown) { + expect(ex).toBeInstanceOf(FailToGetOpenIdToken); + expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); + const mxError = (ex as Error).cause as MatrixError; + expect(mxError.message).toEqual( + "MatrixError: [500] Failed to look up user info from homeserver", + ); + + return; + } + expect.fail("Expected test to throw;"); + }); + + it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => { + let callCount = 0; + + fetchMock.post( + "https://sfu.example.org/sfu/get", + (url, opts) => { + callCount++; + const body = JSON.parse(opts.body as string); + + // First call: check if it has delay parts and return 400 + if (callCount === 1) { + expect(body).toHaveProperty("delay_id", "mock_delay_id"); + return { + status: 400, + body: { errcode: "M_BAD_JSON", error: "Unsupported parameters" }, + }; + } + + // Second call: check if delay parts were stripped and return success + expect(body).not.toHaveProperty("delay_id"); + expect(body).not.toHaveProperty("delay_timeout"); + expect(body).not.toHaveProperty("delay_cs_api_url"); + + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }, + { overwriteRoutes: true }, + ); + + // Note: Assuming getSFUConfig eventually calls getLiveKitJWT + const config = await getSFUConfig({ ...params, ...delayParams }); + + expect(config.jwt).toBe(testJWTToken); + expect(callCount).toBe(2); + }); + + it("should successfully send delay parameters to the JWT service legacy endpoint", async () => { + fetchMock.post( + "https://sfu.example.org/sfu/get", + (url, opts) => { + const body = JSON.parse(opts.body as string); + + // Verify, that the request contains the expected delay parameters + if ( + body.delay_id === "mock_delay_id" && + body.delay_timeout === 3600000 && + body.delay_cs_api_url === "https://matrix.homeserverserver.org" + ) { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + } + return { + status: 400, + body: { error: "Missing expected delay params" }, + }; + }, + { overwriteRoutes: true }, + ); + + const config = await getSFUConfig({ ...params, ...delayParams }); + + // Prüfe das Ergebnis + expect(config).toMatchObject({ + jwt: testJWTToken, + url: serviceUrl, + }); + }); + + it("should use the default endpoint when subscribing", async () => { + fetchMock.post("https://sfu.example.org/get_token", (url, opts) => { + const body = JSON.parse(opts.body as string); + expect(body).toMatchObject({ + room_id: "!example_room_id", + slot_id: "m.call#ROOM", + member: { + id: ownMemberMock.memberId, + claimed_user_id: ownMemberMock.userId, + claimed_device_id: ownMemberMock.deviceId, + }, + }); + return { status: 200, body: { url: serviceUrl, jwt: testJWTToken } }; + }); + const config = await getSFUConfig({ ...params, role: "subscriber" }); + expect(config.jwt).toBe(testJWTToken); + expect(fetchMock.called("https://sfu.example.org/sfu/get")).toBe(false); + }); + + it("should fail when subscribing if the default endpoint is missing", async () => { + // Nothing but the catch-all 404 answers the default endpoint. Subscribers + // must not fall back to the legacy endpoint, which would hand them a + // legacy identity. + await expect( + getSFUConfig({ ...params, role: "subscriber" }), + ).rejects.toThrow(FailToGetOpenIdToken); + expect(fetchMock.called("https://sfu.example.org/get_token")).toBe(true); + expect(fetchMock.called("https://sfu.example.org/sfu/get")).toBe(false); + }); + + it("should retry fetching the openid token", async () => { + let count = 0; + matrixClient.getOpenIdToken.mockImplementation(async () => { + count++; + if (count < 2) { + throw Error("Test failure"); + } + return Promise.resolve({ + token_type: "Bearer", + access_token: "foobar", + matrix_server_name: "example.org", + expires_in: 30, + }); + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfig({ ...params, ...delayParams }); + expect(config).toEqual({ + jwt: testJWTToken, + url: serviceUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + }); +}); diff --git a/src/livekit/auth/openID.ts b/src/livekit/auth/openID.ts new file mode 100644 index 000000000..4479d4261 --- /dev/null +++ b/src/livekit/auth/openID.ts @@ -0,0 +1,262 @@ +/* +Copyright 2023, 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type IOpenIDToken, + type MatrixClient, + parseErrorResponse, +} from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { FailToGetOpenIdToken } from "../../utils/errors"; +import { doNetworkOperationWithRetry } from "../../utils/matrix"; +import { Config } from "../../config/Config"; +import { extractFullConfigFromToken, type SFUConfig } from "./types"; + +// The bits we need from MatrixClient +export type ClientOpenIDParts = Pick; + +export interface GetSFUConfigParams { + client: ClientOpenIDParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The base URL of the LiveKit JWT service. + */ + serviceUrl: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * Whether we want to publish or only subscribe on the {@link transport}. + */ + role: "publisher" | "subscriber"; + /** + * The base URL of the Matrix homeserver, for delayed leave delegation. + */ + delayEndpointBaseUrl?: string; + /** + * The delay ID of the leave event to be delegated to the JWT service. + */ + delayId?: string; + logger: Logger; +} + +/** + * Gets an {@link SFUConfig} appropriate for connecting to the SFU behind a + * given service URL, using the legacy OpenID flow. + * @throws {@link FailToGetOpenIdToken} + */ +export async function getSFUConfig({ + client, + membership, + serviceUrl, + roomId, + slotId, + role, + delayEndpointBaseUrl, + delayId, + logger, +}: GetSFUConfigParams): Promise { + let openIdToken: IOpenIDToken; + try { + openIdToken = await doNetworkOperationWithRetry(async () => + client.getOpenIdToken(), + ); + } catch (error) { + throw new FailToGetOpenIdToken( + error instanceof Error ? error : new Error("Unknown error"), + ); + } + logger.debug("Got openID token", openIdToken); + let sfuConfig: { url: string; jwt: string } | undefined; + + let endpoint: "default" | "legacy"; + if (role === "publisher") { + // When publishing a legacy transport (one with a `livekit_service_url`), + // subscribers will expect our participant identity to use the legacy + // `@user_id:device_id` format. Only the legacy JWT service endpoint assigns + // identities in this format, so we must use it. + endpoint = "legacy"; + } else { + // Use the default endpoint as it affords us slightly better metadata + // protection (pseudonymous participant identities). + endpoint = "default"; + } + + try { + logger.info( + `Trying to get JWT with ${endpoint} endpoint for focus ${serviceUrl}...`, + ); + sfuConfig = await ( + endpoint === "legacy" ? getLiveKitJWTLegacy : getLiveKitJWT + )( + membership, + serviceUrl, + roomId, + slotId, + openIdToken, + delayEndpointBaseUrl, + delayId, + ); + logger.info(`Got JWT from call's active focus URL.`); + return extractFullConfigFromToken(sfuConfig); + } catch (e) { + logger.error(`Failed fetching jwt with ${endpoint} endpoint:`, e); + throw new FailToGetOpenIdToken( + e instanceof Error ? e : new Error(`Unknown error ${e}`), + ); + } +} + +/** + * Gets a JWT token appropriate for connecting to the SFU behind a given service + * URL, using the legacy `/sfu/get` endpoint, which assigns us a legacy + * (`:`) participant identity. + */ +async function getLiveKitJWTLegacy( + membership: CallMembershipIdentityParts, + livekitServiceURL: string, + matrixRoomId: string, + _slotId: string, // Unused - the legacy endpoint is from before slots existed. + openIDToken: IOpenIDToken, + delayEndpointBaseUrl?: string, + delayId?: string, +): Promise<{ url: string; jwt: string }> { + interface IDelayParams { + delay_id?: string; + delay_timeout?: number; + delay_cs_api_url?: string; + } + let bodyDalayParts: IDelayParams = {}; + // Also check for empty string + if (delayId && delayEndpointBaseUrl) { + bodyDalayParts = { + delay_id: delayId, + delay_timeout: + Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, + delay_cs_api_url: delayEndpointBaseUrl, + }; + } + + const makeRequest = async (delayParts: IDelayParams): Promise => { + return await fetch(livekitServiceURL + "/sfu/get", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + // The legacy JWT endpoint uses only the matrix room id to calculate the livekit room alias. + // However, the livekit room alias is provided as part of the JWT payload. + room: matrixRoomId, + openid_token: openIDToken, + device_id: membership.deviceId, + ...delayParts, + }), + }); + }; + + const res = await doNetworkOperationWithRetry(async () => { + let response = await makeRequest(bodyDalayParts); + + // Old service compatibility check + const oldServiceDoesNotSupportDelayParts = + response.status === 400 && Object.keys(bodyDalayParts).length > 0; + // If http status 400 with M_BAD_JSON and we sent delay parts, retry without them + if (oldServiceDoesNotSupportDelayParts) { + try { + const errorBody = await response.json(); + if (errorBody.errcode === "M_BAD_JSON") { + response = await makeRequest({}); + } + } catch { + // If we can't parse the error, treat as real error + } + } + + return response; + }); + + if (!res.ok) { + throw parseErrorResponse(res, await res.text()); + } + return await res.json(); +} + +class NotSupportedError extends Error { + public constructor(message: string) { + super(message); + this.name = "NotSupported"; + } +} + +/** + * Gets a JWT token appropriate for connecting to the SFU behind a given service + * URL, using the default `/get_token` endpoint, which assigns us a hashed + * (pseudonymised) participant identity. + */ +async function getLiveKitJWT( + membership: CallMembershipIdentityParts, + livekitServiceURL: string, + matrixRoomId: string, + slotId: string, + openIDToken: IOpenIDToken, + delayEndpointBaseUrl?: string, + delayId?: string, +): Promise<{ url: string; jwt: string }> { + const { userId, deviceId, memberId } = membership; + + const body = { + room_id: matrixRoomId, + slot_id: slotId, + openid_token: openIDToken, + member: { + id: memberId, + claimed_user_id: userId, + claimed_device_id: deviceId, + }, + }; + + let bodyDalayParts = {}; + // Also check for empty string + if (delayId && delayEndpointBaseUrl) { + bodyDalayParts = { + delay_id: delayId, + delay_timeout: + Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, + delay_cs_api_url: delayEndpointBaseUrl, + }; + } + + const res = await doNetworkOperationWithRetry(async () => { + return await fetch(livekitServiceURL + "/get_token", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ ...body, ...bodyDalayParts }), + }); + }); + + if (!res.ok) { + const msg = "SFU Config fetch failed with status code " + res.status; + if (res.status === 404) { + throw new NotSupportedError(msg); + } else { + throw parseErrorResponse(res, await res.text()); + } + } + return await res.json(); +} diff --git a/src/livekit/auth/types.ts b/src/livekit/auth/types.ts new file mode 100644 index 000000000..409d88a6b --- /dev/null +++ b/src/livekit/auth/types.ts @@ -0,0 +1,90 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; + +/** + * Data identifying a transport and the homeserver to which it belongs, so + * consumers know how to reach it. + */ +export interface TransportLocator { + transport: UnstableLivekitTransport; + serverName: string; +} + +/** + * Configuration and access tokens provided by the SFU on successful authentication. + */ +export interface SFUConfig { + /** + * The WebSocket URL of the LiveKit SFU. This is what we connect the LiveKit + * room to. Note that this is NOT the JWT service URL + * (`livekit_service_url`): the legacy JWT service returns this URL as part of + * its response, while the CS-Api based flow requires us to know it upfront. + */ + url: string; + jwt: string; + livekitAlias: string; + // NOTE: Currently unused. + livekitIdentity: string; +} + +/** + * Decoded details from the JWT. + */ +interface SFUJWTPayload { + /** + * Expiration time for the JWT. + * Note: This value is in seconds since Unix epoch. + */ + exp: number; + /** + * Name of the instance which authored the JWT + */ + iss: string; + /** + * Time at which the JWT can start to be used. + * Note: This value is in seconds since Unix epoch. + */ + nbf: number; + /** + * Subject. The Livekit alias in this context. + */ + sub: string; + /** + * The set of permissions for the user. + */ + video: { + canPublish: boolean; + canSubscribe: boolean; + room: string; + roomJoin: boolean; + }; +} + +/** + * Complements the SFU websocket url and the JWT with the information encoded in + * the JWT payload itself. + * @param sfuConfig The SFU websocket url and the JWT to connect with. + * @returns The full SFU config, including the LiveKit alias and identity. + */ +export function extractFullConfigFromToken(sfuConfig: { + url: string; + jwt: string; +}): SFUConfig { + const [, payloadStr] = sfuConfig.jwt.split("."); + const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; + return { + jwt: sfuConfig.jwt, + url: sfuConfig.url, + livekitAlias: payload.video.room, + // NOTE: Currently unused. + // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. + // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) + livekitIdentity: payload.sub, + }; +} diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts deleted file mode 100644 index f759a3aac..000000000 --- a/src/livekit/openIDSFU.test.ts +++ /dev/null @@ -1,329 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - beforeEach, - afterEach, - describe, - expect, - it, - type MockedObject, - vitest, -} from "vitest"; -import fetchMock from "fetch-mock"; -import { MatrixError } from "matrix-js-sdk"; - -import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU"; -import { testJWTToken } from "../utils/test-fixtures"; -import { ownMemberMock } from "../utils/test"; -import { FailToGetOpenIdToken } from "../utils/errors"; - -const sfuUrl = "https://sfu.example.org"; - -describe("getSFUConfigWithOpenID", () => { - let matrixClient: MockedObject; - beforeEach(() => { - fetchMock.catch(404); - matrixClient = { - getOpenIdToken: vitest.fn(), - getDeviceId: vitest.fn(), - }; - }); - afterEach(() => { - vitest.clearAllMocks(); - fetchMock.reset(); - }); - - it("should handle fetching a token", async () => { - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - expect(config).toEqual({ - jwt: testJWTToken, - url: sfuUrl, - livekitIdentity: "@me:example.org:ABCDEF", - livekitAlias: "!example_room_id", - }); - void (await fetchMock.flush()); - }); - - it("should fail if the SFU errors", async () => { - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - } catch (ex: unknown) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); - const mxError = (ex as Error).cause as MatrixError; - expect(mxError.message).toEqual( - "MatrixError: [500] Failed to look up user info from homeserver", - ); - - void (await fetchMock.flush()); - return; - } - expect.fail("Expected test to throw;"); - }); - - it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => { - let callCount = 0; - - fetchMock.post( - "https://sfu.example.org/sfu/get", - (url, opts) => { - callCount++; - const body = JSON.parse(opts.body as string); - - // First call: check if it has delay parts and return 400 - if (callCount === 1) { - expect(body).toHaveProperty("delay_id", "mock_delay_id"); - return { - status: 400, - body: { errcode: "M_BAD_JSON", error: "Unsupported parameters" }, - }; - } - - // Second call: check if delay parts were stripped and return success - expect(body).not.toHaveProperty("delay_id"); - expect(body).not.toHaveProperty("delay_timeout"); - expect(body).not.toHaveProperty("delay_cs_api_url"); - - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }, - { overwriteRoutes: true }, - ); - - // Note: Assuming getSFUConfigWithOpenID eventually calls getLiveKitJWT - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - }, - ); - - expect(config.jwt).toBe(testJWTToken); - expect(callCount).toBe(2); - void (await fetchMock.flush()); - }); - - it("should successfully send delay parameters to the JWT service legacy endpoint", async () => { - fetchMock.post( - "https://sfu.example.org/sfu/get", - (url, opts) => { - const body = JSON.parse(opts.body as string); - - // Verify, that the request contains the expected delay parameters - if ( - body.delay_id === "mock_delay_id" && - body.delay_timeout === 3600000 && - body.delay_cs_api_url === "https://homeserverserver.org/cs_api" - ) { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - } - return { - status: 400, - body: { error: "Missing expected delay params" }, - }; - }, - { overwriteRoutes: true }, - ); - - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://homeserverserver.org/cs_api", - delayId: "mock_delay_id", - }, - ); - - // Prüfe das Ergebnis - expect(config).toMatchObject({ - jwt: testJWTToken, - url: sfuUrl, - }); - - void (await fetchMock.flush()); - }); - - it("should try legacy and then new endpoint with delay delegation", async () => { - fetchMock.post("https://sfu.example.org/get_token", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - }, - ); - } catch (ex) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); - const mxError = (ex as Error).cause as MatrixError; - expect(mxError.message).toEqual( - "MatrixError: [500] Failed to look up user info from homeserver", - ); - void (await fetchMock.flush()); - } - const calls = fetchMock.calls(); - expect(calls.length).toBe(2); - - expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); - expect(calls[0][1]).toStrictEqual({ - // check if it uses correct delayID! - body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - method: "POST", - headers: { - "Content-Type": "application/json", - }, - }); - - expect(calls[1][0]).toStrictEqual("https://sfu.example.org/sfu/get"); - - expect(calls[1][1]).toStrictEqual({ - body: '{"room":"!example_room_id","device_id":"DEVICE","delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - headers: { - "Content-Type": "application/json", - }, - method: "POST", - }); - }); - - it("dont try legacy if endpoint with delay delegation is sucessful", async () => { - fetchMock.post("https://sfu.example.org/get_token", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { error: "Test failure" }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - }, - ); - } catch (ex) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toEqual( - new Error("SFU Config fetch failed with status code 500"), - ); - void (await fetchMock.flush()); - } - const calls = fetchMock.calls(); - expect(calls.length).toBe(1); - - expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); - expect(calls[0][1]).toStrictEqual({ - // check if it uses correct delayID! - body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - method: "POST", - headers: { - "Content-Type": "application/json", - }, - }); - }); - - it("should retry fetching the openid token", async () => { - let count = 0; - matrixClient.getOpenIdToken.mockImplementation(async () => { - count++; - if (count < 2) { - throw Error("Test failure"); - } - return Promise.resolve({ - token_type: "Bearer", - access_token: "foobar", - matrix_server_name: "example.org", - expires_in: 30, - }); - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - expect(config).toEqual({ - jwt: testJWTToken, - url: sfuUrl, - livekitIdentity: "@me:example.org:ABCDEF", - livekitAlias: "!example_room_id", - }); - void (await fetchMock.flush()); - }); -}); diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts deleted file mode 100644 index cf1f3c951..000000000 --- a/src/livekit/openIDSFU.ts +++ /dev/null @@ -1,315 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type IOpenIDToken, - type MatrixClient, - parseErrorResponse, -} from "matrix-js-sdk"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { type Logger } from "matrix-js-sdk/lib/logger"; - -import { - FailToGetOpenIdToken, - NoMatrix2AuthorizationService, -} from "../utils/errors"; -import { doNetworkOperationWithRetry } from "../utils/matrix"; -import { Config } from "../config/Config"; -import { MatrixRTCMode } from "../config/ConfigOptions"; - -/** - * Configuration and access tokens provided by the SFU on successful authentication. - */ -export interface SFUConfig { - url: string; - jwt: string; - livekitAlias: string; - // NOTE: Currently unused. - livekitIdentity: string; -} - -/** - * Decoded details from the JWT. - */ -interface SFUJWTPayload { - /** - * Expiration time for the JWT. - * Note: This value is in seconds since Unix epoch. - */ - exp: number; - /** - * Name of the instance which authored the JWT - */ - iss: string; - /** - * Time at which the JWT can start to be used. - * Note: This value is in seconds since Unix epoch. - */ - nbf: number; - /** - * Subject. The Livekit alias in this context. - */ - sub: string; - /** - * The set of permissions for the user. - */ - video: { - canPublish: boolean; - canSubscribe: boolean; - room: string; - roomJoin: boolean; - }; -} - -// The bits we need from MatrixClient -export type OpenIDClientParts = Pick< - MatrixClient, - "getOpenIdToken" | "getDeviceId" ->; - -/** - * Gets a bearer token from the homeserver and then use it to authenticate - * to the matrix RTC backend in order to get acces to the SFU. - * It has built-in retry for calls to the homeserver with a backoff policy. - * @param client The Matrix client - * @param membership Our own membership identity parts used to send to jwt service. - * @param serviceUrl The URL of the livekit SFU service - * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. - * @param opts Additional options to modify which endpoint with which data will be used to acquire the jwt token. - * @param opts.matrixRTCMode Determines which version of the JWT endpoint to use, which affects whether the - * RTC backend identity is based on string concatenation (legacy) or a hash (Matrix 2.0). - * This function by default uses whatever is possible with the current jwt service installed next to the SFU. - * For remote connections this does not matter, since we will not publish there we can rely on the newest option. - * @param opts.delayEndpointBaseUrl The URL of the matrix homeserver. - * @param opts.delayId The delay id used for the jwt service to manage. - * @param logger optional logger. - * @returns Object containing the token information - * @throws FailToGetOpenIdToken - */ -export async function getSFUConfigWithOpenID( - client: OpenIDClientParts, - membership: CallMembershipIdentityParts, - serviceUrl: string, - roomId: string, - opts?: { - matrixRTCMode?: MatrixRTCMode; - delayEndpointBaseUrl?: string; - delayId?: string; - }, - logger?: Logger, -): Promise { - let openIdToken: IOpenIDToken; - try { - openIdToken = await doNetworkOperationWithRetry(async () => - client.getOpenIdToken(), - ); - } catch (error) { - throw new FailToGetOpenIdToken( - error instanceof Error ? error : new Error("Unknown error"), - ); - } - logger?.debug("Got openID token", openIdToken); - let sfuConfig: { url: string; jwt: string } | undefined; - - const tryBothJwtEndpoints = opts?.matrixRTCMode === undefined; // This is for SFUs where we do not publish. - - const forceMatrix2Jwt = opts?.matrixRTCMode === MatrixRTCMode.Matrix_2_0; - - // We want to start using the new endpoint (with optional delay delegation) - // if we can use both or if we are forced to use the new one. - if (tryBothJwtEndpoints || forceMatrix2Jwt) { - try { - logger?.info( - `Trying to get JWT via default endpoint for focus ${serviceUrl}...`, - ); - const sfuConfig = await getLiveKitJWT( - membership, - serviceUrl, - roomId, - openIdToken, - opts?.delayEndpointBaseUrl, - opts?.delayId, - ); - - return extractFullConfigFromToken(sfuConfig); - } catch (e) { - logger?.debug(`Failed fetching jwt with matrix 2.0 endpoint:`, e); - // Make this throw a hard error in case we force the matrix2.0 endpoint. - if (forceMatrix2Jwt) { - throw new NoMatrix2AuthorizationService(e as Error); - } - } - } - - // DEPRECATED - // here we either have a sfuConfig or we already exited because of `if (forceMatrix2) throw ...` - // The only case we can get into this condition is, if `forceMatrix2` is `false` - try { - logger?.info( - `Trying to get JWT with legacy endpoint for focus ${serviceUrl}...`, - ); - sfuConfig = await getLiveKitJWTLegacy( - membership.deviceId, - serviceUrl, - roomId, - openIdToken, - opts?.delayEndpointBaseUrl, - opts?.delayId, - ); - logger?.info(`Got JWT from call's active focus URL.`); - return extractFullConfigFromToken(sfuConfig); - } catch (ex) { - throw new FailToGetOpenIdToken( - ex instanceof Error ? ex : new Error(`Unknown error ${ex}`), - ); - } -} - -function extractFullConfigFromToken(sfuConfig: { - url: string; - jwt: string; -}): SFUConfig { - const [, payloadStr] = sfuConfig.jwt.split("."); - const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; - return { - jwt: sfuConfig.jwt, - url: sfuConfig.url, - livekitAlias: payload.video.room, - // NOTE: Currently unused. - // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. - // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) - livekitIdentity: payload.sub, - }; -} - -async function getLiveKitJWTLegacy( - deviceId: string, - livekitServiceURL: string, - matrixRoomId: string, - openIDToken: IOpenIDToken, - delayEndpointBaseUrl?: string, - delayId?: string, -): Promise<{ url: string; jwt: string }> { - interface IDelayParams { - delay_id?: string; - delay_timeout?: number; - delay_cs_api_url?: string; - } - let bodyDalayParts: IDelayParams = {}; - // Also check for empty string - if (delayId && delayEndpointBaseUrl) { - bodyDalayParts = { - delay_id: delayId, - delay_timeout: - Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, - delay_cs_api_url: delayEndpointBaseUrl, - }; - } - - const makeRequest = async (delayParts: IDelayParams): Promise => { - return await fetch(livekitServiceURL + "/sfu/get", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - // The legacy JWT endpoint uses only the matrix room id to calculate the livekit room alias. - // However, the livekit room alias is provided as part of the JWT payload. - room: matrixRoomId, - openid_token: openIDToken, - device_id: deviceId, - ...delayParts, - }), - }); - }; - - const res = await doNetworkOperationWithRetry(async () => { - let response = await makeRequest(bodyDalayParts); - - // Old service compatibility check - const oldServiceDoesNotSupportDelayParts = - response.status === 400 && Object.keys(bodyDalayParts).length > 0; - // If http status 400 with M_BAD_JSON and we sent delay parts, retry without them - if (oldServiceDoesNotSupportDelayParts) { - try { - const errorBody = await response.json(); - if (errorBody.errcode === "M_BAD_JSON") { - response = await makeRequest({}); - } - } catch { - // If we can't parse the error, treat as real error - } - } - - return response; - }); - - if (!res.ok) { - throw parseErrorResponse(res, await res.text()); - } - return await res.json(); -} - -class NotSupportedError extends Error { - public constructor(message: string) { - super(message); - this.name = "NotSupported"; - } -} - -export async function getLiveKitJWT( - membership: CallMembershipIdentityParts, - livekitServiceURL: string, - matrixRoomId: string, - openIDToken: IOpenIDToken, - delayEndpointBaseUrl?: string, - delayId?: string, -): Promise<{ url: string; jwt: string }> { - const { userId, deviceId, memberId } = membership; - - const body = { - room_id: matrixRoomId, - slot_id: "m.call#ROOM", - openid_token: openIDToken, - member: { - id: memberId, - claimed_user_id: userId, - claimed_device_id: deviceId, - }, - }; - - let bodyDalayParts = {}; - // Also check for empty string - if (delayId && delayEndpointBaseUrl) { - bodyDalayParts = { - delay_id: delayId, - delay_timeout: - Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, - delay_cs_api_url: delayEndpointBaseUrl, - }; - } - - const res = await doNetworkOperationWithRetry(async () => { - return await fetch(livekitServiceURL + "/get_token", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ ...body, ...bodyDalayParts }), - }); - }); - - if (!res.ok) { - const msg = "SFU Config fetch failed with status code " + res.status; - if (res.status === 404) { - throw new NotSupportedError(msg); - } else { - throw parseErrorResponse(res, await res.text()); - } - } - return await res.json(); -} diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 9ab64dfd6..1d8fd33d2 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -17,7 +17,7 @@ import { import { act } from "react"; import { type RoomMember } from "matrix-js-sdk"; import { - type LivekitTransport, + type UnstableLivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; @@ -46,7 +46,7 @@ vitest.mock("../useAudioContext"); vitest.mock("../soundUtils"); vitest.mock("../rtcSessionHelpers", async (importOriginal) => ({ ...(await importOriginal()), - makeTransport: (): [LivekitTransport] => [exampleTransport], + makeTransport: (): [UnstableLivekitTransport] => [exampleTransport], })); afterEach(() => { diff --git a/src/room/CallView.tsx b/src/room/CallView.tsx index 3c34dc845..5e0471b74 100644 --- a/src/room/CallView.tsx +++ b/src/room/CallView.tsx @@ -242,12 +242,7 @@ const LoadedCallView: FC = ({ const { displayName, avatarUrl } = useProfile(client); const roomName = useRoomName(room); const roomAvatar = useRoomAvatar(room); - const { - perParticipantE2EE, - returnToLobby, - password: passwordFromUrl, - header, - } = useUrlParams(); + const { returnToLobby, password: passwordFromUrl, header } = useUrlParams(); const e2eeSystem = useRoomEncryptionSystem(room.roomId); // Save the password once we start the groupCallView @@ -365,7 +360,6 @@ const LoadedCallView: FC = ({ rtcSession, preload, skipLobby, - perParticipantE2EE, mediaDevices, latestMuteStates, setJoined, diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b8d7ab89d..bc0d68c4e 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -665,10 +665,10 @@ export const InCallView: FC = ({ onPointerOut={onPointerOut} > {header} - {audioParticipants.map(({ livekitRoom, url, participants }) => ( + {audioParticipants.map(({ livekitRoom, transport, participants }) => ( = ({ tab={settingsTab} onTabChange={setSettingsTab} developerSettingsVm={developerSettingsVm} - livekitRooms={allConnections - .getConnections() - .map((connectionItem) => ({ - room: connectionItem.livekitRoom, - livekitAlias: connectionItem.livekitAlias, - // TODO compute is local or tag it in the livekit room items already - isLocal: undefined, - url: connectionItem.transport.livekit_service_url, - }))} + livekitRooms={allConnections.map( + ({ livekitRoom: room, livekitAlias, transport, role }) => ({ + room, + livekitAlias, + role, + url: + // It should be relatively clear to the user which kind of URL + // this is, based on the scheme (ws:// vs. http://) + "url" in transport + ? transport.url + : transport.livekit_service_url, + }), + )} /> )} diff --git a/src/room/__snapshots__/InCallView.test.tsx.snap b/src/room/__snapshots__/InCallView.test.tsx.snap index b7f8b3fe3..fd0ff4661 100644 --- a/src/room/__snapshots__/InCallView.test.tsx.snap +++ b/src/room/__snapshots__/InCallView.test.tsx.snap @@ -181,7 +181,7 @@ exports[`InCallView > rendering > renders 1`] = ` tabindex="0" >
renders with AppBar android 1`] = ` tabindex="0" >
renders with AppBar android 1`] = ` tabindex="0" >
renders with AppBar ios 1`] = ` tabindex="0" >
renders with AppBar ios 1`] = ` tabindex="0" >
renders with header and participant count 1`] = ` tabindex="0" >
({ @@ -36,8 +36,8 @@ vi.mock("../UrlParams", () => ({ })); // IMPORTANT: mock the same specifier used by DeveloperSettingsTab -vi.mock("../livekit/openIDSFU", () => ({ - getSFUConfigWithOpenID: vi.fn().mockResolvedValue({ +vi.mock("../livekit/auth", () => ({ + getSFUConfig: vi.fn().mockResolvedValue({ url: "mock-url", jwt: "mock-jwt", }), @@ -77,6 +77,21 @@ function createMockMatrixClient(): MatrixClient { } as unknown as MatrixClient; } +/** + * Types the JSON representation of an object into the given element. + */ +async function typeJson( + user: UserEvent, + element: Element, + object: object, +): Promise { + // `user.type` requires us to escape brackets by doubling them + await user.type( + element, + JSON.stringify(object).replace(/([{[\]])/g, (c) => c + c), + ); +} + describe("DeveloperSettingsTab", () => { it("renders and matches snapshot", async () => { const client = createMockMatrixClient(); @@ -124,17 +139,18 @@ describe("DeveloperSettingsTab", () => { expect(container).toMatchSnapshot(); }); - describe("custom livekit url", () => { + describe("custom transport", () => { afterEach(() => { - customLivekitUrlSetting.setValue(null); + customTransportSetting.setValue(null); }); const client = { doesServerSupportUnstableFeature: vi.fn().mockResolvedValue(true), getCrypto: () => ({ getVersion: (): string => "x" }), getUserId: () => "@u:hs", getDeviceId: () => "DEVICE", + getDomain: () => "hs", } as unknown as MatrixClient; - it("will not update custom livekit url without roomId", async () => { + it("will not update custom transport without roomId", async () => { const user = userEvent.setup(); render( @@ -147,17 +163,17 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); - await user.type(input, "wss://example.livekit.invalid"); + await typeJson(user, input, exampleTransport); const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfigWithOpenID).not.toHaveBeenCalled(); + expect(getSFUConfig).not.toHaveBeenCalled(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); - it("will not update custom livekit url without text in input", async () => { + it("will not update custom transport without text in input", async () => { const user = userEvent.setup(); render( @@ -171,16 +187,16 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfigWithOpenID).not.toHaveBeenCalled(); + expect(getSFUConfig).not.toHaveBeenCalled(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); - it("will not update custom livekit url when pressing cancel", async () => { + it("will not update custom transport when pressing cancel", async () => { const user = userEvent.setup(); render( @@ -194,7 +210,7 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); await user.type(input, "wss://example.livekit.invalid"); @@ -202,11 +218,11 @@ describe("DeveloperSettingsTab", () => { name: "Reset overwrite", }); await user.click(cancelButton); - expect(getSFUConfigWithOpenID).not.toHaveBeenCalled(); + expect(getSFUConfig).not.toHaveBeenCalled(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); - it("will update custom livekit url", async () => { + it("will update custom transport", async () => { const user = userEvent.setup(); render( @@ -220,24 +236,28 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); - await user.type(input, "wss://example.livekit.valid"); + await typeJson(user, input, { + type: "livekit", + url: "wss://example.livekit.valid", + }); const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfigWithOpenID).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - "wss://example.livekit.valid", - "#testRoom", + expect(getSFUConfig).toHaveBeenCalledWith( + expect.objectContaining({ + transport: { type: "livekit", url: "wss://example.livekit.valid" }, + roomId: "#testRoom", + }), ); - expect(customLivekitUrlSetting.getValue()).toBe( - "wss://example.livekit.valid", - ); + expect(customTransportSetting.getValue()).toEqual({ + type: "livekit", + url: "wss://example.livekit.valid", + }); }); - it("will show error on invalid url", async () => { + it("will show error on unreachable transport", async () => { const user = userEvent.setup(); render( @@ -251,24 +271,27 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); - await user.type(input, "wss://example.livekit.valid"); + await typeJson(user, input, { + type: "livekit", + url: "wss://example.livekit.valid", + }); const saveButton = screen.getByRole("button", { name: "Save" }); - (getSFUConfigWithOpenID as Mock).mockImplementation(() => { - throw new Error("Invalid URL"); + (getSFUConfig as Mock).mockImplementation(() => { + throw new Error("Failed to get SFU config"); }); await user.click(saveButton); expect( - screen.getByText("invalid URL (did not update)"), + screen.getByText("invalid transport (did not update)"), ).toBeInTheDocument(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); }); // Add this test inside the describe("DeveloperSettingsTab", () => { block, - // after the custom livekit url tests: + // after the custom transport tests: describe("enable extended livekit logs", () => { afterEach(() => { diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index ec5bf7b08..e31282e81 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -45,7 +45,7 @@ import { muteAllAudio as muteAllAudioSetting, alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, matrixRTCMode as matrixRTCModeSetting, - customLivekitUrl as customLivekitUrlSetting, + customTransport as customTransportSetting, advancedScreenShare as advancedScreenShareSetting, screenShareResolution as screenShareResolutionSetting, screenShareFramerate as screenShareFramerateSetting, @@ -67,9 +67,10 @@ import styles from "./DeveloperSettingsTab.module.css"; import settingsStyles from "./SettingsModal.module.css"; import { Slider } from "../Slider"; import { useUrlParams } from "../UrlParams"; -import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { useBehavior } from "../useBehavior"; import { type ViewModel } from "../state/ViewModel.ts"; +import { getSFUConfig } from "../livekit/auth/index.ts"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; /** * The state of MatrixRTC's media key rotation. @@ -167,16 +168,18 @@ export const DeveloperSettingsTab: FC = ({ enableExtendedLivekitLogsSetting, ); - const [customLivekitUrlUpdateError, setCustomLivekitUrlUpdateError] = - useState(null); - const [customLivekitUrl, setCustomLivekitUrl] = useSetting( - customLivekitUrlSetting, + const [customTransportUpdateError, setCustomLivekitUrlUpdateError] = useState< + string | null + >(null); + const [customTransport, setCustomTransport] = useSetting( + customTransportSetting, + ); + const [customTransportTextBuffer, setCustomLivekitUrlTextBuffer] = useState( + () => transportText(customTransport), ); - const [customLivekitUrlTextBuffer, setCustomLivekitUrlTextBuffer] = - useState(customLivekitUrl); useEffect(() => { - setCustomLivekitUrlTextBuffer(customLivekitUrl); - }, [customLivekitUrl]); + setCustomLivekitUrlTextBuffer(transportText(customTransport)); + }, [customTransport]); const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting); @@ -491,23 +494,23 @@ export const DeveloperSettingsTab: FC = ({ e.preventDefault()} helpLabel={ - customLivekitUrl === null - ? t("developer_mode.custom_livekit_url.from_config") - : t("developer_mode.custom_livekit_url.current_url") + - customLivekitUrl + customTransport === null + ? t("developer_mode.custom_transport.from_config") + : t("developer_mode.custom_transport.current") + + JSON.stringify(customTransport) } - label={t("developer_mode.custom_livekit_url.label")} - saveButtonLabel={t("developer_mode.custom_livekit_url.save")} - savingLabel={t("developer_mode.custom_livekit_url.saving")} - cancelButtonLabel={t("developer_mode.custom_livekit_url.reset")} + label={t("developer_mode.custom_transport.label")} + saveButtonLabel={t("developer_mode.custom_transport.save")} + savingLabel={t("developer_mode.custom_transport.saving")} + cancelButtonLabel={t("developer_mode.custom_transport.reset")} onSave={useCallback( async (e: React.FormEvent): Promise => { if ( roomId === undefined || - customLivekitUrlTextBuffer === "" || - customLivekitUrlTextBuffer === null + customTransportTextBuffer === "" || + customTransportTextBuffer === null ) { - setCustomLivekitUrl(null); + setCustomTransport(null); return; } @@ -518,21 +521,29 @@ export const DeveloperSettingsTab: FC = ({ if (userId === null || deviceId === null) { throw new Error("Invalid user or device ID"); } - await getSFUConfigWithOpenID( + const transport = JSON.parse(customTransportTextBuffer); + // Validate the transport by attempting to get a token for it + await getSFUConfig({ client, - { userId, deviceId, memberId: "" }, - customLivekitUrlTextBuffer, + membership: { userId, deviceId, memberId: "" }, + transport, + serverName: client.getDomain()!, roomId, - ); + slotId: "org.example.fake-slot#room", + role: "publisher", + logger, + }); setCustomLivekitUrlUpdateError(null); - setCustomLivekitUrl(customLivekitUrlTextBuffer); + setCustomTransport(transport); } catch { - setCustomLivekitUrlUpdateError("invalid URL (did not update)"); + setCustomLivekitUrlUpdateError( + "invalid transport (did not update)", + ); } }, - [customLivekitUrlTextBuffer, setCustomLivekitUrl, client, roomId], + [customTransportTextBuffer, setCustomTransport, client, roomId], )} - value={customLivekitUrlTextBuffer ?? ""} + value={customTransportTextBuffer ?? ""} onChange={useCallback( (event: ChangeEvent): void => { setCustomLivekitUrlTextBuffer(event.target.value); @@ -541,14 +552,14 @@ export const DeveloperSettingsTab: FC = ({ )} onCancel={useCallback( (e: React.FormEvent) => { - setCustomLivekitUrl(null); + setCustomTransport(null); }, - [setCustomLivekitUrl], + [setCustomTransport], )} - serverInvalid={customLivekitUrlUpdateError !== null} + serverInvalid={customTransportUpdateError !== null} > - {customLivekitUrlUpdateError !== null && ( - {customLivekitUrlUpdateError} + {customTransportUpdateError !== null && ( + {customTransportUpdateError} )} @@ -686,3 +697,10 @@ export const DeveloperSettingsTab: FC = ({ ); }; + +/** + * The text shown in the custom transport field for a given setting value. + */ +function transportText(transport: UnstableLivekitTransport | null): string { + return transport === null ? "" : JSON.stringify(transport); +} diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 933ac66f5..733068acc 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -59,7 +59,7 @@ interface Props { livekitRooms?: { room: LivekitRoom; url: string; - isLocal?: boolean; + role: "publisher" | "subscriber"; }[]; /** Only available while in a call. Used by the developer tab. */ developerSettingsVm?: ViewModel; diff --git a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap index 63a68ac4b..54d1ea478 100644 --- a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap +++ b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap @@ -239,7 +239,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_label_1o4d9_60" for="radix-_r_7_" > - Custom Livekit-url + Custom transport JSON
{ public constructor( @@ -153,8 +154,8 @@ export const matrixRTCMode = new Setting( MatrixRTCMode.Compatibility, ); -export const customLivekitUrl = new Setting( - "custom-livekit-url", +export const customTransport = new Setting( + "custom-transport", null, ); diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index c7783d152..1b1c38d4a 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -31,7 +31,7 @@ import { type CallMembership, type IRTCNotificationContent, MatrixRTCSessionEvent, - type LivekitTransport, + type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { deepCompare } from "matrix-js-sdk/lib/utils"; @@ -97,7 +97,7 @@ vi.mock( "../state/CallViewModel/localMember/localTransport", async (importOriginal) => ({ ...(await importOriginal()), - makeTransport: async (): Promise => + makeTransport: async (): Promise => Promise.resolve(exampleTransport), }), ); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index a58b0837f..169abb98c 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -12,6 +12,7 @@ import { ExternalE2EEKeyProvider, type Room as LivekitRoom, type RoomOptions, + type RemoteParticipant, } from "livekit-client"; import { type Room as MatrixRoom } from "matrix-js-sdk"; import { @@ -38,11 +39,14 @@ import { timer, takeUntil, from, + concatWith, + catchError, + switchAll, } from "rxjs"; import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { MembershipManagerEvent, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, type RTCCallIntent, type RTCNotificationType, @@ -55,6 +59,7 @@ import { createToggle$, filterBehavior, generateItems, + mapScoped, pauseWhen, } from "../../utils/observable"; import { @@ -104,17 +109,14 @@ import { type OneOnOneMedia, } from "../layoutMedia.ts"; import { ElementCallError, UnknownCallError } from "../../utils/errors.ts"; -import { type Epoch, type ObservableScope } from "../ObservableScope.ts"; +import { Epoch, type ObservableScope } from "../ObservableScope.ts"; import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts"; import { createLocalMembership$, enterRTCSession, TransportState, } from "./localMember/LocalMember.ts"; -import { - getLocalTransport, - type LocalTransport, -} from "./localMember/LocalTransport.ts"; +import { getLocalTransport } from "./localMember/LocalTransport.ts"; import { createKeyRotationSuppressed$, createMemberships$, @@ -125,9 +127,9 @@ import { ECConnectionFactory, } from "./remoteMembers/ConnectionFactory.ts"; import { - type ConnectionManagerData, - createConnectionManager$, -} from "./remoteMembers/ConnectionManager.ts"; + ConnectionMap, + createConnectionMap$, +} from "./remoteMembers/ConnectionMap.ts"; import { createRemoteMatrixLivekitMembers$, type LocalMatrixLivekitMember, @@ -163,6 +165,7 @@ import { type RingingMediaViewModel, } from "../media/RingingMediaViewModel.ts"; import { type GridTileViewModel } from "../TileViewModel.ts"; +import { type TransportLocator } from "../../livekit/auth"; //TODO // Larger rename @@ -215,7 +218,7 @@ export interface CallViewModelOptions { */ windowSize$: Behavior<{ width: number; height: number }>; /** Optional value overriding the local transport, for testing purposes. */ - localTransport?: LocalTransport; + localTransport?: TransportLocator; /** Optional value overriding the connection factory, for testing purposes. */ connectionFactory?: ConnectionFactory; /** The version & compatibility mode of MatrixRTC that we should use. */ @@ -276,7 +279,7 @@ interface LayoutScanState { export type LivekitRoomItem = { livekitRoom: LivekitRoom; participants: string[]; - url: string; + transport: TransportLocator; }; /** @@ -373,7 +376,7 @@ export interface CallViewModel { * key is generated when someone joins or leaves. */ keyRotationSuppressed$: Behavior; - allConnections$: Behavior; + allConnections$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ livekitRoomItems$: Behavior; /** use the layout instead, this is just for the sdk export. */ @@ -564,20 +567,14 @@ export function createCallViewModel$( const localTransport$ = options.localTransport ? constant(options.localTransport) - : from( - getLocalTransport({ - ownMembershipIdentity, - client, - roomId: matrixRoom.roomId, - matrixRTCMode, - }), - ); + : from(getLocalTransport(client)); const connectionFactory = options.connectionFactory ?? new ECConnectionFactory( client, matrixRoom.roomId, + matrixRTCSession.slotId, mediaDevices, trackProcessorState$, livekitKeyProvider, @@ -585,14 +582,24 @@ export function createCallViewModel$( options.livekitRoomFactory, ); - const connectionManager = createConnectionManager$({ - scope: scope, - connectionFactory: connectionFactory, - localTransport$, - remoteTransports$: membershipsAndTransports.transports$, - logger: logger, - ownMembershipIdentity, - }); + const connectionMap$ = scope.behavior>( + localTransport$.pipe( + catchError(() => NEVER), // Swallow errors + concatWith(NEVER), // So the Observable doesn't complete prematurely + mapScoped("connectionMap$", (localTransport, scope) => + createConnectionMap$({ + scope, + connectionFactory, + localTransport, + remoteTransports$: membershipsAndTransports.transports$, + logger, + ownMembershipIdentity, + }), + ), + switchAll(), + ), + new Epoch(new ConnectionMap()), + ); const remoteMatrixLivekitMembers$: Behavior< Epoch @@ -600,7 +607,7 @@ export function createCallViewModel$( scope: scope, membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, - connectionManager: connectionManager, + connectionMap$, localUser: { userId, deviceId }, }); @@ -613,7 +620,7 @@ export function createCallViewModel$( ), muteStates, joinMatrixRTC: ( - transport: LivekitTransport, + transport: UnstableLivekitTransport, delayedLeaveTimings: ResolvedDelayedLeaveTimings, ) => { return enterRTCSession( @@ -635,13 +642,11 @@ export function createCallViewModel$( mediaDevices, muteStates, trackProcessorState$, - logger.getChild( - "[Publisher " + connection.transport.livekit_service_url + "]", - ), + logger.getChild(`[Publisher ${JSON.stringify(connection.transport)}]`), controlledAudioDevices, ); }, - connectionManager, + connectionMap$, client, matrixRTCSession, localTransport$, @@ -660,7 +665,6 @@ export function createCallViewModel$( ).pipe(map(([delayId]) => delayId ?? null)), matrixRTCSession.delayId ?? null, ), - matrixRTCMode, logger: logger.getChild(`[${Date.now()}]`), }); @@ -728,51 +732,47 @@ export function createCallViewModel$( }); const allConnections$ = scope.behavior( - connectionManager.connectionManagerData$.pipe(map((d) => d.value)), + connectionMap$.pipe(map((d) => d.value.getConnections())), ); - const livekitRoomItems$ = scope.behavior( - remoteMatrixLivekitMembers$.pipe( - switchMap((members) => { - const a$ = combineLatest( - members.value.map((member) => - combineLatest([member.connection$, member.participant.value$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - return { - url, - livekitRoom, - participant: participant.identity, - }; - }), + const livekitRoomItems$ = scope.behavior( + remoteMatrixLivekitMembers$.pipe( + // Flatten the members to a snapshot of their current connections and + // participant objects + switchMap(({ value: members }) => { + if (members.length === 0) return of([]); + return combineLatest( + members.map((m) => + combineLatest( + [m.connection$, m.participant.value$], + (connection, participant) => ({ connection, participant }), ), ), ); - return a$; }), + // Keep only those with a defined connection and participant map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), + members.filter( + ( + m, + ): m is { connection: Connection; participant: RemoteParticipant } => + m.connection !== null && m.participant !== null, + ), + ), + // Group members by connection + map((members) => + [...Map.groupBy(members, (m) => m.connection)].map( + ([connection, members]) => ({ + transport: { + transport: connection.transport, + serverName: connection.serverName, + }, + livekitRoom: connection.livekitRoom, + participants: members.map((m) => m.participant.identity), + }), + ), ), ), - [], ); const handsRaised$ = scope.behavior( @@ -829,8 +829,8 @@ export function createCallViewModel$( livekitRoom$: scope.behavior( connection$.pipe(map((c) => c?.livekitRoom)), ), - focusUrl$: scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), + transport$: scope.behavior( + connection$.pipe(map((c) => c?.transport)), ), mediaDevices, pretendToBeDisconnected$: localMembership.reconnecting$, @@ -1662,9 +1662,9 @@ export function createCallViewModel$( returnObject.publishError = value.media; else if ( typeof value.media === "object" && - value.media.connection instanceof ElementCallError + value.media.connection.state === "error" ) - returnObject.connectionError = value.media.connection; + returnObject.connectionError = value.media.connection.error; return returnObject; }), ), diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 0863ea2b7..93c190ace 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -30,8 +30,7 @@ import { type CallViewModelOptions, } from "./CallViewModel"; import { - exampleSfuConfig, - exampleTransport, + exampleTransportLocator, mockConfig, MockConnection, mockLivekitRoom, @@ -123,6 +122,8 @@ export function withCallViewModel(mode: MatrixRTCMode) { }; const room = mockMatrixRoom({ client: new (class extends EventEmitter { + public readonly baseUrl = "https://matrix.example.com"; + public getUserId(): string | undefined { return localRtcMember.userId; } @@ -209,26 +210,26 @@ export function withCallViewModel(mode: MatrixRTCMode) { livekitRoomFactory, connectionState$, windowSize$, - localTransport: { - transport: exampleTransport, - sfuConfig: exampleSfuConfig, - }, + localTransport: exampleTransportLocator, connectionFactory: { createConnection( scope, + role, transport, + serverName, ownMembershipIdentity, logger, - sfuConfig, ) { return new MockConnection( { scope, + role, transport, + serverName, ownMembershipIdentity, - existingSFUConfig: sfuConfig, client: room.client, roomId: room.roomId, + slotId: "m.call#ROOM", livekitRoomFactory, }, logger, diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 282759fdd..f8b4e6b6b 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { Status as RTCMemberStatus, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { @@ -21,9 +21,13 @@ import { beforeEach, afterEach, } from "vitest"; -import { BehaviorSubject, map, of, Subject } from "rxjs"; +import { BehaviorSubject, of, Subject } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type LocalParticipant, type LocalTrack } from "livekit-client"; +import { + ConnectionState as LivekitConnectionState, + type LocalParticipant, + type LocalTrack, +} from "livekit-client"; import fetchMock from "fetch-mock"; import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics"; @@ -53,13 +57,15 @@ import { import { MatrixRTCTransportMissingError } from "../../../utils/errors"; import { Epoch, ObservableScope } from "../../ObservableScope"; import { constant } from "../../Behavior"; -import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; -import { ConnectionState, type Connection } from "../remoteMembers/Connection"; +import { ConnectionMap } from "../remoteMembers/ConnectionMap"; +import { + type ConnectionState, + type Connection, +} from "../remoteMembers/Connection"; import { type Publisher } from "./Publisher"; import { initializeWidget } from "../../../widget"; import { nullHostBridge } from "../../../HostBridge"; -import { type LocalTransport } from "./LocalTransport"; -import * as openIDSFU from "../../../livekit/openIDSFU"; +import * as livekitAuth from "../../../livekit/auth"; initializeWidget(); @@ -110,6 +116,7 @@ const delegatedTimings: ResolvedDelayedLeaveTimings = { }; const mockedClient = { + baseUrl: "https://matrix.example.org", getDomain: vi.fn().mockReturnValue("example.org"), getDeviceId: vi.fn().mockReturnValue("AAAA"), getOpenIdToken: vi.fn().mockResolvedValue({ @@ -118,10 +125,11 @@ const mockedClient = { matrix_server_name: "localhost", expires_in: 10000, }), + _unstable_delegateDelayedLeave: vi.fn().mockResolvedValue({}), }; describe("enterRTCSession", () => { - const transport: LivekitTransport = { + const transport: UnstableLivekitTransport = { livekit_service_url: "http://my-livekit-service-url.com", type: "livekit", }; @@ -214,7 +222,10 @@ describe("LocalMembership", () => { processor: undefined, }), logger: logger, - createPublisherFactory: vi.fn(), + createPublisherFactory: vi.fn().mockReturnValue({ + destroy: vi.fn(), + createAndSetupTracks: vi.fn().mockResolvedValue(undefined), + }), joinMatrixRTC: async (): Promise => {}, homeserverConnected: { combined$: constant<[boolean, HomeserverDisconnectReason | null]>([ @@ -248,29 +259,16 @@ describe("LocalMembership", () => { afterEach(async () => { void (await fetchMock.flush()); fetchMock.reset(); + vi.clearAllMocks(); }); it("throws error on missing RTC config error", () => { withTestScheduler(({ scope, hot, expectObservable }) => { - const localTransport$ = scope.behavior( - hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), - null, - ); - - // we do not need any connection data since we want to fail before reaching that. - const mockConnectionManager = { - transports$: scope.behavior( - localTransport$.pipe(map((t) => new Epoch([t]))), - ), - connectionManagerData$: constant( - new Epoch(new ConnectionManagerData()), - ), - }; - const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: mockConnectionManager, + // we do not need any connection data since we want to fail before reaching that. + connectionMap$: constant(new Epoch(new ConnectionMap())), localTransport$: hot( "1ms #", {}, @@ -286,24 +284,19 @@ describe("LocalMembership", () => { }); it("logs if callIntent cannot be updated", async () => { - const scope = new ObservableScope(); - - const mockConnectionManager = { - transports$: constant(new Epoch([])), - connectionManagerData$: constant(new Epoch(new ConnectionManagerData())), - }; async function reject(): Promise { return Promise.reject(new Error("Not connected yet")); } const localMembership = createLocalMembership$({ - scope, + scope: testScope(), ...defaultCreateLocalMemberValues, matrixRTCSession: { + slotId: "m.call#ROOM", updateCallIntent: vi.fn().mockImplementation(reject), leave: vi.fn(), }, - connectionManager: mockConnectionManager, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(new ConnectionMap())), + localTransport$: constant(mockTransportLocator), }); const expextedLog = "'not connected yet' while updating the call intent (this is expected on startup)"; @@ -312,22 +305,18 @@ describe("LocalMembership", () => { await flushPromises(); defaultCreateLocalMemberValues.muteStates.video.setEnabled$.value?.(true); expect(internalLogger).toHaveBeenCalledWith(expextedLog); - scope.end(); }); - const mockTransportConfig = { - livekit_service_url: "a", - } as LivekitTransport; + const mockTransport: UnstableLivekitTransport = { + type: "livekit", + url: "https://sfu.example.org", + livekit_service_url: "https://jwt.example.org", + }; - const mockTransport = { - transport: mockTransportConfig, - sfuConfig: { - jwt: "foo", - livekitAlias: "bar", - livekitIdentity: "baz", - url: "bro", - }, - } as LocalTransport; + const mockTransportLocator: livekitAuth.TransportLocator = { + transport: mockTransport, + serverName: "example.org", + }; const connectionTransportAConnected = { livekitRoom: mockLivekitRoom({ @@ -336,31 +325,52 @@ describe("LocalMembership", () => { trackPublications: [], } as unknown as LocalParticipant, }), - state$: constant(ConnectionState.LivekitConnected), - transport: mockTransportConfig, + state$: constant({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }), + transport: mockTransport, + serverName: "example.org", } as Connection; const connectionTransportAConnecting = { ...connectionTransportAConnected, - state$: constant(ConnectionState.LivekitConnecting), + state$: constant({ + state: "authenticated", + livekitState: LivekitConnectionState.Connecting, + }), livekitRoom: mockLivekitRoom({}), } as unknown as Connection; - const authCallSpy = vi - .spyOn(openIDSFU, "getSFUConfigWithOpenID") - .mockImplementation(() => mockedClient.getOpenIdToken()); - afterEach(() => authCallSpy.mockClear()); + const delegationSpy = vi + .spyOn(livekitAuth, "delegateDelayedLeave") + .mockResolvedValue(); + afterEach(() => delegationSpy.mockClear()); + const urlOnlyTransport: UnstableLivekitTransport = { + type: "livekit", + url: "https://sfu.example.org", + }; + + // With a `url` only we publish through the CS API, so the homeserver + // delegates. With a `livekit_service_url` we publish through the JWT + // service, so that delegates. it.each([ - ["no", null, timings], + ["no", mockTransport, null, timings], [ "homeserver", + urlOnlyTransport, "https://matrix.example.org/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave", delegatedTimings, ], - ["transport", "/a/delegate_delayed_leave", delegatedTimings], + [ + "transport", + mockTransport, + "https://jwt.example.org/delegate_delayed_leave", + delegatedTimings, + ], ])( "joins session with %s delegation support", - async (_serviceName, delegationUrl, delayedLeaveTimings) => { + async (_serviceName, transport, delegationUrl, delayedLeaveTimings) => { const scope = testScope(); const joinMatrixRTC = vi.fn(); const delayId$ = new BehaviorSubject(null); @@ -368,16 +378,17 @@ describe("LocalMembership", () => { if (delegationUrl !== null) fetchMock.post(delegationUrl, () => ({ status: 401, body: {} })); + const connectionMap = new ConnectionMap(); + connectionMap.add( + { ...connectionTransportAConnected, transport } as Connection, + [], + ); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$: constant( - new Epoch(new ConnectionManagerData()), - ), - }, + connectionMap$: constant(new Epoch(connectionMap)), joinMatrixRTC, - localTransport$: constant(mockTransport), + localTransport$: constant({ transport, serverName: "example.org" }), delayId$, }); @@ -386,31 +397,27 @@ describe("LocalMembership", () => { await flushPromises(); // Joins with timings appropriate for the level of delegation support expect(joinMatrixRTC).toHaveBeenCalledWith( - mockTransportConfig, + transport, delayedLeaveTimings, ); - expect(authCallSpy).not.toHaveBeenCalled(); + expect(delegationSpy).not.toHaveBeenCalled(); delayId$.next("leave1"); await flushPromises(); if (delegationUrl === null) { - expect(authCallSpy).not.toHaveBeenCalled(); + expect(delegationSpy).not.toHaveBeenCalled(); } else { // Delegation is supported in this test case, so go on to check that // LocalMember actually performs delegation const expectDelegation = (delayId: string) => - expect(authCallSpy).toHaveBeenLastCalledWith( - mockedClient, - ownMemberMock, - "a", - "!test-room-id:example.org", - { - matrixRTCMode: MATRIX_RTC_MODE, - delayEndpointBaseUrl: "https://matrix.example.org", - delayId, - }, - expect.anything(), - ); + expect(delegationSpy).toHaveBeenLastCalledWith({ + client: mockedClient, + membership: ownMemberMock, + transport, + roomId: "!test-room-id:example.org", + delayId, + logger: expect.anything(), + }); expectDelegation("leave1"); delayId$.next("leave2"); // Can change delegated leaves @@ -449,16 +456,14 @@ describe("LocalMembership", () => { typeof vi.fn >; - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); - // connectionManagerData.add(connectionTransportB, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); + // connectionMap.add(connectionTransportB, []); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); expect(publisherFactory).toHaveBeenCalledOnce(); @@ -479,12 +484,10 @@ describe("LocalMembership", () => { // it("tracks livekit state correctly", async () => { const scope = new ObservableScope(); - const connectionManagerData = new ConnectionManagerData(); - const localTransport$ = new Subject(); + const connectionMap = new ConnectionMap(); + const localTransport$ = new Subject(); - const connectionManagerData$ = new BehaviorSubject( - new Epoch(connectionManagerData), - ); + const connectionMap$ = new BehaviorSubject(new Epoch(connectionMap)); const publishers: Publisher[] = []; const publishing$ = new BehaviorSubject(false); @@ -518,9 +521,7 @@ describe("LocalMembership", () => { const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$, - }, + connectionMap$: connectionMap$, localTransport$, }); @@ -528,38 +529,50 @@ describe("LocalMembership", () => { expect(localMembership.localMemberState$.value).toStrictEqual( TransportState.Waiting, ); - localTransport$.next(mockTransport); + localTransport$.next(mockTransportLocator); await flushPromises(); expect(localMembership.localMemberState$.value).toStrictEqual({ matrix: RTCMemberStatus.Connected, - media: { connection: null, tracks: TrackState.WaitingForUser }, + media: { + connection: { state: "initializing" }, + tracks: TrackState.WaitingForUser, + }, }); - const connectionManagerData2 = new ConnectionManagerData(); - connectionManagerData2.add( + const connectionMap2 = new ConnectionMap(); + connectionMap2.add( // clone because we will mutate this later. { ...connectionTransportAConnecting } as unknown as Connection, [], ); - connectionManagerData$.next(new Epoch(connectionManagerData2)); + connectionMap$.next(new Epoch(connectionMap2)); await flushPromises(); expect(localMembership.localMemberState$.value).toStrictEqual({ matrix: RTCMemberStatus.Connected, media: { - connection: ConnectionState.LivekitConnecting, + connection: { + state: "authenticated", + livekitState: LivekitConnectionState.Connecting, + }, tracks: TrackState.WaitingForUser, }, }); ( - connectionManagerData2.getConnectionForTransport(mockTransportConfig)! + connectionMap2.getConnectionForTransport(mockTransportLocator)! .state$ as BehaviorSubject - ).next(ConnectionState.LivekitConnected); + ).next({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); expect(localMembership.localMemberState$.value).toStrictEqual({ matrix: RTCMemberStatus.Connected, media: { - connection: ConnectionState.LivekitConnected, + connection: { + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }, tracks: TrackState.WaitingForUser, }, }); @@ -643,8 +656,8 @@ describe("LocalMembership", () => { [boolean, HomeserverDisconnectReason | null] >([false, "membership"]); - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); createLocalMembership$({ scope, @@ -653,10 +666,8 @@ describe("LocalMembership", () => { combined$: hsReason$, rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); @@ -680,8 +691,8 @@ describe("LocalMembership", () => { [boolean, HomeserverDisconnectReason | null] >([true, null]); - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); createLocalMembership$({ scope, @@ -690,10 +701,8 @@ describe("LocalMembership", () => { combined$: hsReason$, rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); @@ -717,16 +726,17 @@ describe("LocalMembership", () => { "track", ); - const connectionState$ = new BehaviorSubject( - ConnectionState.LivekitConnected, - ); + const connectionState$ = new BehaviorSubject({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); const mutableConnection = { ...connectionTransportAConnected, state$: connectionState$, } as unknown as Connection; - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(mutableConnection, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(mutableConnection, []); createLocalMembership$({ scope, @@ -738,16 +748,20 @@ describe("LocalMembership", () => { ]), rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); - connectionState$.next(ConnectionState.LivekitDisconnected); - connectionState$.next(ConnectionState.LivekitConnected); + connectionState$.next({ + state: "authenticated", + livekitState: LivekitConnectionState.Disconnected, + }); + connectionState$.next({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); expect(trackSpy).toHaveBeenCalledWith( defaultCreateLocalMemberValues.roomId, @@ -769,8 +783,8 @@ describe("LocalMembership", () => { [boolean, HomeserverDisconnectReason | null] >([true, null]); - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); createLocalMembership$({ scope, @@ -779,10 +793,8 @@ describe("LocalMembership", () => { combined$: hsReason$, rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); @@ -840,15 +852,13 @@ describe("LocalMembership", () => { localMembership: ReturnType; } => { const scope = new ObservableScope(); - const connectionManagerData = new ConnectionManagerData(); - if (connection) connectionManagerData.add(connection, []); + const connectionMap = new ConnectionMap(); + if (connection) connectionMap.add(connection, []); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); return { scope, localMembership }; }; @@ -857,8 +867,12 @@ describe("LocalMembership", () => { const error = new Error("NotReadableError"); const setScreenShareEnabled = vi.fn().mockRejectedValue(error); const connection = { - state$: constant(ConnectionState.LivekitConnected), - transport: mockTransportConfig, + state$: constant({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }), + transport: mockTransport, + serverName: "example.org", livekitRoom: mockLivekitRoom({ localParticipant: mockLocalParticipant({ isScreenShareEnabled: false, diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index feb49554c..baa6b473d 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { + ConnectionState as LivekitConnectionState, type Participant, ParticipantEvent, type LocalParticipant, @@ -15,10 +16,9 @@ import { MediaDeviceFailure, } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; -import { type MatrixClient } from "matrix-js-sdk"; import { Status as RTCSessionStatus, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, type RTCCallIntent, type RTCNotificationType, @@ -47,8 +47,8 @@ import { deepCompare } from "matrix-js-sdk/lib/utils"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Behavior } from "../../Behavior.ts"; -import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts"; -import { type ObservableScope } from "../../ObservableScope.ts"; +import { type ConnectionMap } from "../remoteMembers/ConnectionMap.ts"; +import { type Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { type Publisher } from "./Publisher.ts"; import { type MuteStates } from "../../MuteStates.ts"; import { @@ -74,13 +74,17 @@ import { } from "../../../config/ConfigOptions.ts"; import { Config } from "../../../config/Config.ts"; import { - ConnectionState, + type ConnectionState, type Connection, - type FailedToStartError, } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; -import { type LocalTransport } from "./LocalTransport.ts"; -import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts"; +import { + type ClientDelegationParts, + delegateDelayedLeave, + type ClientOpenIDParts, + chooseAuthFlow, + type TransportLocator, +} from "../../../livekit/auth"; export enum TransportState { /** Not even a transport is available to the LocalMembership */ @@ -113,7 +117,7 @@ export enum TrackState { export type LocalMemberMediaState = | { tracks: TrackState; - connection: ConnectionState | FailedToStartError; + connection: ConnectionState; } | PublishState | ElementCallError; @@ -140,25 +144,27 @@ interface Props { // that the inputs for those createSomething$() functions should NOT contain any js-sdk objectes scope: ObservableScope; muteStates: MuteStates; - connectionManager: IConnectionManager; + connectionMap$: Behavior>; createPublisherFactory: (connection: Connection) => Publisher; joinMatrixRTC: ( - transport: LivekitTransport, + transport: UnstableLivekitTransport, delayedLeaveTimings: ResolvedDelayedLeaveTimings, ) => void; homeserverConnected: HomeserverConnected; roomId: string; ownMembershipIdentity: CallMembershipIdentityParts; - localTransport$: Observable; - client: Pick; - matrixRTCSession: Pick; + localTransport$: Observable; + client: ClientDelegationParts & ClientOpenIDParts; + matrixRTCSession: Pick< + MatrixRTCSession, + "slotId" | "updateCallIntent" | "leave" + >; /** Whether to hide the screen-sharing button. */ hideScreensharing: boolean; /** The application hosting Element Call, to be kept informed of join/leave. */ hostBridge: HostBridge; baseUrl: string; delayId$: Behavior; - matrixRTCMode: MatrixRTCMode; logger: Logger; } @@ -169,7 +175,7 @@ interface Props { * - * @param props The properties required to create the local membership. * @param props.scope The observable scope to use. - * @param props.connectionManager The connection manager to get connections from. + * @param props.connectionMap$ The connection map to get connections from. * @param props.createPublisherFactory Factory to create a publisher once we have a connection. * @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport. * @param props.homeserverConnected The homeserver connected state. @@ -191,7 +197,7 @@ interface Props { */ export const createLocalMembership$ = ({ scope, - connectionManager, + connectionMap$, localTransport$: localTransportWithErrors$, homeserverConnected, createPublisherFactory, @@ -206,7 +212,6 @@ export const createLocalMembership$ = ({ hostBridge, ownMembershipIdentity, delayId$, - matrixRTCMode, }: Props): { /** * This request to start audio and video tracks. @@ -308,16 +313,17 @@ export const createLocalMembership$ = ({ const joinParams$ = scope.behavior( localTransport$.pipe( switchMap(async ({ transport }) => { - const transportSupportsDelegation = checkDelegationSupport( - transport.livekit_service_url + "/delegate_delayed_leave", - `transport ${transport.livekit_service_url}`, - ); - return { - transport, - delegationSupported: - (await homeserverSupportsDelegation) || - (await transportSupportsDelegation), - }; + // Delegation goes through whichever service authenticates us as a + // publisher, so only that service's support counts + const flow = chooseAuthFlow(transport, "publisher"); + const delegationSupported = + flow.kind === "openID" + ? await checkDelegationSupport( + flow.serviceUrl + "/delegate_delayed_leave", + `transport ${flow.serviceUrl}`, + ) + : await homeserverSupportsDelegation; + return { transport, delegationSupported }; }), ), null, @@ -325,16 +331,14 @@ export const createLocalMembership$ = ({ // Drop Epoch data here since we will not combine this anymore const localConnection$ = scope.behavior( - combineLatest([ - connectionManager.connectionManagerData$, - localTransport$, - ]).pipe( - map(([{ value: connectionData }, { transport }]) => - connectionData.getConnectionForTransport(transport), - ), + combineLatest( + [connectionMap$, localTransport$], + ({ value: connectionMap }, transport) => + connectionMap.getConnectionForTransport(transport), + ).pipe( tap((connection) => { logger.info( - `Local connection updated: ${connection?.transport?.livekit_service_url}`, + `Local connection updated: ${JSON.stringify(connection?.transport)}`, ); }), ), @@ -406,7 +410,7 @@ export const createLocalMembership$ = ({ scope.reconcile(localConnection$, async (connection) => { logger.info( "reconcile based on new localConnection:", - connection?.transport.livekit_service_url, + JSON.stringify(connection?.transport), ); if (connection !== null) { const publisher = createPublisherFactory(connection); @@ -475,7 +479,13 @@ export const createLocalMembership$ = ({ }; const localConnectionState$ = localConnection$.pipe( - switchMap((connection) => (connection ? connection.state$ : of(null))), + switchMap((connection) => + connection ? connection.state$ : of({ state: "initializing" as const }), + ), + ); + + const authenticated$ = scope.behavior( + localConnectionState$.pipe(map(({ state }) => state === "authenticated")), ); const mediaState$: Behavior = scope.behavior( @@ -493,7 +503,9 @@ export const createLocalMembership$ = ({ : TrackState.WaitingForUser; if ( - localConnectionState !== ConnectionState.LivekitConnected || + localConnectionState.state !== "authenticated" || + localConnectionState.livekitState !== + LivekitConnectionState.Connected || trackState !== TrackState.Ready ) return { @@ -561,7 +573,11 @@ export const createLocalMembership$ = ({ combineLatest([ homeserverConnected.combined$, localConnectionState$.pipe( - map((state) => state === ConnectionState.LivekitConnected), + map( + (state) => + state.state === "authenticated" && + state.livekitState === LivekitConnectionState.Connected, + ), ), ]).pipe( map(([[hsConnected, hsReason], livekitConnected]) => { @@ -656,34 +672,38 @@ export const createLocalMembership$ = ({ // Join and leave the session as needed scope.reconcile( - scope.behavior(combineLatest([joinParams$, joinAndPublishRequested$])), - async ([joinParams, shouldConnect]) => { - if (!joinParams) return; - // if shouldConnect=false we will do the disconnect as the cleanup from the previous reconcile iteration. - if (!shouldConnect) return; + scope.behavior( + combineLatest([joinParams$, joinAndPublishRequested$, authenticated$]), + ), + async ([joinParams, shouldConnect, authenticated]) => { const sessionConfig = Config.get().matrix_rtc_session; - try { - joinMatrixRTC( - joinParams.transport, - joinParams.delegationSupported - ? sessionConfig.delegated_delayed_leave - : sessionConfig.delayed_leave, - ); - } catch (error) { - logger.error("Error entering RTC session", error); - if (error instanceof Error) - setMatrixError(new MembershipManagerError(error)); - } - - return Promise.resolve(async (): Promise => { + // Only join once authenticated, to ensure the LiveKit room exists first. + // Otherwise, subscribers authenticating with the OpenID flow may fail to + // join (they lack permission to create the room) and never retry. + if (joinParams && shouldConnect && authenticated) { try { - // TODO Update matrixRTCSession to allow udpating the transport without leaving the session! - await matrixRTCSession.leave(1000); - } catch (e) { - logger.error("Error leaving RTC session", e); + joinMatrixRTC( + joinParams.transport, + joinParams.delegationSupported + ? sessionConfig.delegated_delayed_leave + : sessionConfig.delayed_leave, + ); + } catch (error) { + logger.error("Error entering RTC session", error); + if (error instanceof Error) + setMatrixError(new MembershipManagerError(error)); } - }); + + // Clean-up callback to leave the session + return Promise.resolve(async (): Promise => { + try { + await matrixRTCSession.leave(1000); + } catch (e) { + logger.error("Error leaving RTC session", e); + } + }); + } }, ); @@ -702,23 +722,22 @@ export const createLocalMembership$ = ({ async ([joinParams, delayId]) => { if (joinParams?.delegationSupported && delayId !== null) { try { - // This will technically cause the service to issue a new JWT token, - // but it's safe to discard. We're only interested in triggering - // delegation. - await getSFUConfigWithOpenID( - client, - ownMembershipIdentity, - joinParams.transport.livekit_service_url, - roomId, - { matrixRTCMode, delayEndpointBaseUrl: baseUrl, delayId }, - logger, + logger.info( + `Delegating delayed leave to ${JSON.stringify(joinParams.transport)}…`, ); + await delegateDelayedLeave({ + client, + membership: ownMembershipIdentity, + transport: joinParams.transport, + slotId: matrixRTCSession.slotId, + roomId, + delayId, + logger, + }); + logger.info("Delayed leave successfully delegated"); } catch (e) { // TODO: Surface this to the user as a service interruption? - logger.error( - `Failed to delegate leave to ${joinParams.transport.livekit_service_url}`, - e, - ); + logger.error(`Failed to delegate leave`, e); } } }, @@ -951,7 +970,7 @@ interface EnterRTCSessionOptions { * * @param rtcSession - The MatrixRTCSession to join. * @param ownMembershipIdentity - Options for entering the RTC session. - * @param transport - The LivekitTransport to use for this session. + * @param transport - The UnstableLivekitTransport to use for this session. * @param options - `encryptMedia`: Whether to encrypt media. `matrixRTCMode`: The * Matrix RTC mode to use. `delayedLeaveTimings`: The preferred timings for * delayed leave events. `sendNotificationType`: Whether and what kind of @@ -962,7 +981,7 @@ interface EnterRTCSessionOptions { export function enterRTCSession( rtcSession: MatrixRTCSession, ownMembershipIdentity: CallMembershipIdentityParts, - transport: LivekitTransport, + transport: UnstableLivekitTransport, { encryptMedia, matrixRTCMode, diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 90d0d15e5..ed6654e2f 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -16,130 +16,54 @@ import { } from "vitest"; import fetchMock from "fetch-mock"; -import { mockConfig, ownMemberMock } from "../../../utils/test"; +import { mockConfig } from "../../../utils/test"; import { getLocalTransport } from "./LocalTransport"; -import { - MatrixRTCTransportMissingError, - FailToGetOpenIdToken, -} from "../../../utils/errors"; -import * as openIDSFU from "../../../livekit/openIDSFU"; -import { customLivekitUrl } from "../../../settings/settings"; -import { testJWTToken } from "../../../utils/test-fixtures"; -import { MatrixRTCMode } from "../../../config/ConfigOptions"; +import { MatrixRTCTransportMissingError } from "../../../utils/errors"; +import { customTransport } from "../../../settings/settings"; +import { type MatrixClient } from "matrix-js-sdk"; describe("LocalTransport", () => { - const openIdResponse: openIDSFU.SFUConfig = { - url: "https://lk.example.org", - jwt: testJWTToken, - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - }; - beforeEach(() => vi.clearAllMocks()); it("throws if config is missing", async () => { await expect( getLocalTransport({ - roomId: "!room:example.org", - client: { - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - getDomain: () => "example.org", - // These won't be called in this error path but satisfy the type - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, - ownMembershipIdentity: ownMemberMock, - matrixRTCMode: MatrixRTCMode.Compatibility, + _unstable_getRTCTransports: async () => Promise.resolve([]), + getDomain: () => "example.org", }), ).rejects.toThrow(new MatrixRTCTransportMissingError("example.org")); }); - it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => { - // Provide a valid config so makeTransportInternal resolves a transport - mockConfig({ - livekit: { livekit_service_url: "https://lk.example.org" }, - }); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockRejectedValue( - new FailToGetOpenIdToken(new Error("no openid")), - ); - - await expect( - getLocalTransport({ - roomId: "!example_room_id", - client: { - getDomain: () => "example.org", - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, - ownMembershipIdentity: ownMemberMock, - matrixRTCMode: MatrixRTCMode.Compatibility, - }), - ).rejects.toThrow(new FailToGetOpenIdToken(new Error("no openid"))); - }); - it("returns preferred transport", async () => { - // Use config so transport discovery succeeds, but delay OpenID JWT fetch + // Use config so transport discovery succeeds mockConfig({ livekit: { livekit_service_url: "https://lk.example.org" }, }); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue({ - url: "https://lk.example.org", - jwt: "jwt", - livekitAlias: "Akph4alDMhen", - livekitIdentity: ownMemberMock.userId + ":" + ownMemberMock.deviceId, - }); - expect( await getLocalTransport({ - roomId: "!room:example.org", - client: { - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - getDomain: () => "example.org", - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, - ownMembershipIdentity: ownMemberMock, - matrixRTCMode: MatrixRTCMode.Compatibility, + _unstable_getRTCTransports: async () => Promise.resolve([]), + getDomain: () => "example.org", }), ).toStrictEqual({ transport: { livekit_service_url: "https://lk.example.org", type: "livekit", }, - sfuConfig: { - jwt: "jwt", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@alice:example.org:DEVICE", - url: "https://lk.example.org", - }, + serverName: "example.org", }); }); - type LocalTransportProps = Parameters[0]; - describe("transport configuration mechanisms", () => { - let localTransportOpts: LocalTransportProps & { - client: MockedObject; - }; + let client: MockedObject< + Pick + >; beforeEach(() => { mockConfig({}); - customLivekitUrl.setValue(customLivekitUrl.defaultValue); - localTransportOpts = { - ownMembershipIdentity: ownMemberMock, - roomId: "!example_room_id", - matrixRTCMode: MatrixRTCMode.Compatibility, - client: { - getDomain: vi.fn().mockReturnValue("example.org"), - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, + customTransport.setValue(customTransport.defaultValue); + client = { + getDomain: vi.fn().mockReturnValue("example.org"), + _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), }; }); @@ -151,93 +75,50 @@ describe("LocalTransport", () => { mockConfig({ livekit: { livekit_service_url: "https://lk.example.org" }, }); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue( - openIdResponse, - ); - expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ + expect(await getLocalTransport(client)).toStrictEqual({ transport: { livekit_service_url: "https://lk.example.org", type: "livekit", }, - sfuConfig: { - jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - url: "https://lk.example.org", - }, + serverName: "example.org", }); }); it("supports getting transport via user settings", async () => { - customLivekitUrl.setValue("https://lk.example.org"); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue( - openIdResponse, - ); + customTransport.setValue({ + type: "livekit", + livekit_service_url: "https://lk.example.org", + }); - expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ + expect(await getLocalTransport(client)).toStrictEqual({ transport: { livekit_service_url: "https://lk.example.org", type: "livekit", }, - sfuConfig: { - jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - url: "https://lk.example.org", - }, + serverName: "example.org", }); }); it("supports getting transport via backend", async () => { - localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ + client._unstable_getRTCTransports.mockResolvedValue([ { type: "livekit", livekit_service_url: "https://lk.example.org" }, ]); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue( - openIdResponse, - ); - expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ + expect(await getLocalTransport(client)).toStrictEqual({ transport: { livekit_service_url: "https://lk.example.org", type: "livekit", }, - sfuConfig: { - jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - url: "https://lk.example.org", - }, + serverName: "example.org", }); }); - it("fails fast if the openID request fails for backend config", async () => { - localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ - { type: "livekit", livekit_service_url: "https://lk.example.org" }, - ]); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockRejectedValue( - new FailToGetOpenIdToken(new Error("Test driven error")), - ); - - await expect(getLocalTransport(localTransportOpts)).rejects.toThrow( - expect.any(FailToGetOpenIdToken), - ); - }); - it("throws if no options are available", async () => { await expect( getLocalTransport({ - ownMembershipIdentity: ownMemberMock, - roomId: "!example_room_id", - matrixRTCMode: MatrixRTCMode.Compatibility, - client: { - getDomain: () => "example.org", - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - // These won't be called in this error path but satisfy the type - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, + getDomain: () => "example.org", + _unstable_getRTCTransports: async () => Promise.resolve([]), }), ).rejects.toThrow(new MatrixRTCTransportMissingError("example.org")); }); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 3534b3fec..3b43f8112 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -5,142 +5,37 @@ SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type MatrixClient } from "matrix-js-sdk"; -import { logger as rootLogger, type Logger } from "matrix-js-sdk/lib/logger"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { Config } from "../../../config/Config.ts"; -import { - FailToGetOpenIdToken, - MatrixRTCTransportMissingError, - NoMatrix2AuthorizationService, -} from "../../../utils/errors.ts"; -import { - getSFUConfigWithOpenID, - type SFUConfig, - type OpenIDClientParts, -} from "../../../livekit/openIDSFU.ts"; -import { customLivekitUrl } from "../../../settings/settings.ts"; +import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; +import { customTransport as customTransportSetting } from "../../../settings/settings.ts"; import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts"; -import { type MatrixRTCMode } from "../../../config/ConfigOptions.ts"; - -interface Props { - ownMembershipIdentity: CallMembershipIdentityParts; - client: Pick & - OpenIDClientParts; - // Used by the jwt service to create the livekit room and compute the livekit alias. - roomId: string; - matrixRTCMode: MatrixRTCMode; -} - -export interface LocalTransport { - transport: LivekitTransport; - sfuConfig: SFUConfig; -} - -export function isLocalTransport( - obj: LivekitTransport | LocalTransport, -): obj is LocalTransport { - return "transport" in obj && "sfuConfig" in obj; -} +import { type TransportLocator } from "../../../livekit/auth/types.ts"; /** - * Connects to the JWT service and determines the transport that the local member should use. - * - * @prop useOldJwtEndpoint Whether to set forceOldJwtEndpoint on the returned transport and to use the old JWT endpoint. - * This is used when the connection manager needs to know if it has to use the legacy endpoint which implies a string concatenated rtcBackendIdentity. - * (which is expected for non sticky event based rtc member events) - * @returns The transport to advertise in our MatrixRTC membership and publish media on. - * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken + * Determines the transport to advertise in our MatrixRTC membership and publish + * media on. */ -export async function getLocalTransport({ - ownMembershipIdentity, - client, - roomId, - matrixRTCMode, -}: Props): Promise { - const logger = rootLogger.getChild("[LocalTransport]"); +export async function getLocalTransport( + client: Pick, +): Promise { + const serverName = client.getDomain(); + if (serverName === null) throw new Error("No server name"); + const discovery = new RtcTransportAutoDiscovery({ - client: client, - resolvedConfig: Config.get(), - logger: logger, - }); - const customUrl = customLivekitUrl.value$.value; - - // Respect the user's custom URL, if set - const transport: LivekitTransport | null = customUrl - ? { type: "livekit", livekit_service_url: customUrl } - : await discovery.discoverPreferredTransport(); - - if (transport === null) - throw new MatrixRTCTransportMissingError(client.getDomain() ?? ""); - - try { - return await doOpenIdAndJWTFromUrl( - transport, - matrixRTCMode, - ownMembershipIdentity, - roomId, - client, - logger, - ); - } catch (e) { - logger.error( - `Failed to authenticate to transport ${transport.livekit_service_url}`, - e, - ); - throw mapAuthErrorToUserFriendlyError(e); - } -} - -/** - * Utility to ensure the user can authenticate with the SFU. - * We will call `getSFUConfigWithOpenID` once per transport here as it's our - * only mechanism of validation. This means we will also ask the - * homeserver for a OpenID token a few times. Since OpenID tokens are single - * use we don't want to risk any issues by re-using a token. - * - * @param transport The transport to authenticate with. - * @param matrixRTCMode Whether to force the JWT endpoint to be used. - * @param membership The identity of the local member. - * @param roomId The room ID to use for the JWT. - * @param client The client to use for the OpenID token. - * - * @throws FailToGetOpenIdToken, NoMatrix2AuthorizationService - */ -async function doOpenIdAndJWTFromUrl( - transport: LivekitTransport, - matrixRTCMode: MatrixRTCMode, - membership: CallMembershipIdentityParts, - roomId: string, - client: Pick & OpenIDClientParts, - logger?: Logger, -): Promise { - const sfuConfig = await getSFUConfigWithOpenID( client, - membership, - transport.livekit_service_url, - roomId, - { matrixRTCMode }, - logger, - ); - return { - transport, - sfuConfig, - }; -} + resolvedConfig: Config.get(), + logger: rootLogger.getChild("[LocalTransport]"), + }); -function mapAuthErrorToUserFriendlyError(e: unknown): Error { - if ( - e instanceof FailToGetOpenIdToken || - e instanceof NoMatrix2AuthorizationService - ) { - // rethrow as is - return e; - } - // Catch others and rethrow as FailToGetOpenIdToken that has user friendly message. - return new FailToGetOpenIdToken( - e instanceof Error ? e : new Error(String(e)), - ); + // Respect the user's custom transport, if set + const transport = + customTransportSetting.value$.value ?? + (await discovery.discoverPreferredTransport()); + + if (transport === null) throw new MatrixRTCTransportMissingError(serverName); + + return { transport, serverName }; } diff --git a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts index 61df88a73..d3f246c97 100644 --- a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts +++ b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts @@ -16,7 +16,7 @@ import { import { MatrixError } from "matrix-js-sdk"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { - type LivekitTransport, + type UnstableLivekitTransport, type Transport, } from "matrix-js-sdk/lib/matrixrtc"; @@ -28,12 +28,12 @@ import { type DiscoveryClient = RtcTransportAutoDiscoveryProps["client"]; -const backendTransport: LivekitTransport = { +const backendTransport: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://backend.example.org", }; -const configTransport: LivekitTransport = { +const configTransport: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://config.example.org", }; diff --git a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts index aeb6b051a..a5bedcd80 100644 --- a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts +++ b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts @@ -5,8 +5,8 @@ SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { - isLivekitTransport, - type LivekitTransport, + isUnstableLivekitTransport, + type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { type MatrixClient } from "matrix-js-sdk"; import { type Logger } from "matrix-js-sdk/lib/logger"; @@ -40,12 +40,12 @@ export class RtcTransportAutoDiscovery { this.logger = logger.getChild("[RtcTransportAutoDiscovery]"); } - public async discoverPreferredTransport(): Promise { + public async discoverPreferredTransport(): Promise { // 1) backend transports const backendTransport = await this.tryBackendTransports(); if (backendTransport) { this.logger.info( - `Found backend transport: ${backendTransport.livekit_service_url}`, + `Found backend transport: ${JSON.stringify(backendTransport)}`, ); return backendTransport; } @@ -54,7 +54,7 @@ export class RtcTransportAutoDiscovery { const configTransport = this.tryConfigTransport(); if (configTransport) { this.logger.info( - `Found app config transport: ${configTransport.livekit_service_url}`, + `Found app config transport: ${JSON.stringify(configTransport)}`, ); return configTransport; } @@ -67,7 +67,7 @@ export class RtcTransportAutoDiscovery { * This will not throw errors, but instead just log them and return null if the expected config is not found or malformed. * @private */ - private async tryBackendTransports(): Promise { + private async tryBackendTransports(): Promise { const client = this.client; // MSC4143: Attempt to fetch transports from backend. this.logger.info("First try to use getRTCTransports end point ..."); @@ -75,7 +75,7 @@ export class RtcTransportAutoDiscovery { const transportList = await doNetworkOperationWithRetry(async () => client._unstable_getRTCTransports(), ); - const first = transportList.find(isLivekitTransport); + const first = transportList.find(isUnstableLivekitTransport); if (first) { return first; } else { @@ -90,7 +90,7 @@ export class RtcTransportAutoDiscovery { return null; } - private tryConfigTransport(): LivekitTransport | null { + private tryConfigTransport(): UnstableLivekitTransport | null { const url = this.resolvedConfig.livekit?.livekit_service_url; if (url) { return { diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 62dc17a31..bdab8e7af 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -28,27 +28,30 @@ import { } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; -import { type IOpenIDToken } from "matrix-js-sdk"; +import { MatrixError } from "matrix-js-sdk"; import { logger, type Logger } from "matrix-js-sdk/lib/logger"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { + type LivekitGetTokenResponse, + type UnstableLivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; import { Connection, - ConnectionState, + type ConnectionState, type ConnectionOpts, } from "./Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; -import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; -import { - ElementCallError, - FailToGetOpenIdToken, -} from "../../../utils/errors.ts"; +import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; import { testJWTToken } from "../../../utils/test-fixtures.ts"; import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; +import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; let testScope: ObservableScope; -let client: MockedObject; +let client: MockedObject; let fakeLivekitRoom: MockedObject; @@ -56,23 +59,24 @@ let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; const ROOM_ID = "!roomID:example.org"; +const SLOT_ID = "m.call#ROOM"; -const livekitFocus: LivekitTransport = { - livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", +const transport: UnstableLivekitTransport = { type: "livekit", + url: "wss://matrix-rtc.m.localhost/livekit/sfu", }; function setupTest(): void { testScope = new ObservableScope(); - client = vi.mocked({ + client = vi.mocked({ getOpenIdToken: vi.fn().mockResolvedValue({ access_token: "rYsmGUEwNjKgJYyeNUkZseJN", token_type: "Bearer", matrix_server_name: "example.org", expires_in: 3600, }), - getDeviceId: vi.fn().mockReturnValue("ABCDEF"), - } as unknown as OpenIDClientParts); + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), + }); localParticipantEventEmiter = new EventEmitter(); @@ -116,24 +120,17 @@ function setupTest(): void { function setupRemoteConnection(): Connection { const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { - return { - status: 200, - body: { - url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: testJWTToken, - }, - }; - }); - fakeLivekitRoom.connect.mockImplementation(async (): Promise => { const changeEv = RoomEvent.ConnectionStateChanged; @@ -172,26 +169,32 @@ describe("Start connection states", () => { setupTest(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; const connection = new Connection(opts, logger); - expect(connection.state$.getValue()).toEqual("Initialized"); + expect(connection.state$.getValue()).toEqual({ state: "initialized" }); }); - it("fail to getOpenId token then error state", async () => { + it("fail to get SFU token then error state", async () => { setupTest(); vi.useFakeTimers(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, @@ -199,16 +202,16 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); - const deferred = Promise.withResolvers(); + const deferred = Promise.withResolvers(); - client.getOpenIdToken.mockImplementation( - async (): Promise => { + client._unstable_getLivekitToken.mockImplementation( + async (): Promise => { return await deferred.promise; }, ); @@ -219,18 +222,20 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState!).toEqual("FetchingConfig"); + expect(capturedState).toEqual({ state: "authenticating" }); deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); await vi.runAllTimersAsync(); capturedState = capturedStates.pop(); - if (capturedState instanceof Error) { - expect(capturedState.message).toEqual("Something went wrong"); - expect(connection.transport).toEqual(livekitFocus); + if (capturedState?.state === "error") { + expect(capturedState.error.message).toEqual("Something went wrong"); + expect(connection.transport).toEqual(transport); } else { - expect.fail("Expected FailedToStart state but got " + capturedState); + expect.fail( + "Expected FailedToStart state but got " + JSON.stringify(capturedState), + ); } }); @@ -239,9 +244,12 @@ describe("Start connection states", () => { vi.useFakeTimers(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, @@ -249,23 +257,22 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); - // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => { + client._unstable_getLivekitToken.mockImplementation(async () => { await deferredSFU.promise; - return { - status: 500, - body: { + throw new MatrixError( + { errcode: "M_LOOKUP_FAILED", error: "Failed to look up user info from homeserver", }, - }; + 500, + ); }); connection.start().catch(() => { @@ -274,7 +281,7 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState).toEqual(ConnectionState.FetchingConfig); + expect(capturedState).toEqual({ state: "authenticating" }); deferredSFU.resolve(); await vi.runAllTimersAsync(); @@ -282,15 +289,17 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if ( - capturedState instanceof ElementCallError && - capturedState.cause instanceof Error + capturedState?.state === "error" && + capturedState.error.cause instanceof Error ) { - expect(capturedState.cause.message).toContain( + expect(capturedState.error.cause.message).toContain( "Failed to look up user info from homeserver", ); - expect(connection.transport).toEqual(livekitFocus); + expect(connection.transport).toEqual(transport); } else { - expect.fail("Expected FailedToStart state but got " + capturedState); + expect.fail( + "Expected FailedToStart state but got " + JSON.stringify(capturedState), + ); } }); @@ -299,9 +308,12 @@ describe("Start connection states", () => { vi.useFakeTimers(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, @@ -309,24 +321,13 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); - // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { - return { - status: 200, - body: { - url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: testJWTToken, - }, - }; - }); - fakeLivekitRoom.connect.mockImplementation(async () => { await deferredSFU.promise; throw new Error("Failed to connect to livekit"); @@ -339,7 +340,7 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState).toEqual(ConnectionState.FetchingConfig); + expect(capturedState).toEqual({ state: "authenticating" }); deferredSFU.resolve(); await vi.runAllTimersAsync(); @@ -347,13 +348,13 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if ( - capturedState instanceof ElementCallError && - capturedState.cause instanceof Error + capturedState?.state === "error" && + capturedState.error.cause instanceof Error ) { - expect(capturedState.cause.message).toContain( + expect(capturedState.error.cause.message).toContain( "Failed to connect to livekit", ); - expect(connection.transport).toEqual(livekitFocus); + expect(connection.transport).toEqual(transport); } else { expect.fail( "Expected FailedToStart state but got " + JSON.stringify(capturedState), @@ -367,7 +368,7 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -377,22 +378,31 @@ describe("Start connection states", () => { await vi.runAllTimersAsync(); const initialState = capturedStates.shift(); - expect(initialState).toEqual(ConnectionState.Initialized); + expect(initialState).toEqual({ state: "initialized" }); const fetchingState = capturedStates.shift(); - expect(fetchingState).toEqual(ConnectionState.FetchingConfig); + expect(fetchingState).toEqual({ state: "authenticating" }); const disconnectedState = capturedStates.shift(); - expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected); + expect(disconnectedState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Disconnected, + }); const connectingState = capturedStates.shift(); - expect(connectingState).toEqual(ConnectionState.LivekitConnecting); + expect(connectingState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Connecting, + }); const connectedState = capturedStates.shift(); - expect(connectedState).toEqual(ConnectionState.LivekitConnected); + expect(connectedState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); }); it("stopping while connecting does not report an error", async () => { setupTest(); const connection = setupRemoteConnection(); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -409,10 +419,10 @@ describe("Start connection states", () => { pendingConnect.reject(new Error("Client initiated disconnect")); await stopping; - // start() resolves rather than rejecting (it is not awaited by the - // ConnectionManager, so a rejection would be unhandled). + // start() resolves rather than rejecting (it is not awaited by the caller, + // so a rejection would be unhandled). await expect(started).resolves.toBeUndefined(); - expect(capturedStates.at(-1)).toEqual(ConnectionState.Stopped); + expect(capturedStates.at(-1)).toEqual({ state: "stopped" }); expect(capturedStates.some((st) => st instanceof Error)).toBe(false); }); @@ -550,7 +560,10 @@ describe("remote track logging", () => { { client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index cc84142d9..7e94b39f1 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -13,6 +13,7 @@ import { import { ConnectionError, ConnectionErrorReason, + type ConnectionState as LivekitConnectionState, type Participant, type RemoteParticipant, type RemoteTrackPublication, @@ -22,16 +23,11 @@ import { type Track, type TrackPublication, } from "livekit-client"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, map } from "rxjs"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { - getSFUConfigWithOpenID, - type OpenIDClientParts, - type SFUConfig, -} from "../../../livekit/openIDSFU.ts"; import { type Behavior } from "../../Behavior.ts"; import { type ObservableScope } from "../../ObservableScope.ts"; import { @@ -42,51 +38,43 @@ import { SFURoomCreationRestrictedError, UnknownCallError, } from "../../../utils/errors.ts"; +import { + type ClientGetTokenParts, + getSFUConfig, + type ClientOpenIDParts, + type SFUConfig, +} from "../../../livekit/auth"; export interface ConnectionOpts { - /** - * For the local transport we already do know the jwt token and url. We can reuse it. - * On top the local transport will send additional data to the jwt server to use delayed event delegation. - */ - existingSFUConfig?: SFUConfig; /** The identity parts to use on this connection */ ownMembershipIdentity: CallMembershipIdentityParts; + /** Whether we want to publish or only subscribe on this connection. */ + role: "publisher" | "subscriber"; /** The media transport to connect to. */ - transport: LivekitTransport; + transport: UnstableLivekitTransport; + /** The name of the homeserver to which the {@link transport} belongs. */ + serverName: string; /** The Matrix client to use for OpenID and SFU config requests. */ - client: OpenIDClientParts; - /** The room ID this connection is associated with. */ + client: ClientGetTokenParts & ClientOpenIDParts; + /** The ID of the Matrix room in which the session takes place. */ roomId: string; + /** The ID of the MatrixRTC slot in which the session takes place. */ + slotId: string; /** The observable scope to use for this connection. */ scope: ObservableScope; /** Optional factory to create the LiveKit room, mainly for testing purposes. */ livekitRoomFactory: () => LivekitRoom; } -export class FailedToStartError extends Error { - public constructor(message: string) { - super(message); - this.name = "FailedToStartError"; - } -} -export enum ConnectionState { - /** The start state of a connection. It has been created but nothing has loaded yet. */ - Initialized = "Initialized", - /** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room */ - FetchingConfig = "FetchingConfig", - Stopped = "Stopped", - /** The same as ConnectionState.Disconnected from `livekit-client` */ - LivekitDisconnected = "disconnected", - /** The same as ConnectionState.Connecting from `livekit-client` */ - LivekitConnecting = "connecting", - /** The same as ConnectionState.Connected from `livekit-client` */ - LivekitConnected = "connected", - /** The same as ConnectionState.Reconnecting from `livekit-client` */ - LivekitReconnecting = "reconnecting", - /** The same as ConnectionState.SignalReconnecting from `livekit-client` */ - LivekitSignalReconnecting = "signalReconnecting", -} +export type ConnectionState = + // The start state of a connection. It has been created but nothing has loaded yet. + | { state: "initialized" } + // `start` has been called on the connection. It acquires the JWT token to connect to the LK room. + | { state: "authenticating" } + | { state: "authenticated"; livekitState: LivekitConnectionState } + | { state: "error"; error: ElementCallError } + | { state: "stopped" }; /** * A connection to a Matrix RTC LiveKit backend. @@ -95,19 +83,29 @@ export enum ConnectionState { */ export class Connection { // Private Behavior - private readonly _state$ = new BehaviorSubject< - ConnectionState | ElementCallError - >(ConnectionState.Initialized); + private readonly _state$ = new BehaviorSubject({ + state: "initialized", + }); /** * The current state of the connection to the media transport. */ - public readonly state$: Behavior = this._state$; + public readonly state$: Behavior = this._state$; + + /** + * Whether we want to publish or only subscribe on this connection. + */ + public readonly role: "publisher" | "subscriber"; /** * The media transport to connect to. */ - public readonly transport: LivekitTransport; + public readonly transport: UnstableLivekitTransport; + + /** + * The name of the homeserver to which the {@link transport} belongs. + */ + public readonly serverName: string; public readonly livekitRoom: LivekitRoom; @@ -137,11 +135,11 @@ export class Connection { protected stopped = false; // TODO: can we just keep the ConnectionOpts object instead of spreading? - private readonly client: OpenIDClientParts; + private readonly client: ClientGetTokenParts & ClientOpenIDParts; private readonly roomId: string; + private readonly slotId: string; private readonly logger: Logger; private readonly ownMembershipIdentity: CallMembershipIdentityParts; - private readonly existingSFUConfig?: SFUConfig; /** * Creates a new connection to a matrix RTC LiveKit backend. * @@ -151,19 +149,19 @@ export class Connection { */ public constructor(opts: ConnectionOpts, logger: Logger) { this.ownMembershipIdentity = opts.ownMembershipIdentity; - this.existingSFUConfig = opts.existingSFUConfig; this.roomId = opts.roomId; + this.slotId = opts.slotId; this.logger = logger.getChild( - "[Connection " + opts.transport.livekit_service_url + "]", + `[Connection ${JSON.stringify(opts.transport)}]`, ); - this.logger.info( - `constructor: ${opts.transport.livekit_service_url} roomId: ${this.roomId} withSfuConfig?: ${opts.existingSFUConfig ? JSON.stringify(opts.existingSFUConfig) : "undefined"}`, - ); - const { transport, client, scope } = opts; + this.logger.debug(`constructor called`); + const { role, transport, serverName, client, scope } = opts; this.scope = scope; this.livekitRoom = opts.livekitRoomFactory(); + this.role = role; this.transport = transport; + this.serverName = serverName; this.client = client; this.remoteParticipants$ = scope.behavior( @@ -304,16 +302,10 @@ export class Connection { this.logger.debug("Starting Connection"); this.stopped = false; try { - this._state$.next(ConnectionState.FetchingConfig); - // We should already have this information after creating the localTransport. - // only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined) - const { url, jwt, livekitAlias } = - this.existingSFUConfig ?? - (await this.getSFUConfigForRemoteConnection()); + this._state$.next({ state: "authenticating" }); + const { url, jwt, livekitAlias } = await this.getSFUConfig(); this.logger.debug( - "Starting Connection to: ", - this.transport.livekit_service_url, - "jwt: ", + "Starting Connection - jwt: ", jwt, "wss: ", url, @@ -324,16 +316,12 @@ export class Connection { // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - // Setup observer once we are done with getSFUConfigWithOpenID + // Setup observer once we are done with getSFUConfig connectionStateObserver(this.livekitRoom) - .pipe( - this.scope.bind(), - map((s) => s as unknown as ConnectionState), - ) - .subscribe((lkState) => { - // It is save to cast lkState to ConnectionState as they are fully overlapping. - this._state$.next(lkState); - }); + .pipe(this.scope.bind()) + .subscribe((livekitState) => + this._state$.next({ state: "authenticated", livekitState }), + ); try { this.logger.info(`livekitRoom.connect ${url}`); @@ -377,36 +365,37 @@ export class Connection { // stop() was called while we were connecting, which makes the pending // connect reject. That is the abort we asked for, not a failure, so // don't record an error state on a stopped connection or rethrow it - // (start() is not awaited by the ConnectionManager, so a throw here + // (start() is not necessarily awaited by the caller, so a throw here // becomes an unhandled promise rejection). this.logger.debug(`Connect aborted because the connection was stopped`); return; } this.logger.debug(`Failed to connect to LiveKit room: ${error}`); - this._state$.next( - error instanceof ElementCallError - ? error - : error instanceof Error - ? new UnknownCallError(error) - : new UnknownCallError(new Error(`${error}`)), - ); + this._state$.next({ + state: "error", + error: + error instanceof ElementCallError + ? error + : error instanceof Error + ? new UnknownCallError(error) + : new UnknownCallError(new Error(`${error}`)), + }); // Its okay to ignore the throw. The error is part of the state. throw error; } } - protected async getSFUConfigForRemoteConnection(): Promise { - // This will only be called for sfu's where we do not publish ourselves. - // For the local connection we will use the existingJwtTokenData - return await getSFUConfigWithOpenID( - this.client, - this.ownMembershipIdentity, - this.transport.livekit_service_url, - this.roomId, - // dont pass any custom opts for the subscribe only connections - {}, - this.logger, - ); + protected async getSFUConfig(): Promise { + return await getSFUConfig({ + client: this.client, + membership: this.ownMembershipIdentity, + role: this.role, + transport: this.transport, + serverName: this.serverName, + roomId: this.roomId, + slotId: this.slotId, + logger: this.logger, + }); } /** @@ -416,17 +405,13 @@ export class Connection { * If the connection is already stopped, this is a no-op. */ public async stop(): Promise { - this.logger.debug( - `stop: disconnecing from lk room ${this.transport.livekit_service_url}`, - ); + this.logger.debug("stop: disconnecing from lk room"); if (this.stopped) return; // Mark as stopped before disconnecting so that a connect() aborted by the // disconnect sees the flag and does not report the abort as an error. this.stopped = true; await this.livekitRoom.disconnect(); - this._state$.next(ConnectionState.Stopped); - this.logger.debug( - `stop: DONE disconnecing from lk room ${this.transport.livekit_service_url}`, - ); + this._state$.next({ state: "stopped" }); + this.logger.debug("stop: DONE disconnecing from lk room"); } } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 96a28d5ad..e5240f096 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -16,14 +16,10 @@ import { logger, type Logger } from "matrix-js-sdk/lib/logger"; // imported as inline to support worker when loaded from a cdn (cross domain) import E2EEWorker from "livekit-client/e2ee-worker?worker&inline"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; -import type { - OpenIDClientParts, - SFUConfig, -} from "../../../livekit/openIDSFU.ts"; import type { MediaDevices } from "../../MediaDevices.ts"; import type { Behavior } from "../../Behavior.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; @@ -39,15 +35,20 @@ import { noiseSuppressionSetting, autoGainControlSetting, } from "../../../settings/settings.ts"; +import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { createConnection( scope: ObservableScope, - transport: LivekitTransport, + role: "publisher" | "subscriber", + transport: UnstableLivekitTransport, + serverName: string, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - sfuConfig?: SFUConfig, ): Connection; } @@ -58,7 +59,8 @@ export class ECConnectionFactory implements ConnectionFactory { * Creates a ConnectionFactory for LiveKit connections. * * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens. - * @param roomId - The current room ID. + * @param roomId - The ID of the Matrix room in which the session takes place. + * @param slotId - The ID of the MatrixRTC slot in which the session takes place. * @param devices - Used for video/audio out/in capture options. * @param processorState$ - Effects like background blur (only for publishing connection?) * @param livekitKeyProvider - Optional key provider for end-to-end encryption. @@ -66,8 +68,9 @@ export class ECConnectionFactory implements ConnectionFactory { * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used. */ public constructor( - private client: OpenIDClientParts, + private client: ClientGetTokenParts & ClientOpenIDParts, private readonly roomId: string, + private readonly slotId: string, private devices: MediaDevices, private processorState$: Behavior, livekitKeyProvider: BaseKeyProvider | undefined, @@ -96,6 +99,7 @@ export class ECConnectionFactory implements ConnectionFactory { * * @param scope The observable scope (used for clean-up) * @param transport The transport to use for this connection. + * @param serverName The name of the homeserver to which the {@link transport} belongs. * @param ownMembershipIdentity required to connect (using the jwt service) with the SFU. * @param logger The logger instance to use for this connection. * @param sfuConfig optional config in case we already have a token for this connection. @@ -103,16 +107,19 @@ export class ECConnectionFactory implements ConnectionFactory { */ public createConnection( scope: ObservableScope, - transport: LivekitTransport, + role: "publisher" | "subscriber", + transport: UnstableLivekitTransport, + serverName: string, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - sfuConfig?: SFUConfig, ): Connection { return new Connection( { - existingSFUConfig: sfuConfig, roomId: this.roomId, + slotId: this.slotId, + role, transport, + serverName, client: this.client, scope: scope, livekitRoomFactory: this.livekitRoomFactory, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts deleted file mode 100644 index 18f54f021..000000000 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ /dev/null @@ -1,282 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { - combineLatest, - map, - type Observable, - of, - switchMap, - startWith, - catchError, - NEVER, -} from "rxjs"; -import { type Logger } from "matrix-js-sdk/lib/logger"; -import { type RemoteParticipant } from "livekit-client"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; - -import { type Behavior } from "../../Behavior.ts"; -import { type Connection } from "./Connection.ts"; -import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; -import { generateItemsWithEpoch } from "../../../utils/observable.ts"; -import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; -import { type ConnectionFactory } from "./ConnectionFactory.ts"; -import { - isLocalTransport, - type LocalTransport, -} from "../localMember/LocalTransport.ts"; -import { type SFUConfig } from "../../../livekit/openIDSFU.ts"; - -export class ConnectionManagerData { - private readonly store: Map< - string, - { connection: Connection; participants: RemoteParticipant[] } - > = new Map(); - - public constructor(private readonly logger?: Logger) {} - - public add(connection: Connection, participants: RemoteParticipant[]): void { - const key = this.getKey(connection.transport); - const existing = this.store.get(key); - if (!existing) { - this.store.set(key, { connection, participants }); - } else { - // Transports are deduplicated by URL upstream, so this should never - // happen; if it does, members may be matched against the wrong room. - this.logger?.warn( - `Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`, - ); - existing.participants.push(...participants); - } - } - - private getKey(transport: LivekitTransport): string { - // This is enough as a key because the ConnectionManager is already scoped by room. - // We also do not need to consider the slotId at this point since each `MatrixRTCSession` is already scoped by `slotDescription: {id, application}`. - return transport.livekit_service_url; - } - - public getConnections(): Connection[] { - return Array.from(this.store.values()).map(({ connection }) => connection); - } - - public getConnectionForTransport( - transport: LivekitTransport, - ): Connection | null { - return this.store.get(this.getKey(transport))?.connection ?? null; - } - - public getParticipantsForTransport( - transport: LivekitTransport, - ): RemoteParticipant[] { - const key = this.getKey(transport); - const existing = this.store.get(key); - if (existing) { - return existing.participants; - } - return []; - } -} - -interface Props { - scope: ObservableScope; - connectionFactory: ConnectionFactory; - localTransport$: Observable; - remoteTransports$: Behavior>; - - logger: Logger; - ownMembershipIdentity: CallMembershipIdentityParts; -} - -// TODO - write test for scopes (do we really need to bind scope) -export interface IConnectionManager { - connectionManagerData$: Behavior>; -} - -/** - * Crete a `ConnectionManager` - * @param props - Configuration object - * @param props.scope - The observable scope used by this object - * @param props.connectionFactory - Used to create new connections - * @param props.localTransport$ - The transport to publish local media on. (deduplicated with remoteTransports$) - * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$) - * @param props.ownMembershipIdentity - The own membership identity to use. - * @param props.logger - The logger to use. - - * - * Each of these behaviors can be interpreted as subscribed list of transports. - * - * Using `registerTransports` independent external modules can control what connections - * are created by the ConnectionManager. - * - * The connection manager will remove all duplicate transports in each subscibed list. - * - * See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe. - */ -export function createConnectionManager$({ - scope, - connectionFactory, - localTransport$, - remoteTransports$, - logger: parentLogger, - ownMembershipIdentity, -}: Props): IConnectionManager { - const logger = parentLogger.getChild("[ConnectionManager]"); - // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing - - const localTransportAsArray$ = localTransport$.pipe( - // LocalMember already surfaces local transport errors properly in the UI, - // here we can just swallow them - catchError(() => NEVER), - map((transport) => [transport]), - startWith([]), - ); - - /** - * All transports currently managed by the ConnectionManager. - * - * This list does not include duplicate transports. - * - * It is build based on the list of subscribed transports (`transportsSubscriptions$`). - * externally this is modified via `registerTransports()`. - */ - const localAndRemoteTransports$: Behavior< - Epoch<(LivekitTransport | LocalTransport)[]> - > = scope.behavior( - combineLatest([localTransportAsArray$, remoteTransports$]).pipe( - // Combine local and remote transports into one transport array - // and set the forceOldJwtEndpoint property on the local transport - map(([localTransportAsArray, remoteTransports]) => { - const dedupedRemote = removeDuplicateTransports(remoteTransports.value); - const remoteWithoutLocal = dedupedRemote.filter( - (transport) => - !localTransportAsArray.find((l) => - areLivekitTransportsEqual(l.transport, transport), - ), - ); - logger.debug( - "remoteWithoutLocal", - remoteWithoutLocal, - "localTransportAsArray", - localTransportAsArray, - ); - return new Epoch( - [...localTransportAsArray, ...remoteWithoutLocal], - remoteTransports.epoch, - ); - }), - ), - ); - - /** - * Connections for each transport in use by one or more session members. - */ - const connections$ = scope.behavior( - localAndRemoteTransports$.pipe( - generateItemsWithEpoch( - "ConnectionManager connections$", - function* (transports) { - for (const transport of transports) { - if (isLocalTransport(transport)) { - // This is the local transport; only the `LocalTransport` - // interface has a `sfuConfig` field. - yield { - keys: [ - transport.transport.livekit_service_url, - transport.sfuConfig, - ], - data: undefined, - }; - } else { - yield { - keys: [ - transport.livekit_service_url, - undefined as SFUConfig | undefined, - ], - data: undefined, - }; - } - } - }, - (scope, _data$, serviceUrl, sfuConfig) => { - const connection = connectionFactory.createConnection( - scope, - { - type: "livekit", - livekit_service_url: serviceUrl, - }, - ownMembershipIdentity, - logger, - // TODO: This whole optional SFUConfig parameter is not particularly elegant. - // I would like it if connections always fetched the SFUConfig by themselves. - sfuConfig, - ); - // Start the connection immediately - // Use connection state to track connection progress - void connection.start(); - // TODO subscribe to connection state to retry or log issues? - return connection; - }, - ), - ), - ); - - const connectionManagerData$ = scope.behavior( - connections$.pipe( - switchMap((connections) => { - const epoch = connections.epoch; - - // Map the connections to list of {connection, participants}[] - const listOfConnectionsWithRemoteParticipants = connections.value.map( - (connection) => { - return connection.remoteParticipants$.pipe( - map((participants) => ({ - connection, - participants, - })), - ); - }, - ); - - // probably not required - - if (listOfConnectionsWithRemoteParticipants.length === 0) { - return of(new Epoch(new ConnectionManagerData(), epoch)); - } - - // combineLatest the several streams into a single stream with the ConnectionManagerData - return combineLatest(listOfConnectionsWithRemoteParticipants).pipe( - map( - (lists) => - new Epoch( - lists.reduce((data, { connection, participants }) => { - data.add(connection, participants); - return data; - }, new ConnectionManagerData(logger)), - epoch, - ), - ), - ); - }), - ), - new Epoch(new ConnectionManagerData(), -1), - ); - - return { connectionManagerData$ }; -} - -function removeDuplicateTransports( - transports: T[], -): T[] { - return transports.reduce((acc, transport) => { - if (!acc.some((t) => areLivekitTransportsEqual(t, transport))) - acc.push(transport); - return acc; - }, [] as T[]); -} diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts similarity index 63% rename from src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts rename to src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts index e7d91c633..2344c1e55 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts @@ -6,34 +6,49 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { BehaviorSubject, NEVER } from "rxjs"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject } from "rxjs"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type RemoteParticipant } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts"; import { - createConnectionManager$, - ConnectionManagerData, -} from "./ConnectionManager.ts"; + createConnectionMap$, + ConnectionMap, + keyFor, +} from "./ConnectionMap.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts"; -import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type Behavior } from "../../Behavior.ts"; +import { type TransportLocator } from "../../../livekit/auth"; // Some test constants -const TRANSPORT_1: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.example.org", +const TRANSPORT_1: TransportLocator = { + transport: { + type: "livekit", + livekit_service_url: "https://lk.example.org", + }, + serverName: "example.org", }; -const TRANSPORT_2: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.sample.com", +const TRANSPORT_2: TransportLocator = { + transport: { + type: "livekit", + livekit_service_url: "https://lk.sample.com", + }, + serverName: "sample.com", }; +function findConnection( + transport: TransportLocator, + connections: Connection[], +): Connection | undefined { + const key = keyFor(transport); + return connections.find((c) => keyFor(c) === key); +} + let fakeConnectionFactory: ConnectionFactory; let testScope: ObservableScope; @@ -47,9 +62,15 @@ beforeEach(() => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (scope: ObservableScope, transport: LivekitTransport) => { + ( + scope: ObservableScope, + _role: unknown, + transport: UnstableLivekitTransport, + serverName: string, + ) => { const mockConnection = { transport, + serverName, remoteParticipants$: new BehaviorSubject([]), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); @@ -71,10 +92,10 @@ afterEach(() => { describe("connections$ stream", () => { test("Should create and start new connections for each transports", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), @@ -83,7 +104,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("a", { a: expect.toSatisfy((e: Epoch) => { const connections = e.value; @@ -93,15 +114,11 @@ describe("connections$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection, ).toHaveBeenCalledTimes(2); - const conn1 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_1), - ); + const conn1 = findConnection(TRANSPORT_1, connections); expect(conn1).toBeDefined(); expect(conn1!.start).toHaveBeenCalled(); - const conn2 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), - ); + const conn2 = findConnection(TRANSPORT_2, connections); expect(conn2).toBeDefined(); expect(conn2!.start).toHaveBeenCalled(); return true; @@ -110,12 +127,88 @@ describe("connections$ stream", () => { }); }); - test("Should start connection only once", () => { + test("Shares one connection between homeservers advertising the same JWT service", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const sameServiceElsewhere: TransportLocator = { + transport: TRANSPORT_1.transport, + serverName: "elsewhere.org", + }; + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, + remoteTransports$: behavior("a", { + a: new Epoch([sameServiceElsewhere], 0), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable( + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), + ).toBe("a", { + a: expect.toSatisfy((e: Epoch) => { + expect(e.value.length).toBe(1); + // The JWT service is reached directly, so the server name plays no + // part and the local locator, which came first, is the one used + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledExactlyOnceWith( + expect.anything(), + "publisher", + TRANSPORT_1.transport, + "example.org", + expect.anything(), + expect.anything(), + ); + return true; + }), + }); + }); + }); + + test("Keeps one connection per homeserver for transports with a url", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const urlTransport: TransportLocator = { + transport: { type: "livekit", url: "wss://sfu.example.org" }, + serverName: "example.org", + }; + const sameUrlElsewhere: TransportLocator = { + transport: urlTransport.transport, + serverName: "elsewhere.org", + }; + const connectionMap$ = createConnectionMap$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport: urlTransport, + remoteTransports$: behavior("a", { + a: new Epoch([sameUrlElsewhere], 0), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable( + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), + ).toBe("a", { + a: expect.toSatisfy((e: Epoch) => { + // Our homeserver asks the named server for the token, so each + // server name needs its own connection + expect(e.value.length).toBe(2); + expect(findConnection(urlTransport, e.value)).toBeDefined(); + expect(findConnection(sameUrlElsewhere, e.value)).toBeDefined(); + return true; + }), + }); + }); + }); + + test("Should start connection only once", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const connectionMap$ = createConnectionMap$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport: TRANSPORT_1, remoteTransports$: behavior("abcdef", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1], 1), @@ -129,7 +222,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("xxxxxa", { x: expect.anything(), a: expect.toSatisfy((e: Epoch) => { @@ -140,14 +233,10 @@ describe("connections$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection, ).toHaveBeenCalledTimes(2); - const conn2 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), - ); + const conn2 = findConnection(TRANSPORT_2, connections); expect(conn2).toBeDefined(); - const conn1 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_1), - ); + const conn1 = findConnection(TRANSPORT_1, connections); expect(conn1).toBeDefined(); expect(conn1!.start).toHaveBeenCalledOnce(); @@ -159,10 +248,10 @@ describe("connections$ stream", () => { test("Should cleanup connections when not needed anymore", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("abc", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), @@ -173,7 +262,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("xab", { x: expect.anything(), a: expect.toSatisfy((e: Epoch) => { @@ -186,8 +275,9 @@ describe("connections$ stream", () => { expect(connections.length).toBe(1); // The second connection should have been stopped has it is no longer needed. - const connection2 = allCreatedConnections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), + const connection2 = findConnection( + TRANSPORT_2, + allCreatedConnections, ); expect(connection2).toBeDefined(); expect(connection2!.stop).toHaveBeenCalled(); @@ -203,40 +293,36 @@ describe("connections$ stream", () => { }); }); -describe("ConnectionManagerData", () => { +describe("ConnectionMap", () => { test("warns when a second connection to the same URL is merged", () => { const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); - const data = new ConnectionManagerData(logger); - const connection = { transport: TRANSPORT_1 } as unknown as Connection; + const data = new ConnectionMap(logger); + const connection = TRANSPORT_1 as unknown as Connection; const p = (identity: string): RemoteParticipant => ({ identity }) as unknown as RemoteParticipant; data.add(connection, [p("a")]); data.add({ ...connection } as Connection, [p("b")]); expect(warn).toHaveBeenCalledWith( expect.stringContaining( - "second connection to https://lk.example.org: existing [a], adding [b]", + 'second connection to [null,"https://lk.example.org",null]: existing [a], adding [b]', ), ); expect(data.getParticipantsForTransport(TRANSPORT_1)).toHaveLength(2); }); }); -describe("connectionManagerData$ stream", () => { +describe("connectionMap$", () => { // Used in test to control fake connections' remoteParticipants$ streams let fakeRemoteParticipantsStreams: Map>; - function keyForTransport(transport: LivekitTransport): string { - return `${transport.livekit_service_url}`; - } - beforeEach(() => { fakeRemoteParticipantsStreams = new Map(); function getRemoteParticipantsFor( - transport: LivekitTransport, + transport: TransportLocator, ): Behavior { return ( - fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ?? + fakeRemoteParticipantsStreams.get(keyFor(transport)) ?? new BehaviorSubject([]) ); } @@ -245,13 +331,22 @@ describe("connectionManagerData$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (scope: ObservableScope, transport: LivekitTransport) => { + ( + scope: ObservableScope, + _role: unknown, + transport: UnstableLivekitTransport, + serverName: string, + ) => { const fakeRemoteParticipants$ = new BehaviorSubject< RemoteParticipant[] >([]); const mockConnection = { transport, - remoteParticipants$: getRemoteParticipantsFor(transport), + serverName, + remoteParticipants$: getRemoteParticipantsFor({ + transport, + serverName, + }), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); @@ -261,7 +356,7 @@ describe("connectionManagerData$ stream", () => { }); fakeRemoteParticipantsStreams.set( - keyForTransport(transport), + keyFor({ transport, serverName }), fakeRemoteParticipants$, ); return mockConnection; @@ -274,7 +369,7 @@ describe("connectionManagerData$ stream", () => { // Setup the fake participants streams behavior // ============================== fakeRemoteParticipantsStreams.set( - keyForTransport(TRANSPORT_1), + keyFor(TRANSPORT_1), behavior("oa-b", { o: [], a: [{ identity: "user1A" } as RemoteParticipant], @@ -286,7 +381,7 @@ describe("connectionManagerData$ stream", () => { ); fakeRemoteParticipantsStreams.set( - keyForTransport(TRANSPORT_2), + keyFor(TRANSPORT_2), behavior("o-a", { o: [], a: [{ identity: "user2A" } as RemoteParticipant], @@ -294,10 +389,10 @@ describe("connectionManagerData$ stream", () => { ); // ============================== - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), @@ -305,16 +400,16 @@ describe("connectionManagerData$ stream", () => { ownMembershipIdentity: ownMemberMock, }); - expectObservable(connectionManagerData$).toBe("abcd", { + expectObservable(connectionMap$).toBe("abcd", { a: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); return true; }), b: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); @@ -324,7 +419,7 @@ describe("connectionManagerData$ stream", () => { return true; }), c: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); @@ -337,7 +432,7 @@ describe("connectionManagerData$ stream", () => { return true; }), d: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionMap.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts new file mode 100644 index 000000000..586e8d49b --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts @@ -0,0 +1,219 @@ +/* +Copyright 2025-2026 Element Creations Ltd. +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, map, of, switchMap, tap } from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type RemoteParticipant } from "livekit-client"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { type Behavior } from "../../Behavior.ts"; +import { type Connection } from "./Connection.ts"; +import { + Epoch, + mapEpoch, + type ObservableScope, +} from "../../ObservableScope.ts"; +import { generateItemsWithEpoch } from "../../../utils/observable.ts"; +import { type ConnectionFactory } from "./ConnectionFactory.ts"; +import { type TransportLocator } from "../../../livekit/auth/types.ts"; + +// Branded string type, so that only values returned by `keyFor` can be used +// as keys +export type TransportKey = string & { __transportKey: true }; + +/** + * Converts a {@link TransportLocator} to a key suitable for identifying it in a + * {@link Map} or {@link Set}. Locators with the same key share a connection. + */ +export function keyFor({ + transport, + serverName, +}: TransportLocator): TransportKey { + // The server name only takes part in authentication for a transport with a + // `url`, where our homeserver asks that server for the token. A JWT service + // is reached directly, so members of several homeservers advertising the + // same one must share a connection rather than each get their own. + const hasUrl = "url" in transport; + // Serialize the fields in a consistent order + return JSON.stringify([ + hasUrl ? transport.url : null, + "livekit_service_url" in transport ? transport.livekit_service_url : null, + hasUrl ? serverName : null, + ]) as TransportKey; +} + +function removeDuplicateTransports( + transports: TransportLocator[], +): TransportLocator[] { + // The first locator wins, so the local one stays first in the list + const map = new Map(); + for (const transport of transports) { + const key = keyFor(transport); + if (!map.has(key)) map.set(key, transport); + } + return [...map.values()]; +} + +export class ConnectionMap { + private readonly map = new Map< + TransportKey, + { connection: Connection; participants: RemoteParticipant[] } + >(); + + public constructor(private readonly logger?: Logger) {} + + public add(connection: Connection, participants: RemoteParticipant[]): void { + const key = keyFor(connection); + const existing = this.map.get(key); + if (!existing) { + this.map.set(key, { connection, participants }); + } else { + // Transports are deduplicated by key upstream, so this should never + // happen; if it does, members may be matched against the wrong room. + this.logger?.warn( + `Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`, + ); + existing.participants.push(...participants); + } + } + + public getConnections(): Connection[] { + return Array.from(this.map.values()).map(({ connection }) => connection); + } + + public getConnectionForTransport( + transport: TransportLocator, + ): Connection | null { + return this.map.get(keyFor(transport))?.connection ?? null; + } + + public getParticipantsForTransport( + transport: TransportLocator, + ): RemoteParticipant[] { + const existing = this.map.get(keyFor(transport)); + if (existing) { + return existing.participants; + } + return []; + } +} + +interface Props { + scope: ObservableScope; + connectionFactory: ConnectionFactory; + localTransport: TransportLocator; + remoteTransports$: Behavior>; + + logger: Logger; + ownMembershipIdentity: CallMembershipIdentityParts; +} + +/** + * Create a dynamic map of connections and their associated backend participants. + * @param props - Configuration object + * @param props.scope - The observable scope used by this object + * @param props.connectionFactory - Used to create new connections + * @param props.localTransport - The transport to publish local media on. (deduplicated with remoteTransports$) + * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport) + * @param props.ownMembershipIdentity - The own membership identity to use. + * @param props.logger - The logger to use. + */ +export function createConnectionMap$({ + scope, + connectionFactory, + localTransport, + remoteTransports$, + logger: parentLogger, + ownMembershipIdentity, +}: Props): Behavior> { + const logger = parentLogger.getChild("[ConnectionMap]"); + // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing + + /** + * All transports currently represented in the connection map. This list does + * not include duplicate transports. + */ + const localAndRemoteTransports$: Behavior> = + scope.behavior( + remoteTransports$.pipe( + mapEpoch((remoteTransports) => + removeDuplicateTransports([localTransport, ...remoteTransports]), + ), + tap((transports) => + logger.debug("localAndRemoteTransports$ = ", transports), + ), + ), + ); + + const localTransportKey = keyFor(localTransport); + + /** + * Connections for each transport in use by one or more session members. + */ + const connections$ = scope.behavior( + localAndRemoteTransports$.pipe( + generateItemsWithEpoch( + "ConnectionMap connections$", + function* (transports) { + for (const transport of transports) { + const role = + keyFor(transport) === localTransportKey + ? ("publisher" as const) + : ("subscriber" as const); + yield { + // The locator is an object, so a stable key stands in for it + keys: [role, keyFor(transport)], + data: transport, + }; + } + }, + (scope, locator$, role, _key) => { + const { transport, serverName } = locator$.value; + const connection = connectionFactory.createConnection( + scope, + role, + transport, + serverName, + ownMembershipIdentity, + logger, + ); + // Start the connection immediately + // Use connection state to track connection progress + void connection.start(); + // TODO subscribe to connection state to retry or log issues? + return connection; + }, + ), + ), + ); + + return scope.behavior( + connections$.pipe( + switchMap(({ value: connections, epoch }) => { + if (connections.length === 0) + return of(new Epoch(new ConnectionMap(), epoch)); + + return combineLatest( + // Map the connections to list of Observable<{connection, participants}> + connections.map((connection) => + connection.remoteParticipants$.pipe( + map((participants) => ({ connection, participants })), + ), + ), + // Collect into a single ConnectionMap + (...list) => { + const map = new ConnectionMap(logger); + for (const item of list) + map.add(item.connection, item.participants); + return new Epoch(map, epoch); + }, + ); + }), + ), + ); +} diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index 411ae2001..be6c149f4 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -14,7 +14,10 @@ import EventEmitter from "events"; import { ObservableScope } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; -import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; import { exampleTransport, mockMediaDevices, @@ -32,6 +35,7 @@ import { cameraBitrate, cameraCodec, } from "../../../settings/settings.ts"; +import { testJWTToken } from "../../../utils/test-fixtures.ts"; // At the top of your test file, after imports vi.mock("livekit-client", async (importOriginal) => { @@ -51,13 +55,13 @@ vi.mock("livekit-client", async (importOriginal) => { }); let testScope: ObservableScope; -let mockClient: OpenIDClientParts; +let mockClient: ClientGetTokenParts & ClientOpenIDParts; beforeEach(() => { testScope = new ObservableScope(); mockClient = { getOpenIdToken: vi.fn().mockReturnValue(""), - getDeviceId: vi.fn().mockReturnValue("DEV000"), + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), }; }); @@ -79,6 +83,7 @@ describe("ECConnectionFactory - Audio inputs options", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -89,7 +94,9 @@ describe("ECConnectionFactory - Audio inputs options", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -121,6 +128,7 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({ audioOutput: { available$: constant(new Map()), @@ -137,7 +145,9 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -162,6 +172,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -172,7 +183,9 @@ describe("ECConnectionFactory - Camera quality settings", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -199,6 +212,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -209,7 +223,9 @@ describe("ECConnectionFactory - Camera quality settings", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -237,6 +253,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -247,7 +264,9 @@ describe("ECConnectionFactory - Camera quality settings", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index 2e0051928..1c6a32790 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -9,22 +9,16 @@ import { describe, test, expect, beforeEach, afterEach, vi } from "vitest"; import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, - type LivekitTransport, + type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs"; -import { type IConnectionManager } from "./ConnectionManager.ts"; import { type RemoteMatrixLivekitMember, createRemoteMatrixLivekitMembers$, } from "./MatrixLivekitMembers.ts"; -import { - Epoch, - mapEpoch, - ObservableScope, - trackEpoch, -} from "../../ObservableScope.ts"; -import { ConnectionManagerData } from "./ConnectionManager.ts"; +import { Epoch, ObservableScope, trackEpoch } from "../../ObservableScope.ts"; +import { ConnectionMap } from "./ConnectionMap.ts"; import { flushPromises, mockRtcMembership, @@ -33,18 +27,20 @@ import { import { type Connection } from "./Connection.ts"; import { constant } from "../../Behavior.ts"; import { localRtcMember } from "../../../utils/test-fixtures.ts"; +import { type TransportLocator } from "../../../livekit/auth"; +import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; let testScope: ObservableScope; const fallbackMemberId = (userId: string, deviceId: string): string => `${userId}:${deviceId}`; -const transportA: LivekitTransport = { +const transportA: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", }; -const transportB: LivekitTransport = { +const transportB: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://lk.sample.com", }; @@ -78,23 +74,18 @@ function epochMeWith$( test("should signal participant not yet connected to livekit", async () => { const info = vi.spyOn(logger, "info"); const mockedMemberships$ = new BehaviorSubject([bobMembership]); - const mockConnectionManagerData$ = new BehaviorSubject( - new ConnectionManagerData(), - ); + const mockConnectionMap$ = new BehaviorSubject(new ConnectionMap()); const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(mockedMemberships$); - const connectionManagerData$ = epochMeWith$( - memberships$, - mockConnectionManagerData$, + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, mockConnectionMap$), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); @@ -117,21 +108,15 @@ test("should signal participant not yet connected to livekit", async () => { function createEpochedMemberships$(m$: Observable): { memberships$: Observable>; membershipsWithTransport$: Observable< - Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + Epoch< + { membership: CallMembership; transport: TransportLocator | undefined }[] + > >; } { - const memberships$ = m$.pipe(trackEpoch()); - const membershipsWithTransport$ = memberships$.pipe( - mapEpoch((members) => { - return members.map((m) => { - const tr = m.getTransport(); - return { - membership: m, - transport: - tr?.type === "livekit" ? (tr as LivekitTransport) : undefined, - }; - }); - }), + const memberships$ = testScope.behavior(m$.pipe(trackEpoch())); + const { membershipsWithTransport$ } = membershipsAndTransports$( + testScope, + memberships$, ); return { memberships$, @@ -151,23 +136,21 @@ test("should signal participant on a connection that is publishing", async () => const connection = { transport: bobMembership.getTransport(), + serverName: "example.org", } as unknown as Connection; - const dataWithPublisher = new ConnectionManagerData(); + const dataWithPublisher = new ConnectionMap(); dataWithPublisher.add(connection, [ mockRemoteParticipant({ identity: bobParticipantId }), ]); - const connectionManagerData$ = epochMeWith$( - memberships$, - constant(dataWithPublisher), + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, constant(dataWithPublisher)), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); @@ -187,6 +170,90 @@ test("should signal participant on a connection that is publishing", async () => ); }); +// A member advertising both a `url` and a `livekit_service_url` may have +// authenticated through either flow, so it has two candidate identities. +const bothTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://sfu.example.org", + livekit_service_url: "https://lk.example.org", +}; +const bobLegacyId = fallbackMemberId("@bob:example.org", "DEV000"); +const bobHashedId = "2qkXfWz1S7jQmU0bJ6yN8aVtR3pLcDgHe9oZxKwMiA4"; +const bobWithBothIdentities = mockRtcMembership("@bob:example.org", "DEV000", { + fociPreferred: [bothTransport], + backendIdentities: [bobHashedId, bobLegacyId], +}); + +test.each([ + ["legacy", bobLegacyId], + ["hashed", bobHashedId], +])( + "matches a member with two candidate identities publishing under the %s one", + async (_name, identity) => { + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(constant([bobWithBothIdentities])); + + const connection = { + transport: bothTransport, + serverName: "example.org", + } as unknown as Connection; + const dataWithPublisher = new ConnectionMap(); + dataWithPublisher.add(connection, [mockRemoteParticipant({ identity })]); + + const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionMap$: testScope.behavior( + epochMeWith$(memberships$, constant(dataWithPublisher)), + ), + localUser: localRtcMember, + }); + + await flushPromises(); + expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].participant.value$.value?.identity).toEqual(identity); + expect(data[0].connection$.value).toBe(connection); + return true; + }, + ); + }, +); + +test("does not match a member with two candidate identities to anyone else", async () => { + const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( + constant([bobWithBothIdentities]), + ); + + const connection = { + transport: bothTransport, + serverName: "example.org", + } as unknown as Connection; + const dataWithStranger = new ConnectionMap(); + dataWithStranger.add(connection, [ + mockRemoteParticipant({ identity: "@stranger:example.org:DEV999" }), + ]); + + const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionMap$: testScope.behavior( + epochMeWith$(memberships$, constant(dataWithStranger)), + ), + localUser: localRtcMember, + }); + + await flushPromises(); + expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].participant.value$.value).toBe(null); + return true; + }, + ); +}); + test("should signal participant on a connection that is not publishing", async () => { const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( constant([bobMembership]), @@ -194,21 +261,19 @@ test("should signal participant on a connection that is not publishing", async ( const connection = { transport: bobMembership.getTransport(), + serverName: "example.org", } as unknown as Connection; - const dataWithPublisher = new ConnectionManagerData(); + const dataWithPublisher = new ConnectionMap(); dataWithPublisher.add(connection, []); - const connectionManagerData$ = epochMeWith$( - memberships$, - constant(dataWithPublisher), + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, constant(dataWithPublisher)), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); await flushPromises(); @@ -228,16 +293,18 @@ describe("Publication edge case", () => { const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(constant([bobMembership, carlMembership])); - const connectionWithPublisher = new ConnectionManagerData(); + const connectionWithPublisher = new ConnectionMap(); const bobParticipantId = fallbackMemberId( bobMembership.userId, bobMembership.deviceId, ); const connectionA = { transport: transportA, + serverName: "example.org", } as unknown as Connection; const connectionB = { transport: transportB, + serverName: "sample.com", } as unknown as Connection; connectionWithPublisher.add(connectionA, [ @@ -247,17 +314,14 @@ describe("Publication edge case", () => { mockRemoteParticipant({ identity: bobParticipantId }), ]); - const connectionManagerData$ = epochMeWith$( - memberships$, - constant(connectionWithPublisher), + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, constant(connectionWithPublisher)), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); await flushPromises(); @@ -287,14 +351,20 @@ test("bob is publishing in the wrong connection", async () => { const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(mockedMemberships$); - const connectionWithPublisher = new ConnectionManagerData(); + const connectionWithPublisher = new ConnectionMap(); const bobParticipantId = fallbackMemberId( bobMembership.userId, bobMembership.deviceId, ); - const connectionA = { transport: transportA } as unknown as Connection; - const connectionB = { transport: transportB } as unknown as Connection; + const connectionA = { + transport: transportA, + serverName: "example.org", + } as unknown as Connection; + const connectionB = { + transport: transportB, + serverName: "sample.com", + } as unknown as Connection; // Bob is not publishing on A connectionWithPublisher.add(connectionA, []); @@ -306,17 +376,14 @@ test("bob is publishing in the wrong connection", async () => { const connectionsWithPublisher$ = new BehaviorSubject( connectionWithPublisher, ); - const connectionManagerData$ = epochMeWith$( - memberships$, - connectionsWithPublisher$, + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, connectionsWithPublisher$), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 6c9fff026..2f2aa8e0c 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -6,18 +6,16 @@ Please see LICENSE in the repository root for full details. */ import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; -import { - type CallMembership, - type LivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, filter, map } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; -import { type IConnectionManager } from "./ConnectionManager"; +import { type ConnectionMap } from "./ConnectionMap"; import { Epoch, type ObservableScope } from "../../ObservableScope"; import { type Connection } from "./Connection"; import { generateItemsWithEpoch } from "../../../utils/observable"; +import { type TransportLocator } from "../../../livekit/auth"; interface LocalTaggedParticipant { type: "local"; @@ -60,9 +58,11 @@ export interface RemoteMatrixLivekitMember extends MatrixLivekitMember { interface Props { scope: ObservableScope; membershipsWithTransport$: Behavior< - Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + Epoch< + { membership: CallMembership; transport: TransportLocator | undefined }[] + > >; - connectionManager: IConnectionManager; + connectionMap$: Behavior>; localUser: { deviceId: string; userId: string }; } @@ -72,24 +72,21 @@ interface Props { * It has a small public interface: * - in (via constructor): * - an observable of CallMembership[] to track the call members (The matrix side) - * - a `ConnectionManager` for the lk rooms (The livekit side) + * - a `ConnectionMap` for the lk rooms (The livekit side) * - out (via public Observable): * - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data. */ export function createRemoteMatrixLivekitMembers$({ scope, membershipsWithTransport$, - connectionManager, + connectionMap$, localUser, }: Props): Behavior> { /** * Behavior of all the remote call members and their associated livekit data (if available). */ return scope.behavior( - combineLatest([ - membershipsWithTransport$, - connectionManager.connectionManagerData$, - ]).pipe( + combineLatest([membershipsWithTransport$, connectionMap$]).pipe( filter((values) => values.every((value) => value.epoch === values[0].epoch), ), @@ -111,8 +108,8 @@ export function createRemoteMatrixLivekitMembers$({ const participants = transport ? managerData.getParticipantsForTransport(transport) : []; - const matches = participants.filter( - (p) => p.identity == membership.rtcBackendIdentity, + const matches = participants.filter((p) => + membership.backendIdentities.includes(p.identity), ); const participant = matches[0] ?? null; const connection = transport @@ -143,9 +140,9 @@ export function createRemoteMatrixLivekitMembers$({ // Log whether the member could be matched to a LiveKit participant, // since a tile shows "waiting for media" for as long as it cannot. participant$.pipe(scope.bind()).subscribe((p) => { - const url = data$.value.connection?.transport.livekit_service_url; + const transport = data$.value.connection?.transport; logger.info( - `[RemoteMatrixLivekitMembers] ${rtcBackendIdentity}: LiveKit participant ${p ? `matched (${p.sid})` : "missing"} on ${url ?? "no connection"}`, + `[RemoteMatrixLivekitMembers] ${rtcBackendIdentity}: LiveKit participant ${p ? `matched (${p.sid})` : "missing"} on ${transport ? JSON.stringify(transport) : "no connection"}`, ); }); // will only get called once per backend identity. @@ -163,14 +160,3 @@ export function createRemoteMatrixLivekitMembers$({ } // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) - -// TODO add this to the JS-SDK -export function areLivekitTransportsEqual( - t1: T | null, - t2: T | null, -): boolean { - if (t1 && t2) { - return t1.livekit_service_url === t2.livekit_service_url; - } - return !t1 && !t2; -} diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 4c0f4fea7..bf90e9eb4 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -6,11 +6,10 @@ Please see LICENSE in the repository root for full details. */ import { test, vi, expect, beforeEach, afterEach } from "vitest"; -import { BehaviorSubject, NEVER } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { type Room as LivekitRoom } from "livekit-client"; import EventEmitter from "events"; import fetchMock from "fetch-mock"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { @@ -19,8 +18,12 @@ import { trackEpoch, } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; -import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; +import { + exampleTransportLocator, mockMediaDevices, mockRtcMembership, ownMemberMock, @@ -28,19 +31,19 @@ import { } from "../../../utils/test.ts"; import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; import { - areLivekitTransportsEqual, createRemoteMatrixLivekitMembers$, type RemoteMatrixLivekitMember, } from "./MatrixLivekitMembers.ts"; -import { createConnectionManager$ } from "./ConnectionManager.ts"; +import { createConnectionMap$ } from "./ConnectionMap.ts"; import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; import { localRtcMember, testJWTToken } from "../../../utils/test-fixtures.ts"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; -// Test the integration of ConnectionManager and MatrixLivekitMerger +// Test the integration of ConnectionMap and MatrixLivekitMembers let testScope: ObservableScope; let ecConnectionFactory: ECConnectionFactory; -let mockClient: OpenIDClientParts; +let mockClient: ClientGetTokenParts & ClientOpenIDParts; let lkRoomFactory: () => LivekitRoom; const createdMockLivekitRooms: Map = new Map(); @@ -49,7 +52,7 @@ beforeEach(() => { testScope = new ObservableScope(); mockClient = { getOpenIdToken: vi.fn().mockReturnValue(""), - getDeviceId: vi.fn().mockReturnValue("DEV000"), + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), }; lkRoomFactory = vi.fn().mockImplementation(() => { @@ -71,6 +74,7 @@ beforeEach(() => { ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -120,10 +124,10 @@ test("bob, carl, then bob joining no tracks yet", () => { memberships$, ); - const connectionManager = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: ecConnectionFactory, - localTransport$: NEVER, + localTransport: exampleTransportLocator, remoteTransports$: membershipsAndTransports.transports$, logger: logger, ownMembershipIdentity: ownMemberMock, @@ -133,7 +137,7 @@ test("bob, carl, then bob joining no tracks yet", () => { scope: testScope, membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, - connectionManager, + connectionMap$, localUser: localRtcMember, }); @@ -147,10 +151,7 @@ test("bob, carl, then bob joining no tracks yet", () => { }); expectObservable(item.connection$).toBe("a", { a: expect.toSatisfy((co) => - areLivekitTransportsEqual( - co.transport, - bobMembership.transports[0]! as LivekitTransport, - ), + deepCompare(co.transport, bobMembership.transports[0]), ), }); expectObservable(item.participant.value$).toBe("a", { @@ -182,15 +183,9 @@ test("bob, carl, then bob joining no tracks yet", () => { a: null, }); expectObservable(item.connection$).toBe("a", { - a: expect.toSatisfy((connection) => { - expect( - areLivekitTransportsEqual( - connection.transport, - carlMembership.transports[0]! as LivekitTransport, - ), - ).toBe(true); - return true; - }), + a: expect.toSatisfy((connection) => + deepCompare(connection.transport, carlMembership.transports[0]), + ), }); } return true; @@ -212,15 +207,9 @@ test("bob, carl, then bob joining no tracks yet", () => { a: daveMembership, }); expectObservable(item.connection$).toBe("a", { - a: expect.toSatisfy((connection) => { - expect( - areLivekitTransportsEqual( - connection.transport, - daveMembership.transports[0]! as LivekitTransport, - ), - ).toBe(true); - return true; - }), + a: expect.toSatisfy((connection) => + deepCompare(connection.transport, daveMembership.transports[0]), + ), }); expectObservable(item.participant.value$).toBe("a", { a: null, diff --git a/src/state/SessionBehaviors.test.ts b/src/state/SessionBehaviors.test.ts index fca24767c..d9da6f830 100644 --- a/src/state/SessionBehaviors.test.ts +++ b/src/state/SessionBehaviors.test.ts @@ -1,16 +1,25 @@ /* -Copyright 2025 Element Creations Ltd. +Copyright 2025, 2026 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { describe, expect, it, vi } from "vitest"; -import { MatrixRTCSessionEvent } from "matrix-js-sdk/lib/matrixrtc"; +import { + MatrixRTCSessionEvent, + type Transport, + type UnstableLivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; import { EventEmitter } from "events"; -import { createKeyRotationSuppressed$ } from "./SessionBehaviors"; -import { testScope } from "../utils/test"; +import { + createKeyRotationSuppressed$, + membershipsAndTransports$, +} from "./SessionBehaviors"; +import { Epoch } from "./ObservableScope"; +import { constant } from "./Behavior"; +import { mockRtcMembership, testScope } from "../utils/test"; describe("SessionBehaviors", () => { describe("createKeyRotationSuppressed$", () => { @@ -38,4 +47,54 @@ describe("SessionBehaviors", () => { expect(keyRotationSuppressed$.value).toBe(false); }); }); + + describe("membershipsAndTransports$", () => { + const serviceTransport: UnstableLivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", + }; + const urlTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://sfu.sample.com", + }; + + it("locates each member's transport on the member's homeserver", () => { + const bob = mockRtcMembership("@bob:example.org", "DEV000", { + fociPreferred: [serviceTransport], + }); + const carl = mockRtcMembership("@carl:sample.com", "DEV111", { + fociPreferred: [urlTransport], + }); + + const { membershipsWithTransport$, transports$ } = + membershipsAndTransports$( + testScope(), + constant(new Epoch([bob, carl])), + ); + + expect(transports$.value.value).toEqual([ + { transport: serviceTransport, serverName: "example.org" }, + { transport: urlTransport, serverName: "sample.com" }, + ]); + expect( + membershipsWithTransport$.value.value.map( + (m) => m.transport?.serverName, + ), + ).toEqual(["example.org", "sample.com"]); + }); + + it("leaves out members without a LiveKit transport", () => { + const dave = mockRtcMembership("@dave:example.org", "DEV222", { + fociPreferred: [{ type: "something-else" } as Transport], + }); + + const { membershipsWithTransport$, transports$ } = + membershipsAndTransports$(testScope(), constant(new Epoch([dave]))); + + expect(transports$.value.value).toEqual([]); + expect(membershipsWithTransport$.value.value).toEqual([ + { membership: dave, transport: undefined }, + ]); + }); + }); }); diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index 4ca1c5861..6620350cf 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -7,10 +7,9 @@ Please see LICENSE in the repository root for full details. import { type CallMembership, - type LivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, - isLivekitTransport, + isUnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { fromEvent } from "rxjs"; @@ -21,6 +20,7 @@ import { type ObservableScope, } from "./ObservableScope"; import { type Behavior } from "./Behavior"; +import { type TransportLocator } from "../livekit/auth"; /** * Tracks the transports used by ourselves, plus all other MatrixRTC session @@ -31,16 +31,15 @@ export const membershipsAndTransports$ = ( memberships$: Behavior>, ): { membershipsWithTransport$: Behavior< - Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + Epoch< + { membership: CallMembership; transport: TransportLocator | undefined }[] + > >; - transports$: Behavior>; + transports$: Behavior>; } => { const membershipsWithTransport$: Behavior< Epoch< - { - membership: CallMembership; - transport: LivekitTransport | undefined; - }[] + { membership: CallMembership; transport: TransportLocator | undefined }[] > > = scope.behavior( memberships$.pipe( @@ -49,14 +48,19 @@ export const membershipsAndTransports$ = ( const transport = membership.getTransport(); return { membership, - transport: isLivekitTransport(transport) ? transport : undefined, + transport: isUnstableLivekitTransport(transport) + ? { + transport, + serverName: membership.userId.replace(/^.*?:/, ""), + } + : undefined, }; }); }), ), ); - const transports$: Behavior> = scope.behavior( + const transports$: Behavior> = scope.behavior( membershipsWithTransport$.pipe( mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), ), diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts index a7f4612db..e57dcc636 100644 --- a/src/state/media/MemberMediaViewModel.ts +++ b/src/state/media/MemberMediaViewModel.ts @@ -40,6 +40,7 @@ import { E2eeType } from "../../e2ee/e2eeType"; import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats"; import { type UserMediaViewModel } from "./UserMediaViewModel"; import { type ScreenShareViewModel } from "./ScreenShareViewModel"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; // TODO: Encryption status is kinda broken and thus unused right now. Remove? export enum EncryptionStatus { @@ -59,10 +60,10 @@ export interface BaseMemberMediaViewModel extends BaseMediaViewModel { */ video$: Behavior; /** - * The URL of the LiveKit focus on which this member should be publishing. - * Exposed for debugging. + * The LiveKit transport on which this member should be publishing. Exposed + * for debugging. */ - focusUrl$: Behavior; + transport$: Behavior; /** * Whether there should be a warning that this media is unencrypted. */ @@ -75,7 +76,7 @@ export interface MemberMediaInputs extends BaseMediaViewModel { livekitRoom$: Behavior; audioSource: AudioSource; videoSource: VideoSource; - focusUrl$: Behavior; + transport$: Behavior; encryptionSystem: EncryptionSystem; } @@ -86,7 +87,7 @@ export function createMemberMedia( livekitRoom$, audioSource, videoSource, - focusUrl$, + transport$, encryptionSystem, ...inputs }: MemberMediaInputs, @@ -109,7 +110,7 @@ export function createMemberMedia( return { ...createBaseMedia(inputs), video$, - focusUrl$, + transport$, unencryptedWarning$: scope.behavior( combineLatest( [audio$, video$], diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index e00186320..b0625a4d7 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -51,6 +51,7 @@ import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaVie import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; import { RingingStatus } from "./RingingStatus"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface TileProps { ref?: Ref; @@ -103,7 +104,7 @@ interface UserMediaTileProps extends TileProps { playbackMuted: boolean; waitingForMedia?: boolean; primaryButton?: ReactNode; - focusUrl: string | undefined; + transport: UnstableLivekitTransport | undefined; } /** @@ -120,7 +121,7 @@ const UserMediaTileInner: FC = ({ primaryButton, menu, className, - focusUrl, + transport, displayName, mxcAvatarUrl, focusable, @@ -214,7 +215,7 @@ const UserMediaTileInner: FC = ({ currentReaction={reaction ?? undefined} raisedHandOnClick={raisedHandOnClick} waitingForMedia={waitingForMedia} - focusUrl={focusUrl} + transport={transport} setVideoAspectRatio={vm.setVideoAspectRatio} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} @@ -279,7 +280,7 @@ const LocalUserMediaTile: FC = ({ const mirror = useBehavior(vm.mirror$); const alwaysShow = useBehavior(vm.alwaysShow$); const switchCamera = useBehavior(vm.switchCamera$); - const focusUrl = useBehavior(vm.focusUrl$); + const transport = useBehavior(vm.transport$); const latestAlwaysShow = useLatest(alwaysShow); const onSelectAlwaysShow = useCallback( @@ -334,7 +335,7 @@ const LocalUserMediaTile: FC = ({ menuStart={menuStart} menuEnd={menuEnd} focusable={focusable} - focusUrl={focusUrl} + transport={transport} {...props} /> ); @@ -356,7 +357,7 @@ const RemoteUserMediaTile: FC = ({ const waitingForMedia = useBehavior(vm.waitingForMedia$); const playbackMuted = useBehavior(vm.playbackMuted$); const playbackVolume = useBehavior(vm.playbackVolume$); - const focusUrl = useBehavior(vm.focusUrl$); + const transport = useBehavior(vm.transport$); const onSelectMute = useCallback( (e: Event) => { @@ -398,7 +399,7 @@ const RemoteUserMediaTile: FC = ({ } - focusUrl={focusUrl} + transport={transport} {...props} /> ); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index d57441687..6776317df 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -33,6 +33,7 @@ import { ReactionIndicator } from "../reactions/ReactionIndicator"; import { RTCConnectionStats } from "../RTCConnectionStats"; import videoPlaceholder from "../graphics/video-placeholder.gif"; import { autoVideoFit } from "../utils/videoFit"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface Props extends ComponentProps { className?: string; @@ -67,9 +68,9 @@ interface Props extends ComponentProps { videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; rtcBackendIdentity?: string; /** - * The focus url, mainly for debugging purposes. + * The LiveKit transport, mainly for debugging purposes. */ - focusUrl?: string; + transport?: UnstableLivekitTransport; /** * Called whenever the aspect ratio of the video content becomes known or * otherwise changes. @@ -106,7 +107,7 @@ export const MediaView: FC = ({ audioStreamStats, videoStreamStats, rtcBackendIdentity, - focusUrl, + transport, setVideoAspectRatio: setTheirVideoAspectRatio, ...props }) => { @@ -227,7 +228,7 @@ export const MediaView: FC = ({ diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 71b98e8d8..344eed00e 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -54,6 +54,7 @@ import { platform } from "../Platform"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; import { RingingStatus } from "./RingingStatus"; import { useRootElement } from "../RootElementContext"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface SpotlightItemBaseProps { ref?: Ref; @@ -74,7 +75,7 @@ interface SpotlightItemBaseProps { interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps { video: TrackReferenceOrPlaceholder | undefined; unencryptedWarning: boolean; - focusUrl: string | undefined; + transport: UnstableLivekitTransport | undefined; } interface SpotlightUserMediaItemBaseProps extends SpotlightMemberMediaItemBaseProps { @@ -177,13 +178,13 @@ const SpotlightMemberMediaItem: FC = ({ }) => { const video = useBehavior(vm.video$); const unencryptedWarning = useBehavior(vm.unencryptedWarning$); - const focusUrl = useBehavior(vm.focusUrl$); + const transport = useBehavior(vm.transport$); const baseProps: SpotlightMemberMediaItemBaseProps & RefAttributes = { video: video ?? undefined, unencryptedWarning, - focusUrl, + transport, ...props, }; diff --git a/src/utils/matrix.ts b/src/utils/matrix.ts index 4e3ae3c32..737590a34 100644 --- a/src/utils/matrix.ts +++ b/src/utils/matrix.ts @@ -254,9 +254,6 @@ export async function createRoom( "m.sticker": 50, "org.matrix.msc3401.call.member": 0, }, - users: { - [client.getUserId()!]: 100, - }, }, }); @@ -341,9 +338,13 @@ export function getRelativeRoomUrl( * Perform a network operation with retries on ConnectionError. * If the error is not retryable, or the max number of retries is reached, the error is rethrown. * Supports handling of matrix quotas. + * @param operation The operation to perform. + * @param shouldRetry Rules out errors that retrying cannot fix, on top of the + * standard backoff rule. Retries everything by default. */ export async function doNetworkOperationWithRetry( operation: () => Promise, + shouldRetry: (e: unknown) => boolean = () => true, ): Promise { let currentRetryCount = 0; @@ -353,7 +354,9 @@ export async function doNetworkOperationWithRetry( return await operation(); } catch (e) { currentRetryCount++; - const backoff = calculateRetryBackoff(e, currentRetryCount, true); + const backoff = shouldRetry(e) + ? calculateRetryBackoff(e, currentRetryCount, true) + : -1; if (backoff < 0) { // Max number of retries reached, or error is not retryable. rethrow the error throw e; diff --git a/src/utils/observable.ts b/src/utils/observable.ts index ca51ad196..b35e523d6 100644 --- a/src/utils/observable.ts +++ b/src/utils/observable.ts @@ -218,6 +218,28 @@ export function filterBehavior( ); } +/** + * Applies a given projection to each value emitted by the source Observable. + * The `project` function additionally receives an {@link ObservableScope}. + */ +export function mapScoped( + name: string, + project: (a: A, scope: ObservableScope) => B, +): OperatorFunction { + return (a$) => + a$.pipe( + generateItemsInternal( + name, + function* (a) { + yield { keys: [a], data: undefined }; + }, + (scope, _data$, a) => project(a, scope), + (items) => items, + ), + map(([item]) => item), + ); +} + function generateItemsInternal< Input, Keys extends [unknown, ...unknown[]], diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index dcdb9f9c9..dac0c6972 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -64,16 +64,23 @@ export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "\u202eevaD", }); -export const testJWTToken = [ - {}, // header - { - // payload - sub: "@me:example.org:ABCDEF", - video: { - room: "!example_room_id", +export function createJWTToken(sub: string, room: string): string { + return [ + {}, // header + { + // payload + sub, + video: { + room, + }, }, - }, - {}, // signature -] - .map((d) => global.btoa(JSON.stringify(d))) - .join("."); + {}, // signature + ] + .map((d) => global.btoa(JSON.stringify(d))) + .join("."); +} + +export const testJWTToken = createJWTToken( + "@me:example.org:ABCDEF", + "!example_room_id", +); diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 9903b99f8..df4f21bb1 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -69,7 +69,7 @@ export function getBasicRTCSession( getUserId: () => localRtcMember.userId, getDeviceId: () => localRtcMember.deviceId, getSyncState: () => SyncState.Syncing, - getDomain: () => null, + getDomain: () => "example.com", getAccessToken: () => "fake-token", sendEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }), redactEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }), diff --git a/src/utils/test.ts b/src/utils/test.ts index 6af9fd8c7..5ab16f2e5 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -26,7 +26,7 @@ import { import { CallMembership, type LivekitFocusSelection, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -78,7 +78,7 @@ import { type RemoteScreenShareViewModel, } from "../state/media/RemoteScreenShareViewModel"; import { Connection } from "../state/CallViewModel/remoteMembers/Connection"; -import { type SFUConfig } from "../livekit/openIDSFU"; +import { type TransportLocator } from "../livekit/auth"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); @@ -209,16 +209,14 @@ export function mockEmitter(): EmitterMock { }; } -export const exampleTransport: LivekitTransport = { +export const exampleTransport: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", }; -export const exampleSfuConfig: SFUConfig = { - jwt: "foo", - livekitAlias: "bar", - livekitIdentity: "baz", - url: "bro", +export const exampleTransportLocator: TransportLocator = { + transport: exampleTransport, + serverName: "example.org", }; export function mockRtcMembership( @@ -226,6 +224,8 @@ export function mockRtcMembership( deviceId: string, customOverwrites?: { rtcBackendIdentity?: string; + /** Every identity the member may have in LiveKit, when more than one. */ + backendIdentities?: string[]; callId?: string; fociPreferred?: Transport[]; focusActive?: LivekitFocusSelection; @@ -261,11 +261,12 @@ export function mockRtcMembership( content: data, }); - const membershipData = CallMembership.membershipDataFromMatrixEvent(event); + const backendIdentity = rtcBackendIdentity ?? `${userId}:${deviceId}`; const cms = new CallMembership( event, - membershipData, - rtcBackendIdentity ?? `${userId}:${deviceId}`, + CallMembership.membershipDataFromMatrixEvent(event), + customOverwrites?.backendIdentities ?? [backendIdentity], + backendIdentity, ); vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); @@ -357,7 +358,7 @@ export function mockLocalMedia( participant$: constant(localParticipant), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, livekitRoom$: constant(mockLivekitRoom({ localParticipant })), - focusUrl$: constant("https://rtc-example.org"), + transport$: constant(exampleTransport), mediaDevices, displayName$: constant(member.rawDisplayName ?? "nodisplayname"), mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), @@ -400,7 +401,7 @@ export function mockRemoteMedia( participant$: constant(participant), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, livekitRoom$: constant(livekitRoom), - focusUrl$: constant("https://rtc-example.org"), + transport$: constant(exampleTransport), pretendToBeDisconnected$: constant(false), displayName$: constant(member.rawDisplayName ?? "nodisplayname"), mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), @@ -427,7 +428,7 @@ export function mockRemoteScreenShare( participant$: constant(participant), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, livekitRoom$: constant(livekitRoom), - focusUrl$: constant("https://rtc-example.org"), + transport$: constant(exampleTransport), pretendToBeDisconnected$: constant(false), displayName$: constant(member.rawDisplayName ?? "nodisplayname"), mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), diff --git a/vite-component.config.ts b/vite-component.config.ts index 573fb3729..fa6ce4168 100644 --- a/vite-component.config.ts +++ b/vite-component.config.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { defineConfig } from "vite"; import { vitePluginsConfig } from "./vite.config.ts"; -import { scopeStylesToRoot } from "./component/build/scopeStylesToRoot"; +import { scopeStylesToRoot } from "./component/build/scopeStylesToRoot.ts"; // Config for Element Call as a React component, to be imported by an // application embedding it rather than served as a page of its own.