diff --git a/backend/app-service-othersite.yaml b/backend/app-service-othersite.yaml index ef2a7ada7..a80322baf 100644 --- a/backend/app-service-othersite.yaml +++ b/backend/app-service-othersite.yaml @@ -5,7 +5,7 @@ sender_localpart: "_lk_jwt_service" namespaces: users: - exclusive: false - regex: "@.*:homeserver-1" # Cover all local users + regex: "@.*:synapse\\.othersite\\.m\\.localhost" # Cover all local users # Required for Synapse to call POST /_matrix/app/v1/ping on this service. url: "http://auth-server-1:16080" # Enable membership look-ups via /is_joined. diff --git a/backend/app-service.yaml b/backend/app-service.yaml index b480df64e..75716ca59 100644 --- a/backend/app-service.yaml +++ b/backend/app-service.yaml @@ -5,7 +5,7 @@ sender_localpart: "_lk_jwt_service" namespaces: users: - exclusive: false - regex: "@.*:homeserver" # Cover all local users + regex: "@.*:synapse\\.m\\.localhost" # Cover all local users # Required for Synapse to call POST /_matrix/app/v1/ping on this service. url: "http://auth-server:6080" # Enable membership look-ups via /is_joined. diff --git a/backend/dev_livekit-othersite.yaml b/backend/dev_livekit-othersite.yaml index 53fc9ce94..476ecc45c 100644 --- a/backend/dev_livekit-othersite.yaml +++ b/backend/dev_livekit-othersite.yaml @@ -6,6 +6,11 @@ rtc: port_range_start: 50300 port_range_end: 50400 use_external_ip: false + # Advertise the loopback address instead of the container IP, which is not + # routable from the host when Docker runs in a VM (colima, Docker Desktop). + # The browser then reaches the SFU through the published ports. Firefox + # ignores loopback candidates, so this only works in Chromium-based browsers. + node_ip: 127.0.0.1 turn: enabled: false domain: localhost diff --git a/backend/dev_livekit.yaml b/backend/dev_livekit.yaml index 6cef4241a..576c67589 100644 --- a/backend/dev_livekit.yaml +++ b/backend/dev_livekit.yaml @@ -6,6 +6,11 @@ rtc: port_range_start: 50100 port_range_end: 50200 use_external_ip: false + # Advertise the loopback address instead of the container IP, which is not + # routable from the host when Docker runs in a VM (colima, Docker Desktop). + # The browser then reaches the SFU through the published ports. Firefox + # ignores loopback candidates, so this only works in Chromium-based browsers. + node_ip: 127.0.0.1 turn: enabled: false domain: localhost diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 3de89ec38..2db9a63a0 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -3,7 +3,7 @@ networks: services: auth-service: - image: ghcr.io/element-hq/lk-jwt-service:0.7.0 + image: ghcr.io/element-hq/lk-jwt-service:0.8.0 pull_policy: always hostname: auth-server environment: @@ -33,7 +33,7 @@ services: - ecbackend auth-service-1: - image: ghcr.io/element-hq/lk-jwt-service:0.7.0 + image: ghcr.io/element-hq/lk-jwt-service:0.8.0 pull_policy: always hostname: auth-server-1 environment: diff --git a/locales/en/app.json b/locales/en/app.json index e31c7d50a..5be1bfff6 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -59,10 +59,10 @@ "developer_mode": { "always_show_iphone_earpiece": "Show iPhone earpiece option on all platforms", "crypto_version": "Crypto version: {{version}}", - "custom_livekit_url": { - "current_url": "Currently set to: ", + "custom_transport": { + "current": "Currently set to: ", "from_config": "Currently, no overwrite is set. Url from config is used.", - "label": "Custom Livekit-url", + "label": "Custom transport JSON", "reset": "Reset overwrite", "save": "Save", "saving": "Saving..." diff --git a/playwright.config.ts b/playwright.config.ts index 7be29e072..182eca567 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -38,15 +38,20 @@ export default defineConfig({ retries: process.env.CI ? 2 : 0, /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, - /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - // On CI, `list` prints a line per test as it finishes, which keeps the job - // log moving; `github` annotates failures on the PR; `html` is uploaded as - // an artifact and must never try to open a browser there. + /** + * Reporter to use. See https://playwright.dev/docs/test-reporters + * + * On CI, `list` prints a line per test as it finishes, which keeps the job + * log moving; `github` annotates failures on the PR; `html` is uploaded as + * an artifact and must never try to open a browser there. + */ reporter: process.env.CI ? [["list"], ["github"], ["html", { open: "never" }]] : [["list"], ["html"]], /* Stop a broken run early on CI instead of waiting out every retry. */ maxFailures: process.env.CI ? 5 : undefined, + /* Under the job's 60 minute limit, so a hung run still ends with a report. */ + globalTimeout: process.env.CI ? 50 * 60 * 1000 : undefined, /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ diff --git a/playwright/errors.spec.ts b/playwright/errors.spec.ts index 23f6e29f3..e7eff6592 100644 --- a/playwright/errors.spec.ts +++ b/playwright/errors.spec.ts @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { expect, test } from "@playwright/test"; -import { createJTWToken } from "./fixtures/jwt-token"; +import { createJWTToken } from "./fixtures/jwt-token"; test("Should show error screen if fails to get JWT token", async ({ page }) => { await page.goto("/"); @@ -95,7 +95,7 @@ test("Should show error screen if call creation is restricted", async ({ contentType: "application/json", body: JSON.stringify({ url: "wss://badurltotricktest/livekit/sfu", - jwt: createJTWToken("@fake:user", "!fake:room"), + jwt: createJWTToken("@fake:user", "!fake:room"), }), }), ); diff --git a/playwright/fixtures/jwt-token.ts b/playwright/fixtures/jwt-token.ts index 18119c7e0..dff715f85 100644 --- a/playwright/fixtures/jwt-token.ts +++ b/playwright/fixtures/jwt-token.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -export function createJTWToken(sub: string, room: string): string { +export function createJWTToken(sub: string, room: string): string { return [ {}, // header { diff --git a/playwright/url-transport.spec.ts b/playwright/url-transport.spec.ts new file mode 100644 index 000000000..77fe00827 --- /dev/null +++ b/playwright/url-transport.spec.ts @@ -0,0 +1,493 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Page, test } from "@playwright/test"; + +import { SpaHelpers } from "./spa-helpers"; +import { type RtcMode } from "./widget/test-helpers.ts"; + +const sfuUrl = "wss://matrix-rtc.m.localhost/livekit/sfu"; +const serviceUrl = "https://matrix-rtc.m.localhost/livekit/jwt"; + +const otherHomeserver = { + base_url: "https://synapse.othersite.m.localhost", + server_name: "synapse.othersite.m.localhost", +}; +const otherSfuUrl = "wss://matrix-rtc.othersite.m.localhost/livekit/sfu"; + +interface Transport { + type: "livekit"; + url?: string; + livekit_service_url?: string; +} + +/** + * The transport shapes a homeserver may advertise while migrating to the + * latest MSC4195. `legacy` is what the e2e Synapse advertises on its own. + */ +const shapes = { + url: { type: "livekit", url: sfuUrl }, + both: { type: "livekit", url: sfuUrl, livekit_service_url: serviceUrl }, + legacy: null, +} satisfies Record; +type Shape = keyof typeof shapes; + +/** Which endpoint a page asks for its SFU tokens. */ +type Flow = "cs-api" | "service-legacy" | "service-default"; + +const cases: { + hostShape: Shape; + guestShape: Shape; + /** The MatrixRTC mode both sides use; the default is compatibility mode. */ + mode?: RtcMode; + hostFlows: Flow[]; + guestFlows: Flow[]; +}[] = [ + { + hostShape: "url", + guestShape: "url", + hostFlows: ["cs-api"], + guestFlows: ["cs-api"], + }, + { + // Publishers keep the legacy participant identity that clients predating + // MSC4195 expect, which only the JWT service assigns. + hostShape: "both", + guestShape: "both", + hostFlows: ["service-legacy"], + guestFlows: ["service-legacy"], + }, + { + // The MatrixRTC mode has no bearing on the choice. + hostShape: "both", + guestShape: "both", + mode: "2_0", + hostFlows: ["service-legacy"], + guestFlows: ["service-legacy"], + }, + { + hostShape: "url", + guestShape: "legacy", + hostFlows: ["cs-api", "service-default"], + guestFlows: ["service-legacy", "cs-api"], + }, +]; + +for (const { hostShape, guestShape, mode, hostFlows, guestFlows } of cases) { + test(`One to one call with ${hostShape} host and ${guestShape} guest transport${mode === "2_0" ? " in Matrix 2.0 mode" : ""}`, async ({ + browser, + page, + browserName, + }) => { + skipOnFirefox(browserName); + + // The host starts the call. + await advertiseTransport(page, shapes[hostShape]); + const hostRequests = recordAuthRequests(page); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true, mode); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest joins. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await advertiseTransport(guestPage, shapes[guestShape]); + const guestRequests = recordAuthRequests(guestPage); + await SpaHelpers.joinCallFromInviteLink( + guestPage, + inviteLink, + "Pevara", + mode, + ); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // The host and guest ran through the expected flows. + expect(flowsOf(hostRequests)).toEqual(new Set(hostFlows)); + expect(flowsOf(guestRequests)).toEqual(new Set(guestFlows)); + + // All requests succeeded and the SFU URL was set on C-S requests. + for (const request of [...hostRequests, ...guestRequests]) { + expect(request.status).toBe(200); + if (request.flow === "cs-api") expect(request.sfuUrl).toBe(sfuUrl); + // Clients predating MSC4195 match a publisher by the legacy identity, + // which only the JWT service's legacy endpoint assigns. The other flows + // hash it. + expect(request.identity).toMatch( + request.flow === "service-legacy" ? legacyIdentity : hashedIdentity, + ); + } + }); +} + +test("Subscriber falls back to the JWT service when the homeserver lacks MSC4195", async ({ + browser, + page, + browserName, +}) => { + skipOnFirefox(browserName); + + // The host starts the call with both transport shapes. + await advertiseTransport(page, shapes.both); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest joins but their C-S requests fail. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await rejectCsApiTokens(guestPage); + const guestRequests = recordAuthRequests(guestPage); + await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink, "Pevara"); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // The guest ran through all flows. + expect(flowsOf(guestRequests)).toEqual( + new Set(["cs-api", "service-legacy", "service-default"]), + ); + + // The C-S token requests failed. + for (const request of guestRequests) { + expect(request.status).toBe(request.flow === "cs-api" ? 404 : 200); + } + + // The C-S token requests preceeded the fallback request. + const flows = guestRequests.map((r) => r.flow); + expect(flows.indexOf("service-default")).toBeGreaterThan( + flows.indexOf("cs-api"), + ); +}); + +test("Publisher shows an error when the homeserver lacks MSC4195", async ({ + page, + browserName, +}) => { + skipOnFirefox(browserName); + + // The host tries to start a call with only the C-S transport shape available but + // their C-S requests fail. + await advertiseTransport(page, shapes.url); + await rejectCsApiTokens(page); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + + // We land on an error page. + await expect( + page.getByRole("heading", { name: "Something went wrong" }), + ).toBeVisible(); + await expect( + page.getByText( + "The authorization service for your media server (SFU) is out of date.", + ), + ).toBeVisible(); +}); + +/** + * Delegation follows the flow that authenticated the publisher: the homeserver + * for a `url`-only transport, the JWT service wherever there is one. + */ +const delegationCases: { shape: Shape; via: Delegation["via"] }[] = [ + { shape: "url", via: "homeserver" }, + { shape: "legacy", via: "service" }, + { shape: "both", via: "service" }, +]; + +for (const { shape, via } of delegationCases) { + test(`Delayed leave event delegated through the ${via} for a ${shape} transport gets sent on disconnect`, async ({ + browser, + page, + browserName, + }) => { + skipOnFirefox(browserName); + + // The host starts the call. + await advertiseTransport(page, shapes[shape]); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest joins. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await advertiseTransport(guestPage, shapes[shape]); + const delayedLeave = recordDelayedLeave(guestPage); + const delegations = recordDelegations(guestPage); + await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink, "Pevara"); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // The guest handed its delayed leave over, after probing for delegation + // support with bodiless requests, which count as supported unless they + // 404. The homeserver is always probed; the JWT service only when there + // is one. + await expect + .poll(() => delegations.at(-1)) + .toEqual({ + via, + status: 200, + delayId: delayedLeave.delayId, + ...(via === "homeserver" && { sfuUrl }), + }); + const probes = delegations.slice(0, -1); + expect(probes.some((probe) => probe.via === via)).toBe(true); + for (const probe of probes) { + expect(probe.status).toBeGreaterThanOrEqual(400); + expect(probe.status).not.toBe(404); + } + + // The delay is long enough that a leave within the window below cannot + // come from the delayed event expiring on its own. + expect(delayedLeave.delayMs).toBeGreaterThan(60_000); + + // The guest drops without leaving. + await guestPage.reload(); + + // The host eventually sees the guest go once the delayed event fires. + await expect(page.getByTestId("videoTile")).toHaveCount(1, { + timeout: 30_000, + }); + }); +} + +test("Subscribers get tokens for a remote homeserver's SFU over federation", async ({ + browser, + page, + browserName, +}) => { + skipOnFirefox(browserName); + + // The host lives on the other homeserver and publishes on its SFU. + await useHomeserver(page, otherHomeserver); + await advertiseTransport(page, { type: "livekit", url: otherSfuUrl }); + const hostRequests = recordAuthRequests(page); + await page.goto("/"); + await SpaHelpers.createCall(page, "Androl", "HelloCall", true); + const inviteLink = await SpaHelpers.getCallInviteLink(page); + + // The guest lives on the default homeserver and joins over federation. + const guestContext = await browser.newContext({ reducedMotion: "reduce" }); + const guestPage = await guestContext.newPage(); + await advertiseTransport(guestPage, shapes.url); + const guestRequests = recordAuthRequests(guestPage); + await SpaHelpers.joinCallFromInviteLink( + guestPage, + `${inviteLink}&viaServers=${otherHomeserver.server_name}`, + "Pevara", + ); + + // Wait for the call to connect and render. + await SpaHelpers.expectVideoTilesCount(page, 2); + await SpaHelpers.expectVideoTilesCount(guestPage, 2); + + // Each side got a token for its own SFU and for the other side's SFU, all + // through its own homeserver. + for (const requests of [hostRequests, guestRequests]) { + expect(flowsOf(requests)).toEqual(new Set(["cs-api"])); + expect(new Set(requests.map((r) => r.sfuUrl))).toEqual( + new Set([sfuUrl, otherSfuUrl]), + ); + for (const request of requests) expect(request.status).toBe(200); + } +}); + +/** Skips the calling test if we're running on Firefox. */ +function skipOnFirefox(browserName: string): void { + test.skip( + browserName === "firefox", + "The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled", + ); +} + +/** Points this page's Element Call at another homeserver. */ +async function useHomeserver( + page: Page, + homeserver: typeof otherHomeserver, +): Promise { + await page.route("**/config.json", async (route) => { + const response = await route.fetch(); + const config = (await response.json()) as Record; + await route.fulfill({ + response, + json: { + ...config, + default_server_config: { "m.homeserver": homeserver }, + }, + }); + }); +} + +/** + * Maps recorded requests to a set of flows. Publisher and subscriber connections + * start concurrently so using arrays for comparison isn't convenient. + */ +function flowsOf(requests: AuthRequest[]): Set { + return new Set(requests.map((r) => r.flow)); +} + +/** + * Replaces the transports the homeserver advertises to this page. The real + * response is fetched first so its CORS headers carry over. + */ +async function advertiseTransport( + page: Page, + transport: Transport | null, +): Promise { + if (transport === null) return; + await page.route("**/org.matrix.msc4143/rtc/transports", async (route) => { + if (route.request().method() !== "GET") return route.continue(); + const response = await route.fetch(); + await route.fulfill({ response, json: { rtc_transports: [transport] } }); + }); +} + +/** + * Makes the homeserver look like one without MSC4195: its `get_token` + * endpoint answers as Synapse does for an unknown route. + */ +async function rejectCsApiTokens(page: Page): Promise { + await page.route("**/rtc/livekit/get_token", async (route) => { + if (route.request().method() !== "POST") return route.continue(); + await route.fulfill({ + status: 404, + json: { errcode: "M_UNRECOGNIZED", error: "Unrecognized request" }, + headers: { "Access-Control-Allow-Origin": "*" }, + }); + }); +} + +/** A recorded delayed leave event. */ +interface DelayedLeave { + delayMs?: number; + delayId?: string; +} + +/** Records the delayed leave event the page schedules on joining. */ +function recordDelayedLeave(page: Page): DelayedLeave { + const delayedLeave: DelayedLeave = {}; + page.on("response", (response) => { + const delay = new URL(response.request().url()).searchParams.get( + "org.matrix.msc4140.delay", + ); + if (delay === null || response.status() !== 200) return; + delayedLeave.delayMs = Number(delay); + response + .json() + .then((body: { delay_id: string }) => { + delayedLeave.delayId = body.delay_id; + }) + .catch(() => {}); + }); + return delayedLeave; +} + +/** A recorded delegation request for a delayed leave event. */ +interface Delegation { + /** Who was asked to send the leave event on the guest's behalf. */ + via: "homeserver" | "service"; + status: number; + sfuUrl?: string; + delayId?: string; +} + +/** + * Records requests that delegate a delayed leave event, or probe for support: + * the MSC4195 `delegate_delayed_leave` endpoint of the homeserver, and the JWT + * service's `delegate_delayed_leave` probe and `/sfu/get` with delay fields. + */ +function recordDelegations(page: Page): Delegation[] { + const delegations: Delegation[] = []; + page.on("response", (response) => { + const request = response.request(); + if (request.method() !== "POST") return; + const url = request.url(); + const body = request.postDataJSON() as { + url?: string; + delay_id?: string; + } | null; + if (url.endsWith("/rtc/livekit/delegate_delayed_leave")) { + delegations.push({ + via: "homeserver", + status: response.status(), + ...(body && { sfuUrl: body.url, delayId: body.delay_id }), + }); + } else if (url.endsWith("/livekit/jwt/delegate_delayed_leave")) { + delegations.push({ via: "service", status: response.status() }); + } else if (url.endsWith("/livekit/jwt/sfu/get") && body?.delay_id) { + delegations.push({ + via: "service", + status: response.status(), + delayId: body.delay_id, + }); + } + }); + return delegations; +} + +/** A recorded token request. */ +interface AuthRequest { + flow: Flow; + status: number; + /** The SFU a request to the MSC4195 `get_token` endpoint asked for. */ + sfuUrl?: string; + /** The participant identity the issued token carries. */ + identity?: string; +} + +/** The legacy `:` participant identity. */ +const legacyIdentity = /^@[^:]+:[^:]+:[A-Z]+$/; +/** A hashed participant identity: unpadded base64 of a SHA-256. */ +const hashedIdentity = /^[A-Za-z0-9+/]{43}$/; + +/** Reads the subject of a JWT without verifying it. */ +function jwtSubject(jwt: string): string { + const [, payload] = jwt.split("."); + return ( + JSON.parse(Buffer.from(payload, "base64url").toString()) as { + sub: string; + } + ).sub; +} + +/** Records token requests, including the identity each token carries. */ +function recordAuthRequests(page: Page): AuthRequest[] { + const requests: AuthRequest[] = []; + page.on("response", (response) => { + const request = response.request(); + if (request.method() !== "POST") return; + const url = request.url(); + let entry: AuthRequest; + if (url.endsWith("/rtc/livekit/get_token")) { + entry = { + flow: "cs-api", + status: response.status(), + sfuUrl: (request.postDataJSON() as { url: string }).url, + }; + } else if (url.endsWith("/livekit/jwt/sfu/get")) { + entry = { flow: "service-legacy", status: response.status() }; + } else if (url.endsWith("/livekit/jwt/get_token")) { + entry = { flow: "service-default", status: response.status() }; + } else { + return; + } + requests.push(entry); + if (entry.status !== 200) return; + response + .json() + .then((body: { jwt: string }) => { + entry.identity = jwtSubject(body.jwt); + }) + .catch(() => {}); + }); + return requests; +} diff --git a/playwright/widget/federated-call.test.ts b/playwright/widget/federated-call.test.ts index 61f4750b3..8359dd233 100644 --- a/playwright/widget/federated-call.test.ts +++ b/playwright/widget/federated-call.test.ts @@ -37,8 +37,6 @@ modePairs.forEach(([rtcMode1, rtcMode2]) => { await TestHelpers.acceptRoomInvite(roomName, timo.page); - await florian.page.pause(); - await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget( florian.page, rtcMode1, @@ -74,8 +72,6 @@ modePairs.forEach(([rtcMode1, rtcMode2]) => { await TestHelpers.expectVisibleVideoCount(frame, 2); } - - // await florian.page.pause(); }, ); }); diff --git a/playwright/widget/url-transport.test.ts b/playwright/widget/url-transport.test.ts new file mode 100644 index 000000000..c4e39cfdd --- /dev/null +++ b/playwright/widget/url-transport.test.ts @@ -0,0 +1,108 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, type Page, test } from "@playwright/test"; + +import { widgetTest } from "../fixtures/widget-user.ts"; +import { HOST1, TestHelpers } from "./test-helpers.ts"; + +const sfuUrl = "wss://matrix-rtc.m.localhost/livekit/sfu"; +const serviceUrl = "https://matrix-rtc.m.localhost/livekit/jwt"; + +/** Which endpoint a page asks for its SFU tokens. */ +type Flow = "cs-api" | "service-legacy" | "service-default"; + +widgetTest( + "Call between a url + livekit_service_url transport and a legacy one", + async ({ addUser, browserName }) => { + test.skip( + browserName === "firefox", + "The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled", + ); + test.slow(); + + const [host, guest] = await Promise.all([ + addUser("Androl", HOST1), + addUser("Pevara", HOST1), + ]); + + // The host's homeserver has started advertising a `url` next to its JWT + // service; the guest's still advertises the JWT service alone. + await advertiseTransport(host.page, { + type: "livekit", + url: sfuUrl, + livekit_service_url: serviceUrl, + }); + // Element Web answers the widget from a cache it filled at login, so make + // it fetch the transports again now that they are rerouted + await host.clientHandle.evaluate(async (client) => { + await client.cachedRtcTransports.refresh(); + }); + const hostFlows = recordFlows(host.page); + const guestFlows = recordFlows(guest.page); + + const roomName = "Transport Room"; + await TestHelpers.createRoom(roomName, host.page, [guest.mxId]); + await TestHelpers.acceptRoomInvite(roomName, guest.page); + + await TestHelpers.startCallInCurrentRoom(host.page, false); + await TestHelpers.joinCallFromLobby(host.page); + await TestHelpers.joinCallInCurrentRoom(guest.page); + + for (const user of [host, guest]) { + const frame = user.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + await expect(frame.getByTestId("videoTile")).toHaveCount(2, { + timeout: 10000, + }); + await expect(frame.getByText("Waiting for media...")).not.toBeVisible({ + timeout: 10000, + }); + await TestHelpers.expectVisibleVideoCount(frame, 2); + } + + // Element Web does not expose the MSC4195 endpoints to widgets yet, so + // neither side reached the CS API. The host published through the JWT + // service's legacy endpoint and subscribed to the guest through its + // default one; the guest published the same way and fell back to the + // default endpoint for the host's transport. + expect(hostFlows).toEqual(new Set(["service-legacy", "service-default"])); + expect(guestFlows).toEqual(new Set(["service-legacy", "service-default"])); + }, +); + +/** + * Replaces the transports the homeserver advertises to this page, including + * to the widget host, which fetches them on the widget's behalf. The real + * response is fetched first so its CORS headers carry over. + */ +async function advertiseTransport( + page: Page, + transport: { type: "livekit"; url?: string; livekit_service_url?: string }, +): Promise { + await page.route("**/org.matrix.msc4143/rtc/transports", async (route) => { + if (route.request().method() !== "GET") return route.continue(); + const response = await route.fetch(); + await route.fulfill({ response, json: { rtc_transports: [transport] } }); + }); +} + +/** Records which token endpoints the page's widget asks. */ +function recordFlows(page: Page): Set { + const flows = new Set(); + page.on("response", (response) => { + const request = response.request(); + if (request.method() !== "POST") return; + const url = request.url(); + if (url.endsWith("/rtc/livekit/get_token")) flows.add("cs-api"); + else if (url.endsWith("/livekit/jwt/sfu/get")) flows.add("service-legacy"); + else if (url.endsWith("/livekit/jwt/get_token")) + flows.add("service-default"); + }); + return flows; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 17276a233..cc0d93beb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -192,10 +192,10 @@ importers: version: 1.9.2 matrix-js-sdk: specifier: github:matrix-org/matrix-js-sdk#develop - version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999 + version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/4df2f2bfb5826c25f05d9535a8556acecaf2e570 matrix-widget-api: specifier: ^1.18.0 - version: 1.19.0 + version: 1.20.0 node-stdlib-browser: specifier: ^1.3.1 version: 1.3.1 @@ -4673,13 +4673,13 @@ packages: matrix-events-sdk@0.0.1: resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==} - matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999: - resolution: {gitHosted: true, integrity: sha512-qF8++74MX36KS1KEw6Lc8HHXgZ6nHD1oUU4yBZUj5vVPvZTIspk1lOGztkFmizE5vmLAHd5tdadL9Xu1Asy4bA==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999} - version: 42.3.0 + matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/4df2f2bfb5826c25f05d9535a8556acecaf2e570: + resolution: {gitHosted: true, integrity: sha512-b1o6ywaPY9jakj+3GIgx3samM9AbqohzPQ5nZKbyOGYB16IHQtNMyjEo/PY0fW/s0uwBVApEHNb/MMtN40c/rQ==, tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/4df2f2bfb5826c25f05d9535a8556acecaf2e570} + version: 43.0.0 engines: {node: '>=22.0.0'} - matrix-widget-api@1.19.0: - resolution: {integrity: sha512-Qe7L6G1mQJphBC47/Bzp5ljPIK1UARgLiuGgS+/KmEs/eaNHkQ3+ArD0/0kvMV+d6kCGAZ9OKRqsnuPjizI+tw==} + matrix-widget-api@1.20.0: + resolution: {integrity: sha512-qwFP37rgns7mPesM5/vvZgknzb+kvyRB5BISv5sCuBxsGMceMr+Kogf7OkjRxBXdMCPBFuf6gIuYRe5S+lpzQw==} md5.js@1.3.5: resolution: {integrity: sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==} @@ -10227,7 +10227,7 @@ snapshots: matrix-events-sdk@0.0.1: {} - matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fa4772a4ddd335a5a8bfbf8bb21df49b70f7a999: + matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/4df2f2bfb5826c25f05d9535a8556acecaf2e570: dependencies: '@babel/runtime': 8.0.0 '@matrix-org/matrix-sdk-crypto-wasm': 18.9.0 @@ -10236,12 +10236,12 @@ snapshots: content-type: 3.1.1 loglevel: 1.9.2 matrix-events-sdk: 0.0.1 - matrix-widget-api: 1.19.0 + matrix-widget-api: 1.20.0 p-retry: 8.0.1 sdp-transform: 3.0.0 unhomoglyph: 1.0.6 - matrix-widget-api@1.19.0: + matrix-widget-api@1.20.0: dependencies: '@types/events': 3.0.3 events: 3.3.0 diff --git a/sdk/main.ts b/sdk/main.ts index 6cd37032d..3932d1190 100644 --- a/sdk/main.ts +++ b/sdk/main.ts @@ -231,7 +231,7 @@ export async function createMatrixRTCSdk( complete: () => { logger.info("Livekit room items subscription completed"); for (const item of callViewModel.livekitRoomItems$.value) { - logger.info("unregistering room item from room", item.url); + logger.info("unregistering room item from room", item.transport); item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC); } }, diff --git a/src/RTCConnectionStats.tsx b/src/RTCConnectionStats.tsx index ea9df3f5e..a19cdd977 100644 --- a/src/RTCConnectionStats.tsx +++ b/src/RTCConnectionStats.tsx @@ -16,30 +16,21 @@ import classNames from "classnames"; import { Modal } from "./Modal"; import styles from "./RTCConnectionStats.module.css"; import mediaViewStyles from "../src/tile/MediaView.module.css"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; + interface Props { audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; - focusUrl?: string; + transport?: UnstableLivekitTransport; rtcBackendIdentity?: string; } -const extractDomain = (url: string): string => { - try { - const parsedUrl = new URL(url); - return parsedUrl.hostname; // Returns "kdk.cpm" - } catch (error) { - console.error("Invalid URL:", error); - return url; - } -}; - // This is only used in developer mode for debugging purposes, so we don't need full localization export const RTCConnectionStats: FC = ({ audio, video, - focusUrl, + transport, rtcBackendIdentity, - ...rest }) => { const [showModal, setShowModal] = useState(false); const [modalContents, setModalContents] = useState< @@ -76,10 +67,10 @@ export const RTCConnectionStats: FC = ({ rtcBackendIdentity:{rtcBackendIdentity} - {focusUrl && ( + {transport && (
- -  {extractDomain(focusUrl)} + +  {JSON.stringify(transport)}
)} diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index 5d18695a9..5f9910662 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -97,11 +97,19 @@ export interface ConfigOptions { }; }; - // Describes the LiveKit configuration to be used. + /** + * Describes the LiveKit configuration to be used. + */ livekit?: { - // The link to the service that returns a livekit url and token to use it. - // This is a fallback link in case the homeserver in use does not advertise - // a livekit service url over the transports endpoint. + /** + * The link to the service that returns a livekit url and token to use it. + * This is a fallback link in case the homeserver in use does not advertise + * a livekit service url over the transports endpoint. + * + * @deprecated Homeservers should advertise available transports over the + * /_matrix/client/unstable/org.matrix.msc4143/rtc/transports endpoint + * instead. + */ livekit_service_url: string; }; diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index e4573cda9..ab252a0e7 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -24,6 +24,7 @@ import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; import { + exampleTransportLocator, mockLocalParticipant, mockMediaDevices, mockRemoteParticipant, @@ -118,7 +119,7 @@ function renderTestComponent( p.identity)} livekitRoom={livekitRoom} - url={""} + transport={exampleTransportLocator} /> , ); @@ -150,7 +151,7 @@ it("should not render or warn for the local participant", () => { , ); @@ -177,7 +178,7 @@ it("should warn only once per unexpected participant", () => { , ); diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 95d68c056..4ca1d3bae 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -19,12 +19,10 @@ import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; +import { type TransportLocator } from "./auth"; export interface MatrixAudioRendererProps { - /** - * The service URL of the LiveKit room. - */ - url: string; + transport: TransportLocator; livekitRoom: LivekitRoom; /** * The list of participant identities to render audio for. @@ -54,7 +52,7 @@ export interface MatrixAudioRendererProps { * @public */ export function LivekitRoomAudioRenderer({ - url, + transport, livekitRoom, validIdentities, muted, @@ -88,7 +86,7 @@ export function LivekitRoomAudioRenderer({ warnedIdentities.current.add(identity); // Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call. logger.warn( - `Audio track ${identity} from ${url} has no matching matrix call member`, + `Audio track ${identity} from ${JSON.stringify(transport)} has no matching matrix call member`, `current members: ${validIdentities.join()}`, `track will not get rendered`, ); diff --git a/src/livekit/auth/csApi.ts b/src/livekit/auth/csApi.ts new file mode 100644 index 000000000..d62111f02 --- /dev/null +++ b/src/livekit/auth/csApi.ts @@ -0,0 +1,137 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { ConnectionError, HTTPError, type MatrixClient } from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { extractFullConfigFromToken, type SFUConfig } from "./types.ts"; +import { doNetworkOperationWithRetry } from "../../utils/matrix.ts"; + +/** + * In widget mode the host may refuse an action before any request is made + * which surfaces as a plain error. Asking again cannot change its answer, + * so only errors from the network are worth retrying. + */ +function isNetworkError(e: unknown): boolean { + return e instanceof HTTPError || e instanceof ConnectionError; +} + +export type ClientGetTokenParts = Pick< + MatrixClient, + "_unstable_getLivekitToken" +>; + +export interface GetSFUConfigParams { + /** + * The Matrix client. + */ + client: ClientGetTokenParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The WebSocket URL of the SFU for which we wish to get an access token. + */ + url: string; + /** + * The name of the homeserver to which the SFU belongs. + */ + serverName: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; +} + +/** + * Gets an {@link SFUConfig} appropriate for connecting to a given SFU, using + * the MSC4195 LiveKit client-server API endpoints. + */ +export async function getSFUConfig({ + client, + membership, + url, + serverName, + roomId, + slotId, +}: GetSFUConfigParams): Promise { + const res = await doNetworkOperationWithRetry( + async () => + client._unstable_getLivekitToken({ + url, + server_name: serverName, + room_id: roomId, + slot_id: slotId, + member_id: membership.memberId, + }), + isNetworkError, + ); + return extractFullConfigFromToken({ url, jwt: res.jwt }); +} + +export type ClientDelegationParts = Pick< + MatrixClient, + "_unstable_delegateDelayedLeave" | "baseUrl" +>; + +export interface DelegateDelayedLeaveParams { + /** + * The Matrix client. + */ + client: ClientDelegationParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The WebSocket URL of the SFU to which we are publishing and wish to + * delegate the delayed leave event. + */ + url: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * The delay ID of the leave event to be delegated. + */ + delayId: string; +} + +/** + * Delegates a delayed leave event to a given SFU, so that the event will + * be sent on our behalf whenever we disconnect from the SFU. + */ +export async function delegateDelayedLeave({ + client, + membership, + url, + roomId, + slotId, + delayId, +}: DelegateDelayedLeaveParams): Promise { + await doNetworkOperationWithRetry( + async () => + client._unstable_delegateDelayedLeave({ + url, + room_id: roomId, + slot_id: slotId, + member_id: membership.memberId, + delay_id: delayId, + }), + isNetworkError, + ); +} diff --git a/src/livekit/auth/index.test.ts b/src/livekit/auth/index.test.ts new file mode 100644 index 000000000..6c48c5032 --- /dev/null +++ b/src/livekit/auth/index.test.ts @@ -0,0 +1,306 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, expect, type Mocked, vi, describe, it } from "vitest"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { + aliceDeviceId, + aliceId, + aliceUserId, + createJWTToken, +} from "../../utils/test-fixtures"; +import type * as csApi from "./csApi"; +import * as openID from "./openID"; +import { delegateDelayedLeave, getSFUConfig } from "."; +import { mocked } from "storybook/test"; +import { ConnectionError, MatrixError } from "matrix-js-sdk"; +import { NoMatrix2AuthorizationService } from "../../utils/errors"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +vi.mock("./openID", () => ({ getSFUConfig: vi.fn() })); +vi.useFakeTimers(); + +const csApiJWTToken = createJWTToken(aliceId, "via CS API"); +const openIDJWTToken = createJWTToken(aliceId, "via OpenID"); + +mocked(openID).getSFUConfig.mockResolvedValue({ + url: "mock-openID-url", + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + livekitIdentity: aliceId, +}); + +const client: Mocked< + csApi.ClientGetTokenParts & + csApi.ClientDelegationParts & + openID.ClientOpenIDParts +> = { + baseUrl: "https://matrix.example.org", + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: csApiJWTToken }), + _unstable_delegateDelayedLeave: vi.fn().mockResolvedValue({}), + getOpenIdToken: vi + .fn() + .mockResolvedValue({ access_token: "mock-openID-token" }), +}; + +const membership: CallMembershipIdentityParts = { + userId: aliceUserId, + deviceId: aliceDeviceId, + memberId: "123", +}; + +const roomId = "!room:example.org"; +const slotId = "m.call#ROOM"; + +const unsupportedError = new MatrixError({ errcode: "M_UNSUPPORTED" }, 404); + +const urlTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://livekit.example.org", +}; + +const serviceUrlTransport: UnstableLivekitTransport = { + type: "livekit", + livekit_service_url: "https://livekit-jwt.example.org", +}; + +const urlAndServiceUrlTransport: UnstableLivekitTransport = { + ...urlTransport, + ...serviceUrlTransport, +}; + +afterEach(() => vi.clearAllMocks()); + +describe("getSFUConfig", () => { + const defaultParams = { + client, + membership, + serverName: "example.org", + role: "subscriber" as const, + roomId, + slotId, + logger, + }; + + it("uses CS API with url transport", async () => { + expect( + await getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).toEqual( + expect.objectContaining({ + jwt: csApiJWTToken, + livekitAlias: "via CS API", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("does not fall back to OpenID with url transport", async () => { + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + await expect( + getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(expect.any(NoMatrix2AuthorizationService)); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("uses OpenID with livekit_service_url transport", async () => { + expect( + await getSFUConfig({ ...defaultParams, transport: serviceUrlTransport }), + ).toEqual( + expect.objectContaining({ + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + }), + ); + expect(client._unstable_getLivekitToken).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("uses CS API with url + livekit_service_url transport", async () => { + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual( + expect.objectContaining({ + jwt: csApiJWTToken, + livekitAlias: "via CS API", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("falls back to OpenID with url + livekit_service_url transport", async () => { + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual( + expect.objectContaining({ + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + }), + ); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("falls back to OpenID when the widget host refuses the CS API", async () => { + // In widget mode a host without MSC4533 rejects the request before it + // reaches the homeserver, with a plain error rather than a Matrix error + client._unstable_getLivekitToken.mockRejectedValueOnce( + new Error( + "The rtc_livekit_get_token action is not supported by the client.", + ), + ); + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }), + ).toEqual(expect.objectContaining({ jwt: openIDJWTToken })); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("surfaces the widget host's refusal with url transport", async () => { + const error = new Error( + "The rtc_livekit_get_token action is not supported by the client.", + ); + client._unstable_getLivekitToken.mockRejectedValueOnce(error); + await expect( + getSFUConfig({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(error); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("uses CS API when publishing with url transport", async () => { + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlTransport, + role: "publisher", + }), + ).toEqual(expect.objectContaining({ jwt: csApiJWTToken })); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("does not fall back to OpenID when publishing", async () => { + client._unstable_getLivekitToken.mockRejectedValueOnce(unsupportedError); + await expect( + getSFUConfig({ + ...defaultParams, + transport: urlTransport, + role: "publisher", + }), + ).rejects.toThrow(expect.any(NoMatrix2AuthorizationService)); + expect(client._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("uses OpenID when publishing with url + livekit_service_url transport", async () => { + // Subscribers predating MSC4195 expect a legacy participant identity from + // anyone advertising a state event, which only the JWT service assigns + expect( + await getSFUConfig({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + role: "publisher", + }), + ).toEqual( + expect.objectContaining({ + jwt: openIDJWTToken, + livekitAlias: "via OpenID", + }), + ); + expect(client._unstable_getLivekitToken).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalledWith( + expect.objectContaining({ role: "publisher" }), + ); + }); + + it("does not fall back to OpenID on connection error", async () => { + const client_ = { + ...client, + _unstable_getLivekitToken: vi + .fn() + .mockRejectedValue(new ConnectionError("No connection")), + }; + await Promise.all([ + expect( + getSFUConfig({ + ...defaultParams, + client: client_, + transport: urlAndServiceUrlTransport, + }), + ).rejects.toThrow(expect.any(ConnectionError)), + vi.runAllTimersAsync(), // Allow request to be retried + ]); + expect(client_._unstable_getLivekitToken).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); +}); + +describe("delegateDelayedLeave", () => { + const defaultParams = { + client, + membership, + roomId, + slotId, + delayId: "delay123", + logger, + }; + + it("uses CS API with url transport", async () => { + await delegateDelayedLeave({ ...defaultParams, transport: urlTransport }); + expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("does not fall back to OpenID with url transport", async () => { + client._unstable_delegateDelayedLeave.mockRejectedValueOnce( + unsupportedError, + ); + await expect( + delegateDelayedLeave({ ...defaultParams, transport: urlTransport }), + ).rejects.toThrow(); + expect(client._unstable_delegateDelayedLeave).toHaveBeenCalled(); + expect(openID.getSFUConfig).not.toHaveBeenCalled(); + }); + + it("uses OpenID with livekit_service_url transport", async () => { + await delegateDelayedLeave({ + ...defaultParams, + transport: serviceUrlTransport, + }); + expect(client._unstable_delegateDelayedLeave).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalled(); + }); + + it("uses OpenID with url + livekit_service_url transport", async () => { + // Must match the flow that got the token, so the identities line up + await delegateDelayedLeave({ + ...defaultParams, + transport: urlAndServiceUrlTransport, + }); + expect(client._unstable_delegateDelayedLeave).not.toHaveBeenCalled(); + expect(openID.getSFUConfig).toHaveBeenCalledWith( + expect.objectContaining({ + role: "publisher", + delayId: "delay123", + delayEndpointBaseUrl: client.baseUrl, + }), + ); + }); +}); diff --git a/src/livekit/auth/index.ts b/src/livekit/auth/index.ts new file mode 100644 index 000000000..292dac4d4 --- /dev/null +++ b/src/livekit/auth/index.ts @@ -0,0 +1,253 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { ConnectionError, MatrixError } from "matrix-js-sdk"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import * as openID from "./openID"; +import * as csApi from "./csApi"; +import { type SFUConfig } from "./types"; +import { NoMatrix2AuthorizationService } from "../../utils/errors"; + +export type { ClientOpenIDParts } from "./openID"; +export type { ClientGetTokenParts, ClientDelegationParts } from "./csApi"; +export type * from "./types"; + +/** + * Checks whether an error means "this homeserver does not implement the + * (unstable) MSC4195 endpoints". + * + * MSC4195 documents `M_NOT_FOUND` for this, but a homeserver without support + * answers with whatever it uses for unknown endpoints (Synapse: 404 with + * `M_UNRECOGNIZED`), so we accept any 404 as "unsupported". + * @param e The error to check. + */ +function isEndpointUnsupported(e: unknown): boolean { + return ( + e instanceof MatrixError && + (e.httpStatus === 404 || + e.errcode === "M_NOT_FOUND" || + e.errcode === "M_UNRECOGNIZED") + ); +} + +/** + * Checks whether an error means the MSC4195 CS API cannot be used from here: + * the homeserver does not implement it, or the request never reached the + * homeserver because in widget mode the host does not expose the endpoint + * (MSC4533), which the widget API reports as a plain error rather than a + * Matrix error. A connection error is neither, as the endpoint may well exist. + * @param e The error to check. + */ +function isCsApiUnavailable(e: unknown): boolean { + if (isEndpointUnsupported(e)) return true; + return !(e instanceof MatrixError) && !(e instanceof ConnectionError); +} + +export interface GetSFUConfigParams { + /** + * The Matrix client. + */ + client: csApi.ClientGetTokenParts & openID.ClientOpenIDParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The transport for which we wish to get an access token. + */ + transport: UnstableLivekitTransport; + /** + * The name of the homeserver to which the {@link transport} belongs. + */ + serverName: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * Whether we want to publish or only subscribe on the {@link transport}. + */ + role: "publisher" | "subscriber"; + logger: Logger; +} + +/** + * Gets an {@link SFUConfig} appropriate for connecting to a given transport. + */ +export async function getSFUConfig({ + client, + membership, + transport, + serverName, + roomId, + slotId, + role, + logger, +}: GetSFUConfigParams): Promise { + const viaOpenID = async (serviceUrl: string): Promise => + await openID.getSFUConfig({ + client, + membership, + serviceUrl, + roomId, + slotId, + role, + logger, + }); + + const flow = chooseAuthFlow(transport, role); + if (flow.kind === "openID") return await viaOpenID(flow.serviceUrl); + + try { + return await csApi.getSFUConfig({ + client, + membership, + url: flow.url, + serverName, + roomId, + slotId, + }); + } catch (e) { + // Publishers should never fall back. If a homeserver advertises a `url` + // but doesn't support MSC4195 itself, that's a misconfiguration which + // the admin should know about so they can prepare for the future + // deprecation of `livekit_service_url`. + const mayFallBack = + role === "subscriber" && "livekit_service_url" in transport; + if (mayFallBack && isCsApiUnavailable(e)) { + logger.warn( + `Cannot use the MSC4195 get_token endpoint (${e}), falling back to the OpenID flow with service ${transport.livekit_service_url}`, + ); + return await viaOpenID(transport.livekit_service_url); + } + if (isEndpointUnsupported(e)) { + logger.error( + `Homeserver does not support the MSC4195 get_token endpoint, cannot get token for transport ${flow.url}`, + ); + throw new NoMatrix2AuthorizationService(e as Error); + } + throw e; + } +} + +export interface DelegateDelayedLeaveParams { + /** + * The Matrix client. + */ + client: csApi.ClientDelegationParts & openID.ClientOpenIDParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The transport to which we are publishing and wish to delegate the delayed + * leave event. + */ + transport: UnstableLivekitTransport; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * The delay ID of the leave event to be delegated. + */ + delayId: string; + logger: Logger; +} + +/** + * Delegates a delayed leave event to a given transport, so that the event will + * be sent on our behalf whenever we disconnect from the transport's SFU. + */ +export async function delegateDelayedLeave({ + client, + membership, + transport, + roomId, + slotId, + delayId, + logger, +}: DelegateDelayedLeaveParams): Promise { + // It's important that we use the same flow (CS API or OpenID) as whatever we + // would've used to get a JWT token for this transport, since the participant + // identity computed by the service needs to match. + const flow = chooseAuthFlow(transport, "publisher"); + if (flow.kind === "csApi") { + try { + await csApi.delegateDelayedLeave({ + client, + membership, + url: flow.url, + roomId, + slotId, + delayId, + }); + } catch (e) { + if (isEndpointUnsupported(e)) + logger.error( + "Server misconfigured: transport contains a 'url' field but server does not support the MSC4195 delegate_delayed_leave endpoint", + ); + throw e; + } + } else { + // This will technically cause the service to issue a new JWT token, but + // it's safe to discard. We're only interested in triggering delegation. + await openID.getSFUConfig({ + client, + membership, + serviceUrl: flow.serviceUrl, + roomId, + slotId, + role: "publisher", + delayEndpointBaseUrl: client.baseUrl, + delayId, + logger, + }); + } +} + +/** + * The flow with which the local member authenticates against a transport. + */ +export type AuthFlow = + | { kind: "csApi"; url: string } + | { kind: "openID"; serviceUrl: string }; + +/** + * Chooses between the MSC4195 CS API and the legacy OpenID flow against the + * transport's JWT service. + * + * Only the JWT service's `/sfu/get` endpoint assigns the legacy participant + * identity that clients predating MSC4195 expect from anyone advertising a + * membership state event. So a publisher keeps using the JWT service wherever + * the transport still offers one, even if it also has a `url`. Subscribers may + * use the CS API regardless, as nobody matches their identity. + * @param transport The transport to authenticate against. + * @param role Whether we want to publish or only subscribe on the transport. + */ +export function chooseAuthFlow( + transport: UnstableLivekitTransport, + role: "publisher" | "subscriber", +): AuthFlow { + if ("url" in transport) { + const useCsApi = + !("livekit_service_url" in transport) || role === "subscriber"; + if (useCsApi) return { kind: "csApi", url: transport.url }; + } + return { kind: "openID", serviceUrl: transport.livekit_service_url }; +} diff --git a/src/livekit/auth/openID.test.ts b/src/livekit/auth/openID.test.ts new file mode 100644 index 000000000..7498b8fe1 --- /dev/null +++ b/src/livekit/auth/openID.test.ts @@ -0,0 +1,230 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + beforeEach, + afterEach, + describe, + expect, + it, + type MockedObject, + vitest, +} from "vitest"; +import fetchMock from "fetch-mock"; +import { MatrixError } from "matrix-js-sdk"; + +import { + getSFUConfig, + type GetSFUConfigParams, + type ClientOpenIDParts, +} from "./openID"; +import { testJWTToken } from "../../utils/test-fixtures"; +import { ownMemberMock } from "../../utils/test"; +import { FailToGetOpenIdToken } from "../../utils/errors"; +import { logger } from "matrix-js-sdk/lib/logger"; + +const serviceUrl = "https://sfu.example.org"; + +const delayParams = { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", +}; + +describe("getSFUConfig", () => { + let matrixClient: MockedObject; + let params: GetSFUConfigParams; + beforeEach(() => { + fetchMock.catch(404); + matrixClient = { getOpenIdToken: vitest.fn() }; + params = { + client: matrixClient, + membership: ownMemberMock, + serviceUrl: "https://sfu.example.org", + roomId: "!example_room_id", + slotId: "m.call#ROOM", + role: "publisher", + logger, + }; + }); + afterEach(async () => { + void (await fetchMock.flush()); + vitest.clearAllMocks(); + fetchMock.reset(); + }); + + it("should handle fetching a token", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfig(params); + expect(config).toEqual({ + jwt: testJWTToken, + url: serviceUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + }); + + it("should fail if the SFU errors", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { + errcode: "M_LOOKUP_FAILED", + error: "Failed to look up user info from homeserver", + }, + }; + }); + try { + await getSFUConfig(params); + } catch (ex: unknown) { + expect(ex).toBeInstanceOf(FailToGetOpenIdToken); + expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); + const mxError = (ex as Error).cause as MatrixError; + expect(mxError.message).toEqual( + "MatrixError: [500] Failed to look up user info from homeserver", + ); + + return; + } + expect.fail("Expected test to throw;"); + }); + + it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => { + let callCount = 0; + + fetchMock.post( + "https://sfu.example.org/sfu/get", + (url, opts) => { + callCount++; + const body = JSON.parse(opts.body as string); + + // First call: check if it has delay parts and return 400 + if (callCount === 1) { + expect(body).toHaveProperty("delay_id", "mock_delay_id"); + return { + status: 400, + body: { errcode: "M_BAD_JSON", error: "Unsupported parameters" }, + }; + } + + // Second call: check if delay parts were stripped and return success + expect(body).not.toHaveProperty("delay_id"); + expect(body).not.toHaveProperty("delay_timeout"); + expect(body).not.toHaveProperty("delay_cs_api_url"); + + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }, + { overwriteRoutes: true }, + ); + + // Note: Assuming getSFUConfig eventually calls getLiveKitJWT + const config = await getSFUConfig({ ...params, ...delayParams }); + + expect(config.jwt).toBe(testJWTToken); + expect(callCount).toBe(2); + }); + + it("should successfully send delay parameters to the JWT service legacy endpoint", async () => { + fetchMock.post( + "https://sfu.example.org/sfu/get", + (url, opts) => { + const body = JSON.parse(opts.body as string); + + // Verify, that the request contains the expected delay parameters + if ( + body.delay_id === "mock_delay_id" && + body.delay_timeout === 3600000 && + body.delay_cs_api_url === "https://matrix.homeserverserver.org" + ) { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + } + return { + status: 400, + body: { error: "Missing expected delay params" }, + }; + }, + { overwriteRoutes: true }, + ); + + const config = await getSFUConfig({ ...params, ...delayParams }); + + // Prüfe das Ergebnis + expect(config).toMatchObject({ + jwt: testJWTToken, + url: serviceUrl, + }); + }); + + it("should use the default endpoint when subscribing", async () => { + fetchMock.post("https://sfu.example.org/get_token", (url, opts) => { + const body = JSON.parse(opts.body as string); + expect(body).toMatchObject({ + room_id: "!example_room_id", + slot_id: "m.call#ROOM", + member: { + id: ownMemberMock.memberId, + claimed_user_id: ownMemberMock.userId, + claimed_device_id: ownMemberMock.deviceId, + }, + }); + return { status: 200, body: { url: serviceUrl, jwt: testJWTToken } }; + }); + const config = await getSFUConfig({ ...params, role: "subscriber" }); + expect(config.jwt).toBe(testJWTToken); + expect(fetchMock.called("https://sfu.example.org/sfu/get")).toBe(false); + }); + + it("should fail when subscribing if the default endpoint is missing", async () => { + // Nothing but the catch-all 404 answers the default endpoint. Subscribers + // must not fall back to the legacy endpoint, which would hand them a + // legacy identity. + await expect( + getSFUConfig({ ...params, role: "subscriber" }), + ).rejects.toThrow(FailToGetOpenIdToken); + expect(fetchMock.called("https://sfu.example.org/get_token")).toBe(true); + expect(fetchMock.called("https://sfu.example.org/sfu/get")).toBe(false); + }); + + it("should retry fetching the openid token", async () => { + let count = 0; + matrixClient.getOpenIdToken.mockImplementation(async () => { + count++; + if (count < 2) { + throw Error("Test failure"); + } + return Promise.resolve({ + token_type: "Bearer", + access_token: "foobar", + matrix_server_name: "example.org", + expires_in: 30, + }); + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: serviceUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfig({ ...params, ...delayParams }); + expect(config).toEqual({ + jwt: testJWTToken, + url: serviceUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + }); +}); diff --git a/src/livekit/auth/openID.ts b/src/livekit/auth/openID.ts new file mode 100644 index 000000000..4479d4261 --- /dev/null +++ b/src/livekit/auth/openID.ts @@ -0,0 +1,262 @@ +/* +Copyright 2023, 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type IOpenIDToken, + type MatrixClient, + parseErrorResponse, +} from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { FailToGetOpenIdToken } from "../../utils/errors"; +import { doNetworkOperationWithRetry } from "../../utils/matrix"; +import { Config } from "../../config/Config"; +import { extractFullConfigFromToken, type SFUConfig } from "./types"; + +// The bits we need from MatrixClient +export type ClientOpenIDParts = Pick; + +export interface GetSFUConfigParams { + client: ClientOpenIDParts; + /** + * Data identifying the local user's session membership. + */ + membership: CallMembershipIdentityParts; + /** + * The base URL of the LiveKit JWT service. + */ + serviceUrl: string; + /** + * The ID of the Matrix room in which the session takes place. + */ + roomId: string; + /** + * The ID of the MatrixRTC slot in which the session takes place. + */ + slotId: string; + /** + * Whether we want to publish or only subscribe on the {@link transport}. + */ + role: "publisher" | "subscriber"; + /** + * The base URL of the Matrix homeserver, for delayed leave delegation. + */ + delayEndpointBaseUrl?: string; + /** + * The delay ID of the leave event to be delegated to the JWT service. + */ + delayId?: string; + logger: Logger; +} + +/** + * Gets an {@link SFUConfig} appropriate for connecting to the SFU behind a + * given service URL, using the legacy OpenID flow. + * @throws {@link FailToGetOpenIdToken} + */ +export async function getSFUConfig({ + client, + membership, + serviceUrl, + roomId, + slotId, + role, + delayEndpointBaseUrl, + delayId, + logger, +}: GetSFUConfigParams): Promise { + let openIdToken: IOpenIDToken; + try { + openIdToken = await doNetworkOperationWithRetry(async () => + client.getOpenIdToken(), + ); + } catch (error) { + throw new FailToGetOpenIdToken( + error instanceof Error ? error : new Error("Unknown error"), + ); + } + logger.debug("Got openID token", openIdToken); + let sfuConfig: { url: string; jwt: string } | undefined; + + let endpoint: "default" | "legacy"; + if (role === "publisher") { + // When publishing a legacy transport (one with a `livekit_service_url`), + // subscribers will expect our participant identity to use the legacy + // `@user_id:device_id` format. Only the legacy JWT service endpoint assigns + // identities in this format, so we must use it. + endpoint = "legacy"; + } else { + // Use the default endpoint as it affords us slightly better metadata + // protection (pseudonymous participant identities). + endpoint = "default"; + } + + try { + logger.info( + `Trying to get JWT with ${endpoint} endpoint for focus ${serviceUrl}...`, + ); + sfuConfig = await ( + endpoint === "legacy" ? getLiveKitJWTLegacy : getLiveKitJWT + )( + membership, + serviceUrl, + roomId, + slotId, + openIdToken, + delayEndpointBaseUrl, + delayId, + ); + logger.info(`Got JWT from call's active focus URL.`); + return extractFullConfigFromToken(sfuConfig); + } catch (e) { + logger.error(`Failed fetching jwt with ${endpoint} endpoint:`, e); + throw new FailToGetOpenIdToken( + e instanceof Error ? e : new Error(`Unknown error ${e}`), + ); + } +} + +/** + * Gets a JWT token appropriate for connecting to the SFU behind a given service + * URL, using the legacy `/sfu/get` endpoint, which assigns us a legacy + * (`:`) participant identity. + */ +async function getLiveKitJWTLegacy( + membership: CallMembershipIdentityParts, + livekitServiceURL: string, + matrixRoomId: string, + _slotId: string, // Unused - the legacy endpoint is from before slots existed. + openIDToken: IOpenIDToken, + delayEndpointBaseUrl?: string, + delayId?: string, +): Promise<{ url: string; jwt: string }> { + interface IDelayParams { + delay_id?: string; + delay_timeout?: number; + delay_cs_api_url?: string; + } + let bodyDalayParts: IDelayParams = {}; + // Also check for empty string + if (delayId && delayEndpointBaseUrl) { + bodyDalayParts = { + delay_id: delayId, + delay_timeout: + Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, + delay_cs_api_url: delayEndpointBaseUrl, + }; + } + + const makeRequest = async (delayParts: IDelayParams): Promise => { + return await fetch(livekitServiceURL + "/sfu/get", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + // The legacy JWT endpoint uses only the matrix room id to calculate the livekit room alias. + // However, the livekit room alias is provided as part of the JWT payload. + room: matrixRoomId, + openid_token: openIDToken, + device_id: membership.deviceId, + ...delayParts, + }), + }); + }; + + const res = await doNetworkOperationWithRetry(async () => { + let response = await makeRequest(bodyDalayParts); + + // Old service compatibility check + const oldServiceDoesNotSupportDelayParts = + response.status === 400 && Object.keys(bodyDalayParts).length > 0; + // If http status 400 with M_BAD_JSON and we sent delay parts, retry without them + if (oldServiceDoesNotSupportDelayParts) { + try { + const errorBody = await response.json(); + if (errorBody.errcode === "M_BAD_JSON") { + response = await makeRequest({}); + } + } catch { + // If we can't parse the error, treat as real error + } + } + + return response; + }); + + if (!res.ok) { + throw parseErrorResponse(res, await res.text()); + } + return await res.json(); +} + +class NotSupportedError extends Error { + public constructor(message: string) { + super(message); + this.name = "NotSupported"; + } +} + +/** + * Gets a JWT token appropriate for connecting to the SFU behind a given service + * URL, using the default `/get_token` endpoint, which assigns us a hashed + * (pseudonymised) participant identity. + */ +async function getLiveKitJWT( + membership: CallMembershipIdentityParts, + livekitServiceURL: string, + matrixRoomId: string, + slotId: string, + openIDToken: IOpenIDToken, + delayEndpointBaseUrl?: string, + delayId?: string, +): Promise<{ url: string; jwt: string }> { + const { userId, deviceId, memberId } = membership; + + const body = { + room_id: matrixRoomId, + slot_id: slotId, + openid_token: openIDToken, + member: { + id: memberId, + claimed_user_id: userId, + claimed_device_id: deviceId, + }, + }; + + let bodyDalayParts = {}; + // Also check for empty string + if (delayId && delayEndpointBaseUrl) { + bodyDalayParts = { + delay_id: delayId, + delay_timeout: + Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, + delay_cs_api_url: delayEndpointBaseUrl, + }; + } + + const res = await doNetworkOperationWithRetry(async () => { + return await fetch(livekitServiceURL + "/get_token", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ ...body, ...bodyDalayParts }), + }); + }); + + if (!res.ok) { + const msg = "SFU Config fetch failed with status code " + res.status; + if (res.status === 404) { + throw new NotSupportedError(msg); + } else { + throw parseErrorResponse(res, await res.text()); + } + } + return await res.json(); +} diff --git a/src/livekit/auth/types.ts b/src/livekit/auth/types.ts new file mode 100644 index 000000000..409d88a6b --- /dev/null +++ b/src/livekit/auth/types.ts @@ -0,0 +1,90 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; + +/** + * Data identifying a transport and the homeserver to which it belongs, so + * consumers know how to reach it. + */ +export interface TransportLocator { + transport: UnstableLivekitTransport; + serverName: string; +} + +/** + * Configuration and access tokens provided by the SFU on successful authentication. + */ +export interface SFUConfig { + /** + * The WebSocket URL of the LiveKit SFU. This is what we connect the LiveKit + * room to. Note that this is NOT the JWT service URL + * (`livekit_service_url`): the legacy JWT service returns this URL as part of + * its response, while the CS-Api based flow requires us to know it upfront. + */ + url: string; + jwt: string; + livekitAlias: string; + // NOTE: Currently unused. + livekitIdentity: string; +} + +/** + * Decoded details from the JWT. + */ +interface SFUJWTPayload { + /** + * Expiration time for the JWT. + * Note: This value is in seconds since Unix epoch. + */ + exp: number; + /** + * Name of the instance which authored the JWT + */ + iss: string; + /** + * Time at which the JWT can start to be used. + * Note: This value is in seconds since Unix epoch. + */ + nbf: number; + /** + * Subject. The Livekit alias in this context. + */ + sub: string; + /** + * The set of permissions for the user. + */ + video: { + canPublish: boolean; + canSubscribe: boolean; + room: string; + roomJoin: boolean; + }; +} + +/** + * Complements the SFU websocket url and the JWT with the information encoded in + * the JWT payload itself. + * @param sfuConfig The SFU websocket url and the JWT to connect with. + * @returns The full SFU config, including the LiveKit alias and identity. + */ +export function extractFullConfigFromToken(sfuConfig: { + url: string; + jwt: string; +}): SFUConfig { + const [, payloadStr] = sfuConfig.jwt.split("."); + const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; + return { + jwt: sfuConfig.jwt, + url: sfuConfig.url, + livekitAlias: payload.video.room, + // NOTE: Currently unused. + // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. + // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) + livekitIdentity: payload.sub, + }; +} diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts deleted file mode 100644 index f759a3aac..000000000 --- a/src/livekit/openIDSFU.test.ts +++ /dev/null @@ -1,329 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - beforeEach, - afterEach, - describe, - expect, - it, - type MockedObject, - vitest, -} from "vitest"; -import fetchMock from "fetch-mock"; -import { MatrixError } from "matrix-js-sdk"; - -import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU"; -import { testJWTToken } from "../utils/test-fixtures"; -import { ownMemberMock } from "../utils/test"; -import { FailToGetOpenIdToken } from "../utils/errors"; - -const sfuUrl = "https://sfu.example.org"; - -describe("getSFUConfigWithOpenID", () => { - let matrixClient: MockedObject; - beforeEach(() => { - fetchMock.catch(404); - matrixClient = { - getOpenIdToken: vitest.fn(), - getDeviceId: vitest.fn(), - }; - }); - afterEach(() => { - vitest.clearAllMocks(); - fetchMock.reset(); - }); - - it("should handle fetching a token", async () => { - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - expect(config).toEqual({ - jwt: testJWTToken, - url: sfuUrl, - livekitIdentity: "@me:example.org:ABCDEF", - livekitAlias: "!example_room_id", - }); - void (await fetchMock.flush()); - }); - - it("should fail if the SFU errors", async () => { - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - } catch (ex: unknown) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); - const mxError = (ex as Error).cause as MatrixError; - expect(mxError.message).toEqual( - "MatrixError: [500] Failed to look up user info from homeserver", - ); - - void (await fetchMock.flush()); - return; - } - expect.fail("Expected test to throw;"); - }); - - it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => { - let callCount = 0; - - fetchMock.post( - "https://sfu.example.org/sfu/get", - (url, opts) => { - callCount++; - const body = JSON.parse(opts.body as string); - - // First call: check if it has delay parts and return 400 - if (callCount === 1) { - expect(body).toHaveProperty("delay_id", "mock_delay_id"); - return { - status: 400, - body: { errcode: "M_BAD_JSON", error: "Unsupported parameters" }, - }; - } - - // Second call: check if delay parts were stripped and return success - expect(body).not.toHaveProperty("delay_id"); - expect(body).not.toHaveProperty("delay_timeout"); - expect(body).not.toHaveProperty("delay_cs_api_url"); - - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }, - { overwriteRoutes: true }, - ); - - // Note: Assuming getSFUConfigWithOpenID eventually calls getLiveKitJWT - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - }, - ); - - expect(config.jwt).toBe(testJWTToken); - expect(callCount).toBe(2); - void (await fetchMock.flush()); - }); - - it("should successfully send delay parameters to the JWT service legacy endpoint", async () => { - fetchMock.post( - "https://sfu.example.org/sfu/get", - (url, opts) => { - const body = JSON.parse(opts.body as string); - - // Verify, that the request contains the expected delay parameters - if ( - body.delay_id === "mock_delay_id" && - body.delay_timeout === 3600000 && - body.delay_cs_api_url === "https://homeserverserver.org/cs_api" - ) { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - } - return { - status: 400, - body: { error: "Missing expected delay params" }, - }; - }, - { overwriteRoutes: true }, - ); - - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://homeserverserver.org/cs_api", - delayId: "mock_delay_id", - }, - ); - - // Prüfe das Ergebnis - expect(config).toMatchObject({ - jwt: testJWTToken, - url: sfuUrl, - }); - - void (await fetchMock.flush()); - }); - - it("should try legacy and then new endpoint with delay delegation", async () => { - fetchMock.post("https://sfu.example.org/get_token", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { - errcode: "M_LOOKUP_FAILED", - error: "Failed to look up user info from homeserver", - }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - }, - ); - } catch (ex) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); - const mxError = (ex as Error).cause as MatrixError; - expect(mxError.message).toEqual( - "MatrixError: [500] Failed to look up user info from homeserver", - ); - void (await fetchMock.flush()); - } - const calls = fetchMock.calls(); - expect(calls.length).toBe(2); - - expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); - expect(calls[0][1]).toStrictEqual({ - // check if it uses correct delayID! - body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - method: "POST", - headers: { - "Content-Type": "application/json", - }, - }); - - expect(calls[1][0]).toStrictEqual("https://sfu.example.org/sfu/get"); - - expect(calls[1][1]).toStrictEqual({ - body: '{"room":"!example_room_id","device_id":"DEVICE","delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - headers: { - "Content-Type": "application/json", - }, - method: "POST", - }); - }); - - it("dont try legacy if endpoint with delay delegation is sucessful", async () => { - fetchMock.post("https://sfu.example.org/get_token", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 500, - body: { error: "Test failure" }, - }; - }); - try { - await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - { - delayEndpointBaseUrl: "https://matrix.homeserverserver.org", - delayId: "mock_delay_id", - }, - ); - } catch (ex) { - expect(ex).toBeInstanceOf(FailToGetOpenIdToken); - expect((ex as FailToGetOpenIdToken).cause).toEqual( - new Error("SFU Config fetch failed with status code 500"), - ); - void (await fetchMock.flush()); - } - const calls = fetchMock.calls(); - expect(calls.length).toBe(1); - - expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); - expect(calls[0][1]).toStrictEqual({ - // check if it uses correct delayID! - body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', - method: "POST", - headers: { - "Content-Type": "application/json", - }, - }); - }); - - it("should retry fetching the openid token", async () => { - let count = 0; - matrixClient.getOpenIdToken.mockImplementation(async () => { - count++; - if (count < 2) { - throw Error("Test failure"); - } - return Promise.resolve({ - token_type: "Bearer", - access_token: "foobar", - matrix_server_name: "example.org", - expires_in: 30, - }); - }); - fetchMock.post("https://sfu.example.org/sfu/get", () => { - return { - status: 200, - body: { url: sfuUrl, jwt: testJWTToken }, - }; - }); - const config = await getSFUConfigWithOpenID( - matrixClient, - ownMemberMock, - "https://sfu.example.org", - "!example_room_id", - ); - expect(config).toEqual({ - jwt: testJWTToken, - url: sfuUrl, - livekitIdentity: "@me:example.org:ABCDEF", - livekitAlias: "!example_room_id", - }); - void (await fetchMock.flush()); - }); -}); diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts deleted file mode 100644 index cf1f3c951..000000000 --- a/src/livekit/openIDSFU.ts +++ /dev/null @@ -1,315 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type IOpenIDToken, - type MatrixClient, - parseErrorResponse, -} from "matrix-js-sdk"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { type Logger } from "matrix-js-sdk/lib/logger"; - -import { - FailToGetOpenIdToken, - NoMatrix2AuthorizationService, -} from "../utils/errors"; -import { doNetworkOperationWithRetry } from "../utils/matrix"; -import { Config } from "../config/Config"; -import { MatrixRTCMode } from "../config/ConfigOptions"; - -/** - * Configuration and access tokens provided by the SFU on successful authentication. - */ -export interface SFUConfig { - url: string; - jwt: string; - livekitAlias: string; - // NOTE: Currently unused. - livekitIdentity: string; -} - -/** - * Decoded details from the JWT. - */ -interface SFUJWTPayload { - /** - * Expiration time for the JWT. - * Note: This value is in seconds since Unix epoch. - */ - exp: number; - /** - * Name of the instance which authored the JWT - */ - iss: string; - /** - * Time at which the JWT can start to be used. - * Note: This value is in seconds since Unix epoch. - */ - nbf: number; - /** - * Subject. The Livekit alias in this context. - */ - sub: string; - /** - * The set of permissions for the user. - */ - video: { - canPublish: boolean; - canSubscribe: boolean; - room: string; - roomJoin: boolean; - }; -} - -// The bits we need from MatrixClient -export type OpenIDClientParts = Pick< - MatrixClient, - "getOpenIdToken" | "getDeviceId" ->; - -/** - * Gets a bearer token from the homeserver and then use it to authenticate - * to the matrix RTC backend in order to get acces to the SFU. - * It has built-in retry for calls to the homeserver with a backoff policy. - * @param client The Matrix client - * @param membership Our own membership identity parts used to send to jwt service. - * @param serviceUrl The URL of the livekit SFU service - * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. - * @param opts Additional options to modify which endpoint with which data will be used to acquire the jwt token. - * @param opts.matrixRTCMode Determines which version of the JWT endpoint to use, which affects whether the - * RTC backend identity is based on string concatenation (legacy) or a hash (Matrix 2.0). - * This function by default uses whatever is possible with the current jwt service installed next to the SFU. - * For remote connections this does not matter, since we will not publish there we can rely on the newest option. - * @param opts.delayEndpointBaseUrl The URL of the matrix homeserver. - * @param opts.delayId The delay id used for the jwt service to manage. - * @param logger optional logger. - * @returns Object containing the token information - * @throws FailToGetOpenIdToken - */ -export async function getSFUConfigWithOpenID( - client: OpenIDClientParts, - membership: CallMembershipIdentityParts, - serviceUrl: string, - roomId: string, - opts?: { - matrixRTCMode?: MatrixRTCMode; - delayEndpointBaseUrl?: string; - delayId?: string; - }, - logger?: Logger, -): Promise { - let openIdToken: IOpenIDToken; - try { - openIdToken = await doNetworkOperationWithRetry(async () => - client.getOpenIdToken(), - ); - } catch (error) { - throw new FailToGetOpenIdToken( - error instanceof Error ? error : new Error("Unknown error"), - ); - } - logger?.debug("Got openID token", openIdToken); - let sfuConfig: { url: string; jwt: string } | undefined; - - const tryBothJwtEndpoints = opts?.matrixRTCMode === undefined; // This is for SFUs where we do not publish. - - const forceMatrix2Jwt = opts?.matrixRTCMode === MatrixRTCMode.Matrix_2_0; - - // We want to start using the new endpoint (with optional delay delegation) - // if we can use both or if we are forced to use the new one. - if (tryBothJwtEndpoints || forceMatrix2Jwt) { - try { - logger?.info( - `Trying to get JWT via default endpoint for focus ${serviceUrl}...`, - ); - const sfuConfig = await getLiveKitJWT( - membership, - serviceUrl, - roomId, - openIdToken, - opts?.delayEndpointBaseUrl, - opts?.delayId, - ); - - return extractFullConfigFromToken(sfuConfig); - } catch (e) { - logger?.debug(`Failed fetching jwt with matrix 2.0 endpoint:`, e); - // Make this throw a hard error in case we force the matrix2.0 endpoint. - if (forceMatrix2Jwt) { - throw new NoMatrix2AuthorizationService(e as Error); - } - } - } - - // DEPRECATED - // here we either have a sfuConfig or we already exited because of `if (forceMatrix2) throw ...` - // The only case we can get into this condition is, if `forceMatrix2` is `false` - try { - logger?.info( - `Trying to get JWT with legacy endpoint for focus ${serviceUrl}...`, - ); - sfuConfig = await getLiveKitJWTLegacy( - membership.deviceId, - serviceUrl, - roomId, - openIdToken, - opts?.delayEndpointBaseUrl, - opts?.delayId, - ); - logger?.info(`Got JWT from call's active focus URL.`); - return extractFullConfigFromToken(sfuConfig); - } catch (ex) { - throw new FailToGetOpenIdToken( - ex instanceof Error ? ex : new Error(`Unknown error ${ex}`), - ); - } -} - -function extractFullConfigFromToken(sfuConfig: { - url: string; - jwt: string; -}): SFUConfig { - const [, payloadStr] = sfuConfig.jwt.split("."); - const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; - return { - jwt: sfuConfig.jwt, - url: sfuConfig.url, - livekitAlias: payload.video.room, - // NOTE: Currently unused. - // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. - // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) - livekitIdentity: payload.sub, - }; -} - -async function getLiveKitJWTLegacy( - deviceId: string, - livekitServiceURL: string, - matrixRoomId: string, - openIDToken: IOpenIDToken, - delayEndpointBaseUrl?: string, - delayId?: string, -): Promise<{ url: string; jwt: string }> { - interface IDelayParams { - delay_id?: string; - delay_timeout?: number; - delay_cs_api_url?: string; - } - let bodyDalayParts: IDelayParams = {}; - // Also check for empty string - if (delayId && delayEndpointBaseUrl) { - bodyDalayParts = { - delay_id: delayId, - delay_timeout: - Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, - delay_cs_api_url: delayEndpointBaseUrl, - }; - } - - const makeRequest = async (delayParts: IDelayParams): Promise => { - return await fetch(livekitServiceURL + "/sfu/get", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - // The legacy JWT endpoint uses only the matrix room id to calculate the livekit room alias. - // However, the livekit room alias is provided as part of the JWT payload. - room: matrixRoomId, - openid_token: openIDToken, - device_id: deviceId, - ...delayParts, - }), - }); - }; - - const res = await doNetworkOperationWithRetry(async () => { - let response = await makeRequest(bodyDalayParts); - - // Old service compatibility check - const oldServiceDoesNotSupportDelayParts = - response.status === 400 && Object.keys(bodyDalayParts).length > 0; - // If http status 400 with M_BAD_JSON and we sent delay parts, retry without them - if (oldServiceDoesNotSupportDelayParts) { - try { - const errorBody = await response.json(); - if (errorBody.errcode === "M_BAD_JSON") { - response = await makeRequest({}); - } - } catch { - // If we can't parse the error, treat as real error - } - } - - return response; - }); - - if (!res.ok) { - throw parseErrorResponse(res, await res.text()); - } - return await res.json(); -} - -class NotSupportedError extends Error { - public constructor(message: string) { - super(message); - this.name = "NotSupported"; - } -} - -export async function getLiveKitJWT( - membership: CallMembershipIdentityParts, - livekitServiceURL: string, - matrixRoomId: string, - openIDToken: IOpenIDToken, - delayEndpointBaseUrl?: string, - delayId?: string, -): Promise<{ url: string; jwt: string }> { - const { userId, deviceId, memberId } = membership; - - const body = { - room_id: matrixRoomId, - slot_id: "m.call#ROOM", - openid_token: openIDToken, - member: { - id: memberId, - claimed_user_id: userId, - claimed_device_id: deviceId, - }, - }; - - let bodyDalayParts = {}; - // Also check for empty string - if (delayId && delayEndpointBaseUrl) { - bodyDalayParts = { - delay_id: delayId, - delay_timeout: - Config.get().matrix_rtc_session.delegated_delayed_leave.delay_ms, - delay_cs_api_url: delayEndpointBaseUrl, - }; - } - - const res = await doNetworkOperationWithRetry(async () => { - return await fetch(livekitServiceURL + "/get_token", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ ...body, ...bodyDalayParts }), - }); - }); - - if (!res.ok) { - const msg = "SFU Config fetch failed with status code " + res.status; - if (res.status === 404) { - throw new NotSupportedError(msg); - } else { - throw parseErrorResponse(res, await res.text()); - } - } - return await res.json(); -} diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 9ab64dfd6..1d8fd33d2 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -17,7 +17,7 @@ import { import { act } from "react"; import { type RoomMember } from "matrix-js-sdk"; import { - type LivekitTransport, + type UnstableLivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; @@ -46,7 +46,7 @@ vitest.mock("../useAudioContext"); vitest.mock("../soundUtils"); vitest.mock("../rtcSessionHelpers", async (importOriginal) => ({ ...(await importOriginal()), - makeTransport: (): [LivekitTransport] => [exampleTransport], + makeTransport: (): [UnstableLivekitTransport] => [exampleTransport], })); afterEach(() => { diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b8d7ab89d..bc0d68c4e 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -665,10 +665,10 @@ export const InCallView: FC = ({ onPointerOut={onPointerOut} > {header} - {audioParticipants.map(({ livekitRoom, url, participants }) => ( + {audioParticipants.map(({ livekitRoom, transport, participants }) => ( = ({ tab={settingsTab} onTabChange={setSettingsTab} developerSettingsVm={developerSettingsVm} - livekitRooms={allConnections - .getConnections() - .map((connectionItem) => ({ - room: connectionItem.livekitRoom, - livekitAlias: connectionItem.livekitAlias, - // TODO compute is local or tag it in the livekit room items already - isLocal: undefined, - url: connectionItem.transport.livekit_service_url, - }))} + livekitRooms={allConnections.map( + ({ livekitRoom: room, livekitAlias, transport, role }) => ({ + room, + livekitAlias, + role, + url: + // It should be relatively clear to the user which kind of URL + // this is, based on the scheme (ws:// vs. http://) + "url" in transport + ? transport.url + : transport.livekit_service_url, + }), + )} /> )} diff --git a/src/settings/DeveloperSettingsTab.test.tsx b/src/settings/DeveloperSettingsTab.test.tsx index bd2b40b7d..82764d8ec 100644 --- a/src/settings/DeveloperSettingsTab.test.tsx +++ b/src/settings/DeveloperSettingsTab.test.tsx @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, type Mock, vi } from "vitest"; import { render, waitFor, screen } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; +import userEvent, { type UserEvent } from "@testing-library/user-event"; import { TooltipProvider } from "@vector-im/compound-web"; import type { MatrixClient } from "matrix-js-sdk"; @@ -18,14 +18,14 @@ import { } from "./DeveloperSettingsTab"; import { outOfCallDeveloperSettingsTabViewModel } from "./DeveloperSettingsTabViewModel"; import { createStaticViewModel } from "../state/ViewModel"; -import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { - customLivekitUrl as customLivekitUrlSetting, + customTransport as customTransportSetting, enableExtendedLivekitLogs as enableExtendedLivekitLogsSetting, matrixRTCMode as matrixRTCModeSetting, } from "./settings"; import { MatrixRTCMode } from "../config/ConfigOptions"; -import { mockConfig } from "../utils/test"; +import { exampleTransport, mockConfig } from "../utils/test"; +import { getSFUConfig } from "../livekit/auth"; // Mock url params hook to avoid environment-dependent snapshot churn. vi.mock("../UrlParams", () => ({ @@ -36,8 +36,8 @@ vi.mock("../UrlParams", () => ({ })); // IMPORTANT: mock the same specifier used by DeveloperSettingsTab -vi.mock("../livekit/openIDSFU", () => ({ - getSFUConfigWithOpenID: vi.fn().mockResolvedValue({ +vi.mock("../livekit/auth", () => ({ + getSFUConfig: vi.fn().mockResolvedValue({ url: "mock-url", jwt: "mock-jwt", }), @@ -77,6 +77,21 @@ function createMockMatrixClient(): MatrixClient { } as unknown as MatrixClient; } +/** + * Types the JSON representation of an object into the given element. + */ +async function typeJson( + user: UserEvent, + element: Element, + object: object, +): Promise { + // `user.type` requires us to escape brackets by doubling them + await user.type( + element, + JSON.stringify(object).replace(/([{[\]])/g, (c) => c + c), + ); +} + describe("DeveloperSettingsTab", () => { it("renders and matches snapshot", async () => { const client = createMockMatrixClient(); @@ -124,17 +139,18 @@ describe("DeveloperSettingsTab", () => { expect(container).toMatchSnapshot(); }); - describe("custom livekit url", () => { + describe("custom transport", () => { afterEach(() => { - customLivekitUrlSetting.setValue(null); + customTransportSetting.setValue(null); }); const client = { doesServerSupportUnstableFeature: vi.fn().mockResolvedValue(true), getCrypto: () => ({ getVersion: (): string => "x" }), getUserId: () => "@u:hs", getDeviceId: () => "DEVICE", + getDomain: () => "hs", } as unknown as MatrixClient; - it("will not update custom livekit url without roomId", async () => { + it("will not update custom transport without roomId", async () => { const user = userEvent.setup(); render( @@ -147,17 +163,17 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); - await user.type(input, "wss://example.livekit.invalid"); + await typeJson(user, input, exampleTransport); const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfigWithOpenID).not.toHaveBeenCalled(); + expect(getSFUConfig).not.toHaveBeenCalled(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); - it("will not update custom livekit url without text in input", async () => { + it("will not update custom transport without text in input", async () => { const user = userEvent.setup(); render( @@ -171,16 +187,16 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfigWithOpenID).not.toHaveBeenCalled(); + expect(getSFUConfig).not.toHaveBeenCalled(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); - it("will not update custom livekit url when pressing cancel", async () => { + it("will not update custom transport when pressing cancel", async () => { const user = userEvent.setup(); render( @@ -194,7 +210,7 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); await user.type(input, "wss://example.livekit.invalid"); @@ -202,11 +218,11 @@ describe("DeveloperSettingsTab", () => { name: "Reset overwrite", }); await user.click(cancelButton); - expect(getSFUConfigWithOpenID).not.toHaveBeenCalled(); + expect(getSFUConfig).not.toHaveBeenCalled(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); - it("will update custom livekit url", async () => { + it("will update custom transport", async () => { const user = userEvent.setup(); render( @@ -220,24 +236,28 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); - await user.type(input, "wss://example.livekit.valid"); + await typeJson(user, input, { + type: "livekit", + url: "wss://example.livekit.valid", + }); const saveButton = screen.getByRole("button", { name: "Save" }); await user.click(saveButton); - expect(getSFUConfigWithOpenID).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - "wss://example.livekit.valid", - "#testRoom", + expect(getSFUConfig).toHaveBeenCalledWith( + expect.objectContaining({ + transport: { type: "livekit", url: "wss://example.livekit.valid" }, + roomId: "#testRoom", + }), ); - expect(customLivekitUrlSetting.getValue()).toBe( - "wss://example.livekit.valid", - ); + expect(customTransportSetting.getValue()).toEqual({ + type: "livekit", + url: "wss://example.livekit.valid", + }); }); - it("will show error on invalid url", async () => { + it("will show error on unreachable transport", async () => { const user = userEvent.setup(); render( @@ -251,24 +271,27 @@ describe("DeveloperSettingsTab", () => { , ); - const input = screen.getByLabelText("Custom Livekit-url"); + const input = screen.getByLabelText("Custom transport JSON"); await user.clear(input); - await user.type(input, "wss://example.livekit.valid"); + await typeJson(user, input, { + type: "livekit", + url: "wss://example.livekit.valid", + }); const saveButton = screen.getByRole("button", { name: "Save" }); - (getSFUConfigWithOpenID as Mock).mockImplementation(() => { - throw new Error("Invalid URL"); + (getSFUConfig as Mock).mockImplementation(() => { + throw new Error("Failed to get SFU config"); }); await user.click(saveButton); expect( - screen.getByText("invalid URL (did not update)"), + screen.getByText("invalid transport (did not update)"), ).toBeInTheDocument(); - expect(customLivekitUrlSetting.getValue()).toBe(null); + expect(customTransportSetting.getValue()).toBe(null); }); }); // Add this test inside the describe("DeveloperSettingsTab", () => { block, - // after the custom livekit url tests: + // after the custom transport tests: describe("enable extended livekit logs", () => { afterEach(() => { diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index ec5bf7b08..e31282e81 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -45,7 +45,7 @@ import { muteAllAudio as muteAllAudioSetting, alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, matrixRTCMode as matrixRTCModeSetting, - customLivekitUrl as customLivekitUrlSetting, + customTransport as customTransportSetting, advancedScreenShare as advancedScreenShareSetting, screenShareResolution as screenShareResolutionSetting, screenShareFramerate as screenShareFramerateSetting, @@ -67,9 +67,10 @@ import styles from "./DeveloperSettingsTab.module.css"; import settingsStyles from "./SettingsModal.module.css"; import { Slider } from "../Slider"; import { useUrlParams } from "../UrlParams"; -import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { useBehavior } from "../useBehavior"; import { type ViewModel } from "../state/ViewModel.ts"; +import { getSFUConfig } from "../livekit/auth/index.ts"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; /** * The state of MatrixRTC's media key rotation. @@ -167,16 +168,18 @@ export const DeveloperSettingsTab: FC = ({ enableExtendedLivekitLogsSetting, ); - const [customLivekitUrlUpdateError, setCustomLivekitUrlUpdateError] = - useState(null); - const [customLivekitUrl, setCustomLivekitUrl] = useSetting( - customLivekitUrlSetting, + const [customTransportUpdateError, setCustomLivekitUrlUpdateError] = useState< + string | null + >(null); + const [customTransport, setCustomTransport] = useSetting( + customTransportSetting, + ); + const [customTransportTextBuffer, setCustomLivekitUrlTextBuffer] = useState( + () => transportText(customTransport), ); - const [customLivekitUrlTextBuffer, setCustomLivekitUrlTextBuffer] = - useState(customLivekitUrl); useEffect(() => { - setCustomLivekitUrlTextBuffer(customLivekitUrl); - }, [customLivekitUrl]); + setCustomLivekitUrlTextBuffer(transportText(customTransport)); + }, [customTransport]); const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting); @@ -491,23 +494,23 @@ export const DeveloperSettingsTab: FC = ({ e.preventDefault()} helpLabel={ - customLivekitUrl === null - ? t("developer_mode.custom_livekit_url.from_config") - : t("developer_mode.custom_livekit_url.current_url") + - customLivekitUrl + customTransport === null + ? t("developer_mode.custom_transport.from_config") + : t("developer_mode.custom_transport.current") + + JSON.stringify(customTransport) } - label={t("developer_mode.custom_livekit_url.label")} - saveButtonLabel={t("developer_mode.custom_livekit_url.save")} - savingLabel={t("developer_mode.custom_livekit_url.saving")} - cancelButtonLabel={t("developer_mode.custom_livekit_url.reset")} + label={t("developer_mode.custom_transport.label")} + saveButtonLabel={t("developer_mode.custom_transport.save")} + savingLabel={t("developer_mode.custom_transport.saving")} + cancelButtonLabel={t("developer_mode.custom_transport.reset")} onSave={useCallback( async (e: React.FormEvent): Promise => { if ( roomId === undefined || - customLivekitUrlTextBuffer === "" || - customLivekitUrlTextBuffer === null + customTransportTextBuffer === "" || + customTransportTextBuffer === null ) { - setCustomLivekitUrl(null); + setCustomTransport(null); return; } @@ -518,21 +521,29 @@ export const DeveloperSettingsTab: FC = ({ if (userId === null || deviceId === null) { throw new Error("Invalid user or device ID"); } - await getSFUConfigWithOpenID( + const transport = JSON.parse(customTransportTextBuffer); + // Validate the transport by attempting to get a token for it + await getSFUConfig({ client, - { userId, deviceId, memberId: "" }, - customLivekitUrlTextBuffer, + membership: { userId, deviceId, memberId: "" }, + transport, + serverName: client.getDomain()!, roomId, - ); + slotId: "org.example.fake-slot#room", + role: "publisher", + logger, + }); setCustomLivekitUrlUpdateError(null); - setCustomLivekitUrl(customLivekitUrlTextBuffer); + setCustomTransport(transport); } catch { - setCustomLivekitUrlUpdateError("invalid URL (did not update)"); + setCustomLivekitUrlUpdateError( + "invalid transport (did not update)", + ); } }, - [customLivekitUrlTextBuffer, setCustomLivekitUrl, client, roomId], + [customTransportTextBuffer, setCustomTransport, client, roomId], )} - value={customLivekitUrlTextBuffer ?? ""} + value={customTransportTextBuffer ?? ""} onChange={useCallback( (event: ChangeEvent): void => { setCustomLivekitUrlTextBuffer(event.target.value); @@ -541,14 +552,14 @@ export const DeveloperSettingsTab: FC = ({ )} onCancel={useCallback( (e: React.FormEvent) => { - setCustomLivekitUrl(null); + setCustomTransport(null); }, - [setCustomLivekitUrl], + [setCustomTransport], )} - serverInvalid={customLivekitUrlUpdateError !== null} + serverInvalid={customTransportUpdateError !== null} > - {customLivekitUrlUpdateError !== null && ( - {customLivekitUrlUpdateError} + {customTransportUpdateError !== null && ( + {customTransportUpdateError} )} @@ -686,3 +697,10 @@ export const DeveloperSettingsTab: FC = ({ ); }; + +/** + * The text shown in the custom transport field for a given setting value. + */ +function transportText(transport: UnstableLivekitTransport | null): string { + return transport === null ? "" : JSON.stringify(transport); +} diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 933ac66f5..733068acc 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -59,7 +59,7 @@ interface Props { livekitRooms?: { room: LivekitRoom; url: string; - isLocal?: boolean; + role: "publisher" | "subscriber"; }[]; /** Only available while in a call. Used by the developer tab. */ developerSettingsVm?: ViewModel; diff --git a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap index 63a68ac4b..54d1ea478 100644 --- a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap +++ b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap @@ -239,7 +239,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_label_1o4d9_60" for="radix-_r_7_" > - Custom Livekit-url + Custom transport JSON
{ public constructor( @@ -153,8 +154,8 @@ export const matrixRTCMode = new Setting( MatrixRTCMode.Compatibility, ); -export const customLivekitUrl = new Setting( - "custom-livekit-url", +export const customTransport = new Setting( + "custom-transport", null, ); diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index c7783d152..1b1c38d4a 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -31,7 +31,7 @@ import { type CallMembership, type IRTCNotificationContent, MatrixRTCSessionEvent, - type LivekitTransport, + type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { deepCompare } from "matrix-js-sdk/lib/utils"; @@ -97,7 +97,7 @@ vi.mock( "../state/CallViewModel/localMember/localTransport", async (importOriginal) => ({ ...(await importOriginal()), - makeTransport: async (): Promise => + makeTransport: async (): Promise => Promise.resolve(exampleTransport), }), ); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index a58b0837f..169abb98c 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -12,6 +12,7 @@ import { ExternalE2EEKeyProvider, type Room as LivekitRoom, type RoomOptions, + type RemoteParticipant, } from "livekit-client"; import { type Room as MatrixRoom } from "matrix-js-sdk"; import { @@ -38,11 +39,14 @@ import { timer, takeUntil, from, + concatWith, + catchError, + switchAll, } from "rxjs"; import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { MembershipManagerEvent, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, type RTCCallIntent, type RTCNotificationType, @@ -55,6 +59,7 @@ import { createToggle$, filterBehavior, generateItems, + mapScoped, pauseWhen, } from "../../utils/observable"; import { @@ -104,17 +109,14 @@ import { type OneOnOneMedia, } from "../layoutMedia.ts"; import { ElementCallError, UnknownCallError } from "../../utils/errors.ts"; -import { type Epoch, type ObservableScope } from "../ObservableScope.ts"; +import { Epoch, type ObservableScope } from "../ObservableScope.ts"; import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts"; import { createLocalMembership$, enterRTCSession, TransportState, } from "./localMember/LocalMember.ts"; -import { - getLocalTransport, - type LocalTransport, -} from "./localMember/LocalTransport.ts"; +import { getLocalTransport } from "./localMember/LocalTransport.ts"; import { createKeyRotationSuppressed$, createMemberships$, @@ -125,9 +127,9 @@ import { ECConnectionFactory, } from "./remoteMembers/ConnectionFactory.ts"; import { - type ConnectionManagerData, - createConnectionManager$, -} from "./remoteMembers/ConnectionManager.ts"; + ConnectionMap, + createConnectionMap$, +} from "./remoteMembers/ConnectionMap.ts"; import { createRemoteMatrixLivekitMembers$, type LocalMatrixLivekitMember, @@ -163,6 +165,7 @@ import { type RingingMediaViewModel, } from "../media/RingingMediaViewModel.ts"; import { type GridTileViewModel } from "../TileViewModel.ts"; +import { type TransportLocator } from "../../livekit/auth"; //TODO // Larger rename @@ -215,7 +218,7 @@ export interface CallViewModelOptions { */ windowSize$: Behavior<{ width: number; height: number }>; /** Optional value overriding the local transport, for testing purposes. */ - localTransport?: LocalTransport; + localTransport?: TransportLocator; /** Optional value overriding the connection factory, for testing purposes. */ connectionFactory?: ConnectionFactory; /** The version & compatibility mode of MatrixRTC that we should use. */ @@ -276,7 +279,7 @@ interface LayoutScanState { export type LivekitRoomItem = { livekitRoom: LivekitRoom; participants: string[]; - url: string; + transport: TransportLocator; }; /** @@ -373,7 +376,7 @@ export interface CallViewModel { * key is generated when someone joins or leaves. */ keyRotationSuppressed$: Behavior; - allConnections$: Behavior; + allConnections$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ livekitRoomItems$: Behavior; /** use the layout instead, this is just for the sdk export. */ @@ -564,20 +567,14 @@ export function createCallViewModel$( const localTransport$ = options.localTransport ? constant(options.localTransport) - : from( - getLocalTransport({ - ownMembershipIdentity, - client, - roomId: matrixRoom.roomId, - matrixRTCMode, - }), - ); + : from(getLocalTransport(client)); const connectionFactory = options.connectionFactory ?? new ECConnectionFactory( client, matrixRoom.roomId, + matrixRTCSession.slotId, mediaDevices, trackProcessorState$, livekitKeyProvider, @@ -585,14 +582,24 @@ export function createCallViewModel$( options.livekitRoomFactory, ); - const connectionManager = createConnectionManager$({ - scope: scope, - connectionFactory: connectionFactory, - localTransport$, - remoteTransports$: membershipsAndTransports.transports$, - logger: logger, - ownMembershipIdentity, - }); + const connectionMap$ = scope.behavior>( + localTransport$.pipe( + catchError(() => NEVER), // Swallow errors + concatWith(NEVER), // So the Observable doesn't complete prematurely + mapScoped("connectionMap$", (localTransport, scope) => + createConnectionMap$({ + scope, + connectionFactory, + localTransport, + remoteTransports$: membershipsAndTransports.transports$, + logger, + ownMembershipIdentity, + }), + ), + switchAll(), + ), + new Epoch(new ConnectionMap()), + ); const remoteMatrixLivekitMembers$: Behavior< Epoch @@ -600,7 +607,7 @@ export function createCallViewModel$( scope: scope, membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, - connectionManager: connectionManager, + connectionMap$, localUser: { userId, deviceId }, }); @@ -613,7 +620,7 @@ export function createCallViewModel$( ), muteStates, joinMatrixRTC: ( - transport: LivekitTransport, + transport: UnstableLivekitTransport, delayedLeaveTimings: ResolvedDelayedLeaveTimings, ) => { return enterRTCSession( @@ -635,13 +642,11 @@ export function createCallViewModel$( mediaDevices, muteStates, trackProcessorState$, - logger.getChild( - "[Publisher " + connection.transport.livekit_service_url + "]", - ), + logger.getChild(`[Publisher ${JSON.stringify(connection.transport)}]`), controlledAudioDevices, ); }, - connectionManager, + connectionMap$, client, matrixRTCSession, localTransport$, @@ -660,7 +665,6 @@ export function createCallViewModel$( ).pipe(map(([delayId]) => delayId ?? null)), matrixRTCSession.delayId ?? null, ), - matrixRTCMode, logger: logger.getChild(`[${Date.now()}]`), }); @@ -728,51 +732,47 @@ export function createCallViewModel$( }); const allConnections$ = scope.behavior( - connectionManager.connectionManagerData$.pipe(map((d) => d.value)), + connectionMap$.pipe(map((d) => d.value.getConnections())), ); - const livekitRoomItems$ = scope.behavior( - remoteMatrixLivekitMembers$.pipe( - switchMap((members) => { - const a$ = combineLatest( - members.value.map((member) => - combineLatest([member.connection$, member.participant.value$]).pipe( - map(([connection, participant]) => { - // do not render audio for local participant - if (!connection || !participant || participant.isLocal) - return null; - const livekitRoom = connection.livekitRoom; - const url = connection.transport.livekit_service_url; - return { - url, - livekitRoom, - participant: participant.identity, - }; - }), + const livekitRoomItems$ = scope.behavior( + remoteMatrixLivekitMembers$.pipe( + // Flatten the members to a snapshot of their current connections and + // participant objects + switchMap(({ value: members }) => { + if (members.length === 0) return of([]); + return combineLatest( + members.map((m) => + combineLatest( + [m.connection$, m.participant.value$], + (connection, participant) => ({ connection, participant }), ), ), ); - return a$; }), + // Keep only those with a defined connection and participant map((members) => - members.reduce((acc, curr) => { - if (!curr) return acc; - - const existing = acc.find((item) => item.url === curr.url); - if (existing) { - existing.participants.push(curr.participant); - } else { - acc.push({ - livekitRoom: curr.livekitRoom, - participants: [curr.participant], - url: curr.url, - }); - } - return acc; - }, []), + members.filter( + ( + m, + ): m is { connection: Connection; participant: RemoteParticipant } => + m.connection !== null && m.participant !== null, + ), + ), + // Group members by connection + map((members) => + [...Map.groupBy(members, (m) => m.connection)].map( + ([connection, members]) => ({ + transport: { + transport: connection.transport, + serverName: connection.serverName, + }, + livekitRoom: connection.livekitRoom, + participants: members.map((m) => m.participant.identity), + }), + ), ), ), - [], ); const handsRaised$ = scope.behavior( @@ -829,8 +829,8 @@ export function createCallViewModel$( livekitRoom$: scope.behavior( connection$.pipe(map((c) => c?.livekitRoom)), ), - focusUrl$: scope.behavior( - connection$.pipe(map((c) => c?.transport.livekit_service_url)), + transport$: scope.behavior( + connection$.pipe(map((c) => c?.transport)), ), mediaDevices, pretendToBeDisconnected$: localMembership.reconnecting$, @@ -1662,9 +1662,9 @@ export function createCallViewModel$( returnObject.publishError = value.media; else if ( typeof value.media === "object" && - value.media.connection instanceof ElementCallError + value.media.connection.state === "error" ) - returnObject.connectionError = value.media.connection; + returnObject.connectionError = value.media.connection.error; return returnObject; }), ), diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index 0863ea2b7..93c190ace 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -30,8 +30,7 @@ import { type CallViewModelOptions, } from "./CallViewModel"; import { - exampleSfuConfig, - exampleTransport, + exampleTransportLocator, mockConfig, MockConnection, mockLivekitRoom, @@ -123,6 +122,8 @@ export function withCallViewModel(mode: MatrixRTCMode) { }; const room = mockMatrixRoom({ client: new (class extends EventEmitter { + public readonly baseUrl = "https://matrix.example.com"; + public getUserId(): string | undefined { return localRtcMember.userId; } @@ -209,26 +210,26 @@ export function withCallViewModel(mode: MatrixRTCMode) { livekitRoomFactory, connectionState$, windowSize$, - localTransport: { - transport: exampleTransport, - sfuConfig: exampleSfuConfig, - }, + localTransport: exampleTransportLocator, connectionFactory: { createConnection( scope, + role, transport, + serverName, ownMembershipIdentity, logger, - sfuConfig, ) { return new MockConnection( { scope, + role, transport, + serverName, ownMembershipIdentity, - existingSFUConfig: sfuConfig, client: room.client, roomId: room.roomId, + slotId: "m.call#ROOM", livekitRoomFactory, }, logger, diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 282759fdd..f8b4e6b6b 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { Status as RTCMemberStatus, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { @@ -21,9 +21,13 @@ import { beforeEach, afterEach, } from "vitest"; -import { BehaviorSubject, map, of, Subject } from "rxjs"; +import { BehaviorSubject, of, Subject } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type LocalParticipant, type LocalTrack } from "livekit-client"; +import { + ConnectionState as LivekitConnectionState, + type LocalParticipant, + type LocalTrack, +} from "livekit-client"; import fetchMock from "fetch-mock"; import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics"; @@ -53,13 +57,15 @@ import { import { MatrixRTCTransportMissingError } from "../../../utils/errors"; import { Epoch, ObservableScope } from "../../ObservableScope"; import { constant } from "../../Behavior"; -import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; -import { ConnectionState, type Connection } from "../remoteMembers/Connection"; +import { ConnectionMap } from "../remoteMembers/ConnectionMap"; +import { + type ConnectionState, + type Connection, +} from "../remoteMembers/Connection"; import { type Publisher } from "./Publisher"; import { initializeWidget } from "../../../widget"; import { nullHostBridge } from "../../../HostBridge"; -import { type LocalTransport } from "./LocalTransport"; -import * as openIDSFU from "../../../livekit/openIDSFU"; +import * as livekitAuth from "../../../livekit/auth"; initializeWidget(); @@ -110,6 +116,7 @@ const delegatedTimings: ResolvedDelayedLeaveTimings = { }; const mockedClient = { + baseUrl: "https://matrix.example.org", getDomain: vi.fn().mockReturnValue("example.org"), getDeviceId: vi.fn().mockReturnValue("AAAA"), getOpenIdToken: vi.fn().mockResolvedValue({ @@ -118,10 +125,11 @@ const mockedClient = { matrix_server_name: "localhost", expires_in: 10000, }), + _unstable_delegateDelayedLeave: vi.fn().mockResolvedValue({}), }; describe("enterRTCSession", () => { - const transport: LivekitTransport = { + const transport: UnstableLivekitTransport = { livekit_service_url: "http://my-livekit-service-url.com", type: "livekit", }; @@ -214,7 +222,10 @@ describe("LocalMembership", () => { processor: undefined, }), logger: logger, - createPublisherFactory: vi.fn(), + createPublisherFactory: vi.fn().mockReturnValue({ + destroy: vi.fn(), + createAndSetupTracks: vi.fn().mockResolvedValue(undefined), + }), joinMatrixRTC: async (): Promise => {}, homeserverConnected: { combined$: constant<[boolean, HomeserverDisconnectReason | null]>([ @@ -248,29 +259,16 @@ describe("LocalMembership", () => { afterEach(async () => { void (await fetchMock.flush()); fetchMock.reset(); + vi.clearAllMocks(); }); it("throws error on missing RTC config error", () => { withTestScheduler(({ scope, hot, expectObservable }) => { - const localTransport$ = scope.behavior( - hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), - null, - ); - - // we do not need any connection data since we want to fail before reaching that. - const mockConnectionManager = { - transports$: scope.behavior( - localTransport$.pipe(map((t) => new Epoch([t]))), - ), - connectionManagerData$: constant( - new Epoch(new ConnectionManagerData()), - ), - }; - const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: mockConnectionManager, + // we do not need any connection data since we want to fail before reaching that. + connectionMap$: constant(new Epoch(new ConnectionMap())), localTransport$: hot( "1ms #", {}, @@ -286,24 +284,19 @@ describe("LocalMembership", () => { }); it("logs if callIntent cannot be updated", async () => { - const scope = new ObservableScope(); - - const mockConnectionManager = { - transports$: constant(new Epoch([])), - connectionManagerData$: constant(new Epoch(new ConnectionManagerData())), - }; async function reject(): Promise { return Promise.reject(new Error("Not connected yet")); } const localMembership = createLocalMembership$({ - scope, + scope: testScope(), ...defaultCreateLocalMemberValues, matrixRTCSession: { + slotId: "m.call#ROOM", updateCallIntent: vi.fn().mockImplementation(reject), leave: vi.fn(), }, - connectionManager: mockConnectionManager, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(new ConnectionMap())), + localTransport$: constant(mockTransportLocator), }); const expextedLog = "'not connected yet' while updating the call intent (this is expected on startup)"; @@ -312,22 +305,18 @@ describe("LocalMembership", () => { await flushPromises(); defaultCreateLocalMemberValues.muteStates.video.setEnabled$.value?.(true); expect(internalLogger).toHaveBeenCalledWith(expextedLog); - scope.end(); }); - const mockTransportConfig = { - livekit_service_url: "a", - } as LivekitTransport; + const mockTransport: UnstableLivekitTransport = { + type: "livekit", + url: "https://sfu.example.org", + livekit_service_url: "https://jwt.example.org", + }; - const mockTransport = { - transport: mockTransportConfig, - sfuConfig: { - jwt: "foo", - livekitAlias: "bar", - livekitIdentity: "baz", - url: "bro", - }, - } as LocalTransport; + const mockTransportLocator: livekitAuth.TransportLocator = { + transport: mockTransport, + serverName: "example.org", + }; const connectionTransportAConnected = { livekitRoom: mockLivekitRoom({ @@ -336,31 +325,52 @@ describe("LocalMembership", () => { trackPublications: [], } as unknown as LocalParticipant, }), - state$: constant(ConnectionState.LivekitConnected), - transport: mockTransportConfig, + state$: constant({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }), + transport: mockTransport, + serverName: "example.org", } as Connection; const connectionTransportAConnecting = { ...connectionTransportAConnected, - state$: constant(ConnectionState.LivekitConnecting), + state$: constant({ + state: "authenticated", + livekitState: LivekitConnectionState.Connecting, + }), livekitRoom: mockLivekitRoom({}), } as unknown as Connection; - const authCallSpy = vi - .spyOn(openIDSFU, "getSFUConfigWithOpenID") - .mockImplementation(() => mockedClient.getOpenIdToken()); - afterEach(() => authCallSpy.mockClear()); + const delegationSpy = vi + .spyOn(livekitAuth, "delegateDelayedLeave") + .mockResolvedValue(); + afterEach(() => delegationSpy.mockClear()); + const urlOnlyTransport: UnstableLivekitTransport = { + type: "livekit", + url: "https://sfu.example.org", + }; + + // With a `url` only we publish through the CS API, so the homeserver + // delegates. With a `livekit_service_url` we publish through the JWT + // service, so that delegates. it.each([ - ["no", null, timings], + ["no", mockTransport, null, timings], [ "homeserver", + urlOnlyTransport, "https://matrix.example.org/_matrix/client/unstable/io.element.msc4195/rtc/livekit/delegate_delayed_leave", delegatedTimings, ], - ["transport", "/a/delegate_delayed_leave", delegatedTimings], + [ + "transport", + mockTransport, + "https://jwt.example.org/delegate_delayed_leave", + delegatedTimings, + ], ])( "joins session with %s delegation support", - async (_serviceName, delegationUrl, delayedLeaveTimings) => { + async (_serviceName, transport, delegationUrl, delayedLeaveTimings) => { const scope = testScope(); const joinMatrixRTC = vi.fn(); const delayId$ = new BehaviorSubject(null); @@ -368,16 +378,17 @@ describe("LocalMembership", () => { if (delegationUrl !== null) fetchMock.post(delegationUrl, () => ({ status: 401, body: {} })); + const connectionMap = new ConnectionMap(); + connectionMap.add( + { ...connectionTransportAConnected, transport } as Connection, + [], + ); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$: constant( - new Epoch(new ConnectionManagerData()), - ), - }, + connectionMap$: constant(new Epoch(connectionMap)), joinMatrixRTC, - localTransport$: constant(mockTransport), + localTransport$: constant({ transport, serverName: "example.org" }), delayId$, }); @@ -386,31 +397,27 @@ describe("LocalMembership", () => { await flushPromises(); // Joins with timings appropriate for the level of delegation support expect(joinMatrixRTC).toHaveBeenCalledWith( - mockTransportConfig, + transport, delayedLeaveTimings, ); - expect(authCallSpy).not.toHaveBeenCalled(); + expect(delegationSpy).not.toHaveBeenCalled(); delayId$.next("leave1"); await flushPromises(); if (delegationUrl === null) { - expect(authCallSpy).not.toHaveBeenCalled(); + expect(delegationSpy).not.toHaveBeenCalled(); } else { // Delegation is supported in this test case, so go on to check that // LocalMember actually performs delegation const expectDelegation = (delayId: string) => - expect(authCallSpy).toHaveBeenLastCalledWith( - mockedClient, - ownMemberMock, - "a", - "!test-room-id:example.org", - { - matrixRTCMode: MATRIX_RTC_MODE, - delayEndpointBaseUrl: "https://matrix.example.org", - delayId, - }, - expect.anything(), - ); + expect(delegationSpy).toHaveBeenLastCalledWith({ + client: mockedClient, + membership: ownMemberMock, + transport, + roomId: "!test-room-id:example.org", + delayId, + logger: expect.anything(), + }); expectDelegation("leave1"); delayId$.next("leave2"); // Can change delegated leaves @@ -449,16 +456,14 @@ describe("LocalMembership", () => { typeof vi.fn >; - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); - // connectionManagerData.add(connectionTransportB, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); + // connectionMap.add(connectionTransportB, []); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); expect(publisherFactory).toHaveBeenCalledOnce(); @@ -479,12 +484,10 @@ describe("LocalMembership", () => { // it("tracks livekit state correctly", async () => { const scope = new ObservableScope(); - const connectionManagerData = new ConnectionManagerData(); - const localTransport$ = new Subject(); + const connectionMap = new ConnectionMap(); + const localTransport$ = new Subject(); - const connectionManagerData$ = new BehaviorSubject( - new Epoch(connectionManagerData), - ); + const connectionMap$ = new BehaviorSubject(new Epoch(connectionMap)); const publishers: Publisher[] = []; const publishing$ = new BehaviorSubject(false); @@ -518,9 +521,7 @@ describe("LocalMembership", () => { const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$, - }, + connectionMap$: connectionMap$, localTransport$, }); @@ -528,38 +529,50 @@ describe("LocalMembership", () => { expect(localMembership.localMemberState$.value).toStrictEqual( TransportState.Waiting, ); - localTransport$.next(mockTransport); + localTransport$.next(mockTransportLocator); await flushPromises(); expect(localMembership.localMemberState$.value).toStrictEqual({ matrix: RTCMemberStatus.Connected, - media: { connection: null, tracks: TrackState.WaitingForUser }, + media: { + connection: { state: "initializing" }, + tracks: TrackState.WaitingForUser, + }, }); - const connectionManagerData2 = new ConnectionManagerData(); - connectionManagerData2.add( + const connectionMap2 = new ConnectionMap(); + connectionMap2.add( // clone because we will mutate this later. { ...connectionTransportAConnecting } as unknown as Connection, [], ); - connectionManagerData$.next(new Epoch(connectionManagerData2)); + connectionMap$.next(new Epoch(connectionMap2)); await flushPromises(); expect(localMembership.localMemberState$.value).toStrictEqual({ matrix: RTCMemberStatus.Connected, media: { - connection: ConnectionState.LivekitConnecting, + connection: { + state: "authenticated", + livekitState: LivekitConnectionState.Connecting, + }, tracks: TrackState.WaitingForUser, }, }); ( - connectionManagerData2.getConnectionForTransport(mockTransportConfig)! + connectionMap2.getConnectionForTransport(mockTransportLocator)! .state$ as BehaviorSubject - ).next(ConnectionState.LivekitConnected); + ).next({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); expect(localMembership.localMemberState$.value).toStrictEqual({ matrix: RTCMemberStatus.Connected, media: { - connection: ConnectionState.LivekitConnected, + connection: { + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }, tracks: TrackState.WaitingForUser, }, }); @@ -643,8 +656,8 @@ describe("LocalMembership", () => { [boolean, HomeserverDisconnectReason | null] >([false, "membership"]); - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); createLocalMembership$({ scope, @@ -653,10 +666,8 @@ describe("LocalMembership", () => { combined$: hsReason$, rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); @@ -680,8 +691,8 @@ describe("LocalMembership", () => { [boolean, HomeserverDisconnectReason | null] >([true, null]); - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); createLocalMembership$({ scope, @@ -690,10 +701,8 @@ describe("LocalMembership", () => { combined$: hsReason$, rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); @@ -717,16 +726,17 @@ describe("LocalMembership", () => { "track", ); - const connectionState$ = new BehaviorSubject( - ConnectionState.LivekitConnected, - ); + const connectionState$ = new BehaviorSubject({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); const mutableConnection = { ...connectionTransportAConnected, state$: connectionState$, } as unknown as Connection; - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(mutableConnection, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(mutableConnection, []); createLocalMembership$({ scope, @@ -738,16 +748,20 @@ describe("LocalMembership", () => { ]), rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); - connectionState$.next(ConnectionState.LivekitDisconnected); - connectionState$.next(ConnectionState.LivekitConnected); + connectionState$.next({ + state: "authenticated", + livekitState: LivekitConnectionState.Disconnected, + }); + connectionState$.next({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); expect(trackSpy).toHaveBeenCalledWith( defaultCreateLocalMemberValues.roomId, @@ -769,8 +783,8 @@ describe("LocalMembership", () => { [boolean, HomeserverDisconnectReason | null] >([true, null]); - const connectionManagerData = new ConnectionManagerData(); - connectionManagerData.add(connectionTransportAConnected, []); + const connectionMap = new ConnectionMap(); + connectionMap.add(connectionTransportAConnected, []); createLocalMembership$({ scope, @@ -779,10 +793,8 @@ describe("LocalMembership", () => { combined$: hsReason$, rtsSession$: constant(RTCMemberStatus.Connected), }, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); await flushPromises(); @@ -840,15 +852,13 @@ describe("LocalMembership", () => { localMembership: ReturnType; } => { const scope = new ObservableScope(); - const connectionManagerData = new ConnectionManagerData(); - if (connection) connectionManagerData.add(connection, []); + const connectionMap = new ConnectionMap(); + if (connection) connectionMap.add(connection, []); const localMembership = createLocalMembership$({ scope, ...defaultCreateLocalMemberValues, - connectionManager: { - connectionManagerData$: constant(new Epoch(connectionManagerData)), - }, - localTransport$: constant(mockTransport), + connectionMap$: constant(new Epoch(connectionMap)), + localTransport$: constant(mockTransportLocator), }); return { scope, localMembership }; }; @@ -857,8 +867,12 @@ describe("LocalMembership", () => { const error = new Error("NotReadableError"); const setScreenShareEnabled = vi.fn().mockRejectedValue(error); const connection = { - state$: constant(ConnectionState.LivekitConnected), - transport: mockTransportConfig, + state$: constant({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }), + transport: mockTransport, + serverName: "example.org", livekitRoom: mockLivekitRoom({ localParticipant: mockLocalParticipant({ isScreenShareEnabled: false, diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index feb49554c..baa6b473d 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { + ConnectionState as LivekitConnectionState, type Participant, ParticipantEvent, type LocalParticipant, @@ -15,10 +16,9 @@ import { MediaDeviceFailure, } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; -import { type MatrixClient } from "matrix-js-sdk"; import { Status as RTCSessionStatus, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, type RTCCallIntent, type RTCNotificationType, @@ -47,8 +47,8 @@ import { deepCompare } from "matrix-js-sdk/lib/utils"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Behavior } from "../../Behavior.ts"; -import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts"; -import { type ObservableScope } from "../../ObservableScope.ts"; +import { type ConnectionMap } from "../remoteMembers/ConnectionMap.ts"; +import { type Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { type Publisher } from "./Publisher.ts"; import { type MuteStates } from "../../MuteStates.ts"; import { @@ -74,13 +74,17 @@ import { } from "../../../config/ConfigOptions.ts"; import { Config } from "../../../config/Config.ts"; import { - ConnectionState, + type ConnectionState, type Connection, - type FailedToStartError, } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; -import { type LocalTransport } from "./LocalTransport.ts"; -import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts"; +import { + type ClientDelegationParts, + delegateDelayedLeave, + type ClientOpenIDParts, + chooseAuthFlow, + type TransportLocator, +} from "../../../livekit/auth"; export enum TransportState { /** Not even a transport is available to the LocalMembership */ @@ -113,7 +117,7 @@ export enum TrackState { export type LocalMemberMediaState = | { tracks: TrackState; - connection: ConnectionState | FailedToStartError; + connection: ConnectionState; } | PublishState | ElementCallError; @@ -140,25 +144,27 @@ interface Props { // that the inputs for those createSomething$() functions should NOT contain any js-sdk objectes scope: ObservableScope; muteStates: MuteStates; - connectionManager: IConnectionManager; + connectionMap$: Behavior>; createPublisherFactory: (connection: Connection) => Publisher; joinMatrixRTC: ( - transport: LivekitTransport, + transport: UnstableLivekitTransport, delayedLeaveTimings: ResolvedDelayedLeaveTimings, ) => void; homeserverConnected: HomeserverConnected; roomId: string; ownMembershipIdentity: CallMembershipIdentityParts; - localTransport$: Observable; - client: Pick; - matrixRTCSession: Pick; + localTransport$: Observable; + client: ClientDelegationParts & ClientOpenIDParts; + matrixRTCSession: Pick< + MatrixRTCSession, + "slotId" | "updateCallIntent" | "leave" + >; /** Whether to hide the screen-sharing button. */ hideScreensharing: boolean; /** The application hosting Element Call, to be kept informed of join/leave. */ hostBridge: HostBridge; baseUrl: string; delayId$: Behavior; - matrixRTCMode: MatrixRTCMode; logger: Logger; } @@ -169,7 +175,7 @@ interface Props { * - * @param props The properties required to create the local membership. * @param props.scope The observable scope to use. - * @param props.connectionManager The connection manager to get connections from. + * @param props.connectionMap$ The connection map to get connections from. * @param props.createPublisherFactory Factory to create a publisher once we have a connection. * @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport. * @param props.homeserverConnected The homeserver connected state. @@ -191,7 +197,7 @@ interface Props { */ export const createLocalMembership$ = ({ scope, - connectionManager, + connectionMap$, localTransport$: localTransportWithErrors$, homeserverConnected, createPublisherFactory, @@ -206,7 +212,6 @@ export const createLocalMembership$ = ({ hostBridge, ownMembershipIdentity, delayId$, - matrixRTCMode, }: Props): { /** * This request to start audio and video tracks. @@ -308,16 +313,17 @@ export const createLocalMembership$ = ({ const joinParams$ = scope.behavior( localTransport$.pipe( switchMap(async ({ transport }) => { - const transportSupportsDelegation = checkDelegationSupport( - transport.livekit_service_url + "/delegate_delayed_leave", - `transport ${transport.livekit_service_url}`, - ); - return { - transport, - delegationSupported: - (await homeserverSupportsDelegation) || - (await transportSupportsDelegation), - }; + // Delegation goes through whichever service authenticates us as a + // publisher, so only that service's support counts + const flow = chooseAuthFlow(transport, "publisher"); + const delegationSupported = + flow.kind === "openID" + ? await checkDelegationSupport( + flow.serviceUrl + "/delegate_delayed_leave", + `transport ${flow.serviceUrl}`, + ) + : await homeserverSupportsDelegation; + return { transport, delegationSupported }; }), ), null, @@ -325,16 +331,14 @@ export const createLocalMembership$ = ({ // Drop Epoch data here since we will not combine this anymore const localConnection$ = scope.behavior( - combineLatest([ - connectionManager.connectionManagerData$, - localTransport$, - ]).pipe( - map(([{ value: connectionData }, { transport }]) => - connectionData.getConnectionForTransport(transport), - ), + combineLatest( + [connectionMap$, localTransport$], + ({ value: connectionMap }, transport) => + connectionMap.getConnectionForTransport(transport), + ).pipe( tap((connection) => { logger.info( - `Local connection updated: ${connection?.transport?.livekit_service_url}`, + `Local connection updated: ${JSON.stringify(connection?.transport)}`, ); }), ), @@ -406,7 +410,7 @@ export const createLocalMembership$ = ({ scope.reconcile(localConnection$, async (connection) => { logger.info( "reconcile based on new localConnection:", - connection?.transport.livekit_service_url, + JSON.stringify(connection?.transport), ); if (connection !== null) { const publisher = createPublisherFactory(connection); @@ -475,7 +479,13 @@ export const createLocalMembership$ = ({ }; const localConnectionState$ = localConnection$.pipe( - switchMap((connection) => (connection ? connection.state$ : of(null))), + switchMap((connection) => + connection ? connection.state$ : of({ state: "initializing" as const }), + ), + ); + + const authenticated$ = scope.behavior( + localConnectionState$.pipe(map(({ state }) => state === "authenticated")), ); const mediaState$: Behavior = scope.behavior( @@ -493,7 +503,9 @@ export const createLocalMembership$ = ({ : TrackState.WaitingForUser; if ( - localConnectionState !== ConnectionState.LivekitConnected || + localConnectionState.state !== "authenticated" || + localConnectionState.livekitState !== + LivekitConnectionState.Connected || trackState !== TrackState.Ready ) return { @@ -561,7 +573,11 @@ export const createLocalMembership$ = ({ combineLatest([ homeserverConnected.combined$, localConnectionState$.pipe( - map((state) => state === ConnectionState.LivekitConnected), + map( + (state) => + state.state === "authenticated" && + state.livekitState === LivekitConnectionState.Connected, + ), ), ]).pipe( map(([[hsConnected, hsReason], livekitConnected]) => { @@ -656,34 +672,38 @@ export const createLocalMembership$ = ({ // Join and leave the session as needed scope.reconcile( - scope.behavior(combineLatest([joinParams$, joinAndPublishRequested$])), - async ([joinParams, shouldConnect]) => { - if (!joinParams) return; - // if shouldConnect=false we will do the disconnect as the cleanup from the previous reconcile iteration. - if (!shouldConnect) return; + scope.behavior( + combineLatest([joinParams$, joinAndPublishRequested$, authenticated$]), + ), + async ([joinParams, shouldConnect, authenticated]) => { const sessionConfig = Config.get().matrix_rtc_session; - try { - joinMatrixRTC( - joinParams.transport, - joinParams.delegationSupported - ? sessionConfig.delegated_delayed_leave - : sessionConfig.delayed_leave, - ); - } catch (error) { - logger.error("Error entering RTC session", error); - if (error instanceof Error) - setMatrixError(new MembershipManagerError(error)); - } - - return Promise.resolve(async (): Promise => { + // Only join once authenticated, to ensure the LiveKit room exists first. + // Otherwise, subscribers authenticating with the OpenID flow may fail to + // join (they lack permission to create the room) and never retry. + if (joinParams && shouldConnect && authenticated) { try { - // TODO Update matrixRTCSession to allow udpating the transport without leaving the session! - await matrixRTCSession.leave(1000); - } catch (e) { - logger.error("Error leaving RTC session", e); + joinMatrixRTC( + joinParams.transport, + joinParams.delegationSupported + ? sessionConfig.delegated_delayed_leave + : sessionConfig.delayed_leave, + ); + } catch (error) { + logger.error("Error entering RTC session", error); + if (error instanceof Error) + setMatrixError(new MembershipManagerError(error)); } - }); + + // Clean-up callback to leave the session + return Promise.resolve(async (): Promise => { + try { + await matrixRTCSession.leave(1000); + } catch (e) { + logger.error("Error leaving RTC session", e); + } + }); + } }, ); @@ -702,23 +722,22 @@ export const createLocalMembership$ = ({ async ([joinParams, delayId]) => { if (joinParams?.delegationSupported && delayId !== null) { try { - // This will technically cause the service to issue a new JWT token, - // but it's safe to discard. We're only interested in triggering - // delegation. - await getSFUConfigWithOpenID( - client, - ownMembershipIdentity, - joinParams.transport.livekit_service_url, - roomId, - { matrixRTCMode, delayEndpointBaseUrl: baseUrl, delayId }, - logger, + logger.info( + `Delegating delayed leave to ${JSON.stringify(joinParams.transport)}…`, ); + await delegateDelayedLeave({ + client, + membership: ownMembershipIdentity, + transport: joinParams.transport, + slotId: matrixRTCSession.slotId, + roomId, + delayId, + logger, + }); + logger.info("Delayed leave successfully delegated"); } catch (e) { // TODO: Surface this to the user as a service interruption? - logger.error( - `Failed to delegate leave to ${joinParams.transport.livekit_service_url}`, - e, - ); + logger.error(`Failed to delegate leave`, e); } } }, @@ -951,7 +970,7 @@ interface EnterRTCSessionOptions { * * @param rtcSession - The MatrixRTCSession to join. * @param ownMembershipIdentity - Options for entering the RTC session. - * @param transport - The LivekitTransport to use for this session. + * @param transport - The UnstableLivekitTransport to use for this session. * @param options - `encryptMedia`: Whether to encrypt media. `matrixRTCMode`: The * Matrix RTC mode to use. `delayedLeaveTimings`: The preferred timings for * delayed leave events. `sendNotificationType`: Whether and what kind of @@ -962,7 +981,7 @@ interface EnterRTCSessionOptions { export function enterRTCSession( rtcSession: MatrixRTCSession, ownMembershipIdentity: CallMembershipIdentityParts, - transport: LivekitTransport, + transport: UnstableLivekitTransport, { encryptMedia, matrixRTCMode, diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 90d0d15e5..ed6654e2f 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -16,130 +16,54 @@ import { } from "vitest"; import fetchMock from "fetch-mock"; -import { mockConfig, ownMemberMock } from "../../../utils/test"; +import { mockConfig } from "../../../utils/test"; import { getLocalTransport } from "./LocalTransport"; -import { - MatrixRTCTransportMissingError, - FailToGetOpenIdToken, -} from "../../../utils/errors"; -import * as openIDSFU from "../../../livekit/openIDSFU"; -import { customLivekitUrl } from "../../../settings/settings"; -import { testJWTToken } from "../../../utils/test-fixtures"; -import { MatrixRTCMode } from "../../../config/ConfigOptions"; +import { MatrixRTCTransportMissingError } from "../../../utils/errors"; +import { customTransport } from "../../../settings/settings"; +import { type MatrixClient } from "matrix-js-sdk"; describe("LocalTransport", () => { - const openIdResponse: openIDSFU.SFUConfig = { - url: "https://lk.example.org", - jwt: testJWTToken, - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - }; - beforeEach(() => vi.clearAllMocks()); it("throws if config is missing", async () => { await expect( getLocalTransport({ - roomId: "!room:example.org", - client: { - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - getDomain: () => "example.org", - // These won't be called in this error path but satisfy the type - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, - ownMembershipIdentity: ownMemberMock, - matrixRTCMode: MatrixRTCMode.Compatibility, + _unstable_getRTCTransports: async () => Promise.resolve([]), + getDomain: () => "example.org", }), ).rejects.toThrow(new MatrixRTCTransportMissingError("example.org")); }); - it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => { - // Provide a valid config so makeTransportInternal resolves a transport - mockConfig({ - livekit: { livekit_service_url: "https://lk.example.org" }, - }); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockRejectedValue( - new FailToGetOpenIdToken(new Error("no openid")), - ); - - await expect( - getLocalTransport({ - roomId: "!example_room_id", - client: { - getDomain: () => "example.org", - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, - ownMembershipIdentity: ownMemberMock, - matrixRTCMode: MatrixRTCMode.Compatibility, - }), - ).rejects.toThrow(new FailToGetOpenIdToken(new Error("no openid"))); - }); - it("returns preferred transport", async () => { - // Use config so transport discovery succeeds, but delay OpenID JWT fetch + // Use config so transport discovery succeeds mockConfig({ livekit: { livekit_service_url: "https://lk.example.org" }, }); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue({ - url: "https://lk.example.org", - jwt: "jwt", - livekitAlias: "Akph4alDMhen", - livekitIdentity: ownMemberMock.userId + ":" + ownMemberMock.deviceId, - }); - expect( await getLocalTransport({ - roomId: "!room:example.org", - client: { - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - getDomain: () => "example.org", - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, - ownMembershipIdentity: ownMemberMock, - matrixRTCMode: MatrixRTCMode.Compatibility, + _unstable_getRTCTransports: async () => Promise.resolve([]), + getDomain: () => "example.org", }), ).toStrictEqual({ transport: { livekit_service_url: "https://lk.example.org", type: "livekit", }, - sfuConfig: { - jwt: "jwt", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@alice:example.org:DEVICE", - url: "https://lk.example.org", - }, + serverName: "example.org", }); }); - type LocalTransportProps = Parameters[0]; - describe("transport configuration mechanisms", () => { - let localTransportOpts: LocalTransportProps & { - client: MockedObject; - }; + let client: MockedObject< + Pick + >; beforeEach(() => { mockConfig({}); - customLivekitUrl.setValue(customLivekitUrl.defaultValue); - localTransportOpts = { - ownMembershipIdentity: ownMemberMock, - roomId: "!example_room_id", - matrixRTCMode: MatrixRTCMode.Compatibility, - client: { - getDomain: vi.fn().mockReturnValue("example.org"), - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, + customTransport.setValue(customTransport.defaultValue); + client = { + getDomain: vi.fn().mockReturnValue("example.org"), + _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), }; }); @@ -151,93 +75,50 @@ describe("LocalTransport", () => { mockConfig({ livekit: { livekit_service_url: "https://lk.example.org" }, }); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue( - openIdResponse, - ); - expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ + expect(await getLocalTransport(client)).toStrictEqual({ transport: { livekit_service_url: "https://lk.example.org", type: "livekit", }, - sfuConfig: { - jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - url: "https://lk.example.org", - }, + serverName: "example.org", }); }); it("supports getting transport via user settings", async () => { - customLivekitUrl.setValue("https://lk.example.org"); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue( - openIdResponse, - ); + customTransport.setValue({ + type: "livekit", + livekit_service_url: "https://lk.example.org", + }); - expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ + expect(await getLocalTransport(client)).toStrictEqual({ transport: { livekit_service_url: "https://lk.example.org", type: "livekit", }, - sfuConfig: { - jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - url: "https://lk.example.org", - }, + serverName: "example.org", }); }); it("supports getting transport via backend", async () => { - localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ + client._unstable_getRTCTransports.mockResolvedValue([ { type: "livekit", livekit_service_url: "https://lk.example.org" }, ]); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockResolvedValue( - openIdResponse, - ); - expect(await getLocalTransport(localTransportOpts)).toStrictEqual({ + expect(await getLocalTransport(client)).toStrictEqual({ transport: { livekit_service_url: "https://lk.example.org", type: "livekit", }, - sfuConfig: { - jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", - livekitAlias: "Akph4alDMhen", - livekitIdentity: "@lk_user:ABCDEF", - url: "https://lk.example.org", - }, + serverName: "example.org", }); }); - it("fails fast if the openID request fails for backend config", async () => { - localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ - { type: "livekit", livekit_service_url: "https://lk.example.org" }, - ]); - vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockRejectedValue( - new FailToGetOpenIdToken(new Error("Test driven error")), - ); - - await expect(getLocalTransport(localTransportOpts)).rejects.toThrow( - expect.any(FailToGetOpenIdToken), - ); - }); - it("throws if no options are available", async () => { await expect( getLocalTransport({ - ownMembershipIdentity: ownMemberMock, - roomId: "!example_room_id", - matrixRTCMode: MatrixRTCMode.Compatibility, - client: { - getDomain: () => "example.org", - // eslint-disable-next-line @typescript-eslint/naming-convention - _unstable_getRTCTransports: async () => Promise.resolve([]), - // These won't be called in this error path but satisfy the type - getOpenIdToken: vi.fn(), - getDeviceId: vi.fn(), - }, + getDomain: () => "example.org", + _unstable_getRTCTransports: async () => Promise.resolve([]), }), ).rejects.toThrow(new MatrixRTCTransportMissingError("example.org")); }); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 3534b3fec..3b43f8112 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -5,142 +5,37 @@ SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type MatrixClient } from "matrix-js-sdk"; -import { logger as rootLogger, type Logger } from "matrix-js-sdk/lib/logger"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { Config } from "../../../config/Config.ts"; -import { - FailToGetOpenIdToken, - MatrixRTCTransportMissingError, - NoMatrix2AuthorizationService, -} from "../../../utils/errors.ts"; -import { - getSFUConfigWithOpenID, - type SFUConfig, - type OpenIDClientParts, -} from "../../../livekit/openIDSFU.ts"; -import { customLivekitUrl } from "../../../settings/settings.ts"; +import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; +import { customTransport as customTransportSetting } from "../../../settings/settings.ts"; import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts"; -import { type MatrixRTCMode } from "../../../config/ConfigOptions.ts"; - -interface Props { - ownMembershipIdentity: CallMembershipIdentityParts; - client: Pick & - OpenIDClientParts; - // Used by the jwt service to create the livekit room and compute the livekit alias. - roomId: string; - matrixRTCMode: MatrixRTCMode; -} - -export interface LocalTransport { - transport: LivekitTransport; - sfuConfig: SFUConfig; -} - -export function isLocalTransport( - obj: LivekitTransport | LocalTransport, -): obj is LocalTransport { - return "transport" in obj && "sfuConfig" in obj; -} +import { type TransportLocator } from "../../../livekit/auth/types.ts"; /** - * Connects to the JWT service and determines the transport that the local member should use. - * - * @prop useOldJwtEndpoint Whether to set forceOldJwtEndpoint on the returned transport and to use the old JWT endpoint. - * This is used when the connection manager needs to know if it has to use the legacy endpoint which implies a string concatenated rtcBackendIdentity. - * (which is expected for non sticky event based rtc member events) - * @returns The transport to advertise in our MatrixRTC membership and publish media on. - * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken + * Determines the transport to advertise in our MatrixRTC membership and publish + * media on. */ -export async function getLocalTransport({ - ownMembershipIdentity, - client, - roomId, - matrixRTCMode, -}: Props): Promise { - const logger = rootLogger.getChild("[LocalTransport]"); +export async function getLocalTransport( + client: Pick, +): Promise { + const serverName = client.getDomain(); + if (serverName === null) throw new Error("No server name"); + const discovery = new RtcTransportAutoDiscovery({ - client: client, - resolvedConfig: Config.get(), - logger: logger, - }); - const customUrl = customLivekitUrl.value$.value; - - // Respect the user's custom URL, if set - const transport: LivekitTransport | null = customUrl - ? { type: "livekit", livekit_service_url: customUrl } - : await discovery.discoverPreferredTransport(); - - if (transport === null) - throw new MatrixRTCTransportMissingError(client.getDomain() ?? ""); - - try { - return await doOpenIdAndJWTFromUrl( - transport, - matrixRTCMode, - ownMembershipIdentity, - roomId, - client, - logger, - ); - } catch (e) { - logger.error( - `Failed to authenticate to transport ${transport.livekit_service_url}`, - e, - ); - throw mapAuthErrorToUserFriendlyError(e); - } -} - -/** - * Utility to ensure the user can authenticate with the SFU. - * We will call `getSFUConfigWithOpenID` once per transport here as it's our - * only mechanism of validation. This means we will also ask the - * homeserver for a OpenID token a few times. Since OpenID tokens are single - * use we don't want to risk any issues by re-using a token. - * - * @param transport The transport to authenticate with. - * @param matrixRTCMode Whether to force the JWT endpoint to be used. - * @param membership The identity of the local member. - * @param roomId The room ID to use for the JWT. - * @param client The client to use for the OpenID token. - * - * @throws FailToGetOpenIdToken, NoMatrix2AuthorizationService - */ -async function doOpenIdAndJWTFromUrl( - transport: LivekitTransport, - matrixRTCMode: MatrixRTCMode, - membership: CallMembershipIdentityParts, - roomId: string, - client: Pick & OpenIDClientParts, - logger?: Logger, -): Promise { - const sfuConfig = await getSFUConfigWithOpenID( client, - membership, - transport.livekit_service_url, - roomId, - { matrixRTCMode }, - logger, - ); - return { - transport, - sfuConfig, - }; -} + resolvedConfig: Config.get(), + logger: rootLogger.getChild("[LocalTransport]"), + }); -function mapAuthErrorToUserFriendlyError(e: unknown): Error { - if ( - e instanceof FailToGetOpenIdToken || - e instanceof NoMatrix2AuthorizationService - ) { - // rethrow as is - return e; - } - // Catch others and rethrow as FailToGetOpenIdToken that has user friendly message. - return new FailToGetOpenIdToken( - e instanceof Error ? e : new Error(String(e)), - ); + // Respect the user's custom transport, if set + const transport = + customTransportSetting.value$.value ?? + (await discovery.discoverPreferredTransport()); + + if (transport === null) throw new MatrixRTCTransportMissingError(serverName); + + return { transport, serverName }; } diff --git a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts index 61df88a73..d3f246c97 100644 --- a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts +++ b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.test.ts @@ -16,7 +16,7 @@ import { import { MatrixError } from "matrix-js-sdk"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { - type LivekitTransport, + type UnstableLivekitTransport, type Transport, } from "matrix-js-sdk/lib/matrixrtc"; @@ -28,12 +28,12 @@ import { type DiscoveryClient = RtcTransportAutoDiscoveryProps["client"]; -const backendTransport: LivekitTransport = { +const backendTransport: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://backend.example.org", }; -const configTransport: LivekitTransport = { +const configTransport: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://config.example.org", }; diff --git a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts index aeb6b051a..a5bedcd80 100644 --- a/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts +++ b/src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts @@ -5,8 +5,8 @@ SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { - isLivekitTransport, - type LivekitTransport, + isUnstableLivekitTransport, + type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { type MatrixClient } from "matrix-js-sdk"; import { type Logger } from "matrix-js-sdk/lib/logger"; @@ -40,12 +40,12 @@ export class RtcTransportAutoDiscovery { this.logger = logger.getChild("[RtcTransportAutoDiscovery]"); } - public async discoverPreferredTransport(): Promise { + public async discoverPreferredTransport(): Promise { // 1) backend transports const backendTransport = await this.tryBackendTransports(); if (backendTransport) { this.logger.info( - `Found backend transport: ${backendTransport.livekit_service_url}`, + `Found backend transport: ${JSON.stringify(backendTransport)}`, ); return backendTransport; } @@ -54,7 +54,7 @@ export class RtcTransportAutoDiscovery { const configTransport = this.tryConfigTransport(); if (configTransport) { this.logger.info( - `Found app config transport: ${configTransport.livekit_service_url}`, + `Found app config transport: ${JSON.stringify(configTransport)}`, ); return configTransport; } @@ -67,7 +67,7 @@ export class RtcTransportAutoDiscovery { * This will not throw errors, but instead just log them and return null if the expected config is not found or malformed. * @private */ - private async tryBackendTransports(): Promise { + private async tryBackendTransports(): Promise { const client = this.client; // MSC4143: Attempt to fetch transports from backend. this.logger.info("First try to use getRTCTransports end point ..."); @@ -75,7 +75,7 @@ export class RtcTransportAutoDiscovery { const transportList = await doNetworkOperationWithRetry(async () => client._unstable_getRTCTransports(), ); - const first = transportList.find(isLivekitTransport); + const first = transportList.find(isUnstableLivekitTransport); if (first) { return first; } else { @@ -90,7 +90,7 @@ export class RtcTransportAutoDiscovery { return null; } - private tryConfigTransport(): LivekitTransport | null { + private tryConfigTransport(): UnstableLivekitTransport | null { const url = this.resolvedConfig.livekit?.livekit_service_url; if (url) { return { diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 62dc17a31..bdab8e7af 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -28,27 +28,30 @@ import { } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; -import { type IOpenIDToken } from "matrix-js-sdk"; +import { MatrixError } from "matrix-js-sdk"; import { logger, type Logger } from "matrix-js-sdk/lib/logger"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { + type LivekitGetTokenResponse, + type UnstableLivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; import { Connection, - ConnectionState, + type ConnectionState, type ConnectionOpts, } from "./Connection.ts"; import { ObservableScope } from "../../ObservableScope.ts"; -import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; -import { - ElementCallError, - FailToGetOpenIdToken, -} from "../../../utils/errors.ts"; +import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; import { testJWTToken } from "../../../utils/test-fixtures.ts"; import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; +import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; let testScope: ObservableScope; -let client: MockedObject; +let client: MockedObject; let fakeLivekitRoom: MockedObject; @@ -56,23 +59,24 @@ let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; const ROOM_ID = "!roomID:example.org"; +const SLOT_ID = "m.call#ROOM"; -const livekitFocus: LivekitTransport = { - livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", +const transport: UnstableLivekitTransport = { type: "livekit", + url: "wss://matrix-rtc.m.localhost/livekit/sfu", }; function setupTest(): void { testScope = new ObservableScope(); - client = vi.mocked({ + client = vi.mocked({ getOpenIdToken: vi.fn().mockResolvedValue({ access_token: "rYsmGUEwNjKgJYyeNUkZseJN", token_type: "Bearer", matrix_server_name: "example.org", expires_in: 3600, }), - getDeviceId: vi.fn().mockReturnValue("ABCDEF"), - } as unknown as OpenIDClientParts); + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), + }); localParticipantEventEmiter = new EventEmitter(); @@ -116,24 +120,17 @@ function setupTest(): void { function setupRemoteConnection(): Connection { const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { - return { - status: 200, - body: { - url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: testJWTToken, - }, - }; - }); - fakeLivekitRoom.connect.mockImplementation(async (): Promise => { const changeEv = RoomEvent.ConnectionStateChanged; @@ -172,26 +169,32 @@ describe("Start connection states", () => { setupTest(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; const connection = new Connection(opts, logger); - expect(connection.state$.getValue()).toEqual("Initialized"); + expect(connection.state$.getValue()).toEqual({ state: "initialized" }); }); - it("fail to getOpenId token then error state", async () => { + it("fail to get SFU token then error state", async () => { setupTest(); vi.useFakeTimers(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, @@ -199,16 +202,16 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); - const deferred = Promise.withResolvers(); + const deferred = Promise.withResolvers(); - client.getOpenIdToken.mockImplementation( - async (): Promise => { + client._unstable_getLivekitToken.mockImplementation( + async (): Promise => { return await deferred.promise; }, ); @@ -219,18 +222,20 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState!).toEqual("FetchingConfig"); + expect(capturedState).toEqual({ state: "authenticating" }); deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); await vi.runAllTimersAsync(); capturedState = capturedStates.pop(); - if (capturedState instanceof Error) { - expect(capturedState.message).toEqual("Something went wrong"); - expect(connection.transport).toEqual(livekitFocus); + if (capturedState?.state === "error") { + expect(capturedState.error.message).toEqual("Something went wrong"); + expect(connection.transport).toEqual(transport); } else { - expect.fail("Expected FailedToStart state but got " + capturedState); + expect.fail( + "Expected FailedToStart state but got " + JSON.stringify(capturedState), + ); } }); @@ -239,9 +244,12 @@ describe("Start connection states", () => { vi.useFakeTimers(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, @@ -249,23 +257,22 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); - // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => { + client._unstable_getLivekitToken.mockImplementation(async () => { await deferredSFU.promise; - return { - status: 500, - body: { + throw new MatrixError( + { errcode: "M_LOOKUP_FAILED", error: "Failed to look up user info from homeserver", }, - }; + 500, + ); }); connection.start().catch(() => { @@ -274,7 +281,7 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState).toEqual(ConnectionState.FetchingConfig); + expect(capturedState).toEqual({ state: "authenticating" }); deferredSFU.resolve(); await vi.runAllTimersAsync(); @@ -282,15 +289,17 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if ( - capturedState instanceof ElementCallError && - capturedState.cause instanceof Error + capturedState?.state === "error" && + capturedState.error.cause instanceof Error ) { - expect(capturedState.cause.message).toContain( + expect(capturedState.error.cause.message).toContain( "Failed to look up user info from homeserver", ); - expect(connection.transport).toEqual(livekitFocus); + expect(connection.transport).toEqual(transport); } else { - expect.fail("Expected FailedToStart state but got " + capturedState); + expect.fail( + "Expected FailedToStart state but got " + JSON.stringify(capturedState), + ); } }); @@ -299,9 +308,12 @@ describe("Start connection states", () => { vi.useFakeTimers(); const opts: ConnectionOpts = { - client: client, + client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, @@ -309,24 +321,13 @@ describe("Start connection states", () => { const connection = new Connection(opts, logger); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); - // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { - return { - status: 200, - body: { - url: "wss://matrix-rtc.m.localhost/livekit/sfu", - jwt: testJWTToken, - }, - }; - }); - fakeLivekitRoom.connect.mockImplementation(async () => { await deferredSFU.promise; throw new Error("Failed to connect to livekit"); @@ -339,7 +340,7 @@ describe("Start connection states", () => { let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); - expect(capturedState).toEqual(ConnectionState.FetchingConfig); + expect(capturedState).toEqual({ state: "authenticating" }); deferredSFU.resolve(); await vi.runAllTimersAsync(); @@ -347,13 +348,13 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if ( - capturedState instanceof ElementCallError && - capturedState.cause instanceof Error + capturedState?.state === "error" && + capturedState.error.cause instanceof Error ) { - expect(capturedState.cause.message).toContain( + expect(capturedState.error.cause.message).toContain( "Failed to connect to livekit", ); - expect(connection.transport).toEqual(livekitFocus); + expect(connection.transport).toEqual(transport); } else { expect.fail( "Expected FailedToStart state but got " + JSON.stringify(capturedState), @@ -367,7 +368,7 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -377,22 +378,31 @@ describe("Start connection states", () => { await vi.runAllTimersAsync(); const initialState = capturedStates.shift(); - expect(initialState).toEqual(ConnectionState.Initialized); + expect(initialState).toEqual({ state: "initialized" }); const fetchingState = capturedStates.shift(); - expect(fetchingState).toEqual(ConnectionState.FetchingConfig); + expect(fetchingState).toEqual({ state: "authenticating" }); const disconnectedState = capturedStates.shift(); - expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected); + expect(disconnectedState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Disconnected, + }); const connectingState = capturedStates.shift(); - expect(connectingState).toEqual(ConnectionState.LivekitConnecting); + expect(connectingState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Connecting, + }); const connectedState = capturedStates.shift(); - expect(connectedState).toEqual(ConnectionState.LivekitConnected); + expect(connectedState).toEqual({ + state: "authenticated", + livekitState: LivekitConnectionState.Connected, + }); }); it("stopping while connecting does not report an error", async () => { setupTest(); const connection = setupRemoteConnection(); - const capturedStates: (ConnectionState | Error)[] = []; + const capturedStates: ConnectionState[] = []; const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); @@ -409,10 +419,10 @@ describe("Start connection states", () => { pendingConnect.reject(new Error("Client initiated disconnect")); await stopping; - // start() resolves rather than rejecting (it is not awaited by the - // ConnectionManager, so a rejection would be unhandled). + // start() resolves rather than rejecting (it is not awaited by the caller, + // so a rejection would be unhandled). await expect(started).resolves.toBeUndefined(); - expect(capturedStates.at(-1)).toEqual(ConnectionState.Stopped); + expect(capturedStates.at(-1)).toEqual({ state: "stopped" }); expect(capturedStates.some((st) => st instanceof Error)).toBe(false); }); @@ -550,7 +560,10 @@ describe("remote track logging", () => { { client, roomId: ROOM_ID, - transport: livekitFocus, + slotId: SLOT_ID, + role: "subscriber", + transport, + serverName: "m.localhost", scope: testScope, ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index cc84142d9..7e94b39f1 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -13,6 +13,7 @@ import { import { ConnectionError, ConnectionErrorReason, + type ConnectionState as LivekitConnectionState, type Participant, type RemoteParticipant, type RemoteTrackPublication, @@ -22,16 +23,11 @@ import { type Track, type TrackPublication, } from "livekit-client"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, map } from "rxjs"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { - getSFUConfigWithOpenID, - type OpenIDClientParts, - type SFUConfig, -} from "../../../livekit/openIDSFU.ts"; import { type Behavior } from "../../Behavior.ts"; import { type ObservableScope } from "../../ObservableScope.ts"; import { @@ -42,51 +38,43 @@ import { SFURoomCreationRestrictedError, UnknownCallError, } from "../../../utils/errors.ts"; +import { + type ClientGetTokenParts, + getSFUConfig, + type ClientOpenIDParts, + type SFUConfig, +} from "../../../livekit/auth"; export interface ConnectionOpts { - /** - * For the local transport we already do know the jwt token and url. We can reuse it. - * On top the local transport will send additional data to the jwt server to use delayed event delegation. - */ - existingSFUConfig?: SFUConfig; /** The identity parts to use on this connection */ ownMembershipIdentity: CallMembershipIdentityParts; + /** Whether we want to publish or only subscribe on this connection. */ + role: "publisher" | "subscriber"; /** The media transport to connect to. */ - transport: LivekitTransport; + transport: UnstableLivekitTransport; + /** The name of the homeserver to which the {@link transport} belongs. */ + serverName: string; /** The Matrix client to use for OpenID and SFU config requests. */ - client: OpenIDClientParts; - /** The room ID this connection is associated with. */ + client: ClientGetTokenParts & ClientOpenIDParts; + /** The ID of the Matrix room in which the session takes place. */ roomId: string; + /** The ID of the MatrixRTC slot in which the session takes place. */ + slotId: string; /** The observable scope to use for this connection. */ scope: ObservableScope; /** Optional factory to create the LiveKit room, mainly for testing purposes. */ livekitRoomFactory: () => LivekitRoom; } -export class FailedToStartError extends Error { - public constructor(message: string) { - super(message); - this.name = "FailedToStartError"; - } -} -export enum ConnectionState { - /** The start state of a connection. It has been created but nothing has loaded yet. */ - Initialized = "Initialized", - /** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room */ - FetchingConfig = "FetchingConfig", - Stopped = "Stopped", - /** The same as ConnectionState.Disconnected from `livekit-client` */ - LivekitDisconnected = "disconnected", - /** The same as ConnectionState.Connecting from `livekit-client` */ - LivekitConnecting = "connecting", - /** The same as ConnectionState.Connected from `livekit-client` */ - LivekitConnected = "connected", - /** The same as ConnectionState.Reconnecting from `livekit-client` */ - LivekitReconnecting = "reconnecting", - /** The same as ConnectionState.SignalReconnecting from `livekit-client` */ - LivekitSignalReconnecting = "signalReconnecting", -} +export type ConnectionState = + // The start state of a connection. It has been created but nothing has loaded yet. + | { state: "initialized" } + // `start` has been called on the connection. It acquires the JWT token to connect to the LK room. + | { state: "authenticating" } + | { state: "authenticated"; livekitState: LivekitConnectionState } + | { state: "error"; error: ElementCallError } + | { state: "stopped" }; /** * A connection to a Matrix RTC LiveKit backend. @@ -95,19 +83,29 @@ export enum ConnectionState { */ export class Connection { // Private Behavior - private readonly _state$ = new BehaviorSubject< - ConnectionState | ElementCallError - >(ConnectionState.Initialized); + private readonly _state$ = new BehaviorSubject({ + state: "initialized", + }); /** * The current state of the connection to the media transport. */ - public readonly state$: Behavior = this._state$; + public readonly state$: Behavior = this._state$; + + /** + * Whether we want to publish or only subscribe on this connection. + */ + public readonly role: "publisher" | "subscriber"; /** * The media transport to connect to. */ - public readonly transport: LivekitTransport; + public readonly transport: UnstableLivekitTransport; + + /** + * The name of the homeserver to which the {@link transport} belongs. + */ + public readonly serverName: string; public readonly livekitRoom: LivekitRoom; @@ -137,11 +135,11 @@ export class Connection { protected stopped = false; // TODO: can we just keep the ConnectionOpts object instead of spreading? - private readonly client: OpenIDClientParts; + private readonly client: ClientGetTokenParts & ClientOpenIDParts; private readonly roomId: string; + private readonly slotId: string; private readonly logger: Logger; private readonly ownMembershipIdentity: CallMembershipIdentityParts; - private readonly existingSFUConfig?: SFUConfig; /** * Creates a new connection to a matrix RTC LiveKit backend. * @@ -151,19 +149,19 @@ export class Connection { */ public constructor(opts: ConnectionOpts, logger: Logger) { this.ownMembershipIdentity = opts.ownMembershipIdentity; - this.existingSFUConfig = opts.existingSFUConfig; this.roomId = opts.roomId; + this.slotId = opts.slotId; this.logger = logger.getChild( - "[Connection " + opts.transport.livekit_service_url + "]", + `[Connection ${JSON.stringify(opts.transport)}]`, ); - this.logger.info( - `constructor: ${opts.transport.livekit_service_url} roomId: ${this.roomId} withSfuConfig?: ${opts.existingSFUConfig ? JSON.stringify(opts.existingSFUConfig) : "undefined"}`, - ); - const { transport, client, scope } = opts; + this.logger.debug(`constructor called`); + const { role, transport, serverName, client, scope } = opts; this.scope = scope; this.livekitRoom = opts.livekitRoomFactory(); + this.role = role; this.transport = transport; + this.serverName = serverName; this.client = client; this.remoteParticipants$ = scope.behavior( @@ -304,16 +302,10 @@ export class Connection { this.logger.debug("Starting Connection"); this.stopped = false; try { - this._state$.next(ConnectionState.FetchingConfig); - // We should already have this information after creating the localTransport. - // only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined) - const { url, jwt, livekitAlias } = - this.existingSFUConfig ?? - (await this.getSFUConfigForRemoteConnection()); + this._state$.next({ state: "authenticating" }); + const { url, jwt, livekitAlias } = await this.getSFUConfig(); this.logger.debug( - "Starting Connection to: ", - this.transport.livekit_service_url, - "jwt: ", + "Starting Connection - jwt: ", jwt, "wss: ", url, @@ -324,16 +316,12 @@ export class Connection { // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - // Setup observer once we are done with getSFUConfigWithOpenID + // Setup observer once we are done with getSFUConfig connectionStateObserver(this.livekitRoom) - .pipe( - this.scope.bind(), - map((s) => s as unknown as ConnectionState), - ) - .subscribe((lkState) => { - // It is save to cast lkState to ConnectionState as they are fully overlapping. - this._state$.next(lkState); - }); + .pipe(this.scope.bind()) + .subscribe((livekitState) => + this._state$.next({ state: "authenticated", livekitState }), + ); try { this.logger.info(`livekitRoom.connect ${url}`); @@ -377,36 +365,37 @@ export class Connection { // stop() was called while we were connecting, which makes the pending // connect reject. That is the abort we asked for, not a failure, so // don't record an error state on a stopped connection or rethrow it - // (start() is not awaited by the ConnectionManager, so a throw here + // (start() is not necessarily awaited by the caller, so a throw here // becomes an unhandled promise rejection). this.logger.debug(`Connect aborted because the connection was stopped`); return; } this.logger.debug(`Failed to connect to LiveKit room: ${error}`); - this._state$.next( - error instanceof ElementCallError - ? error - : error instanceof Error - ? new UnknownCallError(error) - : new UnknownCallError(new Error(`${error}`)), - ); + this._state$.next({ + state: "error", + error: + error instanceof ElementCallError + ? error + : error instanceof Error + ? new UnknownCallError(error) + : new UnknownCallError(new Error(`${error}`)), + }); // Its okay to ignore the throw. The error is part of the state. throw error; } } - protected async getSFUConfigForRemoteConnection(): Promise { - // This will only be called for sfu's where we do not publish ourselves. - // For the local connection we will use the existingJwtTokenData - return await getSFUConfigWithOpenID( - this.client, - this.ownMembershipIdentity, - this.transport.livekit_service_url, - this.roomId, - // dont pass any custom opts for the subscribe only connections - {}, - this.logger, - ); + protected async getSFUConfig(): Promise { + return await getSFUConfig({ + client: this.client, + membership: this.ownMembershipIdentity, + role: this.role, + transport: this.transport, + serverName: this.serverName, + roomId: this.roomId, + slotId: this.slotId, + logger: this.logger, + }); } /** @@ -416,17 +405,13 @@ export class Connection { * If the connection is already stopped, this is a no-op. */ public async stop(): Promise { - this.logger.debug( - `stop: disconnecing from lk room ${this.transport.livekit_service_url}`, - ); + this.logger.debug("stop: disconnecing from lk room"); if (this.stopped) return; // Mark as stopped before disconnecting so that a connect() aborted by the // disconnect sees the flag and does not report the abort as an error. this.stopped = true; await this.livekitRoom.disconnect(); - this._state$.next(ConnectionState.Stopped); - this.logger.debug( - `stop: DONE disconnecing from lk room ${this.transport.livekit_service_url}`, - ); + this._state$.next({ state: "stopped" }); + this.logger.debug("stop: DONE disconnecing from lk room"); } } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 96a28d5ad..e5240f096 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -16,14 +16,10 @@ import { logger, type Logger } from "matrix-js-sdk/lib/logger"; // imported as inline to support worker when loaded from a cdn (cross domain) import E2EEWorker from "livekit-client/e2ee-worker?worker&inline"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; -import type { - OpenIDClientParts, - SFUConfig, -} from "../../../livekit/openIDSFU.ts"; import type { MediaDevices } from "../../MediaDevices.ts"; import type { Behavior } from "../../Behavior.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; @@ -39,15 +35,20 @@ import { noiseSuppressionSetting, autoGainControlSetting, } from "../../../settings/settings.ts"; +import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { createConnection( scope: ObservableScope, - transport: LivekitTransport, + role: "publisher" | "subscriber", + transport: UnstableLivekitTransport, + serverName: string, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - sfuConfig?: SFUConfig, ): Connection; } @@ -58,7 +59,8 @@ export class ECConnectionFactory implements ConnectionFactory { * Creates a ConnectionFactory for LiveKit connections. * * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens. - * @param roomId - The current room ID. + * @param roomId - The ID of the Matrix room in which the session takes place. + * @param slotId - The ID of the MatrixRTC slot in which the session takes place. * @param devices - Used for video/audio out/in capture options. * @param processorState$ - Effects like background blur (only for publishing connection?) * @param livekitKeyProvider - Optional key provider for end-to-end encryption. @@ -66,8 +68,9 @@ export class ECConnectionFactory implements ConnectionFactory { * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used. */ public constructor( - private client: OpenIDClientParts, + private client: ClientGetTokenParts & ClientOpenIDParts, private readonly roomId: string, + private readonly slotId: string, private devices: MediaDevices, private processorState$: Behavior, livekitKeyProvider: BaseKeyProvider | undefined, @@ -96,6 +99,7 @@ export class ECConnectionFactory implements ConnectionFactory { * * @param scope The observable scope (used for clean-up) * @param transport The transport to use for this connection. + * @param serverName The name of the homeserver to which the {@link transport} belongs. * @param ownMembershipIdentity required to connect (using the jwt service) with the SFU. * @param logger The logger instance to use for this connection. * @param sfuConfig optional config in case we already have a token for this connection. @@ -103,16 +107,19 @@ export class ECConnectionFactory implements ConnectionFactory { */ public createConnection( scope: ObservableScope, - transport: LivekitTransport, + role: "publisher" | "subscriber", + transport: UnstableLivekitTransport, + serverName: string, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - sfuConfig?: SFUConfig, ): Connection { return new Connection( { - existingSFUConfig: sfuConfig, roomId: this.roomId, + slotId: this.slotId, + role, transport, + serverName, client: this.client, scope: scope, livekitRoomFactory: this.livekitRoomFactory, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts deleted file mode 100644 index 18f54f021..000000000 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ /dev/null @@ -1,282 +0,0 @@ -/* -Copyright 2025 Element Creations Ltd. -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { - combineLatest, - map, - type Observable, - of, - switchMap, - startWith, - catchError, - NEVER, -} from "rxjs"; -import { type Logger } from "matrix-js-sdk/lib/logger"; -import { type RemoteParticipant } from "livekit-client"; -import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; - -import { type Behavior } from "../../Behavior.ts"; -import { type Connection } from "./Connection.ts"; -import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; -import { generateItemsWithEpoch } from "../../../utils/observable.ts"; -import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; -import { type ConnectionFactory } from "./ConnectionFactory.ts"; -import { - isLocalTransport, - type LocalTransport, -} from "../localMember/LocalTransport.ts"; -import { type SFUConfig } from "../../../livekit/openIDSFU.ts"; - -export class ConnectionManagerData { - private readonly store: Map< - string, - { connection: Connection; participants: RemoteParticipant[] } - > = new Map(); - - public constructor(private readonly logger?: Logger) {} - - public add(connection: Connection, participants: RemoteParticipant[]): void { - const key = this.getKey(connection.transport); - const existing = this.store.get(key); - if (!existing) { - this.store.set(key, { connection, participants }); - } else { - // Transports are deduplicated by URL upstream, so this should never - // happen; if it does, members may be matched against the wrong room. - this.logger?.warn( - `Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`, - ); - existing.participants.push(...participants); - } - } - - private getKey(transport: LivekitTransport): string { - // This is enough as a key because the ConnectionManager is already scoped by room. - // We also do not need to consider the slotId at this point since each `MatrixRTCSession` is already scoped by `slotDescription: {id, application}`. - return transport.livekit_service_url; - } - - public getConnections(): Connection[] { - return Array.from(this.store.values()).map(({ connection }) => connection); - } - - public getConnectionForTransport( - transport: LivekitTransport, - ): Connection | null { - return this.store.get(this.getKey(transport))?.connection ?? null; - } - - public getParticipantsForTransport( - transport: LivekitTransport, - ): RemoteParticipant[] { - const key = this.getKey(transport); - const existing = this.store.get(key); - if (existing) { - return existing.participants; - } - return []; - } -} - -interface Props { - scope: ObservableScope; - connectionFactory: ConnectionFactory; - localTransport$: Observable; - remoteTransports$: Behavior>; - - logger: Logger; - ownMembershipIdentity: CallMembershipIdentityParts; -} - -// TODO - write test for scopes (do we really need to bind scope) -export interface IConnectionManager { - connectionManagerData$: Behavior>; -} - -/** - * Crete a `ConnectionManager` - * @param props - Configuration object - * @param props.scope - The observable scope used by this object - * @param props.connectionFactory - Used to create new connections - * @param props.localTransport$ - The transport to publish local media on. (deduplicated with remoteTransports$) - * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$) - * @param props.ownMembershipIdentity - The own membership identity to use. - * @param props.logger - The logger to use. - - * - * Each of these behaviors can be interpreted as subscribed list of transports. - * - * Using `registerTransports` independent external modules can control what connections - * are created by the ConnectionManager. - * - * The connection manager will remove all duplicate transports in each subscibed list. - * - * See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe. - */ -export function createConnectionManager$({ - scope, - connectionFactory, - localTransport$, - remoteTransports$, - logger: parentLogger, - ownMembershipIdentity, -}: Props): IConnectionManager { - const logger = parentLogger.getChild("[ConnectionManager]"); - // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing - - const localTransportAsArray$ = localTransport$.pipe( - // LocalMember already surfaces local transport errors properly in the UI, - // here we can just swallow them - catchError(() => NEVER), - map((transport) => [transport]), - startWith([]), - ); - - /** - * All transports currently managed by the ConnectionManager. - * - * This list does not include duplicate transports. - * - * It is build based on the list of subscribed transports (`transportsSubscriptions$`). - * externally this is modified via `registerTransports()`. - */ - const localAndRemoteTransports$: Behavior< - Epoch<(LivekitTransport | LocalTransport)[]> - > = scope.behavior( - combineLatest([localTransportAsArray$, remoteTransports$]).pipe( - // Combine local and remote transports into one transport array - // and set the forceOldJwtEndpoint property on the local transport - map(([localTransportAsArray, remoteTransports]) => { - const dedupedRemote = removeDuplicateTransports(remoteTransports.value); - const remoteWithoutLocal = dedupedRemote.filter( - (transport) => - !localTransportAsArray.find((l) => - areLivekitTransportsEqual(l.transport, transport), - ), - ); - logger.debug( - "remoteWithoutLocal", - remoteWithoutLocal, - "localTransportAsArray", - localTransportAsArray, - ); - return new Epoch( - [...localTransportAsArray, ...remoteWithoutLocal], - remoteTransports.epoch, - ); - }), - ), - ); - - /** - * Connections for each transport in use by one or more session members. - */ - const connections$ = scope.behavior( - localAndRemoteTransports$.pipe( - generateItemsWithEpoch( - "ConnectionManager connections$", - function* (transports) { - for (const transport of transports) { - if (isLocalTransport(transport)) { - // This is the local transport; only the `LocalTransport` - // interface has a `sfuConfig` field. - yield { - keys: [ - transport.transport.livekit_service_url, - transport.sfuConfig, - ], - data: undefined, - }; - } else { - yield { - keys: [ - transport.livekit_service_url, - undefined as SFUConfig | undefined, - ], - data: undefined, - }; - } - } - }, - (scope, _data$, serviceUrl, sfuConfig) => { - const connection = connectionFactory.createConnection( - scope, - { - type: "livekit", - livekit_service_url: serviceUrl, - }, - ownMembershipIdentity, - logger, - // TODO: This whole optional SFUConfig parameter is not particularly elegant. - // I would like it if connections always fetched the SFUConfig by themselves. - sfuConfig, - ); - // Start the connection immediately - // Use connection state to track connection progress - void connection.start(); - // TODO subscribe to connection state to retry or log issues? - return connection; - }, - ), - ), - ); - - const connectionManagerData$ = scope.behavior( - connections$.pipe( - switchMap((connections) => { - const epoch = connections.epoch; - - // Map the connections to list of {connection, participants}[] - const listOfConnectionsWithRemoteParticipants = connections.value.map( - (connection) => { - return connection.remoteParticipants$.pipe( - map((participants) => ({ - connection, - participants, - })), - ); - }, - ); - - // probably not required - - if (listOfConnectionsWithRemoteParticipants.length === 0) { - return of(new Epoch(new ConnectionManagerData(), epoch)); - } - - // combineLatest the several streams into a single stream with the ConnectionManagerData - return combineLatest(listOfConnectionsWithRemoteParticipants).pipe( - map( - (lists) => - new Epoch( - lists.reduce((data, { connection, participants }) => { - data.add(connection, participants); - return data; - }, new ConnectionManagerData(logger)), - epoch, - ), - ), - ); - }), - ), - new Epoch(new ConnectionManagerData(), -1), - ); - - return { connectionManagerData$ }; -} - -function removeDuplicateTransports( - transports: T[], -): T[] { - return transports.reduce((acc, transport) => { - if (!acc.some((t) => areLivekitTransportsEqual(t, transport))) - acc.push(transport); - return acc; - }, [] as T[]); -} diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts similarity index 63% rename from src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts rename to src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts index e7d91c633..2344c1e55 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.test.ts @@ -6,34 +6,49 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { BehaviorSubject, NEVER } from "rxjs"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject } from "rxjs"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type RemoteParticipant } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts"; import { - createConnectionManager$, - ConnectionManagerData, -} from "./ConnectionManager.ts"; + createConnectionMap$, + ConnectionMap, + keyFor, +} from "./ConnectionMap.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts"; -import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type Behavior } from "../../Behavior.ts"; +import { type TransportLocator } from "../../../livekit/auth"; // Some test constants -const TRANSPORT_1: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.example.org", +const TRANSPORT_1: TransportLocator = { + transport: { + type: "livekit", + livekit_service_url: "https://lk.example.org", + }, + serverName: "example.org", }; -const TRANSPORT_2: LivekitTransport = { - type: "livekit", - livekit_service_url: "https://lk.sample.com", +const TRANSPORT_2: TransportLocator = { + transport: { + type: "livekit", + livekit_service_url: "https://lk.sample.com", + }, + serverName: "sample.com", }; +function findConnection( + transport: TransportLocator, + connections: Connection[], +): Connection | undefined { + const key = keyFor(transport); + return connections.find((c) => keyFor(c) === key); +} + let fakeConnectionFactory: ConnectionFactory; let testScope: ObservableScope; @@ -47,9 +62,15 @@ beforeEach(() => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (scope: ObservableScope, transport: LivekitTransport) => { + ( + scope: ObservableScope, + _role: unknown, + transport: UnstableLivekitTransport, + serverName: string, + ) => { const mockConnection = { transport, + serverName, remoteParticipants$: new BehaviorSubject([]), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); @@ -71,10 +92,10 @@ afterEach(() => { describe("connections$ stream", () => { test("Should create and start new connections for each transports", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), @@ -83,7 +104,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("a", { a: expect.toSatisfy((e: Epoch) => { const connections = e.value; @@ -93,15 +114,11 @@ describe("connections$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection, ).toHaveBeenCalledTimes(2); - const conn1 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_1), - ); + const conn1 = findConnection(TRANSPORT_1, connections); expect(conn1).toBeDefined(); expect(conn1!.start).toHaveBeenCalled(); - const conn2 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), - ); + const conn2 = findConnection(TRANSPORT_2, connections); expect(conn2).toBeDefined(); expect(conn2!.start).toHaveBeenCalled(); return true; @@ -110,12 +127,88 @@ describe("connections$ stream", () => { }); }); - test("Should start connection only once", () => { + test("Shares one connection between homeservers advertising the same JWT service", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const sameServiceElsewhere: TransportLocator = { + transport: TRANSPORT_1.transport, + serverName: "elsewhere.org", + }; + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, + remoteTransports$: behavior("a", { + a: new Epoch([sameServiceElsewhere], 0), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable( + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), + ).toBe("a", { + a: expect.toSatisfy((e: Epoch) => { + expect(e.value.length).toBe(1); + // The JWT service is reached directly, so the server name plays no + // part and the local locator, which came first, is the one used + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledExactlyOnceWith( + expect.anything(), + "publisher", + TRANSPORT_1.transport, + "example.org", + expect.anything(), + expect.anything(), + ); + return true; + }), + }); + }); + }); + + test("Keeps one connection per homeserver for transports with a url", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const urlTransport: TransportLocator = { + transport: { type: "livekit", url: "wss://sfu.example.org" }, + serverName: "example.org", + }; + const sameUrlElsewhere: TransportLocator = { + transport: urlTransport.transport, + serverName: "elsewhere.org", + }; + const connectionMap$ = createConnectionMap$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport: urlTransport, + remoteTransports$: behavior("a", { + a: new Epoch([sameUrlElsewhere], 0), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable( + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), + ).toBe("a", { + a: expect.toSatisfy((e: Epoch) => { + // Our homeserver asks the named server for the token, so each + // server name needs its own connection + expect(e.value.length).toBe(2); + expect(findConnection(urlTransport, e.value)).toBeDefined(); + expect(findConnection(sameUrlElsewhere, e.value)).toBeDefined(); + return true; + }), + }); + }); + }); + + test("Should start connection only once", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const connectionMap$ = createConnectionMap$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport: TRANSPORT_1, remoteTransports$: behavior("abcdef", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1], 1), @@ -129,7 +222,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("xxxxxa", { x: expect.anything(), a: expect.toSatisfy((e: Epoch) => { @@ -140,14 +233,10 @@ describe("connections$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection, ).toHaveBeenCalledTimes(2); - const conn2 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), - ); + const conn2 = findConnection(TRANSPORT_2, connections); expect(conn2).toBeDefined(); - const conn1 = connections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_1), - ); + const conn1 = findConnection(TRANSPORT_1, connections); expect(conn1).toBeDefined(); expect(conn1!.start).toHaveBeenCalledOnce(); @@ -159,10 +248,10 @@ describe("connections$ stream", () => { test("Should cleanup connections when not needed anymore", () => { withTestScheduler(({ behavior, expectObservable }) => { - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("abc", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), @@ -173,7 +262,7 @@ describe("connections$ stream", () => { }); expectObservable( - connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + connectionMap$.pipe(mapEpoch((d) => d.getConnections())), ).toBe("xab", { x: expect.anything(), a: expect.toSatisfy((e: Epoch) => { @@ -186,8 +275,9 @@ describe("connections$ stream", () => { expect(connections.length).toBe(1); // The second connection should have been stopped has it is no longer needed. - const connection2 = allCreatedConnections.find((c) => - areLivekitTransportsEqual(c.transport, TRANSPORT_2), + const connection2 = findConnection( + TRANSPORT_2, + allCreatedConnections, ); expect(connection2).toBeDefined(); expect(connection2!.stop).toHaveBeenCalled(); @@ -203,40 +293,36 @@ describe("connections$ stream", () => { }); }); -describe("ConnectionManagerData", () => { +describe("ConnectionMap", () => { test("warns when a second connection to the same URL is merged", () => { const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); - const data = new ConnectionManagerData(logger); - const connection = { transport: TRANSPORT_1 } as unknown as Connection; + const data = new ConnectionMap(logger); + const connection = TRANSPORT_1 as unknown as Connection; const p = (identity: string): RemoteParticipant => ({ identity }) as unknown as RemoteParticipant; data.add(connection, [p("a")]); data.add({ ...connection } as Connection, [p("b")]); expect(warn).toHaveBeenCalledWith( expect.stringContaining( - "second connection to https://lk.example.org: existing [a], adding [b]", + 'second connection to [null,"https://lk.example.org",null]: existing [a], adding [b]', ), ); expect(data.getParticipantsForTransport(TRANSPORT_1)).toHaveLength(2); }); }); -describe("connectionManagerData$ stream", () => { +describe("connectionMap$", () => { // Used in test to control fake connections' remoteParticipants$ streams let fakeRemoteParticipantsStreams: Map>; - function keyForTransport(transport: LivekitTransport): string { - return `${transport.livekit_service_url}`; - } - beforeEach(() => { fakeRemoteParticipantsStreams = new Map(); function getRemoteParticipantsFor( - transport: LivekitTransport, + transport: TransportLocator, ): Behavior { return ( - fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ?? + fakeRemoteParticipantsStreams.get(keyFor(transport)) ?? new BehaviorSubject([]) ); } @@ -245,13 +331,22 @@ describe("connectionManagerData$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (scope: ObservableScope, transport: LivekitTransport) => { + ( + scope: ObservableScope, + _role: unknown, + transport: UnstableLivekitTransport, + serverName: string, + ) => { const fakeRemoteParticipants$ = new BehaviorSubject< RemoteParticipant[] >([]); const mockConnection = { transport, - remoteParticipants$: getRemoteParticipantsFor(transport), + serverName, + remoteParticipants$: getRemoteParticipantsFor({ + transport, + serverName, + }), } as unknown as Connection; vi.mocked(mockConnection).start = vi.fn(); vi.mocked(mockConnection).stop = vi.fn(); @@ -261,7 +356,7 @@ describe("connectionManagerData$ stream", () => { }); fakeRemoteParticipantsStreams.set( - keyForTransport(transport), + keyFor({ transport, serverName }), fakeRemoteParticipants$, ); return mockConnection; @@ -274,7 +369,7 @@ describe("connectionManagerData$ stream", () => { // Setup the fake participants streams behavior // ============================== fakeRemoteParticipantsStreams.set( - keyForTransport(TRANSPORT_1), + keyFor(TRANSPORT_1), behavior("oa-b", { o: [], a: [{ identity: "user1A" } as RemoteParticipant], @@ -286,7 +381,7 @@ describe("connectionManagerData$ stream", () => { ); fakeRemoteParticipantsStreams.set( - keyForTransport(TRANSPORT_2), + keyFor(TRANSPORT_2), behavior("o-a", { o: [], a: [{ identity: "user2A" } as RemoteParticipant], @@ -294,10 +389,10 @@ describe("connectionManagerData$ stream", () => { ); // ============================== - const { connectionManagerData$ } = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: fakeConnectionFactory, - localTransport$: NEVER, + localTransport: TRANSPORT_1, remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), @@ -305,16 +400,16 @@ describe("connectionManagerData$ stream", () => { ownMembershipIdentity: ownMemberMock, }); - expectObservable(connectionManagerData$).toBe("abcd", { + expectObservable(connectionMap$).toBe("abcd", { a: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); return true; }), b: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); @@ -324,7 +419,7 @@ describe("connectionManagerData$ stream", () => { return true; }), c: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); @@ -337,7 +432,7 @@ describe("connectionManagerData$ stream", () => { return true; }), d: expect.toSatisfy((e) => { - const data: ConnectionManagerData = e.value; + const data: ConnectionMap = e.value; expect(data.getConnections().length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2); expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionMap.ts b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts new file mode 100644 index 000000000..586e8d49b --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/ConnectionMap.ts @@ -0,0 +1,219 @@ +/* +Copyright 2025-2026 Element Creations Ltd. +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, map, of, switchMap, tap } from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type RemoteParticipant } from "livekit-client"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { type Behavior } from "../../Behavior.ts"; +import { type Connection } from "./Connection.ts"; +import { + Epoch, + mapEpoch, + type ObservableScope, +} from "../../ObservableScope.ts"; +import { generateItemsWithEpoch } from "../../../utils/observable.ts"; +import { type ConnectionFactory } from "./ConnectionFactory.ts"; +import { type TransportLocator } from "../../../livekit/auth/types.ts"; + +// Branded string type, so that only values returned by `keyFor` can be used +// as keys +export type TransportKey = string & { __transportKey: true }; + +/** + * Converts a {@link TransportLocator} to a key suitable for identifying it in a + * {@link Map} or {@link Set}. Locators with the same key share a connection. + */ +export function keyFor({ + transport, + serverName, +}: TransportLocator): TransportKey { + // The server name only takes part in authentication for a transport with a + // `url`, where our homeserver asks that server for the token. A JWT service + // is reached directly, so members of several homeservers advertising the + // same one must share a connection rather than each get their own. + const hasUrl = "url" in transport; + // Serialize the fields in a consistent order + return JSON.stringify([ + hasUrl ? transport.url : null, + "livekit_service_url" in transport ? transport.livekit_service_url : null, + hasUrl ? serverName : null, + ]) as TransportKey; +} + +function removeDuplicateTransports( + transports: TransportLocator[], +): TransportLocator[] { + // The first locator wins, so the local one stays first in the list + const map = new Map(); + for (const transport of transports) { + const key = keyFor(transport); + if (!map.has(key)) map.set(key, transport); + } + return [...map.values()]; +} + +export class ConnectionMap { + private readonly map = new Map< + TransportKey, + { connection: Connection; participants: RemoteParticipant[] } + >(); + + public constructor(private readonly logger?: Logger) {} + + public add(connection: Connection, participants: RemoteParticipant[]): void { + const key = keyFor(connection); + const existing = this.map.get(key); + if (!existing) { + this.map.set(key, { connection, participants }); + } else { + // Transports are deduplicated by key upstream, so this should never + // happen; if it does, members may be matched against the wrong room. + this.logger?.warn( + `Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`, + ); + existing.participants.push(...participants); + } + } + + public getConnections(): Connection[] { + return Array.from(this.map.values()).map(({ connection }) => connection); + } + + public getConnectionForTransport( + transport: TransportLocator, + ): Connection | null { + return this.map.get(keyFor(transport))?.connection ?? null; + } + + public getParticipantsForTransport( + transport: TransportLocator, + ): RemoteParticipant[] { + const existing = this.map.get(keyFor(transport)); + if (existing) { + return existing.participants; + } + return []; + } +} + +interface Props { + scope: ObservableScope; + connectionFactory: ConnectionFactory; + localTransport: TransportLocator; + remoteTransports$: Behavior>; + + logger: Logger; + ownMembershipIdentity: CallMembershipIdentityParts; +} + +/** + * Create a dynamic map of connections and their associated backend participants. + * @param props - Configuration object + * @param props.scope - The observable scope used by this object + * @param props.connectionFactory - Used to create new connections + * @param props.localTransport - The transport to publish local media on. (deduplicated with remoteTransports$) + * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport) + * @param props.ownMembershipIdentity - The own membership identity to use. + * @param props.logger - The logger to use. + */ +export function createConnectionMap$({ + scope, + connectionFactory, + localTransport, + remoteTransports$, + logger: parentLogger, + ownMembershipIdentity, +}: Props): Behavior> { + const logger = parentLogger.getChild("[ConnectionMap]"); + // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing + + /** + * All transports currently represented in the connection map. This list does + * not include duplicate transports. + */ + const localAndRemoteTransports$: Behavior> = + scope.behavior( + remoteTransports$.pipe( + mapEpoch((remoteTransports) => + removeDuplicateTransports([localTransport, ...remoteTransports]), + ), + tap((transports) => + logger.debug("localAndRemoteTransports$ = ", transports), + ), + ), + ); + + const localTransportKey = keyFor(localTransport); + + /** + * Connections for each transport in use by one or more session members. + */ + const connections$ = scope.behavior( + localAndRemoteTransports$.pipe( + generateItemsWithEpoch( + "ConnectionMap connections$", + function* (transports) { + for (const transport of transports) { + const role = + keyFor(transport) === localTransportKey + ? ("publisher" as const) + : ("subscriber" as const); + yield { + // The locator is an object, so a stable key stands in for it + keys: [role, keyFor(transport)], + data: transport, + }; + } + }, + (scope, locator$, role, _key) => { + const { transport, serverName } = locator$.value; + const connection = connectionFactory.createConnection( + scope, + role, + transport, + serverName, + ownMembershipIdentity, + logger, + ); + // Start the connection immediately + // Use connection state to track connection progress + void connection.start(); + // TODO subscribe to connection state to retry or log issues? + return connection; + }, + ), + ), + ); + + return scope.behavior( + connections$.pipe( + switchMap(({ value: connections, epoch }) => { + if (connections.length === 0) + return of(new Epoch(new ConnectionMap(), epoch)); + + return combineLatest( + // Map the connections to list of Observable<{connection, participants}> + connections.map((connection) => + connection.remoteParticipants$.pipe( + map((participants) => ({ connection, participants })), + ), + ), + // Collect into a single ConnectionMap + (...list) => { + const map = new ConnectionMap(logger); + for (const item of list) + map.add(item.connection, item.participants); + return new Epoch(map, epoch); + }, + ); + }), + ), + ); +} diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index 411ae2001..be6c149f4 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -14,7 +14,10 @@ import EventEmitter from "events"; import { ObservableScope } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; -import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; import { exampleTransport, mockMediaDevices, @@ -32,6 +35,7 @@ import { cameraBitrate, cameraCodec, } from "../../../settings/settings.ts"; +import { testJWTToken } from "../../../utils/test-fixtures.ts"; // At the top of your test file, after imports vi.mock("livekit-client", async (importOriginal) => { @@ -51,13 +55,13 @@ vi.mock("livekit-client", async (importOriginal) => { }); let testScope: ObservableScope; -let mockClient: OpenIDClientParts; +let mockClient: ClientGetTokenParts & ClientOpenIDParts; beforeEach(() => { testScope = new ObservableScope(); mockClient = { getOpenIdToken: vi.fn().mockReturnValue(""), - getDeviceId: vi.fn().mockReturnValue("DEV000"), + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), }; }); @@ -79,6 +83,7 @@ describe("ECConnectionFactory - Audio inputs options", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -89,7 +94,9 @@ describe("ECConnectionFactory - Audio inputs options", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -121,6 +128,7 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({ audioOutput: { available$: constant(new Map()), @@ -137,7 +145,9 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -162,6 +172,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -172,7 +183,9 @@ describe("ECConnectionFactory - Camera quality settings", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -199,6 +212,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -209,7 +223,9 @@ describe("ECConnectionFactory - Camera quality settings", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); @@ -237,6 +253,7 @@ describe("ECConnectionFactory - Camera quality settings", () => { const ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -247,7 +264,9 @@ describe("ECConnectionFactory - Camera quality settings", () => { ); ecConnectionFactory.createConnection( testScope, + "subscriber", exampleTransport, + "example.org", ownMemberMock, logger, ); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index 2e0051928..1c6a32790 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -9,22 +9,16 @@ import { describe, test, expect, beforeEach, afterEach, vi } from "vitest"; import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, - type LivekitTransport, + type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs"; -import { type IConnectionManager } from "./ConnectionManager.ts"; import { type RemoteMatrixLivekitMember, createRemoteMatrixLivekitMembers$, } from "./MatrixLivekitMembers.ts"; -import { - Epoch, - mapEpoch, - ObservableScope, - trackEpoch, -} from "../../ObservableScope.ts"; -import { ConnectionManagerData } from "./ConnectionManager.ts"; +import { Epoch, ObservableScope, trackEpoch } from "../../ObservableScope.ts"; +import { ConnectionMap } from "./ConnectionMap.ts"; import { flushPromises, mockRtcMembership, @@ -33,18 +27,20 @@ import { import { type Connection } from "./Connection.ts"; import { constant } from "../../Behavior.ts"; import { localRtcMember } from "../../../utils/test-fixtures.ts"; +import { type TransportLocator } from "../../../livekit/auth"; +import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; let testScope: ObservableScope; const fallbackMemberId = (userId: string, deviceId: string): string => `${userId}:${deviceId}`; -const transportA: LivekitTransport = { +const transportA: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", }; -const transportB: LivekitTransport = { +const transportB: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://lk.sample.com", }; @@ -78,23 +74,18 @@ function epochMeWith$( test("should signal participant not yet connected to livekit", async () => { const info = vi.spyOn(logger, "info"); const mockedMemberships$ = new BehaviorSubject([bobMembership]); - const mockConnectionManagerData$ = new BehaviorSubject( - new ConnectionManagerData(), - ); + const mockConnectionMap$ = new BehaviorSubject(new ConnectionMap()); const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(mockedMemberships$); - const connectionManagerData$ = epochMeWith$( - memberships$, - mockConnectionManagerData$, + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, mockConnectionMap$), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); @@ -117,21 +108,15 @@ test("should signal participant not yet connected to livekit", async () => { function createEpochedMemberships$(m$: Observable): { memberships$: Observable>; membershipsWithTransport$: Observable< - Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + Epoch< + { membership: CallMembership; transport: TransportLocator | undefined }[] + > >; } { - const memberships$ = m$.pipe(trackEpoch()); - const membershipsWithTransport$ = memberships$.pipe( - mapEpoch((members) => { - return members.map((m) => { - const tr = m.getTransport(); - return { - membership: m, - transport: - tr?.type === "livekit" ? (tr as LivekitTransport) : undefined, - }; - }); - }), + const memberships$ = testScope.behavior(m$.pipe(trackEpoch())); + const { membershipsWithTransport$ } = membershipsAndTransports$( + testScope, + memberships$, ); return { memberships$, @@ -151,23 +136,21 @@ test("should signal participant on a connection that is publishing", async () => const connection = { transport: bobMembership.getTransport(), + serverName: "example.org", } as unknown as Connection; - const dataWithPublisher = new ConnectionManagerData(); + const dataWithPublisher = new ConnectionMap(); dataWithPublisher.add(connection, [ mockRemoteParticipant({ identity: bobParticipantId }), ]); - const connectionManagerData$ = epochMeWith$( - memberships$, - constant(dataWithPublisher), + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, constant(dataWithPublisher)), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); @@ -187,6 +170,90 @@ test("should signal participant on a connection that is publishing", async () => ); }); +// A member advertising both a `url` and a `livekit_service_url` may have +// authenticated through either flow, so it has two candidate identities. +const bothTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://sfu.example.org", + livekit_service_url: "https://lk.example.org", +}; +const bobLegacyId = fallbackMemberId("@bob:example.org", "DEV000"); +const bobHashedId = "2qkXfWz1S7jQmU0bJ6yN8aVtR3pLcDgHe9oZxKwMiA4"; +const bobWithBothIdentities = mockRtcMembership("@bob:example.org", "DEV000", { + fociPreferred: [bothTransport], + backendIdentities: [bobHashedId, bobLegacyId], +}); + +test.each([ + ["legacy", bobLegacyId], + ["hashed", bobHashedId], +])( + "matches a member with two candidate identities publishing under the %s one", + async (_name, identity) => { + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(constant([bobWithBothIdentities])); + + const connection = { + transport: bothTransport, + serverName: "example.org", + } as unknown as Connection; + const dataWithPublisher = new ConnectionMap(); + dataWithPublisher.add(connection, [mockRemoteParticipant({ identity })]); + + const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionMap$: testScope.behavior( + epochMeWith$(memberships$, constant(dataWithPublisher)), + ), + localUser: localRtcMember, + }); + + await flushPromises(); + expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].participant.value$.value?.identity).toEqual(identity); + expect(data[0].connection$.value).toBe(connection); + return true; + }, + ); + }, +); + +test("does not match a member with two candidate identities to anyone else", async () => { + const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( + constant([bobWithBothIdentities]), + ); + + const connection = { + transport: bothTransport, + serverName: "example.org", + } as unknown as Connection; + const dataWithStranger = new ConnectionMap(); + dataWithStranger.add(connection, [ + mockRemoteParticipant({ identity: "@stranger:example.org:DEV999" }), + ]); + + const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionMap$: testScope.behavior( + epochMeWith$(memberships$, constant(dataWithStranger)), + ), + localUser: localRtcMember, + }); + + await flushPromises(); + expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].participant.value$.value).toBe(null); + return true; + }, + ); +}); + test("should signal participant on a connection that is not publishing", async () => { const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( constant([bobMembership]), @@ -194,21 +261,19 @@ test("should signal participant on a connection that is not publishing", async ( const connection = { transport: bobMembership.getTransport(), + serverName: "example.org", } as unknown as Connection; - const dataWithPublisher = new ConnectionManagerData(); + const dataWithPublisher = new ConnectionMap(); dataWithPublisher.add(connection, []); - const connectionManagerData$ = epochMeWith$( - memberships$, - constant(dataWithPublisher), + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, constant(dataWithPublisher)), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); await flushPromises(); @@ -228,16 +293,18 @@ describe("Publication edge case", () => { const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(constant([bobMembership, carlMembership])); - const connectionWithPublisher = new ConnectionManagerData(); + const connectionWithPublisher = new ConnectionMap(); const bobParticipantId = fallbackMemberId( bobMembership.userId, bobMembership.deviceId, ); const connectionA = { transport: transportA, + serverName: "example.org", } as unknown as Connection; const connectionB = { transport: transportB, + serverName: "sample.com", } as unknown as Connection; connectionWithPublisher.add(connectionA, [ @@ -247,17 +314,14 @@ describe("Publication edge case", () => { mockRemoteParticipant({ identity: bobParticipantId }), ]); - const connectionManagerData$ = epochMeWith$( - memberships$, - constant(connectionWithPublisher), + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, constant(connectionWithPublisher)), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); await flushPromises(); @@ -287,14 +351,20 @@ test("bob is publishing in the wrong connection", async () => { const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(mockedMemberships$); - const connectionWithPublisher = new ConnectionManagerData(); + const connectionWithPublisher = new ConnectionMap(); const bobParticipantId = fallbackMemberId( bobMembership.userId, bobMembership.deviceId, ); - const connectionA = { transport: transportA } as unknown as Connection; - const connectionB = { transport: transportB } as unknown as Connection; + const connectionA = { + transport: transportA, + serverName: "example.org", + } as unknown as Connection; + const connectionB = { + transport: transportB, + serverName: "sample.com", + } as unknown as Connection; // Bob is not publishing on A connectionWithPublisher.add(connectionA, []); @@ -306,17 +376,14 @@ test("bob is publishing in the wrong connection", async () => { const connectionsWithPublisher$ = new BehaviorSubject( connectionWithPublisher, ); - const connectionManagerData$ = epochMeWith$( - memberships$, - connectionsWithPublisher$, + const connectionMap$ = testScope.behavior( + epochMeWith$(memberships$, connectionsWithPublisher$), ); const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, + connectionMap$, localUser: localRtcMember, }); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 6c9fff026..2f2aa8e0c 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -6,18 +6,16 @@ Please see LICENSE in the repository root for full details. */ import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; -import { - type CallMembership, - type LivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, filter, map } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; -import { type IConnectionManager } from "./ConnectionManager"; +import { type ConnectionMap } from "./ConnectionMap"; import { Epoch, type ObservableScope } from "../../ObservableScope"; import { type Connection } from "./Connection"; import { generateItemsWithEpoch } from "../../../utils/observable"; +import { type TransportLocator } from "../../../livekit/auth"; interface LocalTaggedParticipant { type: "local"; @@ -60,9 +58,11 @@ export interface RemoteMatrixLivekitMember extends MatrixLivekitMember { interface Props { scope: ObservableScope; membershipsWithTransport$: Behavior< - Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + Epoch< + { membership: CallMembership; transport: TransportLocator | undefined }[] + > >; - connectionManager: IConnectionManager; + connectionMap$: Behavior>; localUser: { deviceId: string; userId: string }; } @@ -72,24 +72,21 @@ interface Props { * It has a small public interface: * - in (via constructor): * - an observable of CallMembership[] to track the call members (The matrix side) - * - a `ConnectionManager` for the lk rooms (The livekit side) + * - a `ConnectionMap` for the lk rooms (The livekit side) * - out (via public Observable): * - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data. */ export function createRemoteMatrixLivekitMembers$({ scope, membershipsWithTransport$, - connectionManager, + connectionMap$, localUser, }: Props): Behavior> { /** * Behavior of all the remote call members and their associated livekit data (if available). */ return scope.behavior( - combineLatest([ - membershipsWithTransport$, - connectionManager.connectionManagerData$, - ]).pipe( + combineLatest([membershipsWithTransport$, connectionMap$]).pipe( filter((values) => values.every((value) => value.epoch === values[0].epoch), ), @@ -111,8 +108,8 @@ export function createRemoteMatrixLivekitMembers$({ const participants = transport ? managerData.getParticipantsForTransport(transport) : []; - const matches = participants.filter( - (p) => p.identity == membership.rtcBackendIdentity, + const matches = participants.filter((p) => + membership.backendIdentities.includes(p.identity), ); const participant = matches[0] ?? null; const connection = transport @@ -143,9 +140,9 @@ export function createRemoteMatrixLivekitMembers$({ // Log whether the member could be matched to a LiveKit participant, // since a tile shows "waiting for media" for as long as it cannot. participant$.pipe(scope.bind()).subscribe((p) => { - const url = data$.value.connection?.transport.livekit_service_url; + const transport = data$.value.connection?.transport; logger.info( - `[RemoteMatrixLivekitMembers] ${rtcBackendIdentity}: LiveKit participant ${p ? `matched (${p.sid})` : "missing"} on ${url ?? "no connection"}`, + `[RemoteMatrixLivekitMembers] ${rtcBackendIdentity}: LiveKit participant ${p ? `matched (${p.sid})` : "missing"} on ${transport ? JSON.stringify(transport) : "no connection"}`, ); }); // will only get called once per backend identity. @@ -163,14 +160,3 @@ export function createRemoteMatrixLivekitMembers$({ } // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) - -// TODO add this to the JS-SDK -export function areLivekitTransportsEqual( - t1: T | null, - t2: T | null, -): boolean { - if (t1 && t2) { - return t1.livekit_service_url === t2.livekit_service_url; - } - return !t1 && !t2; -} diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 4c0f4fea7..bf90e9eb4 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -6,11 +6,10 @@ Please see LICENSE in the repository root for full details. */ import { test, vi, expect, beforeEach, afterEach } from "vitest"; -import { BehaviorSubject, NEVER } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { type Room as LivekitRoom } from "livekit-client"; import EventEmitter from "events"; import fetchMock from "fetch-mock"; -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { @@ -19,8 +18,12 @@ import { trackEpoch, } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; -import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { + type ClientGetTokenParts, + type ClientOpenIDParts, +} from "../../../livekit/auth"; +import { + exampleTransportLocator, mockMediaDevices, mockRtcMembership, ownMemberMock, @@ -28,19 +31,19 @@ import { } from "../../../utils/test.ts"; import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; import { - areLivekitTransportsEqual, createRemoteMatrixLivekitMembers$, type RemoteMatrixLivekitMember, } from "./MatrixLivekitMembers.ts"; -import { createConnectionManager$ } from "./ConnectionManager.ts"; +import { createConnectionMap$ } from "./ConnectionMap.ts"; import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; import { localRtcMember, testJWTToken } from "../../../utils/test-fixtures.ts"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; -// Test the integration of ConnectionManager and MatrixLivekitMerger +// Test the integration of ConnectionMap and MatrixLivekitMembers let testScope: ObservableScope; let ecConnectionFactory: ECConnectionFactory; -let mockClient: OpenIDClientParts; +let mockClient: ClientGetTokenParts & ClientOpenIDParts; let lkRoomFactory: () => LivekitRoom; const createdMockLivekitRooms: Map = new Map(); @@ -49,7 +52,7 @@ beforeEach(() => { testScope = new ObservableScope(); mockClient = { getOpenIdToken: vi.fn().mockReturnValue(""), - getDeviceId: vi.fn().mockReturnValue("DEV000"), + _unstable_getLivekitToken: vi.fn().mockResolvedValue({ jwt: testJWTToken }), }; lkRoomFactory = vi.fn().mockImplementation(() => { @@ -71,6 +74,7 @@ beforeEach(() => { ecConnectionFactory = new ECConnectionFactory( mockClient, "!roomid:example.org", + "m.call#ROOM", mockMediaDevices({}), new BehaviorSubject({ supported: true, @@ -120,10 +124,10 @@ test("bob, carl, then bob joining no tracks yet", () => { memberships$, ); - const connectionManager = createConnectionManager$({ + const connectionMap$ = createConnectionMap$({ scope: testScope, connectionFactory: ecConnectionFactory, - localTransport$: NEVER, + localTransport: exampleTransportLocator, remoteTransports$: membershipsAndTransports.transports$, logger: logger, ownMembershipIdentity: ownMemberMock, @@ -133,7 +137,7 @@ test("bob, carl, then bob joining no tracks yet", () => { scope: testScope, membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, - connectionManager, + connectionMap$, localUser: localRtcMember, }); @@ -147,10 +151,7 @@ test("bob, carl, then bob joining no tracks yet", () => { }); expectObservable(item.connection$).toBe("a", { a: expect.toSatisfy((co) => - areLivekitTransportsEqual( - co.transport, - bobMembership.transports[0]! as LivekitTransport, - ), + deepCompare(co.transport, bobMembership.transports[0]), ), }); expectObservable(item.participant.value$).toBe("a", { @@ -182,15 +183,9 @@ test("bob, carl, then bob joining no tracks yet", () => { a: null, }); expectObservable(item.connection$).toBe("a", { - a: expect.toSatisfy((connection) => { - expect( - areLivekitTransportsEqual( - connection.transport, - carlMembership.transports[0]! as LivekitTransport, - ), - ).toBe(true); - return true; - }), + a: expect.toSatisfy((connection) => + deepCompare(connection.transport, carlMembership.transports[0]), + ), }); } return true; @@ -212,15 +207,9 @@ test("bob, carl, then bob joining no tracks yet", () => { a: daveMembership, }); expectObservable(item.connection$).toBe("a", { - a: expect.toSatisfy((connection) => { - expect( - areLivekitTransportsEqual( - connection.transport, - daveMembership.transports[0]! as LivekitTransport, - ), - ).toBe(true); - return true; - }), + a: expect.toSatisfy((connection) => + deepCompare(connection.transport, daveMembership.transports[0]), + ), }); expectObservable(item.participant.value$).toBe("a", { a: null, diff --git a/src/state/SessionBehaviors.test.ts b/src/state/SessionBehaviors.test.ts index fca24767c..d9da6f830 100644 --- a/src/state/SessionBehaviors.test.ts +++ b/src/state/SessionBehaviors.test.ts @@ -1,16 +1,25 @@ /* -Copyright 2025 Element Creations Ltd. +Copyright 2025, 2026 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { describe, expect, it, vi } from "vitest"; -import { MatrixRTCSessionEvent } from "matrix-js-sdk/lib/matrixrtc"; +import { + MatrixRTCSessionEvent, + type Transport, + type UnstableLivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; import { EventEmitter } from "events"; -import { createKeyRotationSuppressed$ } from "./SessionBehaviors"; -import { testScope } from "../utils/test"; +import { + createKeyRotationSuppressed$, + membershipsAndTransports$, +} from "./SessionBehaviors"; +import { Epoch } from "./ObservableScope"; +import { constant } from "./Behavior"; +import { mockRtcMembership, testScope } from "../utils/test"; describe("SessionBehaviors", () => { describe("createKeyRotationSuppressed$", () => { @@ -38,4 +47,54 @@ describe("SessionBehaviors", () => { expect(keyRotationSuppressed$.value).toBe(false); }); }); + + describe("membershipsAndTransports$", () => { + const serviceTransport: UnstableLivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", + }; + const urlTransport: UnstableLivekitTransport = { + type: "livekit", + url: "wss://sfu.sample.com", + }; + + it("locates each member's transport on the member's homeserver", () => { + const bob = mockRtcMembership("@bob:example.org", "DEV000", { + fociPreferred: [serviceTransport], + }); + const carl = mockRtcMembership("@carl:sample.com", "DEV111", { + fociPreferred: [urlTransport], + }); + + const { membershipsWithTransport$, transports$ } = + membershipsAndTransports$( + testScope(), + constant(new Epoch([bob, carl])), + ); + + expect(transports$.value.value).toEqual([ + { transport: serviceTransport, serverName: "example.org" }, + { transport: urlTransport, serverName: "sample.com" }, + ]); + expect( + membershipsWithTransport$.value.value.map( + (m) => m.transport?.serverName, + ), + ).toEqual(["example.org", "sample.com"]); + }); + + it("leaves out members without a LiveKit transport", () => { + const dave = mockRtcMembership("@dave:example.org", "DEV222", { + fociPreferred: [{ type: "something-else" } as Transport], + }); + + const { membershipsWithTransport$, transports$ } = + membershipsAndTransports$(testScope(), constant(new Epoch([dave]))); + + expect(transports$.value.value).toEqual([]); + expect(membershipsWithTransport$.value.value).toEqual([ + { membership: dave, transport: undefined }, + ]); + }); + }); }); diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index 4ca1c5861..6620350cf 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -7,10 +7,9 @@ Please see LICENSE in the repository root for full details. import { type CallMembership, - type LivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, - isLivekitTransport, + isUnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { fromEvent } from "rxjs"; @@ -21,6 +20,7 @@ import { type ObservableScope, } from "./ObservableScope"; import { type Behavior } from "./Behavior"; +import { type TransportLocator } from "../livekit/auth"; /** * Tracks the transports used by ourselves, plus all other MatrixRTC session @@ -31,16 +31,15 @@ export const membershipsAndTransports$ = ( memberships$: Behavior>, ): { membershipsWithTransport$: Behavior< - Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + Epoch< + { membership: CallMembership; transport: TransportLocator | undefined }[] + > >; - transports$: Behavior>; + transports$: Behavior>; } => { const membershipsWithTransport$: Behavior< Epoch< - { - membership: CallMembership; - transport: LivekitTransport | undefined; - }[] + { membership: CallMembership; transport: TransportLocator | undefined }[] > > = scope.behavior( memberships$.pipe( @@ -49,14 +48,19 @@ export const membershipsAndTransports$ = ( const transport = membership.getTransport(); return { membership, - transport: isLivekitTransport(transport) ? transport : undefined, + transport: isUnstableLivekitTransport(transport) + ? { + transport, + serverName: membership.userId.replace(/^.*?:/, ""), + } + : undefined, }; }); }), ), ); - const transports$: Behavior> = scope.behavior( + const transports$: Behavior> = scope.behavior( membershipsWithTransport$.pipe( mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), ), diff --git a/src/state/media/MemberMediaViewModel.ts b/src/state/media/MemberMediaViewModel.ts index a7f4612db..e57dcc636 100644 --- a/src/state/media/MemberMediaViewModel.ts +++ b/src/state/media/MemberMediaViewModel.ts @@ -40,6 +40,7 @@ import { E2eeType } from "../../e2ee/e2eeType"; import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats"; import { type UserMediaViewModel } from "./UserMediaViewModel"; import { type ScreenShareViewModel } from "./ScreenShareViewModel"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; // TODO: Encryption status is kinda broken and thus unused right now. Remove? export enum EncryptionStatus { @@ -59,10 +60,10 @@ export interface BaseMemberMediaViewModel extends BaseMediaViewModel { */ video$: Behavior; /** - * The URL of the LiveKit focus on which this member should be publishing. - * Exposed for debugging. + * The LiveKit transport on which this member should be publishing. Exposed + * for debugging. */ - focusUrl$: Behavior; + transport$: Behavior; /** * Whether there should be a warning that this media is unencrypted. */ @@ -75,7 +76,7 @@ export interface MemberMediaInputs extends BaseMediaViewModel { livekitRoom$: Behavior; audioSource: AudioSource; videoSource: VideoSource; - focusUrl$: Behavior; + transport$: Behavior; encryptionSystem: EncryptionSystem; } @@ -86,7 +87,7 @@ export function createMemberMedia( livekitRoom$, audioSource, videoSource, - focusUrl$, + transport$, encryptionSystem, ...inputs }: MemberMediaInputs, @@ -109,7 +110,7 @@ export function createMemberMedia( return { ...createBaseMedia(inputs), video$, - focusUrl$, + transport$, unencryptedWarning$: scope.behavior( combineLatest( [audio$, video$], diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index e00186320..b0625a4d7 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -51,6 +51,7 @@ import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaVie import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; import { RingingStatus } from "./RingingStatus"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface TileProps { ref?: Ref; @@ -103,7 +104,7 @@ interface UserMediaTileProps extends TileProps { playbackMuted: boolean; waitingForMedia?: boolean; primaryButton?: ReactNode; - focusUrl: string | undefined; + transport: UnstableLivekitTransport | undefined; } /** @@ -120,7 +121,7 @@ const UserMediaTileInner: FC = ({ primaryButton, menu, className, - focusUrl, + transport, displayName, mxcAvatarUrl, focusable, @@ -214,7 +215,7 @@ const UserMediaTileInner: FC = ({ currentReaction={reaction ?? undefined} raisedHandOnClick={raisedHandOnClick} waitingForMedia={waitingForMedia} - focusUrl={focusUrl} + transport={transport} setVideoAspectRatio={vm.setVideoAspectRatio} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} @@ -279,7 +280,7 @@ const LocalUserMediaTile: FC = ({ const mirror = useBehavior(vm.mirror$); const alwaysShow = useBehavior(vm.alwaysShow$); const switchCamera = useBehavior(vm.switchCamera$); - const focusUrl = useBehavior(vm.focusUrl$); + const transport = useBehavior(vm.transport$); const latestAlwaysShow = useLatest(alwaysShow); const onSelectAlwaysShow = useCallback( @@ -334,7 +335,7 @@ const LocalUserMediaTile: FC = ({ menuStart={menuStart} menuEnd={menuEnd} focusable={focusable} - focusUrl={focusUrl} + transport={transport} {...props} /> ); @@ -356,7 +357,7 @@ const RemoteUserMediaTile: FC = ({ const waitingForMedia = useBehavior(vm.waitingForMedia$); const playbackMuted = useBehavior(vm.playbackMuted$); const playbackVolume = useBehavior(vm.playbackVolume$); - const focusUrl = useBehavior(vm.focusUrl$); + const transport = useBehavior(vm.transport$); const onSelectMute = useCallback( (e: Event) => { @@ -398,7 +399,7 @@ const RemoteUserMediaTile: FC = ({ } - focusUrl={focusUrl} + transport={transport} {...props} /> ); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index d57441687..6776317df 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -33,6 +33,7 @@ import { ReactionIndicator } from "../reactions/ReactionIndicator"; import { RTCConnectionStats } from "../RTCConnectionStats"; import videoPlaceholder from "../graphics/video-placeholder.gif"; import { autoVideoFit } from "../utils/videoFit"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface Props extends ComponentProps { className?: string; @@ -67,9 +68,9 @@ interface Props extends ComponentProps { videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; rtcBackendIdentity?: string; /** - * The focus url, mainly for debugging purposes. + * The LiveKit transport, mainly for debugging purposes. */ - focusUrl?: string; + transport?: UnstableLivekitTransport; /** * Called whenever the aspect ratio of the video content becomes known or * otherwise changes. @@ -106,7 +107,7 @@ export const MediaView: FC = ({ audioStreamStats, videoStreamStats, rtcBackendIdentity, - focusUrl, + transport, setVideoAspectRatio: setTheirVideoAspectRatio, ...props }) => { @@ -227,7 +228,7 @@ export const MediaView: FC = ({ diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 71b98e8d8..344eed00e 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -54,6 +54,7 @@ import { platform } from "../Platform"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; import { RingingStatus } from "./RingingStatus"; import { useRootElement } from "../RootElementContext"; +import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; interface SpotlightItemBaseProps { ref?: Ref; @@ -74,7 +75,7 @@ interface SpotlightItemBaseProps { interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps { video: TrackReferenceOrPlaceholder | undefined; unencryptedWarning: boolean; - focusUrl: string | undefined; + transport: UnstableLivekitTransport | undefined; } interface SpotlightUserMediaItemBaseProps extends SpotlightMemberMediaItemBaseProps { @@ -177,13 +178,13 @@ const SpotlightMemberMediaItem: FC = ({ }) => { const video = useBehavior(vm.video$); const unencryptedWarning = useBehavior(vm.unencryptedWarning$); - const focusUrl = useBehavior(vm.focusUrl$); + const transport = useBehavior(vm.transport$); const baseProps: SpotlightMemberMediaItemBaseProps & RefAttributes = { video: video ?? undefined, unencryptedWarning, - focusUrl, + transport, ...props, }; diff --git a/src/utils/matrix.ts b/src/utils/matrix.ts index 68fa68ae4..737590a34 100644 --- a/src/utils/matrix.ts +++ b/src/utils/matrix.ts @@ -338,9 +338,13 @@ export function getRelativeRoomUrl( * Perform a network operation with retries on ConnectionError. * If the error is not retryable, or the max number of retries is reached, the error is rethrown. * Supports handling of matrix quotas. + * @param operation The operation to perform. + * @param shouldRetry Rules out errors that retrying cannot fix, on top of the + * standard backoff rule. Retries everything by default. */ export async function doNetworkOperationWithRetry( operation: () => Promise, + shouldRetry: (e: unknown) => boolean = () => true, ): Promise { let currentRetryCount = 0; @@ -350,7 +354,9 @@ export async function doNetworkOperationWithRetry( return await operation(); } catch (e) { currentRetryCount++; - const backoff = calculateRetryBackoff(e, currentRetryCount, true); + const backoff = shouldRetry(e) + ? calculateRetryBackoff(e, currentRetryCount, true) + : -1; if (backoff < 0) { // Max number of retries reached, or error is not retryable. rethrow the error throw e; diff --git a/src/utils/observable.ts b/src/utils/observable.ts index ca51ad196..b35e523d6 100644 --- a/src/utils/observable.ts +++ b/src/utils/observable.ts @@ -218,6 +218,28 @@ export function filterBehavior( ); } +/** + * Applies a given projection to each value emitted by the source Observable. + * The `project` function additionally receives an {@link ObservableScope}. + */ +export function mapScoped( + name: string, + project: (a: A, scope: ObservableScope) => B, +): OperatorFunction { + return (a$) => + a$.pipe( + generateItemsInternal( + name, + function* (a) { + yield { keys: [a], data: undefined }; + }, + (scope, _data$, a) => project(a, scope), + (items) => items, + ), + map(([item]) => item), + ); +} + function generateItemsInternal< Input, Keys extends [unknown, ...unknown[]], diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index dcdb9f9c9..dac0c6972 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -64,16 +64,23 @@ export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "\u202eevaD", }); -export const testJWTToken = [ - {}, // header - { - // payload - sub: "@me:example.org:ABCDEF", - video: { - room: "!example_room_id", +export function createJWTToken(sub: string, room: string): string { + return [ + {}, // header + { + // payload + sub, + video: { + room, + }, }, - }, - {}, // signature -] - .map((d) => global.btoa(JSON.stringify(d))) - .join("."); + {}, // signature + ] + .map((d) => global.btoa(JSON.stringify(d))) + .join("."); +} + +export const testJWTToken = createJWTToken( + "@me:example.org:ABCDEF", + "!example_room_id", +); diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 9903b99f8..df4f21bb1 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -69,7 +69,7 @@ export function getBasicRTCSession( getUserId: () => localRtcMember.userId, getDeviceId: () => localRtcMember.deviceId, getSyncState: () => SyncState.Syncing, - getDomain: () => null, + getDomain: () => "example.com", getAccessToken: () => "fake-token", sendEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }), redactEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }), diff --git a/src/utils/test.ts b/src/utils/test.ts index 6af9fd8c7..5ab16f2e5 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -26,7 +26,7 @@ import { import { CallMembership, type LivekitFocusSelection, - type LivekitTransport, + type UnstableLivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -78,7 +78,7 @@ import { type RemoteScreenShareViewModel, } from "../state/media/RemoteScreenShareViewModel"; import { Connection } from "../state/CallViewModel/remoteMembers/Connection"; -import { type SFUConfig } from "../livekit/openIDSFU"; +import { type TransportLocator } from "../livekit/auth"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); @@ -209,16 +209,14 @@ export function mockEmitter(): EmitterMock { }; } -export const exampleTransport: LivekitTransport = { +export const exampleTransport: UnstableLivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", }; -export const exampleSfuConfig: SFUConfig = { - jwt: "foo", - livekitAlias: "bar", - livekitIdentity: "baz", - url: "bro", +export const exampleTransportLocator: TransportLocator = { + transport: exampleTransport, + serverName: "example.org", }; export function mockRtcMembership( @@ -226,6 +224,8 @@ export function mockRtcMembership( deviceId: string, customOverwrites?: { rtcBackendIdentity?: string; + /** Every identity the member may have in LiveKit, when more than one. */ + backendIdentities?: string[]; callId?: string; fociPreferred?: Transport[]; focusActive?: LivekitFocusSelection; @@ -261,11 +261,12 @@ export function mockRtcMembership( content: data, }); - const membershipData = CallMembership.membershipDataFromMatrixEvent(event); + const backendIdentity = rtcBackendIdentity ?? `${userId}:${deviceId}`; const cms = new CallMembership( event, - membershipData, - rtcBackendIdentity ?? `${userId}:${deviceId}`, + CallMembership.membershipDataFromMatrixEvent(event), + customOverwrites?.backendIdentities ?? [backendIdentity], + backendIdentity, ); vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); @@ -357,7 +358,7 @@ export function mockLocalMedia( participant$: constant(localParticipant), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, livekitRoom$: constant(mockLivekitRoom({ localParticipant })), - focusUrl$: constant("https://rtc-example.org"), + transport$: constant(exampleTransport), mediaDevices, displayName$: constant(member.rawDisplayName ?? "nodisplayname"), mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), @@ -400,7 +401,7 @@ export function mockRemoteMedia( participant$: constant(participant), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, livekitRoom$: constant(livekitRoom), - focusUrl$: constant("https://rtc-example.org"), + transport$: constant(exampleTransport), pretendToBeDisconnected$: constant(false), displayName$: constant(member.rawDisplayName ?? "nodisplayname"), mxcAvatarUrl$: constant(member.getMxcAvatarUrl()), @@ -427,7 +428,7 @@ export function mockRemoteScreenShare( participant$: constant(participant), encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, livekitRoom$: constant(livekitRoom), - focusUrl$: constant("https://rtc-example.org"), + transport$: constant(exampleTransport), pretendToBeDisconnected$: constant(false), displayName$: constant(member.rawDisplayName ?? "nodisplayname"), mxcAvatarUrl$: constant(member.getMxcAvatarUrl()),