diff --git a/README.md b/README.md index 73505a8d..688a7a7f 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Chat](https://img.shields.io/matrix/webrtc:matrix.org)](https://matrix.to/#/#webrtc:matrix.org) [![Localazy](https://img.shields.io/endpoint?url=https%3A%2F%2Fconnect.localazy.com%2Fstatus%2Felement-call%2Fdata%3Fcontent%3Dall%26title%3Dlocalazy%26logo%3Dtrue)](https://localazy.com/p/element-call) [![License](https://img.shields.io/github/license/element-hq/element-call)](LICENSE-AGPL-3.0) +[![Codecov](https://img.shields.io/codecov/c/github/element-hq/element-call)](https://app.codecov.io/gh/element-hq/element-call) [🎬 Live Demo 🎬](https://call.element.io) diff --git a/backend/dev_homeserver-othersite.yaml b/backend/dev_homeserver-othersite.yaml index 947e33cd..81e775ca 100644 --- a/backend/dev_homeserver-othersite.yaml +++ b/backend/dev_homeserver-othersite.yaml @@ -38,6 +38,8 @@ experimental_features: # MSC4222 needed for syncv2 state_after. This allow clients to # correctly track the state of the room. msc4222_enabled: true + # sticky events for MatrixRTC user state + msc4354_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. Must be a positive value if set. Defaults to no diff --git a/backend/dev_homeserver.yaml b/backend/dev_homeserver.yaml index fe89d95a..dc7b42c8 100644 --- a/backend/dev_homeserver.yaml +++ b/backend/dev_homeserver.yaml @@ -38,7 +38,7 @@ experimental_features: # MSC4222 needed for syncv2 state_after. This allow clients to # correctly track the state of the room. msc4222_enabled: true - # sticky events for matrixRTC user state + # sticky events for MatrixRTC user state msc4354_enabled: true # The maximum allowed duration by which sent events can be delayed, as diff --git a/backend/playwright_homeserver-othersite.yaml b/backend/playwright_homeserver-othersite.yaml index 5cb0dd65..35640ae9 100644 --- a/backend/playwright_homeserver-othersite.yaml +++ b/backend/playwright_homeserver-othersite.yaml @@ -38,6 +38,8 @@ experimental_features: # MSC4222 needed for syncv2 state_after. This allow clients to # correctly track the state of the room. msc4222_enabled: true + # sticky events for MatrixRTC user state + msc4354_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. Must be a positive value if set. Defaults to no diff --git a/backend/playwright_homeserver.yaml b/backend/playwright_homeserver.yaml index 0d7b175c..a83247cd 100644 --- a/backend/playwright_homeserver.yaml +++ b/backend/playwright_homeserver.yaml @@ -38,6 +38,8 @@ experimental_features: # MSC4222 needed for syncv2 state_after. This allow clients to # correctly track the state of the room. msc4222_enabled: true + # sticky events for MatrixRTC user state + msc4354_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. Must be a positive value if set. Defaults to no diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index c7591847..0efefd07 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -3,7 +3,7 @@ networks: services: auth-service: - image: ghcr.io/element-hq/lk-jwt-service:latest-ci + image: ghcr.io/element-hq/lk-jwt-service:pr_139 pull_policy: always hostname: auth-server environment: @@ -25,7 +25,7 @@ services: - ecbackend auth-service-1: - image: ghcr.io/element-hq/lk-jwt-service:latest-ci + image: ghcr.io/element-hq/lk-jwt-service:pr_139 pull_policy: always hostname: auth-server-1 environment: @@ -88,7 +88,7 @@ services: synapse: hostname: homeserver - image: docker.io/matrixdotorg/synapse:latest + image: ghcr.io/element-hq/synapse:pr-18968-dcb7678281bc02d4551043a6338fe5b7e6aa47ce pull_policy: always environment: - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml @@ -106,7 +106,7 @@ services: synapse-1: hostname: homeserver-1 - image: docker.io/matrixdotorg/synapse:latest + image: ghcr.io/element-hq/synapse:pr-18968-dcb7678281bc02d4551043a6338fe5b7e6aa47ce pull_policy: always environment: - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml diff --git a/locales/en/app.json b/locales/en/app.json index 1ff066ea..c70a0a49 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -116,6 +116,7 @@ "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).", "membership_manager": "Membership Manager Error", "membership_manager_description": "The Membership Manager had to shut down. This is caused by many consequtive failed network requests.", + "no_matrix_2_authorization_service": "Your authorization service for you media server (SFU) is not on the newest version", "open_elsewhere": "Opened in another tab", "open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.", "room_creation_restricted": "Failed to create call", diff --git a/package.json b/package.json index 944fc43b..04308630 100644 --- a/package.json +++ b/package.json @@ -125,6 +125,7 @@ "typescript": "^5.8.3", "typescript-eslint-language-service": "^5.0.5", "unique-names-generator": "^4.6.0", + "uuid": "^13.0.0", "vaul": "^1.0.0", "vite": "^7.0.0", "vite-plugin-generate-file": "^0.3.0", diff --git a/sdk/README.md b/sdk/README.md index 03801b83..91337f10 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -6,16 +6,40 @@ It allows to use matrixRTC in combination with livekit without relying on elemen This is done by instantiating the call view model and exposing some useful behaviors (observables) and methods. -This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver ellowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template. +This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver allowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template. + +## Getting started + +To get started run + +``` +yarn +yarn build:sdk +``` + +in the repository root. + +It will create a `dist` folder containing the compiled js file. + +This file needs to be hosted. Locally (via `npx serve -l 81234 --cors`) or on a remote server. + +Now you just need to add the widget to element web via: + +``` +/addwidget http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id +``` ## Widgets -The sdk mode is particularly interesting to be used in widgets where you do not need to pay attention to matrix login/cs api ... -To create a widget see the example index.html file in this folder. And add it to EW via: +The sdk mode is particularly interesting to be used in widgets. In widgets you do not need to pay attention to matrix login/cs api ... +To create a widget see the example `index.html` file in this folder. And add it to EW via: `/addwidget ` (see **url parameters** for more details on ``) ### url parameters +The url parameters are needed to pass initial data to the widget. They will automatically be used +by the matrixRTCSdk to start the postmessage widget api (communication between the client (e.g. Element Web) and the widget) + ``` widgetId = $matrix_widget_id perParticipantE2EE = true diff --git a/sdk/index.html b/sdk/index.html index 1420c3ea..8883b9a3 100644 --- a/sdk/index.html +++ b/sdk/index.html @@ -1,7 +1,7 @@ - Godot MatrixRTC Widget + MatrixRTC Widget - - -
diff --git a/sdk/main.ts b/sdk/main.ts index e8af9a2c..c371587f 100644 --- a/sdk/main.ts +++ b/sdk/main.ts @@ -104,12 +104,7 @@ export async function createMatrixRTCSdk( videoEnabled: true, }); const slot = { application, id }; - const rtcSession = new MatrixRTCSession( - client, - room, - MatrixRTCSession.sessionMembershipsForSlot(room, slot), - slot, - ); + const rtcSession = new MatrixRTCSession(client, room, slot); const callViewModel = createCallViewModel$( scope, rtcSession, diff --git a/src/RTCConnectionStats.tsx b/src/RTCConnectionStats.tsx index d51089cf..f62a1e4d 100644 --- a/src/RTCConnectionStats.tsx +++ b/src/RTCConnectionStats.tsx @@ -20,6 +20,7 @@ interface Props { audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; focusUrl?: string; + rtcBackendIdentity?: string; } const extractDomain = (url: string): string => { @@ -37,6 +38,7 @@ export const RTCConnectionStats: FC = ({ audio, video, focusUrl, + rtcBackendIdentity, ...rest }) => { const [showModal, setShowModal] = useState(false); @@ -71,6 +73,9 @@ export const RTCConnectionStats: FC = ({
+ + rtcBackendIdentity:{rtcBackendIdentity} + {focusUrl && (
diff --git a/src/__snapshots__/AppBar.test.tsx.snap b/src/__snapshots__/AppBar.test.tsx.snap index 25bb54ed..c769ec12 100644 --- a/src/__snapshots__/AppBar.test.tsx.snap +++ b/src/__snapshots__/AppBar.test.tsx.snap @@ -21,7 +21,7 @@ exports[`AppBar > renders 1`] = ` tabindex="0" >
, encryptionKeyIndex: number, - participantId: string, + membershipParts: CallMembershipIdentityParts, + rtcBackendIdentity: string, ): void => { crypto.subtle .importKey("raw", encryptionKey, "HKDF", false, [ @@ -53,17 +56,17 @@ export class MatrixKeyProvider extends BaseKeyProvider { (keyMaterial) => { this.onSetEncryptionKey( keyMaterial, - participantId, + rtcBackendIdentity, encryptionKeyIndex, ); logger.debug( - `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`, + `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}:${membershipParts.deviceId}) encryptionKeyIndex=${encryptionKeyIndex}`, ); }, (e) => { logger.error( - `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`, + `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipParts.userId}:${membershipParts.deviceId} encryptionKeyIndex=${encryptionKeyIndex}`, e, ); }, diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 741529b8..5a4b2257 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -15,7 +15,6 @@ import { type AudioTrackProps, } from "@livekit/components-react"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type ParticipantId } from "matrix-js-sdk/lib/matrixrtc"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; @@ -32,7 +31,7 @@ export interface MatrixAudioRendererProps { * This list needs to be composed based on the matrixRTC members so that we do not play audio from users * that are not expected to be in the rtc session (local user is excluded). */ - validIdentities: ParticipantId[]; + validIdentities: string[]; /** * If set to `true`, mutes all audio tracks rendered by the component. * @remarks diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts index 2a260b01..20820748 100644 --- a/src/livekit/openIDSFU.test.ts +++ b/src/livekit/openIDSFU.test.ts @@ -18,6 +18,7 @@ import fetchMock from "fetch-mock"; import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU"; import { testJWTToken } from "../utils/test-fixtures"; +import { ownMemberMock } from "../utils/test"; const sfuUrl = "https://sfu.example.org"; @@ -33,6 +34,7 @@ describe("getSFUConfigWithOpenID", () => { vitest.clearAllMocks(); fetchMock.reset(); }); + it("should handle fetching a token", async () => { fetchMock.post("https://sfu.example.org/sfu/get", () => { return { @@ -42,6 +44,7 @@ describe("getSFUConfigWithOpenID", () => { }); const config = await getSFUConfigWithOpenID( matrixClient, + ownMemberMock, "https://sfu.example.org", "!example_room_id", ); @@ -53,6 +56,7 @@ describe("getSFUConfigWithOpenID", () => { }); void (await fetchMock.flush()); }); + it("should fail if the SFU errors", async () => { fetchMock.post("https://sfu.example.org/sfu/get", () => { return { @@ -63,11 +67,12 @@ describe("getSFUConfigWithOpenID", () => { try { await getSFUConfigWithOpenID( matrixClient, + ownMemberMock, "https://sfu.example.org", "!example_room_id", ); } catch (ex) { - expect(((ex as Error).cause as Error).message).toEqual( + expect((ex as Error).message).toEqual( "SFU Config fetch failed with status code 500", ); void (await fetchMock.flush()); @@ -76,6 +81,104 @@ describe("getSFUConfigWithOpenID", () => { expect.fail("Expected test to throw;"); }); + it("should try legacy and then new endpoint with delay delegation", async () => { + fetchMock.post("https://sfu.example.org/get_token", () => { + return { + status: 500, + body: { error: "Test failure" }, + }; + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { error: "Test failure" }, + }; + }); + try { + await getSFUConfigWithOpenID( + matrixClient, + ownMemberMock, + "https://sfu.example.org", + "!example_room_id", + { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + }, + ); + } catch (ex) { + expect((ex as Error).message).toEqual( + "SFU Config fetch failed with status code 500", + ); + void (await fetchMock.flush()); + } + const calls = fetchMock.calls(); + expect(calls.length).toBe(2); + + expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); + expect(calls[0][1]).toStrictEqual({ + // check if it uses correct delayID! + body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":1000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', + method: "POST", + headers: { + "Content-Type": "application/json", + }, + }); + + expect(calls[1][0]).toStrictEqual("https://sfu.example.org/sfu/get"); + + expect(calls[1][1]).toStrictEqual({ + body: '{"room":"!example_room_id","device_id":"DEVICE"}', + headers: { + "Content-Type": "application/json", + }, + method: "POST", + }); + }); + + it("dont try legacy if endpoint with delay delegation is sucessful", async () => { + fetchMock.post("https://sfu.example.org/get_token", () => { + return { + status: 200, + body: { url: sfuUrl, jwt: testJWTToken }, + }; + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { error: "Test failure" }, + }; + }); + try { + await getSFUConfigWithOpenID( + matrixClient, + ownMemberMock, + "https://sfu.example.org", + "!example_room_id", + { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + }, + ); + } catch (ex) { + expect((ex as Error).message).toEqual( + "SFU Config fetch failed with status code 500", + ); + void (await fetchMock.flush()); + } + const calls = fetchMock.calls(); + expect(calls.length).toBe(1); + + expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); + expect(calls[0][1]).toStrictEqual({ + // check if it uses correct delayID! + body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":1000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', + method: "POST", + headers: { + "Content-Type": "application/json", + }, + }); + }); + it("should retry fetching the openid token", async () => { let count = 0; matrixClient.getOpenIdToken.mockImplementation(async () => { @@ -98,6 +201,7 @@ describe("getSFUConfigWithOpenID", () => { }); const config = await getSFUConfigWithOpenID( matrixClient, + ownMemberMock, "https://sfu.example.org", "!example_room_id", ); diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 34c98a88..0b7c2c78 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -5,11 +5,21 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk"; -import { logger } from "matrix-js-sdk/lib/logger"; +import { + retryNetworkOperation, + type IOpenIDToken, + type MatrixClient, +} from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; -import { FailToGetOpenIdToken } from "../utils/errors"; +import { + FailToGetOpenIdToken, + NoMatrix2AuthorizationService, +} from "../utils/errors"; import { doNetworkOperationWithRetry } from "../utils/matrix"; +import { Config } from "../config/Config"; +import { JwtEndpointVersion } from "../state/CallViewModel/localMember/LocalTransport"; /** * Configuration and access tokens provided by the SFU on successful authentication. @@ -18,6 +28,7 @@ export interface SFUConfig { url: string; jwt: string; livekitAlias: string; + // NOTE: Currently unused. livekitIdentity: string; } @@ -64,15 +75,32 @@ export type OpenIDClientParts = Pick< * to the matrix RTC backend in order to get acces to the SFU. * It has built-in retry for calls to the homeserver with a backoff policy. * @param client The Matrix client + * @param membership Our own membership identity parts used to send to jwt service. * @param serviceUrl The URL of the livekit SFU service - * @param matrixRoomId The Matrix room ID for which to get the SFU config + * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. + * @param opts Additional options to modify which endpoint with which data will be used to aquire the jwt token. + * @param opts.forceJwtEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination + * instead of a hash. + * This function by default uses whatever is possible with the current jwt service installed next to the SFU. + * For remote connections this does not matter, since we will not publish there we can rely on the newest option. + * For our own connection we can only use the hashed version if we also send the new matrix2.0 sticky events. + * @param opts.delayEndpointBaseUrl The URL of the matrix homeserver. + * @param opts.delayId The delay id used for the jwt service to manage. + * @param logger optional logger. * @returns Object containing the token information * @throws FailToGetOpenIdToken */ export async function getSFUConfigWithOpenID( client: OpenIDClientParts, + membership: CallMembershipIdentityParts, serviceUrl: string, - matrixRoomId: string, + roomId: string, + opts?: { + forceJwtEndpoint?: JwtEndpointVersion; + delayEndpointBaseUrl?: string; + delayId?: string; + }, + logger?: Logger, ): Promise { let openIdToken: IOpenIDToken; try { @@ -84,16 +112,67 @@ export async function getSFUConfigWithOpenID( error instanceof Error ? error : new Error("Unknown error"), ); } - logger.debug("Got openID token", openIdToken); + logger?.debug("Got openID token", openIdToken); - logger.info(`Trying to get JWT for focus ${serviceUrl}...`); - const sfuConfig = await getLiveKitJWT( - client, - serviceUrl, - matrixRoomId, - openIdToken, - ); - logger.info(`Got JWT from call's active focus URL.`); + logger?.info(`Trying to get JWT for focus ${serviceUrl}...`); + + let sfuConfig: { url: string; jwt: string } | undefined; + + const tryBothJwtEndpoints = opts?.forceJwtEndpoint === undefined; // This is for SFUs where we do not publish. + + const forceMatrix2Jwt = + opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0; + + // We want to start using the new endpoint (with optional delay delegation) + // if we can use both or if we are forced to use the new one. + if (tryBothJwtEndpoints || forceMatrix2Jwt) { + try { + sfuConfig = await getLiveKitJWTWithDelayDelegation( + membership, + serviceUrl, + roomId, + openIdToken, + opts?.delayEndpointBaseUrl, + opts?.delayId, + ); + logger?.info(`Got JWT from call's active focus URL.`); + } catch (e) { + if (e instanceof NotSupportedError) { + logger?.warn( + `Failed fetching jwt with matrix 2.0 endpoint (retry with legacy) Not supported`, + e, + ); + sfuConfig = undefined; + } else { + logger?.warn( + `Failed fetching jwt with matrix 2.0 endpoint other issues ->`, + `(not going to try with legacy endpoint: forceOldJwtEndpoint is set to false, we did not get a not supported error from the sfu)`, + e, + ); + // Make this throw a hard error in case we force the matrix2.0 endpoint. + if (forceMatrix2Jwt) + throw new NoMatrix2AuthorizationService(e as Error); + // NEVER get bejond this point if we forceMatrix2 and it failed! + } + } + } + + // DEPRECATED + // here we either have a sfuConfig or we alredy exited because of `if (forceMatrix2) throw ...` + // The only case we can get into this condition is, if `forceMatrix2` is `false` + if (sfuConfig === undefined) { + sfuConfig = await getLiveKitJWT( + membership.deviceId, + serviceUrl, + roomId, + openIdToken, + ); + logger?.info(`Got JWT from call's active focus URL.`); + } + + if (!sfuConfig) { + throw new Error("No `sfuConfig` after trying with old and new endpoints"); + } // Pull the details from the JWT const [, payloadStr] = sfuConfig.jwt.split("."); @@ -104,33 +183,108 @@ export async function getSFUConfigWithOpenID( url: sfuConfig.url, livekitAlias: payload.video.room, // NOTE: Currently unused. + // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. + // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) livekitIdentity: payload.sub, }; } - +const RETRIES = 4; async function getLiveKitJWT( - client: OpenIDClientParts, + deviceId: string, livekitServiceURL: string, - roomName: string, + matrixRoomId: string, openIDToken: IOpenIDToken, ): Promise<{ url: string; jwt: string }> { - try { - const res = await fetch(livekitServiceURL + "/sfu/get", { + let res: Response | undefined; + await retryNetworkOperation(RETRIES, async () => { + res = await fetch(livekitServiceURL + "/sfu/get", { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ - room: roomName, + // This is the actual livekit room alias. For the legacy jwt endpoint simply the room id was used. + room: matrixRoomId, openid_token: openIDToken, - device_id: client.getDeviceId(), + device_id: deviceId, }), }); - if (!res.ok) { - throw new Error("SFU Config fetch failed with status code " + res.status); - } - return await res.json(); - } catch (e) { - throw new Error("SFU Config fetch failed with exception", { cause: e }); + }); + if (!res) { + throw new Error( + `Network error while connecting to jwt service after ${RETRIES} retries`, + ); + } + if (!res.ok) { + throw new Error("SFU Config fetch failed with status code " + res.status); + } + return await res.json(); +} + +class NotSupportedError extends Error { + public constructor(message: string) { + super(message); + this.name = "NotSupported"; } } + +export async function getLiveKitJWTWithDelayDelegation( + membership: CallMembershipIdentityParts, + livekitServiceURL: string, + matrixRoomId: string, + openIDToken: IOpenIDToken, + delayEndpointBaseUrl?: string, + delayId?: string, +): Promise<{ url: string; jwt: string }> { + const { userId, deviceId, memberId } = membership; + + const body = { + room_id: matrixRoomId, + slot_id: "m.call#ROOM", + openid_token: openIDToken, + member: { + id: memberId, + claimed_user_id: userId, + claimed_device_id: deviceId, + }, + }; + + let bodyDalayParts = {}; + // Also check for empty string + if (delayId && delayEndpointBaseUrl) { + const delayTimeoutMs = + Config.get().matrix_rtc_session?.delayed_leave_event_delay_ms ?? 1000; + bodyDalayParts = { + delay_id: delayId, + delay_timeout: delayTimeoutMs, + delay_cs_api_url: delayEndpointBaseUrl, + }; + } + + let res: Response | undefined; + + await retryNetworkOperation(RETRIES, async () => { + res = await fetch(livekitServiceURL + "/get_token", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ ...body, ...bodyDalayParts }), + }); + }); + + if (!res) { + throw new Error( + `Network error while connecting to jwt service after ${RETRIES} retries`, + ); + } + if (!res.ok) { + const msg = "SFU Config fetch failed with status code " + res.status; + if (res.status === 404) { + throw new NotSupportedError(msg); + } else { + throw new Error(msg); + } + } + return await res.json(); +} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index fdbd4461..5b80db3d 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -798,6 +798,8 @@ export const InCallView: FC = ({
); + const allConnections = useBehavior(vm.allConnections$); + return (
= ({ onDismiss={closeSettings} tab={settingsTab} onTabChange={setSettingsTab} - // TODO expose correct data to setttings modal - livekitRooms={[]} + livekitRooms={allConnections + .getConnections() + .map((connectionItem) => ({ + room: connectionItem.livekitRoom, + // TODO compute is local or tag it in the livekit room items already + isLocal: undefined, + url: connectionItem.transport.livekit_service_url, + }))} /> )} diff --git a/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap b/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap index 73a6df12..3f02a49a 100644 --- a/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap +++ b/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap @@ -108,7 +108,7 @@ exports[`ConnectionLostError: Action handling should reset error state 1`] = ` class="error" >
rendering > renders 1`] = ` data-show="false" >
rendering > renders 1`] = ` Only works while using app

+
+

{t("developer_mode.livekit_sfu", { url: livekitRoom.url || "unknown", })} -

+

{livekitRoom.isLocal &&

ws-url: {localSfuUrl?.href}

}

{t("developer_mode.livekit_server_info")}( @@ -321,7 +321,19 @@ export const DeveloperSettingsTab: FC = ({ : "undefined"} {livekitRoom.room.metadata} - +

Local Participant

+ 
+            {livekitRoom.room.localParticipant.identity}
+
+

Remote Participants

+
    + {Array.from(livekitRoom.room.remoteParticipants.keys()).map( + (id) => ( +
  • {id}
    • + ), + )} +
+
))}

{t("developer_mode.environment_variables")}

{JSON.stringify(env, null, 2)}
diff --git a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap index 4a2dada0..2ee3710b 100644 --- a/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap +++ b/src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap @@ -234,12 +234,12 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_inline-field-control_19upo_44" >
renders and matches snapshot 1`] = ` value="legacy" />
@@ -275,19 +275,19 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_inline-field-control_19upo_44" >
@@ -315,11 +315,11 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = ` class="_inline-field-control_19upo_44" >
renders and matches snapshot 1`] = ` value="matrix_2_0" />
@@ -349,46 +349,78 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
-

- LiveKit SFU: wss://local-sfu.example.org -

-

- ws-url: - wss://local-sfu.example.org/ -

-

- LiveKit Server Info - ( - local - ) -

- 
-    {
+    

+      LiveKit SFU: wss://local-sfu.example.org
+    

+    

+      ws-url: 
+      wss://local-sfu.example.org/
+    

+    

+      LiveKit Server Info
+      (
+      local
+      )
+    

+    
+      {
   "region": "local",
   "version": "1.2.3"
 }
-    local-metadata
-  

-  

-    LiveKit SFU: wss://remote-sfu.example.org
-  

-  

-    LiveKit Server Info
-    (
-    remote
-    )
-  

-  
+    

+      Local Participant
+    

+    
+      localParticipantIdentity
+    

+    

+      Remote Participants
+    

+    
    
+
+
- { +

+ LiveKit SFU: wss://remote-sfu.example.org +

+

+ LiveKit Server Info + ( + remote + ) +

+ 
+      {
   "region": "remote",
   "version": "4.5.6"
 }
-    remote-metadata
-
+ remote-metadata + +

+ Local Participant +

+ 
+      localParticipantIdentity
+
+

+ Remote Participants +

+
    +

Environment variables

diff --git a/src/settings/settings.ts b/src/settings/settings.ts index f85e1414..a674f1aa 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -126,7 +126,13 @@ export const alwaysShowIphoneEarpiece = new Setting( export enum MatrixRTCMode { Legacy = "legacy", - Compatibil = "compatibil", + Compatibility = "compatibility", + /** This implies using + * - sticky events + * - hashed RTC backend identity + * - the new endpoint for the jwt token on the local membership (remote memberships will always try the new jwt endpoint first -> then the legacy one) + * - use the hashed identity for the local membership + */ Matrix_2_0 = "matrix_2_0", } diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 3205c07f..376d8986 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -78,11 +78,14 @@ vi.mock("../e2ee/matrixKeyProvider"); const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); vi.mock("../UrlParams", () => ({ getUrlParams })); -vi.mock("../rtcSessionHelpers", async (importOriginal) => ({ - ...(await importOriginal()), - makeTransport: async (): Promise => - Promise.resolve(exampleTransport), -})); +vi.mock( + "../state/CallViewModel/localMember/localTransport", + async (importOriginal) => ({ + ...(await importOriginal()), + makeTransport: async (): Promise => + Promise.resolve(exampleTransport), + }), +); const yesNo = { y: true, @@ -232,7 +235,7 @@ const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; describe.each([ [MatrixRTCMode.Legacy], - [MatrixRTCMode.Compatibil], + [MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0], ])("CallViewModel (%s mode)", (mode) => { const withCallViewModel = withCallViewModelInMode(mode); @@ -1255,11 +1258,6 @@ describe.each([ y: () => { rtcSession.membershipStatus = Status.Connected; }, - n: () => { - // NOTE: This was removed in https://github.com/matrix-org/matrix-js-sdk/pull/5103 accidentally. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - rtcSession.membershipStatus = "Reconnecting" as any; - }, }); schedule(probablyLeftMarbles, { y: () => { diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 5324c65d..0f212101 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -41,10 +41,13 @@ import { } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { + MembershipManagerEvent, type LivekitTransport, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { type IWidgetApiRequest } from "matrix-widget-api"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { v4 as uuidv4 } from "uuid"; import { LocalUserMediaViewModel, @@ -98,7 +101,7 @@ import { type SpotlightLandscapeLayoutMedia, type SpotlightPortraitLayoutMedia, } from "../layout-types.ts"; -import { ElementCallError } from "../../utils/errors.ts"; +import { ElementCallError, UnknownCallError } from "../../utils/errors.ts"; import { type ObservableScope } from "../ObservableScope.ts"; import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts"; import { @@ -106,13 +109,19 @@ import { enterRTCSession, TransportState, } from "./localMember/LocalMember.ts"; -import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; +import { + createLocalTransport$, + JwtEndpointVersion, +} from "./localMember/LocalTransport.ts"; import { createMemberships$, membershipsAndTransports$, } from "../SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; -import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; +import { + type ConnectionManagerData, + createConnectionManager$, +} from "./remoteMembers/ConnectionManager.ts"; import { createMatrixLivekitMembers$, type TaggedParticipant, @@ -261,6 +270,7 @@ export interface CallViewModel { * multiple devices. */ participantCount$: Behavior; + allConnections$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ livekitRoomItems$: Behavior; userMedia$: Behavior; @@ -381,8 +391,11 @@ export function createCallViewModel$( trackProcessorState$: Behavior, ): CallViewModel { const client = matrixRoom.client; - const userId = client.getUserId()!; - const deviceId = client.getDeviceId()!; + const userId = client.getUserId(); + const deviceId = client.getDeviceId(); + if (!(userId && deviceId)) + throw new UnknownCallError(new Error("userId and deviceId are required")); + const livekitKeyProvider = getE2eeKeyProvider( options.encryptionSystem, matrixRTCSession, @@ -415,11 +428,37 @@ export function createCallViewModel$( memberships$, ); + const ownMembershipIdentity: CallMembershipIdentityParts = { + userId, + deviceId, + // This will only be consumed by the sticky membership manager. So it has no impact on legacy calls. + memberId: uuidv4(), + }; + const localTransport$ = createLocalTransport$({ scope: scope, memberships$: memberships$, + ownMembershipIdentity, client, + delayId$: scope.behavior( + ( + fromEvent( + matrixRTCSession, + MembershipManagerEvent.DelayIdChanged, + ) as Observable + ).pipe(map((v) => v ?? null)), + matrixRTCSession.delayId ?? null, + ), roomId: matrixRoom.roomId, + forceJwtEndpoint$: scope.behavior( + matrixRTCMode$.pipe( + map((v) => + v === MatrixRTCMode.Matrix_2_0 + ? JwtEndpointVersion.Matrix_2_0 + : JwtEndpointVersion.Legacy, + ), + ), + ), useOldestMember$: scope.behavior( matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), @@ -439,30 +478,20 @@ export function createCallViewModel$( const connectionManager = createConnectionManager$({ scope: scope, connectionFactory: connectionFactory, - inputTransports$: scope.behavior( - combineLatest( - [ - localTransport$.pipe( - catchError((e: unknown) => { - logger.info( - "dont pass local transport to createConnectionManager$. localTransport$ threw an error", - e, - ); - return of(null); - }), - ), - membershipsAndTransports.transports$, - ], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, + localTransport$: scope.behavior( + localTransport$.pipe( + catchError((e: unknown) => { + logger.info( + "could not pass local transport to createConnectionManager$. localTransport$ threw an error", + e, + ); + return of(null); + }), ), ), - logger, + remoteTransports$: membershipsAndTransports.transports$, + logger: logger, + ownMembershipIdentity, }); const matrixLivekitMembers$ = createMatrixLivekitMembers$({ @@ -493,6 +522,7 @@ export function createCallViewModel$( joinMatrixRTC: (transport: LivekitTransport) => { return enterRTCSession( matrixRTCSession, + ownMembershipIdentity, transport, connectOptions$.value, ); @@ -604,15 +634,14 @@ export function createCallViewModel$( ), ); + const allConnections$ = scope.behavior( + connectionManager.connectionManagerData$.pipe(map((d) => d.value)), + ); const livekitRoomItems$ = scope.behavior( matrixLivekitMembers$.pipe( - tap((val) => { - logger.debug("matrixLivekitMembers$ updated", val.value); - }), - switchMap((membersWithEpoch) => { - const members = membersWithEpoch.value; + switchMap((members) => { const a$ = combineLatest( - members.map((member) => + members.value.map((member) => combineLatest([member.connection$, member.participant.value$]).pipe( map(([connection, participant]) => { // do not render audio for local participant @@ -685,29 +714,29 @@ export function createCallViewModel$( generateItems( function* ([ localMatrixLivekitMember, - { value: matrixLivekitMembers }, + matrixLivekitMembers, duplicateTiles, ]) { - let localParticipantId: string | undefined = undefined; + let localUserMediaId: string | undefined = undefined; // add local member if available if (localMatrixLivekitMember) { const { userId, participant, connection$, membership$ } = localMatrixLivekitMember; - localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional - // const participantId = membership$.value.membershipID; - if (localParticipantId) { - for (let dup = 0; dup < 1 + duplicateTiles; dup++) { - yield { - keys: [ - dup, - localParticipantId, - userId, - participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely - connection$, - ], - data: undefined, - }; - } + + localUserMediaId = `${userId}:${membership$.value.deviceId}`; + const rtcBackendIdentity = membership$.value.rtcBackendIdentity; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [ + dup, + localUserMediaId, + userId, + participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely + connection$, + rtcBackendIdentity, + ], + data: undefined, + }; } } // add remote members that are available @@ -716,13 +745,22 @@ export function createCallViewModel$( participant, connection$, membership$, - } of matrixLivekitMembers) { - const participantId = `${userId}:${membership$.value.deviceId}`; - if (participantId === localParticipantId) continue; - // const participantId = membership$.value?.identity; + } of matrixLivekitMembers.value) { + const userMediaId = `${userId}:${membership$.value.deviceId}`; + const rtcBackendIdentity = membership$.value.rtcBackendIdentity; + // skip local user as we added them manually before + if (userMediaId === localUserMediaId) continue; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [dup, participantId, userId, participant, connection$], + keys: [ + dup, + userMediaId, + userId, + participant, + connection$, + rtcBackendIdentity, + ], data: undefined, }; } @@ -732,10 +770,11 @@ export function createCallViewModel$( scope, _data$, dup, - participantId, + userMediaId, userId, participant, connection$, + rtcBackendIdentity, ) => { const livekitRoom$ = scope.behavior( connection$.pipe(map((c) => c?.livekitRoom)), @@ -751,8 +790,9 @@ export function createCallViewModel$( return new UserMedia( scope, - `${participantId}:${dup}`, + `${userMediaId}:${dup}`, userId, + rtcBackendIdentity, participant, options.encryptionSystem, livekitRoom$, @@ -761,8 +801,8 @@ export function createCallViewModel$( localMembership.reconnecting$, displayName$, matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), - handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), - reactions$.pipe(map((v) => v[participantId] ?? undefined)), + handsRaised$.pipe(map((v) => v[userMediaId]?.time ?? null)), + reactions$.pipe(map((v) => v[userMediaId] ?? undefined)), ); }, ), @@ -1503,6 +1543,7 @@ export function createCallViewModel$( ), null, ), + allConnections$, participantCount$: participantCount$, handsRaised$: handsRaised$, reactions$: reactions$, diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 0d77611b..af12c98b 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -24,6 +24,7 @@ import { mockLivekitRoom, mockMuteStates, withTestScheduler, + ownMemberMock, } from "../../../utils/test"; import { TransportState, @@ -38,6 +39,7 @@ import { constant } from "../../Behavior"; import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; import { ConnectionState, type Connection } from "../remoteMembers/Connection"; import { type Publisher } from "./Publisher"; +import { type LocalTransportWithSFUConfig } from "./LocalTransport"; const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); @@ -103,11 +105,12 @@ describe("LocalMembership", () => { getOldestMembership: vi.fn().mockReturnValue({ getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]), }), - joinRoomSession: vi.fn(), + joinRTCSession: vi.fn(), }) as unknown as MatrixRTCSession; enterRTCSession( mockedSession, + ownMemberMock, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", @@ -119,7 +122,12 @@ describe("LocalMembership", () => { }, ); - expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith( + expect(mockedSession.joinRTCSession).toHaveBeenLastCalledWith( + { + deviceId: "DEVICE", + memberId: "@alice:example.org:DEVICE", + userId: "@alice:example.org", + }, [ { livekit_alias: "roomId", @@ -161,11 +169,12 @@ describe("LocalMembership", () => { }, memberships: [], getFocusInUse: vi.fn(), - joinRoomSession: vi.fn(), + joinRTCSession: vi.fn(), }) as unknown as MatrixRTCSession; enterRTCSession( mockedSession, + ownMemberMock, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", @@ -204,10 +213,11 @@ describe("LocalMembership", () => { it("throws error on missing RTC config error", () => { withTestScheduler(({ scope, hot, expectObservable }) => { - const localTransport$ = scope.behavior( - hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), - null, - ); + const localTransport$ = + scope.behavior( + hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), + null, + ); // we do not need any connection data since we want to fail before reaching that. const mockConnectionManager = { @@ -235,11 +245,23 @@ describe("LocalMembership", () => { }); const aTransport = { - livekit_service_url: "a", - } as LivekitTransport; + transport: { + livekit_service_url: "a", + } as LivekitTransport, + sfuConfig: { + url: "sfu-url", + jwt: "sfu-token", + }, + } as LocalTransportWithSFUConfig; const bTransport = { - livekit_service_url: "b", - } as LivekitTransport; + transport: { + livekit_service_url: "b", + } as LivekitTransport, + sfuConfig: { + url: "sfu-url", + jwt: "sfu-token", + }, + } as LocalTransportWithSFUConfig; const connectionTransportAConnected = { livekitRoom: mockLivekitRoom({ @@ -249,7 +271,7 @@ describe("LocalMembership", () => { } as unknown as LocalParticipant, }), state$: constant(ConnectionState.LivekitConnected), - transport: aTransport, + transport: aTransport.transport, } as unknown as Connection; const connectionTransportAConnecting = { ...connectionTransportAConnected, @@ -258,11 +280,11 @@ describe("LocalMembership", () => { } as unknown as Connection; const connectionTransportBConnected = { state$: constant(ConnectionState.LivekitConnected), - transport: bTransport, + transport: bTransport.transport, livekitRoom: mockLivekitRoom({}), } as unknown as Connection; - it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => { + it("recreates publisher if new connection is used, always unpublish and end tracks", async () => { const scope = new ObservableScope(); const localTransport$ = new BehaviorSubject(aTransport); @@ -310,8 +332,12 @@ describe("LocalMembership", () => { expect(publishers[1].stopTracks).not.toHaveBeenCalled(); expect(publishers[0].stopPublishing).toHaveBeenCalled(); expect(publishers[1].stopPublishing).not.toHaveBeenCalled(); - expect(publisherFactory.mock.calls[0][0].transport).toBe(aTransport); - expect(publisherFactory.mock.calls[1][0].transport).toBe(bTransport); + expect(publisherFactory.mock.calls[0][0].transport).toBe( + aTransport.transport, + ); + expect(publisherFactory.mock.calls[1][0].transport).toBe( + bTransport.transport, + ); scope.end(); await flushPromises(); // stop all tracks after ending scopes @@ -383,7 +409,8 @@ describe("LocalMembership", () => { const scope = new ObservableScope(); const connectionManagerData = new ConnectionManagerData(); - const localTransport$ = new BehaviorSubject(null); + const localTransport$ = + new BehaviorSubject(null); const connectionManagerData$ = new BehaviorSubject( new Epoch(connectionManagerData), ); @@ -460,7 +487,7 @@ describe("LocalMembership", () => { }); ( - connectionManagerData2.getConnectionForTransport(aTransport)! + connectionManagerData2.getConnectionForTransport(aTransport.transport)! .state$ as BehaviorSubject ).next(ConnectionState.LivekitConnected); expect(localMembership.localMemberState$.value).toStrictEqual({ diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index dc22db23..4749e942 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -36,6 +36,7 @@ import { } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { deepCompare } from "matrix-js-sdk/lib/utils"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Behavior } from "../../Behavior.ts"; import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts"; @@ -60,6 +61,7 @@ import { } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; import { and$ } from "../../../utils/observable.ts"; +import { type LocalTransportWithSFUConfig } from "./LocalTransport.ts"; export enum TransportState { /** Not even a transport is available to the LocalMembership */ @@ -125,7 +127,7 @@ interface Props { createPublisherFactory: (connection: Connection) => Publisher; joinMatrixRTC: (transport: LivekitTransport) => void; homeserverConnected: HomeserverConnected; - localTransport$: Behavior; + localTransport$: Behavior; matrixRTCSession: Pick< MatrixRTCSession, "updateCallIntent" | "leaveRoomSession" @@ -233,7 +235,9 @@ export const createLocalMembership$ = ({ return null; } - return connectionData.getConnectionForTransport(localTransport); + return connectionData.getConnectionForTransport( + localTransport.transport, + ); }), tap((connection) => { logger.info( @@ -532,7 +536,7 @@ export const createLocalMembership$ = ({ if (!shouldConnect) return; try { - joinMatrixRTC(transport); + joinMatrixRTC(transport.transport); } catch (error) { logger.error("Error entering RTC session", error); if (error instanceof Error) @@ -551,7 +555,12 @@ export const createLocalMembership$ = ({ ); const participant$ = scope.behavior( - localConnection$.pipe(map((c) => c?.livekitRoom?.localParticipant ?? null)), + localConnection$.pipe( + map((c) => c?.livekitRoom?.localParticipant ?? null), + tap((p) => { + logger.debug("participant$ updated:", p?.identity); + }), + ), ); // Pause upstream of all local media tracks when we're disconnected from @@ -686,18 +695,19 @@ interface EnterRTCSessionOptions { * - Handles retries (fails only after several attempts) * * @param rtcSession - The MatrixRTCSession to join. + * @param ownMembershipIdentity - Options for entering the RTC session. * @param transport - The LivekitTransport to use for this session. - * @param options - Options for entering the RTC session. - * @param options.encryptMedia - Whether to encrypt media. - * @param options.matrixRTCMode - The Matrix RTC mode to use. + * @param options - `encryptMedia`: Whether to encrypt media `matrixRTCMode`: The Matrix RTC mode to use. * @throws If the widget could not send ElementWidgetActions.JoinCall action. */ // Exported for unit testing export function enterRTCSession( rtcSession: MatrixRTCSession, + ownMembershipIdentity: CallMembershipIdentityParts, transport: LivekitTransport, - { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions, + options: EnterRTCSessionOptions, ): void { + const { encryptMedia, matrixRTCMode } = options; PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); @@ -709,10 +719,13 @@ export function enterRTCSession( const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; const { sendNotificationType: notificationType, callIntent } = getUrlParams(); - const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy; + const multiSFU = + matrixRTCMode === MatrixRTCMode.Compatibility || + matrixRTCMode === MatrixRTCMode.Matrix_2_0; // Multi-sfu does not need a preferred foci list. just the focus that is actually used. // TODO where/how do we track errors originating from the ongoing rtcSession? - rtcSession.joinRoomSession( + rtcSession.joinRTCSession( + ownMembershipIdentity, multiSFU ? [] : [transport], multiSFU ? transport : undefined, { diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 3e69bf2c..9199b51e 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -18,8 +18,8 @@ import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, lastValueFrom } from "rxjs"; import fetchMock from "fetch-mock"; -import { mockConfig, flushPromises } from "../../../utils/test"; -import { createLocalTransport$ } from "./LocalTransport"; +import { mockConfig, flushPromises, ownMemberMock } from "../../../utils/test"; +import { createLocalTransport$, JwtEndpointVersion } from "./LocalTransport"; import { constant } from "../../Behavior"; import { Epoch, ObservableScope } from "../../ObservableScope"; import { @@ -39,11 +39,35 @@ describe("LocalTransport", () => { }; let scope: ObservableScope; - beforeEach(() => { - scope = new ObservableScope(); - }); + beforeEach(() => (scope = new ObservableScope())); afterEach(() => scope.end()); + it("throws if config is missing", async () => { + const localTransport$ = createLocalTransport$({ + scope, + roomId: "!room:example.org", + useOldestMember$: constant(false), + memberships$: constant(new Epoch([])), + client: { + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: async () => Promise.resolve([]), + getDomain: () => "", + baseUrl: "example.org", + // These won't be called in this error path but satisfy the type + getOpenIdToken: vi.fn(), + getDeviceId: vi.fn(), + }, + ownMembershipIdentity: ownMemberMock, + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), + delayId$: constant("delay_id_mock"), + }); + await flushPromises(); + + expect(() => localTransport$.value).toThrow( + new MatrixRTCTransportMissingError(""), + ); + }); + it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => { // Provide a valid config so makeTransportInternal resolves a transport const scope = new ObservableScope(); @@ -65,6 +89,7 @@ describe("LocalTransport", () => { useOldestMember$: constant(false), memberships$: constant(new Epoch([])), client: { + baseUrl: "https://lk.example.org", // Use empty domain to skip .well-known and use config directly getDomain: () => "", // eslint-disable-next-line @typescript-eslint/naming-convention @@ -72,6 +97,9 @@ describe("LocalTransport", () => { getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), }, + ownMembershipIdentity: ownMemberMock, + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), + delayId$: constant("delay_id_mock"), }); localTransport$.subscribe( (o) => observations.push(o), @@ -86,6 +114,60 @@ describe("LocalTransport", () => { expect(() => localTransport$.value).toThrow(expectedError); }); + it("emits preferred transport after OpenID resolves", async () => { + // Use config so transport discovery succeeds, but delay OpenID JWT fetch + mockConfig({ + livekit: { livekit_service_url: "https://lk.example.org" }, + }); + + const openIdResolver = Promise.withResolvers(); + + vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue( + openIdResolver.promise, + ); + + const localTransport$ = createLocalTransport$({ + scope, + roomId: "!room:example.org", + useOldestMember$: constant(false), + memberships$: constant(new Epoch([])), + client: { + // eslint-disable-next-line @typescript-eslint/naming-convention + _unstable_getRTCTransports: async () => Promise.resolve([]), + getDomain: () => "", + getOpenIdToken: vi.fn(), + getDeviceId: vi.fn(), + baseUrl: "https://lk.example.org", + }, + ownMembershipIdentity: ownMemberMock, + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), + delayId$: constant("delay_id_mock"), + }); + + openIdResolver.resolve?.({ + url: "https://lk.example.org", + jwt: "jwt", + livekitAlias: "!room:example.org", + livekitIdentity: ownMemberMock.userId + ":" + ownMemberMock.deviceId, + }); + expect(localTransport$.value).toBe(null); + await flushPromises(); + // final + expect(localTransport$.value).toStrictEqual({ + transport: { + livekit_alias: "!room:example.org", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "jwt", + livekitAlias: "!room:example.org", + livekitIdentity: "@alice:example.org:DEVICE", + url: "https://lk.example.org", + }, + }); + }); + it("updates local transport when oldest member changes", async () => { // Use config so transport discovery succeeds, but delay OpenID JWT fetch mockConfig({ @@ -109,7 +191,11 @@ describe("LocalTransport", () => { _unstable_getRTCTransports: async () => Promise.resolve([]), getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), + baseUrl: "https://lk.example.org", }, + ownMembershipIdentity: ownMemberMock, + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), + delayId$: constant("delay_id_mock"), }); openIdResolver.resolve?.(openIdResponse); @@ -117,9 +203,17 @@ describe("LocalTransport", () => { await flushPromises(); // final expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!example_room_id", - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + url: "https://lk.example.org", + }, }); }); @@ -134,11 +228,15 @@ describe("LocalTransport", () => { mockConfig({}); customLivekitUrl.setValue(customLivekitUrl.defaultValue); localTransportOpts = { + ownMembershipIdentity: ownMemberMock, scope, roomId: "!example_room_id", useOldestMember$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), + delayId$: constant(null), memberships$: constant(new Epoch([])), client: { + baseUrl: "https://example.org", getDomain: vi.fn().mockReturnValue(""), // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), @@ -165,9 +263,17 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!example_room_id", - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + url: "https://lk.example.org", + }, }); }); it("supports getting transport via user settings", async () => { @@ -177,9 +283,17 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!example_room_id", - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + url: "https://lk.example.org", + }, }); }); it("supports getting transport via backend", async () => { @@ -191,9 +305,17 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!example_room_id", - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + url: "https://lk.example.org", + }, }); }); it("fails fast if the openID request fails for backend config", async () => { @@ -222,9 +344,17 @@ describe("LocalTransport", () => { expect(localTransport$.value).toBe(null); await flushPromises(); expect(localTransport$.value).toStrictEqual({ - livekit_alias: "!example_room_id", - livekit_service_url: "https://lk.example.org", - type: "livekit", + transport: { + livekit_alias: "!example_room_id", + livekit_service_url: "https://lk.example.org", + type: "livekit", + }, + sfuConfig: { + jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=", + livekitAlias: "!example_room_id", + livekitIdentity: "@lk_user:ABCDEF", + url: "https://lk.example.org", + }, }); expect(fetchMock.done()).toEqual(true); }); @@ -248,11 +378,15 @@ describe("LocalTransport", () => { it("throws if no options are available", async () => { const localTransport$ = createLocalTransport$({ scope, + ownMembershipIdentity: ownMemberMock, roomId: "!example_room_id", useOldestMember$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), + delayId$: constant(null), memberships$: constant(new Epoch([])), client: { getDomain: () => "", + baseUrl: "https://example.org", // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: async () => Promise.resolve([]), // These won't be called in this error path but satisfy the type diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index e72b076f..6e0e56a3 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -23,6 +23,7 @@ import { } from "rxjs"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Behavior } from "../../Behavior.ts"; import { type Epoch, type ObservableScope } from "../../ObservableScope.ts"; @@ -30,9 +31,11 @@ import { Config } from "../../../config/Config.ts"; import { FailToGetOpenIdToken, MatrixRTCTransportMissingError, + NoMatrix2AuthorizationService, } from "../../../utils/errors.ts"; import { getSFUConfigWithOpenID, + type SFUConfig, type OpenIDClientParts, } from "../../../livekit/openIDSFU.ts"; import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts"; @@ -47,11 +50,32 @@ const logger = rootLogger.getChild("[LocalTransport]"); */ interface Props { scope: ObservableScope; + ownMembershipIdentity: CallMembershipIdentityParts; memberships$: Behavior>; - client: Pick & + client: Pick< + MatrixClient, + "getDomain" | "baseUrl" | "_unstable_getRTCTransports" + > & OpenIDClientParts; roomId: string; useOldestMember$: Behavior; + forceJwtEndpoint$: Behavior; + delayId$: Behavior; +} + +export enum JwtEndpointVersion { + Legacy = "legacy", + Matrix_2_0 = "matrix_2_0", +} + +export interface LocalTransportWithSFUConfig { + transport: LivekitTransport; + sfuConfig: SFUConfig; +} +export function isLocalTransportWithSFUConfig( + obj: LivekitTransport | LocalTransportWithSFUConfig, +): obj is LocalTransportWithSFUConfig { + return "transport" in obj && "sfuConfig" in obj; } /** @@ -61,26 +85,53 @@ interface Props { * @prop useOldestMember Whether to use the same transport as the oldest member. * This will only update once the first oldest member appears. Will not recompute if the oldest member leaves. * + * @prop useOldJwtEndpoint$ Whether to set forceOldJwtEndpoint on the returned transport and to use the old JWT endpoint. + * This is used when the connection manager needs to know if it has to use the legacy endpoint which implies a string concatenated rtcBackendIdentity. + * (which is expected for non sticky event based rtc member events) + * @returns The local transport. It will be created using the correct sfu endpoint based on the useOldJwtEndpoint$ value. * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ export const createLocalTransport$ = ({ scope, memberships$, + ownMembershipIdentity, client, roomId, useOldestMember$, -}: Props): Behavior => { + forceJwtEndpoint$, + delayId$, +}: Props): Behavior => { /** * The transport over which we should be actively publishing our media. * undefined when not joined. */ const oldestMemberTransport$ = scope.behavior( - memberships$.pipe( - map( - (memberships) => - memberships.value[0]?.getTransport(memberships.value[0]) ?? null, - ), + combineLatest([memberships$]).pipe( + map(([memberships]) => { + const oldestMember = memberships.value[0]; + const transport = oldestMember?.getTransport(memberships.value[0]); + if (!transport) return null; + return transport; + }), first((t) => t != null && isLivekitTransport(t)), + switchMap((transport) => { + // Get the open jwt token to connect to the sfu + const computeLocalTransportWithSFUConfig = + async (): Promise => { + return { + transport, + sfuConfig: await getSFUConfigWithOpenID( + client, + ownMembershipIdentity, + transport.livekit_service_url, + roomId, + { forceJwtEndpoint: JwtEndpointVersion.Legacy }, + logger, + ), + }; + }; + return from(computeLocalTransportWithSFUConfig()); + }), ), null, ); @@ -91,9 +142,30 @@ export const createLocalTransport$ = ({ * * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ - const preferredTransport$: Behavior = scope.behavior( - customLivekitUrl.value$.pipe( - switchMap((customUrl) => from(makeTransport(client, roomId, customUrl))), + const preferredTransport$ = scope.behavior( + // preferredTransport$ (used for multi sfu) needs to know if we are using the old or new + // jwt endpoint (`get_token` vs `sfu/get`) based on that the jwt endpoint will compute the rtcBackendIdentity + // differently. (sha(`${userId}|${deviceId}|${memberId}`) vs `${userId}|${deviceId}|${memberId}`) + // When using sticky events (we need to use the new endpoint). + combineLatest([customLivekitUrl.value$, delayId$, forceJwtEndpoint$]).pipe( + switchMap(([customUrl, delayId, forceEndpoint]) => { + logger.info( + "Creating preferred transport based on: ", + customUrl, + delayId, + forceEndpoint, + ); + return from( + makeTransport( + client, + ownMembershipIdentity, + roomId, + customUrl, + forceEndpoint, + delayId ?? undefined, + ), + ); + }), ), null, ); @@ -112,7 +184,9 @@ export const createLocalTransport$ = ({ ? (oldestMemberTransport ?? preferredTransport) : preferredTransport, ), - distinctUntilChanged(areLivekitTransportsEqual), + distinctUntilChanged((t1, t2) => + areLivekitTransportsEqual(t1?.transport ?? null, t2?.transport ?? null), + ), ), ); }; @@ -124,25 +198,63 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; * validating auth against the service to ensure it's correct. * Prefers in order: * + * 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw. * 2. The transports returned via the homeserver. * 3. The transports returned via .well-known. * 4. The transport configured in Element Call's config. * * @param client The authenticated Matrix client for the current user + * @param membership The membership identity of the user. * @param roomId The ID of the room to be connected to. * @param urlFromDevSettings Override URL provided by the user's local config. + * @param forceJwtEndpoint Whether to force a specific JWT endpoint + * - `Legacy` / `Matrix_2_0` + * - `get_token` / `sfu/get` + * - not hashing / hashing the backendIdentity + * @param delayId the delay id passed to the jwt service. + * * @returns A fully validated transport config. * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ async function makeTransport( - client: Pick & + client: Pick< + MatrixClient, + "getDomain" | "baseUrl" | "_unstable_getRTCTransports" + > & OpenIDClientParts, + membership: CallMembershipIdentityParts, roomId: string, urlFromDevSettings: string | null, -): Promise { + forceJwtEndpoint: JwtEndpointVersion, + delayId?: string, +): Promise { logger.trace("Searching for a preferred transport"); + async function doOpenIdAndJWTFromUrl( + url: string, + ): Promise { + const sfuConfig = await getSFUConfigWithOpenID( + client, + membership, + url, + roomId, + { + forceJwtEndpoint: forceJwtEndpoint, + delayEndpointBaseUrl: client.baseUrl, + delayId, + }, + logger, + ); + return { + transport: { + type: "livekit", + livekit_service_url: url, + livekit_alias: sfuConfig.livekitAlias, + }, + sfuConfig, + }; + } // We will call `getSFUConfigWithOpenID` once per transport here as it's our // only mechanism of valiation. This means we will also ask the // homeserver for a OpenID token a few times. Since OpenID tokens are single @@ -153,39 +265,29 @@ async function makeTransport( // DEVTOOL: Highest priority: Load from devtool setting if (urlFromDevSettings !== null) { - logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings); // Validate that the SFU is up. Otherwise, we want to fail on this // as we don't permit other SFUs. - const config = await getSFUConfigWithOpenID( - client, - urlFromDevSettings, - roomId, - ); - return { - type: "livekit", - livekit_service_url: urlFromDevSettings, - livekit_alias: config.livekitAlias, - }; + // This will call the jwt/sfu/get endpoint to pre create the livekit room. + logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings); + return await doOpenIdAndJWTFromUrl(urlFromDevSettings); } async function getFirstUsableTransport( transports: Transport[], - ): Promise { + ): Promise { for (const potentialTransport of transports) { if (isLivekitTransportConfig(potentialTransport)) { try { - const { livekitAlias } = await getSFUConfigWithOpenID( - client, + // This will call the jwt/sfu/get endpoint to pre create the livekit room. + return await doOpenIdAndJWTFromUrl( potentialTransport.livekit_service_url, - roomId, ); - return { - ...potentialTransport, - livekit_alias: livekitAlias, - }; } catch (ex) { + // Explictly throw these if (ex instanceof FailToGetOpenIdToken) { - // Explictly throw these + throw ex; + } + if (ex instanceof NoMatrix2AuthorizationService) { throw ex; } logger.debug( @@ -245,18 +347,9 @@ async function makeTransport( const urlFromConf = Config.get().livekit?.livekit_service_url; if (urlFromConf) { try { - const { livekitAlias } = await getSFUConfigWithOpenID( - client, - urlFromConf, - roomId, - ); - const selectedTransport: LivekitTransport = { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; - logger.info("Using config SFU", selectedTransport); - return selectedTransport; + // This will call the jwt/sfu/get endpoint to pre create the livekit room. + logger.info("Using config SFU", urlFromConf); + return await doOpenIdAndJWTFromUrl(urlFromConf); } catch (ex) { if (ex instanceof FailToGetOpenIdToken) { throw ex; @@ -265,5 +358,6 @@ async function makeTransport( } } + // If we do not have returned a transport by now we throw an error throw new MatrixRTCTransportMissingError(domain ?? ""); } diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index c1e24eb4..cc430645 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -26,8 +26,8 @@ import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport"; -import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { Connection, ConnectionState, @@ -40,7 +40,7 @@ import { FailToGetOpenIdToken, } from "../../../utils/errors.ts"; import { testJWTToken } from "../../../utils/test-fixtures.ts"; -import { mockRemoteParticipant } from "../../../utils/test.ts"; +import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; let testScope: ObservableScope; @@ -114,6 +114,7 @@ function setupRemoteConnection(): Connection { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; @@ -155,6 +156,7 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; const connection = new Connection(opts, logger); @@ -170,6 +172,7 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; @@ -220,6 +223,7 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; @@ -259,7 +263,7 @@ describe("Start connection states", () => { capturedState.cause instanceof Error ) { expect(capturedState.cause.message).toContain( - "SFU Config fetch failed with exception", + "SFU Config fetch failed with status code 500", ); expect(connection.transport.livekit_alias).toEqual( livekitFocus.livekit_alias, @@ -277,6 +281,7 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 41dfe665..f286b0cd 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -18,6 +18,7 @@ import { import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, map } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { getSFUConfigWithOpenID, @@ -32,8 +33,21 @@ import { SFURoomCreationRestrictedError, UnknownCallError, } from "../../../utils/errors.ts"; +import { type JwtEndpointVersion } from "../localMember/LocalTransport.ts"; export interface ConnectionOpts { + /** + * For the local transport we already do know the jwt token and url. We can reuse it. + * On top the local transport will send additional data to the jwt server to use delayed event delegation. + */ + existingSFUConfig?: SFUConfig; + /** + * For local connections that use the oldest member pattern. here we have not prefetched the sfuConfig + * and hence we need to let the connection do the jwt token fetching. + */ + forceJwtEndpoint?: JwtEndpointVersion; + /** The identity parts to use on this connection */ + ownMembershipIdentity: CallMembershipIdentityParts; /** The media transport to connect to. */ transport: LivekitTransport; /** The Matrix client to use for OpenID and SFU config requests. */ @@ -129,8 +143,10 @@ export class Connection { try { this._state$.next(ConnectionState.FetchingConfig); // We should already have this information after creating the localTransport. - // It would probably be better to forward this here. - const { url, jwt } = await this.getSFUConfigWithOpenID(); + // only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined) + const { url, jwt } = + this.existingSFUConfig ?? + (await this.getSFUConfigForRemoteConnection()); // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; @@ -186,11 +202,17 @@ export class Connection { } } - protected async getSFUConfigWithOpenID(): Promise { + protected async getSFUConfigForRemoteConnection(): Promise { + // This will only be called for sfu's where we do not publish ourselves. + // For the local connection we will use the existingJwtTokenData return await getSFUConfigWithOpenID( this.client, + this.ownMembershipIdentity, this.transport.livekit_service_url, this.transport.livekit_alias, + // dont pass any custom opts for the subscribe only connections + {}, + this.logger, ); } @@ -212,7 +234,8 @@ export class Connection { private readonly client: OpenIDClientParts; private readonly logger: Logger; - + private readonly ownMembershipIdentity: CallMembershipIdentityParts; + private readonly existingSFUConfig?: SFUConfig; /** * Creates a new connection to a matrix RTC LiveKit backend. * @@ -221,6 +244,8 @@ export class Connection { * @param logger - The logger to use. */ public constructor(opts: ConnectionOpts, logger: Logger) { + this.ownMembershipIdentity = opts.ownMembershipIdentity; + this.existingSFUConfig = opts.existingSFUConfig; this.logger = logger.getChild("[Connection]"); this.logger.info( `Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index c3364059..aa20037c 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -5,7 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { Room as LivekitRoom, type RoomOptions, @@ -16,10 +15,15 @@ import { import { type Logger } from "matrix-js-sdk/lib/logger"; // imported as inline to support worker when loaded from a cdn (cross domain) import E2EEWorker from "livekit-client/e2ee-worker?worker&inline"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport"; import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; -import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import type { + OpenIDClientParts, + SFUConfig, +} from "../../../livekit/openIDSFU.ts"; import type { MediaDevices } from "../../MediaDevices.ts"; import type { Behavior } from "../../Behavior.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; @@ -28,9 +32,11 @@ import { defaultLiveKitOptions } from "../../../livekit/options.ts"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { createConnection( - transport: LivekitTransport, scope: ObservableScope, + transport: LivekitTransport, + ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, + sfuConfig?: SFUConfig, ): Connection; } @@ -78,17 +84,30 @@ export class ECConnectionFactory implements ConnectionFactory { this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; } + /** + * + * @param scope The observable scope (used for clean-up) + * @param transport The transport to use for this connection. + * @param ownMembershipIdentity required to connect (using the jwt service) with the SFU. + * @param logger The logger instance to use for this connection. + * @param sfuConfig optional config in case we already have a token for this connection. + * @returns + */ public createConnection( - transport: LivekitTransport, scope: ObservableScope, + transport: LivekitTransport, + ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, + sfuConfig?: SFUConfig, ): Connection { return new Connection( { + existingSFUConfig: sfuConfig, transport, client: this.client, scope: scope, livekitRoomFactory: this.livekitRoomFactory, + ownMembershipIdentity, }, logger, ); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index 280d8ff7..cf930415 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -18,9 +18,9 @@ import { } from "./ConnectionManager.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; -import { withTestScheduler } from "../../../utils/test.ts"; +import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; -import { type Behavior } from "../../Behavior.ts"; +import { constant, type Behavior } from "../../Behavior.ts"; // Some test constants @@ -49,7 +49,7 @@ beforeEach(() => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (transport: LivekitTransport, scope: ObservableScope) => { + (scope: ObservableScope, transport: LivekitTransport) => { const mockConnection = { transport, remoteParticipants$: new BehaviorSubject([]), @@ -76,10 +76,12 @@ describe("connections$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("a", { + localTransport$: constant(null), + remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), logger: logger, + ownMembershipIdentity: ownMemberMock, }); expectObservable( @@ -115,7 +117,8 @@ describe("connections$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("abcdef", { + localTransport$: constant(null), + remoteTransports$: behavior("abcdef", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1], 1), c: new Epoch([TRANSPORT_1], 2), @@ -124,6 +127,7 @@ describe("connections$ stream", () => { f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5), }), logger: logger, + ownMembershipIdentity: ownMemberMock, }); expectObservable( @@ -160,12 +164,14 @@ describe("connections$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("abc", { + localTransport$: constant(null), + remoteTransports$: behavior("abc", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), c: new Epoch([TRANSPORT_1], 2), }), logger: logger, + ownMembershipIdentity: ownMemberMock, }); expectObservable( @@ -223,7 +229,7 @@ describe("connectionManagerData$ stream", () => { vi.mocked(fakeConnectionFactory).createConnection = vi .fn() .mockImplementation( - (transport: LivekitTransport, scope: ObservableScope) => { + (scope: ObservableScope, transport: LivekitTransport) => { const fakeRemoteParticipants$ = new BehaviorSubject< RemoteParticipant[] >([]); @@ -275,10 +281,12 @@ describe("connectionManagerData$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("a", { + localTransport$: constant(null), + remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), logger, + ownMembershipIdentity: ownMemberMock, }); expectObservable(connectionManagerData$).toBe("abcd", { diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 101e34ed..4295c5f2 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -7,9 +7,10 @@ Please see LICENSE in the repository root for full details. */ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, map, of, switchMap, tap } from "rxjs"; +import { combineLatest, map, of, switchMap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type RemoteParticipant } from "livekit-client"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Behavior } from "../../Behavior.ts"; import { type Connection } from "./Connection.ts"; @@ -17,6 +18,11 @@ import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { generateItemsWithEpoch } from "../../../utils/observable.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; +import { + isLocalTransportWithSFUConfig, + type LocalTransportWithSFUConfig, +} from "../localMember/LocalTransport.ts"; +import { type SFUConfig } from "../../../livekit/openIDSFU.ts"; export class ConnectionManagerData { private readonly store: Map< @@ -65,8 +71,11 @@ export class ConnectionManagerData { interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; - inputTransports$: Behavior>; + localTransport$: Behavior; + remoteTransports$: Behavior>; + logger: Logger; + ownMembershipIdentity: CallMembershipIdentityParts; } // TODO - write test for scopes (do we really need to bind scope) @@ -79,8 +88,12 @@ export interface IConnectionManager { * @param props - Configuration object * @param props.scope - The observable scope used by this object * @param props.connectionFactory - Used to create new connections - * @param props.inputTransports$ - A list of Behaviors each containing a LIST of LivekitTransport. - * @param props.logger - The logger to use + * @param props.localTransport$ - The local transport to use. (deduplicated with remoteTransports$) + * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$) + * @param props.ownMembershipIdentity - The own membership identity to use. + * @param props.logger - The logger to use. + + * * Each of these behaviors can be interpreted as subscribed list of transports. * * Using `registerTransports` independent external modules can control what connections @@ -93,8 +106,10 @@ export interface IConnectionManager { export function createConnectionManager$({ scope, connectionFactory, - inputTransports$, + localTransport$, + remoteTransports$, logger: parentLogger, + ownMembershipIdentity, }: Props): IConnectionManager { const logger = parentLogger.getChild("[ConnectionManager]"); // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing @@ -107,12 +122,33 @@ export function createConnectionManager$({ * It is build based on the list of subscribed transports (`transportsSubscriptions$`). * externally this is modified via `registerTransports()`. */ - const transports$ = scope.behavior( - inputTransports$.pipe( - map((transports) => transports.mapInner(removeDuplicateTransports)), - tap(({ value: transports }) => { - logger.trace( - `Managing transports: ${transports.map((t) => t.livekit_service_url).join(", ")}`, + const localAndRemoteTransports$: Behavior< + Epoch<(LivekitTransport | LocalTransportWithSFUConfig)[]> + > = scope.behavior( + combineLatest([remoteTransports$, localTransport$]).pipe( + // Combine local and remote transports into one transport array + // and set the forceOldJwtEndpoint property on the local transport + map(([remoteTransports, localTransport]) => { + let localTransportAsArray: LocalTransportWithSFUConfig[] = []; + if (localTransport) { + localTransportAsArray = [localTransport]; + } + const dedupedRemote = removeDuplicateTransports(remoteTransports.value); + const remoteWithoutLocal = dedupedRemote.filter( + (transport) => + !localTransportAsArray.find((l) => + areLivekitTransportsEqual(l.transport, transport), + ), + ); + logger.debug( + "remoteWithoutLocal", + remoteWithoutLocal, + "localTransportAsArray", + localTransportAsArray, + ); + return new Epoch( + [...localTransportAsArray, ...remoteWithoutLocal], + remoteTransports.epoch, ); }), ), @@ -122,25 +158,51 @@ export function createConnectionManager$({ * Connections for each transport in use by one or more session members. */ const connections$ = scope.behavior( - transports$.pipe( + localAndRemoteTransports$.pipe( generateItemsWithEpoch( function* (transports) { - for (const transport of transports) - yield { - keys: [transport.livekit_service_url, transport.livekit_alias], - data: undefined, - }; + for (const transportWithOrWithoutSfuConfig of transports) { + if ( + isLocalTransportWithSFUConfig(transportWithOrWithoutSfuConfig) + ) { + // This is the local transport only the `LocalTransportWithSFUConfig` has a `sfuConfig` field + const { transport, sfuConfig } = transportWithOrWithoutSfuConfig; + yield { + keys: [ + transport.livekit_service_url, + transport.livekit_alias, + sfuConfig, + ], + data: undefined, + }; + } else { + const transport = transportWithOrWithoutSfuConfig; + yield { + keys: [ + transport.livekit_service_url, + transport.livekit_alias, + undefined as undefined | SFUConfig, + ], + data: undefined, + }; + } + } }, - (scope, _data$, serviceUrl, alias) => { - logger.debug(`Creating connection to ${serviceUrl} (${alias})`); + (scope, _data$, serviceUrl, alias, sfuConfig) => { + logger.debug( + `Creating connection to ${serviceUrl} (${alias}, withSfuConfig (local connection?): ${JSON.stringify(sfuConfig) ?? "no config->remote connection"})`, + ); + const connection = connectionFactory.createConnection( + scope, { type: "livekit", livekit_service_url: serviceUrl, livekit_alias: alias, }, - scope, + ownMembershipIdentity, logger, + sfuConfig, ); // Start the connection immediately // Use connection state to track connection progress @@ -190,18 +252,18 @@ export function createConnectionManager$({ ); }), ), - new Epoch(new ConnectionManagerData()), + new Epoch(new ConnectionManagerData(), -1), ); return { connectionManagerData$ }; } -function removeDuplicateTransports( - transports: LivekitTransport[], -): LivekitTransport[] { +function removeDuplicateTransports( + transports: T[], +): T[] { return transports.reduce((acc, transport) => { if (!acc.some((t) => areLivekitTransportsEqual(t, transport))) acc.push(transport); return acc; - }, [] as LivekitTransport[]); + }, [] as T[]); } diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index 0c439a6b..b56a17f7 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -15,7 +15,11 @@ import EventEmitter from "events"; import { ObservableScope } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; -import { exampleTransport, mockMediaDevices } from "../../../utils/test.ts"; +import { + exampleTransport, + mockMediaDevices, + ownMemberMock, +} from "../../../utils/test.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; import { constant } from "../../Behavior"; @@ -72,7 +76,12 @@ describe("ECConnectionFactory - Audio inputs options", () => { echo, noise, ); - ecConnectionFactory.createConnection(exampleTransport, testScope, logger); + ecConnectionFactory.createConnection( + testScope, + exampleTransport, + ownMemberMock, + logger, + ); // Check if Room was constructed with expected options expect(RoomConstructor).toHaveBeenCalledWith( @@ -113,7 +122,12 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { false, false, ); - ecConnectionFactory.createConnection(exampleTransport, testScope, logger); + ecConnectionFactory.createConnection( + testScope, + exampleTransport, + ownMemberMock, + logger, + ); // Check if Room was constructed with expected options expect(RoomConstructor).toHaveBeenCalledWith( diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index d26bac37..5d34f7be 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -10,8 +10,7 @@ import { type CallMembership, type LivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; -import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils"; -import { combineLatest, map, type Observable } from "rxjs"; +import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs"; import { type IConnectionManager } from "./ConnectionManager.ts"; import { @@ -26,14 +25,18 @@ import { } from "../../ObservableScope.ts"; import { ConnectionManagerData } from "./ConnectionManager.ts"; import { - mockCallMembership, + flushPromises, + mockRtcMembership, mockRemoteParticipant, - withTestScheduler, } from "../../../utils/test.ts"; import { type Connection } from "./Connection.ts"; +import { constant } from "../../Behavior.ts"; let testScope: ObservableScope; +const fallbackMemberId = (userId: string, deviceId: string): string => + `${userId}:${deviceId}`; + const transportA: LivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", @@ -46,16 +49,12 @@ const transportB: LivekitTransport = { livekit_alias: "!alias:sample.com", }; -const bobMembership = mockCallMembership( - "@bob:example.org", - "DEV000", - transportA, -); -const carlMembership = mockCallMembership( - "@carl:sample.com", - "DEV111", - transportB, -); +const bobMembership = mockRtcMembership("@bob:example.org", "DEV000", { + fociPreferred: [transportA], +}); +const carlMembership = mockRtcMembership("@carl:sample.com", "DEV111", { + fociPreferred: [transportB], +}); beforeEach(() => { testScope = new ObservableScope(); @@ -76,52 +75,41 @@ function epochMeWith$( ); } -test("should signal participant not yet connected to livekit", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const { memberships$, membershipsWithTransport$ } = fromMemberships$( - behavior("a", { - a: [bobMembership], - }), - ); +test("should signal participant not yet connected to livekit", async () => { + const mockedMemberships$ = new BehaviorSubject([bobMembership]); + const mockConnectionManagerData$ = new BehaviorSubject( + new ConnectionManagerData(), + ); + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(mockedMemberships$); - const connectionManagerData$ = epochMeWith$( - memberships$, - behavior("a", { - a: new ConnectionManagerData(), - }), - ); + const connectionManagerData$ = epochMeWith$( + memberships$, + mockConnectionManagerData$, + ); - const matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, - }); - - expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( - "a", - { - a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant.value$).toBe("a", { - a: null, - }); - expectObservable(data[0].connection$).toBe("a", { - a: null, - }); - return true; - }), - }, - ); + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, }); + + await flushPromises(); + expect(matrixLivekitMember$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].participant.value$.value).toBe(null); + expect(data[0].connection$.value).toBe(null); + return true; + }, + ); }); // Helper to create epoch'ed memberships$ and membershipsWithTransport$ from memberships observable. -function fromMemberships$(m$: Observable): { +function createEpochedMemberships$(m$: Observable): { memberships$: Observable>; membershipsWithTransport$: Observable< Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> @@ -146,32 +134,115 @@ function fromMemberships$(m$: Observable): { }; } -test("should signal participant on a connection that is publishing", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const bobParticipantId = getParticipantId( +test("should signal participant on a connection that is publishing", async () => { + const bobParticipantId = fallbackMemberId( + bobMembership.userId, + bobMembership.deviceId, + ); + + const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( + constant([bobMembership]), + ); + + const connection = { + transport: bobMembership.getTransport(bobMembership), + } as unknown as Connection; + const dataWithPublisher = new ConnectionManagerData(); + dataWithPublisher.add(connection, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + + const connectionManagerData$ = epochMeWith$( + memberships$, + constant(dataWithPublisher), + ); + + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + }); + + await flushPromises(); + expect(matrixLivekitMember$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].participant.value$.value).toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }); + expect(data[0].connection$.value).toBe(connection); + return true; + }, + ); +}); + +test("should signal participant on a connection that is not publishing", async () => { + const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( + constant([bobMembership]), + ); + + const connection = { + transport: bobMembership.getTransport(bobMembership), + } as unknown as Connection; + const dataWithPublisher = new ConnectionManagerData(); + dataWithPublisher.add(connection, []); + + const connectionManagerData$ = epochMeWith$( + memberships$, + constant(dataWithPublisher), + ); + + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + }); + await flushPromises(); + expect(matrixLivekitMember$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].participant.value$.value).toBe(null); + expect(data[0].connection$.value).toBe(connection); + return true; + }, + ); +}); + +describe("Publication edge case", () => { + test("bob is publishing in several connections", async () => { + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(constant([bobMembership, carlMembership])); + + const connectionWithPublisher = new ConnectionManagerData(); + const bobParticipantId = fallbackMemberId( bobMembership.userId, bobMembership.deviceId, ); - - const { memberships$, membershipsWithTransport$ } = fromMemberships$( - behavior("a", { - a: [bobMembership], - }), - ); - - const connection = { - transport: bobMembership.getTransport(bobMembership), + const connectionA = { + transport: transportA, } as unknown as Connection; - const dataWithPublisher = new ConnectionManagerData(); - dataWithPublisher.add(connection, [ + const connectionB = { + transport: transportB, + } as unknown as Connection; + + connectionWithPublisher.add(connectionA, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + connectionWithPublisher.add(connectionB, [ mockRemoteParticipant({ identity: bobParticipantId }), ]); const connectionManagerData$ = epochMeWith$( memberships$, - behavior("a", { - a: dataWithPublisher, - }), + constant(connectionWithPublisher), ); const matrixLivekitMembers$ = createMatrixLivekitMembers$({ @@ -181,213 +252,73 @@ test("should signal participant on a connection that is publishing", () => { connectionManagerData$: connectionManagerData$, } as unknown as IConnectionManager, }); - - expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( - "a", - { - a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant.value$).toBe("a", { - a: expect.toSatisfy((participant) => { - expect(participant).toBeDefined(); - expect(participant!.identity).toEqual(bobParticipantId); - return true; - }), - }); - expectObservable(data[0].connection$).toBe("a", { - a: connection, - }); + await flushPromises(); + expect(matrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(2); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].connection$.value).toBe(connectionA); + expect(data[0].participant.value$.value).toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); return true; - }), + }); + + return true; }, ); }); }); -test("should signal participant on a connection that is not publishing", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const { memberships$, membershipsWithTransport$ } = fromMemberships$( - behavior("a", { - a: [bobMembership], - }), - ); +test("bob is publishing in the wrong connection", async () => { + const mockedMemberships$ = new BehaviorSubject([ + bobMembership, + carlMembership, + ]); - const connection = { - transport: bobMembership.getTransport(bobMembership), - } as unknown as Connection; - const dataWithPublisher = new ConnectionManagerData(); - dataWithPublisher.add(connection, []); + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(mockedMemberships$); - const connectionManagerData$ = epochMeWith$( - memberships$, - behavior("a", { - a: dataWithPublisher, - }), - ); + const connectionWithPublisher = new ConnectionManagerData(); - const matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, - }); + const bobParticipantId = fallbackMemberId( + bobMembership.userId, + bobMembership.deviceId, + ); + const connectionA = { transport: transportA } as unknown as Connection; + const connectionB = { transport: transportB } as unknown as Connection; - expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( - "a", - { - a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant.value$).toBe("a", { - a: null, - }); - expectObservable(data[0].connection$).toBe("a", { - a: connection, - }); - return true; - }), - }, - ); - }); -}); - -describe("Publication edge case", () => { - test("bob is publishing in several connections", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const { memberships$, membershipsWithTransport$ } = fromMemberships$( - behavior("a", { - a: [bobMembership, carlMembership], - }), - ); - - const connectionWithPublisher = new ConnectionManagerData(); - const bobParticipantId = getParticipantId( - bobMembership.userId, - bobMembership.deviceId, - ); - const connectionA = { - transport: transportA, - } as unknown as Connection; - const connectionB = { - transport: transportB, - } as unknown as Connection; - - connectionWithPublisher.add(connectionA, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); - connectionWithPublisher.add(connectionB, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); - - const connectionManagerData$ = epochMeWith$( - memberships$, - behavior("a", { - a: connectionWithPublisher, - }), - ); - - const matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior( - membershipsWithTransport$, - ), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, - }); - - expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( - "a", - { - a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(2); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].connection$).toBe("a", { - // The real connection should be from transportA as per the membership - a: connectionA, - }); - expectObservable(data[0].participant.value$).toBe("a", { - a: expect.toSatisfy((participant) => { - expect(participant).toBeDefined(); - expect(participant!.identity).toEqual(bobParticipantId); - return true; - }), - }); - return true; - }), - }, - ); - }); - }); - - test("bob is publishing in the wrong connection", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const { memberships$, membershipsWithTransport$ } = fromMemberships$( - behavior("a", { - a: [bobMembership, carlMembership], - }), - ); - - const connectionWithPublisher = new ConnectionManagerData(); - const bobParticipantId = getParticipantId( - bobMembership.userId, - bobMembership.deviceId, - ); - const connectionA = { transport: transportA } as unknown as Connection; - const connectionB = { transport: transportB } as unknown as Connection; - - // Bob is not publishing on A - connectionWithPublisher.add(connectionA, []); - // Bob is publishing on B but his membership says A - connectionWithPublisher.add(connectionB, [ - mockRemoteParticipant({ identity: bobParticipantId }), - ]); - - const connectionManagerData$ = epochMeWith$( - memberships$, - behavior("a", { - a: connectionWithPublisher, - }), - ); - - const matrixLivekitMembers$ = createMatrixLivekitMembers$({ - scope: testScope, - membershipsWithTransport$: testScope.behavior( - membershipsWithTransport$, - ), - connectionManager: { - connectionManagerData$: connectionManagerData$, - } as unknown as IConnectionManager, - }); - - expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( - "a", - { - a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(2); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].connection$).toBe("a", { - // The real connection should be from transportA as per the membership - a: connectionA, - }); - expectObservable(data[0].participant.value$).toBe("a", { - // No participant as Bob is not publishing on his membership transport - a: null, - }); - return true; - }), - }, - ); - }); + // Bob is not publishing on A + connectionWithPublisher.add(connectionA, []); + // Bob is publishing on B but his membership says A + connectionWithPublisher.add(connectionB, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + + const connectionsWithPublisher$ = new BehaviorSubject( + connectionWithPublisher, + ); + const connectionManagerData$ = epochMeWith$( + memberships$, + connectionsWithPublisher$, + ); + + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, }); + + await flushPromises(); + expect(matrixLivekitMember$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(2); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].connection$.value).toBe(connectionA); + expect(data[0].participant.value$.value).toBe(null); + return true; + }, + ); }); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 6501adb4..10a3e2cb 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -84,7 +84,6 @@ export function createMatrixLivekitMembers$({ /** * Stream of all the call members and their associated livekit data (if available). */ - return scope.behavior( combineLatest([ membershipsWithTransport$, @@ -93,47 +92,39 @@ export function createMatrixLivekitMembers$({ filter((values) => values.every((value) => value.epoch === values[0].epoch), ), - map( - ([ - { value: membershipsWithTransports, epoch }, - { value: managerData }, - ]) => - new Epoch([membershipsWithTransports, managerData] as const, epoch), - ), + map(([ms, data]) => new Epoch([ms.value, data.value] as const, ms.epoch)), generateItemsWithEpoch( // Generator function. // creates an array of `{key, data}[]` // Each change in the keys (new key, missing key) will result in a call to the factory function. - function* ([membershipsWithTransports, managerData]) { - for (const { membership, transport } of membershipsWithTransports) { - // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to - const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; - + function* ([membershipsWithTransport, managerData]) { + for (const { membership, transport } of membershipsWithTransport) { const participants = transport ? managerData.getParticipantsForTransport(transport) : []; const participant = - participants.find((p) => p.identity == participantId) ?? null; + participants.find( + (p) => p.identity == membership.rtcBackendIdentity, + ) ?? null; const connection = transport ? managerData.getConnectionForTransport(transport) : null; yield { - keys: [participantId, membership.userId], + keys: [membership.userId, membership.deviceId], data: { membership, participant, connection }, }; } }, // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory. - (scope, data$, participantId, userId) => { + (scope, data$, userId, deviceId) => { logger.debug( - `Generating member for participantId: ${participantId}, userId: ${userId}`, + `Generating member for livekitIdentity: ${data$.value.membership.rtcBackendIdentity}, userId:deviceId: ${userId}${deviceId}`, ); const { participant$, ...rest } = scope.splitBehavior(data$); // will only get called once per `participantId, userId` pair. // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. return { - participantId, userId, participant: { type: "remote" as const, value$: participant$ }, ...rest, @@ -141,15 +132,16 @@ export function createMatrixLivekitMembers$({ }, ), ), + new Epoch([], -1), ); } // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) // TODO add this to the JS-SDK -export function areLivekitTransportsEqual( - t1: LivekitTransport | null, - t2: LivekitTransport | null, +export function areLivekitTransportsEqual( + t1: T | null, + t2: T | null, ): boolean { if (t1 && t2) return t1.livekit_service_url === t2.livekit_service_url; // In case we have different lk rooms in the same SFU (depends on the livekit authorization service) diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts index 6f392351..f7dd775c 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts @@ -18,7 +18,7 @@ import { it } from "vitest"; import { ObservableScope } from "../../ObservableScope.ts"; import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; import { - mockCallMembership, + mockRtcMembership, mockMatrixRoomMember, withTestScheduler, } from "../../../utils/test.ts"; @@ -111,7 +111,7 @@ describe("MatrixMemberMetadata", () => { rawDisplayName: "it's a me", }); const memberships$ = behavior("a", { - a: [mockCallMembership("@local:example.com", "DEVICE1")], + a: [mockRtcMembership("@local:example.com", "DEVICE1")], }); const metadataStore = createMatrixMemberMetadata$( testScope, @@ -149,8 +149,8 @@ describe("MatrixMemberMetadata", () => { withTestScheduler(({ behavior, expectObservable }) => { const memberships$ = behavior("a", { a: [ - mockCallMembership("@alice:example.com", "DEVICE1"), - mockCallMembership("@bob:example.com", "DEVICE1"), + mockRtcMembership("@alice:example.com", "DEVICE1"), + mockRtcMembership("@bob:example.com", "DEVICE1"), ], }); const metadataStore = createMatrixMemberMetadata$( @@ -179,7 +179,7 @@ describe("MatrixMemberMetadata", () => { setUpBasicRoom(); const memberships$ = behavior("a", { - a: [mockCallMembership("@no-name:foo.bar", "D000")], + a: [mockRtcMembership("@no-name:foo.bar", "D000")], }); const metadataStore = createMatrixMemberMetadata$( testScope, @@ -201,11 +201,11 @@ describe("MatrixMemberMetadata", () => { const memberships$ = behavior("a", { a: [ - mockCallMembership("@bob:example.com", "DEVICE1"), - mockCallMembership("@bob:example.com", "DEVICE2"), - mockCallMembership("@bob:foo.bar", "BOB000"), - mockCallMembership("@carl:example.com", "C000"), - mockCallMembership("@evil:example.com", "E000"), + mockRtcMembership("@bob:example.com", "DEVICE1"), + mockRtcMembership("@bob:example.com", "DEVICE2"), + mockRtcMembership("@bob:foo.bar", "BOB000"), + mockRtcMembership("@carl:example.com", "C000"), + mockRtcMembership("@evil:example.com", "E000"), ], }); @@ -233,10 +233,10 @@ describe("MatrixMemberMetadata", () => { setUpBasicRoom(); const memberships$ = behavior("ab", { - a: [mockCallMembership("@bob:example.com", "DEVICE1")], + a: [mockRtcMembership("@bob:example.com", "DEVICE1")], b: [ - mockCallMembership("@bob:example.com", "DEVICE1"), - mockCallMembership("@bob:foo.bar", "BOB000"), + mockRtcMembership("@bob:example.com", "DEVICE1"), + mockRtcMembership("@bob:foo.bar", "BOB000"), ], }); @@ -262,10 +262,10 @@ describe("MatrixMemberMetadata", () => { const memberships$ = behavior("ab", { a: [ - mockCallMembership("@bob:example.com", "DEVICE1"), - mockCallMembership("@bob:foo.bar", "BOB000"), + mockRtcMembership("@bob:example.com", "DEVICE1"), + mockRtcMembership("@bob:foo.bar", "BOB000"), ], - b: [mockCallMembership("@bob:example.com", "DEVICE1")], + b: [mockRtcMembership("@bob:example.com", "DEVICE1")], }); const metadataStore = createMatrixMemberMetadata$( @@ -292,8 +292,8 @@ describe("MatrixMemberMetadata", () => { const memberships$ = behavior("a", { a: [ - mockCallMembership("@bob:example.com", "B000"), - mockCallMembership("@carl:example.com", "C000"), + mockRtcMembership("@bob:example.com", "B000"), + mockRtcMembership("@carl:example.com", "C000"), ], }); const metadataStore = createMatrixMemberMetadata$( @@ -331,16 +331,16 @@ describe("MatrixMemberMetadata", () => { // - room join/leave // - disambiguate const memberships$ = behavior("ab-d", { - a: [mockCallMembership(CARL, "C000")], + a: [mockRtcMembership(CARL, "C000")], b: [ - mockCallMembership(CARL, "C000"), + mockRtcMembership(CARL, "C000"), // bob joins - mockCallMembership(BOB, "B000"), + mockRtcMembership(BOB, "B000"), ], // c carl gets renamed to BOB d: [ // carl leaves - mockCallMembership(BOB, "B000"), + mockRtcMembership(BOB, "B000"), ], }); schedule("--a-", { @@ -379,8 +379,8 @@ describe("MatrixMemberMetadata", () => { it("should disambiguate users with invisible characters", () => { withTestScheduler(({ behavior, expectObservable }) => { - const bobRtcMember = mockCallMembership("@bob:example.org", "BBBB"); - const bobZeroWidthSpaceRtcMember = mockCallMembership( + const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); + const bobZeroWidthSpaceRtcMember = mockRtcMembership( "@bob2:example.org", "BBBB", ); @@ -397,9 +397,9 @@ describe("MatrixMemberMetadata", () => { fakeMemberWith(bobZeroWidthSpace); fakeMemberWith({ userId: "@carol:example.org" }); const memberships$ = behavior("ab", { - a: [mockCallMembership("@carol:example.org", "1111"), bobRtcMember], + a: [mockRtcMembership("@carol:example.org", "1111"), bobRtcMember], b: [ - mockCallMembership("@carol:example.org", "1111"), + mockRtcMembership("@carol:example.org", "1111"), bobRtcMember, bobZeroWidthSpaceRtcMember, ], @@ -450,8 +450,8 @@ describe("MatrixMemberMetadata", () => { it("should strip RTL characters from displayname", () => { withTestScheduler(({ behavior, expectObservable }) => { - const daveRtcMember = mockCallMembership("@dave:example.org", "DDDD"); - const daveRTLRtcMember = mockCallMembership( + const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); + const daveRTLRtcMember = mockRtcMembership( "@dave2:example.org", "DDDD", ); @@ -466,9 +466,9 @@ describe("MatrixMemberMetadata", () => { fakeMemberWith(daveRTL); fakeMemberWith(dave); const memberships$ = behavior("ab", { - a: [mockCallMembership("@carol:example.org", "DDDD")], + a: [mockRtcMembership("@carol:example.org", "DDDD")], b: [ - mockCallMembership("@carol:example.org", "DDDD"), + mockRtcMembership("@carol:example.org", "DDDD"), daveRtcMember, daveRTLRtcMember, ], @@ -527,8 +527,8 @@ describe("MatrixMemberMetadata", () => { }); const memberships$ = behavior("a", { a: [ - mockCallMembership("@local:example.com", "DEVICE1"), - mockCallMembership("@alice:example.com", "DEVICE1"), + mockRtcMembership("@local:example.com", "DEVICE1"), + mockRtcMembership("@alice:example.com", "DEVICE1"), ], }); const metadataStore = createMatrixMemberMetadata$( @@ -562,12 +562,12 @@ describe("MatrixMemberMetadata", () => { fakeMemberWith({ userId: "@carl:example.com" }); fakeMemberWith({ userId: "@bob:example.com" }); const memberships$ = behavior("ab-d", { - a: [mockCallMembership("@bob:example.com", "B000")], + a: [mockRtcMembership("@bob:example.com", "B000")], b: [ - mockCallMembership("@bob:example.com", "B000"), - mockCallMembership("@carl:example.com", "C000"), + mockRtcMembership("@bob:example.com", "B000"), + mockRtcMembership("@carl:example.com", "C000"), ], - d: [mockCallMembership("@carl:example.com", "C000")], + d: [mockRtcMembership("@carl:example.com", "C000")], }); const metadataStore = createMatrixMemberMetadata$( diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index d885ddc6..c29f07c0 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -21,8 +21,9 @@ import { import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { - mockCallMembership, mockMediaDevices, + mockRtcMembership, + ownMemberMock, withTestScheduler, } from "../../../utils/test.ts"; import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; @@ -33,6 +34,7 @@ import { } from "./MatrixLivekitMembers.ts"; import { createConnectionManager$ } from "./ConnectionManager.ts"; import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; +import { constant } from "../../Behavior.ts"; import { testJWTToken } from "../../../utils/test-fixtures.ts"; // Test the integration of ConnectionManager and MatrixLivekitMerger @@ -99,9 +101,9 @@ afterEach(() => { test("bob, carl, then bob joining no tracks yet", () => { withTestScheduler(({ expectObservable, behavior, scope }) => { - const bobMembership = mockCallMembership("@bob:example.com", "BDEV000"); - const carlMembership = mockCallMembership("@carl:example.com", "CDEV000"); - const daveMembership = mockCallMembership("@dave:foo.bar", "DDEV000"); + const bobMembership = mockRtcMembership("@bob:example.com", "BDEV000"); + const carlMembership = mockRtcMembership("@carl:example.com", "CDEV000"); + const daveMembership = mockRtcMembership("@dave:foo.bar", "DDEV000"); const eMarble = "abc"; const vMarble = "abc"; @@ -121,8 +123,10 @@ test("bob, carl, then bob joining no tracks yet", () => { const connectionManager = createConnectionManager$({ scope: testScope, connectionFactory: ecConnectionFactory, - inputTransports$: membershipsAndTransports.transports$, + localTransport$: constant(null), + remoteTransports$: membershipsAndTransports.transports$, logger: logger, + ownMembershipIdentity: ownMemberMock, }); const matrixLivekitMembers$ = createMatrixLivekitMembers$({ diff --git a/src/state/CallViewModelWidget.test.ts b/src/state/CallViewModelWidget.test.ts index 5d6442f1..76776720 100644 --- a/src/state/CallViewModelWidget.test.ts +++ b/src/state/CallViewModelWidget.test.ts @@ -37,7 +37,7 @@ vi.mock("../widget", () => ({ it.each([ [MatrixRTCMode.Legacy], - [MatrixRTCMode.Compatibil], + [MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0], ])( "expect leave when ElementWidgetActions.HangupCall is called (%s mode)", diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 9888d6bf..3da69c46 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -257,6 +257,7 @@ abstract class BaseMediaViewModel { * The Matrix user to which this media belongs. */ public readonly userId: string, + public readonly rtcBackendIdentity: string, // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through // livekit. protected readonly participant$: Observable< @@ -406,6 +407,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcBackendIdentity: string, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -419,6 +421,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { scope, id, userId, + rtcBackendIdentity, participant$, encryptionSystem, Track.Source.Microphone, @@ -544,6 +547,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcBackendIdentity: string, participant$: Behavior, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -558,6 +562,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { scope, id, userId, + rtcBackendIdentity, participant$, encryptionSystem, livekitRoom$, @@ -671,6 +676,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcBackendIdentity: string, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -685,6 +691,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { scope, id, userId, + rtcBackendIdentity, participant$, encryptionSystem, livekitRoom$, @@ -772,6 +779,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcBackendIdentity: string, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -785,6 +793,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { scope, id, userId, + rtcBackendIdentity, participant$, encryptionSystem, Track.Source.ScreenShareAudio, diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts index 0a241cdf..e4f5de1f 100644 --- a/src/state/ScreenShare.ts +++ b/src/state/ScreenShare.ts @@ -28,6 +28,7 @@ export class ScreenShare { private readonly scope: ObservableScope, id: string, userId: string, + rtcBackendIdentity: string, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -40,6 +41,7 @@ export class ScreenShare { this.scope, id, userId, + rtcBackendIdentity, of(participant), encryptionSystem, livekitRoom$, diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index 690870e6..2a125257 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -75,6 +75,7 @@ export class UserMedia { this.scope, this.id, this.userId, + this.rtcBackendIdentity, this.participant.value$, this.encryptionSystem, this.livekitRoom$, @@ -89,6 +90,7 @@ export class UserMedia { this.scope, this.id, this.userId, + this.rtcBackendIdentity, this.participant.value$, this.encryptionSystem, this.livekitRoom$, @@ -140,6 +142,7 @@ export class UserMedia { scope, `${this.id}:${key}`, this.userId, + this.rtcBackendIdentity, p, this.encryptionSystem, this.livekitRoom$, @@ -191,6 +194,7 @@ export class UserMedia { private readonly scope: ObservableScope, public readonly id: string, private readonly userId: string, + private readonly rtcBackendIdentity: string, private readonly participant: TaggedParticipant, private readonly encryptionSystem: EncryptionSystem, private readonly livekitRoom$: Behavior, diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 2f750c50..92262f05 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -113,6 +113,7 @@ const UserMediaTile: FC = ({ }, [vm], ); + const rtcBackendIdentity = vm.rtcBackendIdentity; const handRaised = useBehavior(vm.handRaised$); const reaction = useBehavior(vm.reaction$); @@ -200,6 +201,7 @@ const UserMediaTile: FC = ({ focusUrl={focusUrl} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} + rtcBackendIdentity={rtcBackendIdentity} {...props} /> ); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 8bb38d94..fadc9d2b 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -18,7 +18,11 @@ import styles from "./MediaView.module.css"; import { Avatar } from "../Avatar"; import { type EncryptionStatus } from "../state/MediaViewModel"; import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator"; -import { showHandRaisedTimer, useSetting } from "../settings/settings"; +import { + showConnectionStats, + showHandRaisedTimer, + useSetting, +} from "../settings/settings"; import { type ReactionOption } from "../reactions"; import { ReactionIndicator } from "../reactions/ReactionIndicator"; import { RTCConnectionStats } from "../RTCConnectionStats"; @@ -46,6 +50,7 @@ interface Props extends ComponentProps { waitingForMedia?: boolean; audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; + rtcBackendIdentity?: string; // The focus url, mainly for debugging purposes focusUrl?: string; } @@ -74,11 +79,13 @@ export const MediaView: FC = ({ waitingForMedia, audioStreamStats, videoStreamStats, + rtcBackendIdentity, focusUrl, ...props }) => { const { t } = useTranslation(); const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer); + const [showConnectioStats] = useSetting(showConnectionStats); const avatarSize = Math.round(Math.min(targetWidth, targetHeight) / 2); @@ -132,14 +139,18 @@ export const MediaView: FC = ({ {waitingForMedia && (
{t("video_tile.waiting_for_media")} + {showConnectioStats ? " " + rtcBackendIdentity : ""}
)} {(audioStreamStats || videoStreamStats) && ( - + <> + + )} {/* TODO: Bring this back once encryption status is less broken */} {/*encryptionStatus !== EncryptionStatus.Okay && ( diff --git a/src/utils/errors.ts b/src/utils/errors.ts index cddf90de..20a282e6 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -19,6 +19,7 @@ export enum ErrorCode { INSUFFICIENT_CAPACITY_ERROR = "INSUFFICIENT_CAPACITY_ERROR", E2EE_NOT_SUPPORTED = "E2EE_NOT_SUPPORTED", OPEN_ID_ERROR = "OPEN_ID_ERROR", + NO_MATRIX_2_AUTHORIZATION_SERVICE = "NO_MATRIX_2_0_AUTHORIZATION_SERVICE", SFU_ERROR = "SFU_ERROR", UNKNOWN_ERROR = "UNKNOWN_ERROR", } @@ -171,6 +172,23 @@ export class FailToGetOpenIdToken extends ElementCallError { } } +export class NoMatrix2AuthorizationService extends ElementCallError { + /** + * Creates an instance of NoMatrix2_0AuthorizationService. + * @param error - The underlying error that caused the failure. + */ + public constructor(error: Error) { + super( + t("error.generic"), + ErrorCode.NO_MATRIX_2_AUTHORIZATION_SERVICE, + ErrorCategory.CONFIGURATION_ISSUE, + t("error.no_matrix_2_authorization_service"), + // Properly set it as a cause for a better reporting on sentry + error, + ); + } +} + /** * Error indicating a failure to start publishing on a LiveKit connection. */ diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index f915bb19..dcdb9f9c 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -17,14 +17,16 @@ export const localRtcMemberDevice2 = mockRtcMembership( "2222", ); export const local = mockMatrixRoomMember(localRtcMember); -// export const localParticipant = mockLocalParticipant({ identity: "" }); + export const localId = `${local.userId}:${localRtcMember.deviceId}`; -export const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); +export const aliceDeviceId = "AAAA"; +export const aliceUserId = "@alice:example.org"; +export const aliceId = `${aliceUserId}:${aliceDeviceId}`; +export const aliceRtcMember = mockRtcMembership(aliceUserId, aliceDeviceId); export const alice = mockMatrixRoomMember(aliceRtcMember, { rawDisplayName: "Alice", }); -export const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`; export const aliceParticipant = mockRemoteParticipant({ identity: aliceId }); export const aliceDoppelgangerRtcMember = mockRtcMembership( @@ -38,11 +40,13 @@ export const aliceDoppelganger = mockMatrixRoomMember( }, ); -export const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); +export const bobDeviceId = "BBBB"; +export const bobUserId = "@bob:example.org"; +export const bobId = `${bobUserId}:${bobDeviceId}`; +export const bobRtcMember = mockRtcMembership(bobUserId, bobDeviceId); export const bob = mockMatrixRoomMember(bobRtcMember, { rawDisplayName: "Bob", }); -export const bobId = `${bob.userId}:${bobRtcMember.deviceId}`; export const bobZeroWidthSpaceRtcMember = mockRtcMembership( "@bob2:example.org", diff --git a/src/utils/test.ts b/src/utils/test.ts index 44ac2257..c99eb77d 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -50,6 +50,7 @@ import { type KeyTransportEvents, type KeyTransportEventsHandlerMap, } from "matrix-js-sdk/lib/matrixrtc/IKeyTransport"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { LocalUserMediaViewModel, @@ -201,40 +202,30 @@ export const exampleTransport: LivekitTransport = { livekit_alias: "!alias:example.org", }; -export function mockCallMembership( - userId: string, - deviceId: string, - transport?: Transport, -): CallMembership { - const t = transport ?? transportForUser(userId); - return { - userId: userId, - deviceId: deviceId, - getTransport: vi.fn().mockReturnValue(t), - transports: [t], - } as unknown as CallMembership; -} - -function transportForUser(userId: string): Transport { - const domain = userId.split(":")[1]; - return { - type: "livekit", - livekit_service_url: `https://lk.${domain}`, - livekit_alias: `!alias:${domain}`, - }; -} - export function mockRtcMembership( user: string | RoomMember, deviceId: string, - callId = "", - fociPreferred: Transport[] = [exampleTransport], - focusActive: LivekitFocusSelection = { - type: "livekit", - focus_selection: "oldest_membership", + customOverwrites?: { + rtcBackendIdentity?: string; + callId?: string; + fociPreferred?: Transport[]; + focusActive?: LivekitFocusSelection; + membership?: Partial; }, - membership: Partial = {}, ): CallMembership { + // setup defaults based on overwrites and fallback values. + const { rtcBackendIdentity, callId, fociPreferred, focusActive, membership } = + { + fociPreferred: [exampleTransport], + focusActive: { + type: "livekit" as const, + focus_selection: "oldest_membership" as const, + }, + callId: "", + membership: {}, + ...customOverwrites, + }; + const data: SessionMembershipData = { application: "m.call", call_id: callId, @@ -243,17 +234,29 @@ export function mockRtcMembership( focus_active: focusActive, ...membership, }; + const userId = typeof user === "string" ? user : user.userId; const event = new MatrixEvent({ - sender: typeof user === "string" ? user : user.userId, + sender: userId, event_id: `$-ev-${randomUUID()}:example.org`, content: data, }); - const cms = new CallMembership(event, data); + const membershipData = CallMembership.membershipDataFromMatrixEvent(event); + const cms = new CallMembership( + event, + membershipData, + rtcBackendIdentity ?? `${userId}:${deviceId}`, + ); vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); + return cms; } +export const ownMemberMock: CallMembershipIdentityParts = { + userId: "@alice:example.org", + deviceId: "DEVICE", + memberId: "@alice:example.org:DEVICE", +}; // Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are // rather simple, but if one util to mock a member is good enough for us, maybe // it's useful for matrix-js-sdk consumers in general. @@ -331,6 +334,7 @@ export function createLocalMedia( testScope(), "local", member.userId, + rtcMember.rtcBackendIdentity, constant(localParticipant), { kind: E2eeType.PER_PARTICIPANT, @@ -376,6 +380,7 @@ export function createRemoteMedia( testScope(), "remote", member.userId, + rtcMember.rtcBackendIdentity, constant(participant), { kind: E2eeType.PER_PARTICIPANT, @@ -478,7 +483,7 @@ export class MockRTCSession extends TypedEventEmitter< if (value !== prev) this.emit(MembershipManagerEvent.ProbablyLeft, value); } - public async joinRoomSession(): Promise { + public async joinRTCSession(): Promise { return Promise.resolve(); } } diff --git a/yarn.lock b/yarn.lock index b350926d..8ef454d1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3300,10 +3300,10 @@ __metadata: languageName: node linkType: hard -"@matrix-org/matrix-sdk-crypto-wasm@npm:^16.0.0": - version: 16.0.0 - resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:16.0.0" - checksum: 10c0/13b4ede3e618da819957abff778afefcf3baf9a2faac04a36bb5a07a44fae2ea05fbfa072eb3408d48b2b7b9aaf27242ce52c594c8ce9bf1fb8b3aade2832be1 +"@matrix-org/matrix-sdk-crypto-wasm@npm:^17.0.0": + version: 17.0.0 + resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:17.0.0" + checksum: 10c0/fa97e3111099057e0953e7550d6556b6e7553f3badd5b25a6988d2fcc94d22288a27e63cb204771b74ff24388d770c83f2cf5aec583f05c6ecf46509b8020570 languageName: node linkType: hard @@ -3838,6 +3838,25 @@ __metadata: languageName: node linkType: hard +"@radix-ui/react-arrow@npm:1.1.7": + version: 1.1.7 + resolution: "@radix-ui/react-arrow@npm:1.1.7" + dependencies: + "@radix-ui/react-primitive": "npm:2.1.3" + peerDependencies: + "@types/react": "*" + "@types/react-dom": "*" + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + "@types/react": + optional: true + "@types/react-dom": + optional: true + checksum: 10c0/c3b46766238b3ee2a394d8806a5141432361bf1425110c9f0dcf480bda4ebd304453a53f294b5399c6ee3ccfcae6fd544921fd01ddc379cf5942acdd7168664b + languageName: node + linkType: hard + "@radix-ui/react-collection@npm:1.1.1": version: 1.1.1 resolution: "@radix-ui/react-collection@npm:1.1.1" @@ -3908,16 +3927,16 @@ __metadata: languageName: node linkType: hard -"@radix-ui/react-context-menu@npm:^2.2.1": - version: 2.2.4 - resolution: "@radix-ui/react-context-menu@npm:2.2.4" +"@radix-ui/react-context-menu@npm:^2.2.16": + version: 2.2.16 + resolution: "@radix-ui/react-context-menu@npm:2.2.16" dependencies: - "@radix-ui/primitive": "npm:1.1.1" - "@radix-ui/react-context": "npm:1.1.1" - "@radix-ui/react-menu": "npm:2.1.4" - "@radix-ui/react-primitive": "npm:2.0.1" - "@radix-ui/react-use-callback-ref": "npm:1.1.0" - "@radix-ui/react-use-controllable-state": "npm:1.1.0" + "@radix-ui/primitive": "npm:1.1.3" + "@radix-ui/react-context": "npm:1.1.2" + "@radix-ui/react-menu": "npm:2.1.16" + "@radix-ui/react-primitive": "npm:2.1.3" + "@radix-ui/react-use-callback-ref": "npm:1.1.1" + "@radix-ui/react-use-controllable-state": "npm:1.2.2" peerDependencies: "@types/react": "*" "@types/react-dom": "*" @@ -3928,7 +3947,7 @@ __metadata: optional: true "@types/react-dom": optional: true - checksum: 10c0/f500590b1300dfcd8a2d0fb51fcada0e7d9a1a354ac239328ffdd32f3736bde888ebf0cd64d9039f7d894e3d13eb549a872359669de8c7ff128ee1afb9cf21a8 + checksum: 10c0/950f7559e65474a19145238cf44d744cb1e49be2221ff18436ba49b496b05ccf93bd3906aaa2c7ab76bc77daf694911a78442801e0053f57d2e57ebbfd281c49 languageName: node linkType: hard @@ -4228,6 +4247,42 @@ __metadata: languageName: node linkType: hard +"@radix-ui/react-menu@npm:2.1.16": + version: 2.1.16 + resolution: "@radix-ui/react-menu@npm:2.1.16" + dependencies: + "@radix-ui/primitive": "npm:1.1.3" + "@radix-ui/react-collection": "npm:1.1.7" + "@radix-ui/react-compose-refs": "npm:1.1.2" + "@radix-ui/react-context": "npm:1.1.2" + "@radix-ui/react-direction": "npm:1.1.1" + "@radix-ui/react-dismissable-layer": "npm:1.1.11" + "@radix-ui/react-focus-guards": "npm:1.1.3" + "@radix-ui/react-focus-scope": "npm:1.1.7" + "@radix-ui/react-id": "npm:1.1.1" + "@radix-ui/react-popper": "npm:1.2.8" + "@radix-ui/react-portal": "npm:1.1.9" + "@radix-ui/react-presence": "npm:1.1.5" + "@radix-ui/react-primitive": "npm:2.1.3" + "@radix-ui/react-roving-focus": "npm:1.1.11" + "@radix-ui/react-slot": "npm:1.2.3" + "@radix-ui/react-use-callback-ref": "npm:1.1.1" + aria-hidden: "npm:^1.2.4" + react-remove-scroll: "npm:^2.6.3" + peerDependencies: + "@types/react": "*" + "@types/react-dom": "*" + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + "@types/react": + optional: true + "@types/react-dom": + optional: true + checksum: 10c0/27516b2b987fa9181c4da8645000af8f60691866a349d7a46b9505fa7d2e9d92b9e364db4f7305d08e9e57d0e1afc8df8354f8ee3c12aa05c0100c16b0e76c27 + languageName: node + linkType: hard + "@radix-ui/react-menu@npm:2.1.4": version: 2.1.4 resolution: "@radix-ui/react-menu@npm:2.1.4" @@ -4292,6 +4347,34 @@ __metadata: languageName: node linkType: hard +"@radix-ui/react-popper@npm:1.2.8": + version: 1.2.8 + resolution: "@radix-ui/react-popper@npm:1.2.8" + dependencies: + "@floating-ui/react-dom": "npm:^2.0.0" + "@radix-ui/react-arrow": "npm:1.1.7" + "@radix-ui/react-compose-refs": "npm:1.1.2" + "@radix-ui/react-context": "npm:1.1.2" + "@radix-ui/react-primitive": "npm:2.1.3" + "@radix-ui/react-use-callback-ref": "npm:1.1.1" + "@radix-ui/react-use-layout-effect": "npm:1.1.1" + "@radix-ui/react-use-rect": "npm:1.1.1" + "@radix-ui/react-use-size": "npm:1.1.1" + "@radix-ui/rect": "npm:1.1.1" + peerDependencies: + "@types/react": "*" + "@types/react-dom": "*" + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + "@types/react": + optional: true + "@types/react-dom": + optional: true + checksum: 10c0/48e3f13eac3b8c13aca8ded37d74db17e1bb294da8d69f142ab6b8719a06c3f90051668bed64520bf9f3abdd77b382ce7ce209d056bb56137cecc949b69b421c + languageName: node + linkType: hard + "@radix-ui/react-portal@npm:1.1.3": version: 1.1.3 resolution: "@radix-ui/react-portal@npm:1.1.3" @@ -4476,6 +4559,33 @@ __metadata: languageName: node linkType: hard +"@radix-ui/react-roving-focus@npm:1.1.11": + version: 1.1.11 + resolution: "@radix-ui/react-roving-focus@npm:1.1.11" + dependencies: + "@radix-ui/primitive": "npm:1.1.3" + "@radix-ui/react-collection": "npm:1.1.7" + "@radix-ui/react-compose-refs": "npm:1.1.2" + "@radix-ui/react-context": "npm:1.1.2" + "@radix-ui/react-direction": "npm:1.1.1" + "@radix-ui/react-id": "npm:1.1.1" + "@radix-ui/react-primitive": "npm:2.1.3" + "@radix-ui/react-use-callback-ref": "npm:1.1.1" + "@radix-ui/react-use-controllable-state": "npm:1.2.2" + peerDependencies: + "@types/react": "*" + "@types/react-dom": "*" + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + "@types/react": + optional: true + "@types/react-dom": + optional: true + checksum: 10c0/2cd43339c36e89a3bf1db8aab34b939113dfbde56bf3a33df2d74757c78c9489b847b1962f1e2441c67e41817d120cb6177943e0f655f47bc1ff8e44fd55b1a2 + languageName: node + linkType: hard + "@radix-ui/react-separator@npm:^1.1.0": version: 1.1.1 resolution: "@radix-ui/react-separator@npm:1.1.1" @@ -4725,6 +4835,21 @@ __metadata: languageName: node linkType: hard +"@radix-ui/react-use-rect@npm:1.1.1": + version: 1.1.1 + resolution: "@radix-ui/react-use-rect@npm:1.1.1" + dependencies: + "@radix-ui/rect": "npm:1.1.1" + peerDependencies: + "@types/react": "*" + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + "@types/react": + optional: true + checksum: 10c0/271711404c05c589c8dbdaa748749e7daf44bcc6bffc9ecd910821c3ebca0ee245616cf5b39653ce690f53f875c3836fd3f36f51ab1c628273b6db599eee4864 + languageName: node + linkType: hard + "@radix-ui/react-use-size@npm:1.1.0": version: 1.1.0 resolution: "@radix-ui/react-use-size@npm:1.1.0" @@ -4781,6 +4906,13 @@ __metadata: languageName: node linkType: hard +"@radix-ui/rect@npm:1.1.1": + version: 1.1.1 + resolution: "@radix-ui/rect@npm:1.1.1" + checksum: 10c0/0dac4f0f15691199abe6a0e067821ddd9d0349c0c05f39834e4eafc8403caf724106884035ae91bbc826e10367e6a5672e7bec4d4243860fa7649de246b1f60b + languageName: node + linkType: hard + "@react-spring/animated@npm:~10.0.3": version: 10.0.3 resolution: "@react-spring/animated@npm:10.0.3" @@ -6026,8 +6158,8 @@ __metadata: linkType: hard "@vector-im/compound-design-tokens@npm:^6.0.0": - version: 6.0.0 - resolution: "@vector-im/compound-design-tokens@npm:6.0.0" + version: 6.6.0 + resolution: "@vector-im/compound-design-tokens@npm:6.6.0" peerDependencies: "@types/react": "*" react: ^17 || ^18 || ^19.0.0 @@ -6036,16 +6168,16 @@ __metadata: optional: true react: optional: true - checksum: 10c0/1af5b2b73a3a55149047cd0716f071b83a4df8a210c9ad432db4cc2f9b9e72e958f93ff850dbaddb88e37a01870c5eb810b03dfb0acc89cc147eaaf6cf1dada1 + checksum: 10c0/93b152dd1de96371f9b6b1f7dbcc381d7ab598031dbc900f52d610f015766c0d4426ae6e47d417e723bfb62d1a53099155b4d788848b78232916ba132c03c2fe languageName: node linkType: hard "@vector-im/compound-web@npm:^8.0.0": - version: 8.2.0 - resolution: "@vector-im/compound-web@npm:8.2.0" + version: 8.3.4 + resolution: "@vector-im/compound-web@npm:8.3.4" dependencies: "@floating-ui/react": "npm:^0.27.0" - "@radix-ui/react-context-menu": "npm:^2.2.1" + "@radix-ui/react-context-menu": "npm:^2.2.16" "@radix-ui/react-dropdown-menu": "npm:^2.1.1" "@radix-ui/react-form": "npm:^0.1.0" "@radix-ui/react-progress": "npm:^1.1.0" @@ -6057,12 +6189,12 @@ __metadata: "@fontsource/inconsolata": ^5 "@fontsource/inter": ^5 "@types/react": "*" - "@vector-im/compound-design-tokens": ">=1.6.1 <6.0.0" + "@vector-im/compound-design-tokens": ">=1.6.1 <7.0.0" react: ^18 || ^19.0.0 peerDependenciesMeta: "@types/react": optional: true - checksum: 10c0/4ac4074dcf9611bdff7de4bf66763397c926d6312f31758bcabe3e7bf704cb76bc2ce1023fe5f2cf0d05e97c9c540fef8b63edea7a521a2f7b4b7fbcb883fb17 + checksum: 10c0/44764fa64b5fce2e7181e25b50ee970eda4d921cf650b92bd5e88df0eb60872f3086b8702d18f55c3e39b3751ac19f10bafda8c4306df65c3605bd44b297d95c languageName: node linkType: hard @@ -8267,6 +8399,7 @@ __metadata: typescript: "npm:^5.8.3" typescript-eslint-language-service: "npm:^5.0.5" unique-names-generator: "npm:^4.6.0" + uuid: "npm:^13.0.0" vaul: "npm:^1.0.0" vite: "npm:^7.0.0" vite-plugin-generate-file: "npm:^0.3.0" @@ -11335,10 +11468,10 @@ __metadata: "matrix-js-sdk@matrix-org/matrix-js-sdk#develop": version: 39.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=174439c2f0c09cf9926c28435ba4db1345df4aee" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4d0d32307eb4f1ce1fb65080fcca704f5bdedc31" dependencies: "@babel/runtime": "npm:^7.12.5" - "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" + "@matrix-org/matrix-sdk-crypto-wasm": "npm:^17.0.0" another-json: "npm:^0.2.0" bs58: "npm:^6.0.0" content-type: "npm:^1.0.4" @@ -11351,7 +11484,7 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/5178de27bb618aed6f80632a72c5582542ceedb51ef15534493360a624b072e0c276693ad9e37d83f2ddb06716f9eb6d02960e158e029f7a005676873778c745 + checksum: 10c0/59c9d81ccf823584dc783502cb5c928562e3490c63f5ce98ee3232a603545d6278e90dc951c1fd0bae2792ba732ec5171e03596fd396bb2150d596cebb7fbac9 languageName: node linkType: hard @@ -15214,7 +15347,7 @@ __metadata: languageName: node linkType: hard -"uuid@npm:13": +"uuid@npm:13, uuid@npm:^13.0.0": version: 13.0.0 resolution: "uuid@npm:13.0.0" bin: