From 31eafb394bc07af28bb0937d22a979ece635ffb1 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Fri, 2 Oct 2026 14:35:45 +0200 Subject: [PATCH] 7000 loc COPIED from element call They are copied for now but will be removed from element call once it adopts the sdk. Those are no real line additions. --- .oxlintrc.json | 1 + eslint/SdkImportBoundary.js | 96 +++ eslint/index.js | 1 + sdk/README.md | 5 +- sdk/SdkArchitecture.md | 17 +- sdk/SdkMigration.md | 68 +- sdk/index.ts | 10 +- sdk/src/api.ts | 19 +- sdk/src/config.ts | 94 +++ sdk/src/encryption.ts | 27 + sdk/src/errors.ts | 169 ++++ sdk/src/media/LivekitMediaTrack.test.ts | 2 +- sdk/src/media/LivekitMediaTrack.ts | 4 +- sdk/src/media/LivekitMemberMedia.ts | 8 +- sdk/src/reactive/Behavior.ts | 29 + sdk/src/reactive/ObservableScope.test.ts | 293 +++++++ sdk/src/reactive/ObservableScope.ts | 329 ++++++++ sdk/src/reactive/observable.test.ts | 104 +++ sdk/src/reactive/observable.ts | 264 +++++++ sdk/src/session/Connection.test.ts | 733 ++++++++++++++++++ sdk/src/session/Connection.ts | 426 ++++++++++ sdk/src/session/ConnectionFactory.ts | 28 +- sdk/src/session/ConnectionManager.test.ts | 358 +++++++++ sdk/src/session/ConnectionManager.ts | 279 +++++++ sdk/src/session/HomeserverConnected.test.ts | 336 ++++++++ sdk/src/session/HomeserverConnected.ts | 143 ++++ sdk/src/session/KeyProvider.ts | 6 +- sdk/src/session/LocalMember.ts | 46 +- sdk/src/session/LocalTransport.ts | 26 +- sdk/src/session/MatrixKeyProvider.test.ts | 72 ++ sdk/src/session/MatrixKeyProvider.ts | 75 ++ sdk/src/session/MatrixLivekitMembers.test.ts | 333 ++++++++ sdk/src/session/MatrixLivekitMembers.ts | 160 ++++ sdk/src/session/MatrixMemberMetadata.test.ts | 632 +++++++++++++++ sdk/src/session/MatrixMemberMetadata.ts | 157 ++++ sdk/src/session/MatrixRTCClient.ts | 16 +- sdk/src/session/Members.ts | 12 +- sdk/src/session/Publisher.ts | 4 +- .../session/RtcTransportAutoDiscovery.test.ts | 142 ++++ sdk/src/session/RtcTransportAutoDiscovery.ts | 103 +++ sdk/src/session/SessionBehaviors.test.ts | 41 + sdk/src/session/SessionBehaviors.ts | 102 +++ sdk/src/session/Transports.ts | 8 +- sdk/src/session/joinJsSdkSession.ts | 24 +- sdk/src/session/livekitOptions.test.ts | 189 +++++ sdk/src/session/livekitOptions.ts | 140 ++++ sdk/src/session/openIDSFU.test.ts | 333 ++++++++ sdk/src/session/openIDSFU.ts | 315 ++++++++ sdk/src/session/status.test.ts | 2 +- sdk/src/utils/displayname.test.ts | 133 ++++ sdk/src/utils/displayname.ts | 114 +++ sdk/src/utils/errors.test.ts | 35 - sdk/src/utils/errors.ts | 27 - sdk/src/utils/mapScoped.test.ts | 2 +- sdk/src/utils/mapScoped.ts | 4 +- sdk/src/utils/network.ts | 31 + sdk/src/utils/test-fixtures.ts | 65 ++ sdk/src/utils/test.ts | 342 ++++++++ 58 files changed, 7330 insertions(+), 204 deletions(-) create mode 100644 eslint/SdkImportBoundary.js create mode 100644 sdk/src/config.ts create mode 100644 sdk/src/encryption.ts create mode 100644 sdk/src/errors.ts create mode 100644 sdk/src/reactive/Behavior.ts create mode 100644 sdk/src/reactive/ObservableScope.test.ts create mode 100644 sdk/src/reactive/ObservableScope.ts create mode 100644 sdk/src/reactive/observable.test.ts create mode 100644 sdk/src/reactive/observable.ts create mode 100644 sdk/src/session/Connection.test.ts create mode 100644 sdk/src/session/Connection.ts create mode 100644 sdk/src/session/ConnectionManager.test.ts create mode 100644 sdk/src/session/ConnectionManager.ts create mode 100644 sdk/src/session/HomeserverConnected.test.ts create mode 100644 sdk/src/session/HomeserverConnected.ts create mode 100644 sdk/src/session/MatrixKeyProvider.test.ts create mode 100644 sdk/src/session/MatrixKeyProvider.ts create mode 100644 sdk/src/session/MatrixLivekitMembers.test.ts create mode 100644 sdk/src/session/MatrixLivekitMembers.ts create mode 100644 sdk/src/session/MatrixMemberMetadata.test.ts create mode 100644 sdk/src/session/MatrixMemberMetadata.ts create mode 100644 sdk/src/session/RtcTransportAutoDiscovery.test.ts create mode 100644 sdk/src/session/RtcTransportAutoDiscovery.ts create mode 100644 sdk/src/session/SessionBehaviors.test.ts create mode 100644 sdk/src/session/SessionBehaviors.ts create mode 100644 sdk/src/session/livekitOptions.test.ts create mode 100644 sdk/src/session/livekitOptions.ts create mode 100644 sdk/src/session/openIDSFU.test.ts create mode 100644 sdk/src/session/openIDSFU.ts create mode 100644 sdk/src/utils/displayname.test.ts create mode 100644 sdk/src/utils/displayname.ts delete mode 100644 sdk/src/utils/errors.test.ts delete mode 100644 sdk/src/utils/errors.ts create mode 100644 sdk/src/utils/network.ts create mode 100644 sdk/src/utils/test-fixtures.ts create mode 100644 sdk/src/utils/test.ts diff --git a/.oxlintrc.json b/.oxlintrc.json index c89ab8a28..dd0df610e 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -35,6 +35,7 @@ ], "element-call/no-observablescope-leak": "error", "element-call/no-top-level-logger-get-child": "error", + "element-call/sdk-import-boundary": "error", "jsdoc/empty-tags": "error", "jsdoc/check-property-names": "error", "jsdoc/require-param-description": "warn", diff --git a/eslint/SdkImportBoundary.js b/eslint/SdkImportBoundary.js new file mode 100644 index 000000000..7dd8a7790 --- /dev/null +++ b/eslint/SdkImportBoundary.js @@ -0,0 +1,96 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { dirname, resolve, sep } from "node:path"; +import { ESLintUtils } from "@typescript-eslint/utils"; + +/** + * Packages a host supplies and the SDK must not know about: UI, i18n and + * Element Call's design system. + */ +const BANNED_PACKAGES = [ + "react", + "react-dom", + "react-i18next", + "i18next", + "@vector-im/", +]; + +/** + * Keeps the SDK free of Element Call: nothing under `sdk/` may import a file + * outside `sdk/` by relative path (that is `src/`, `component/`, + * `playwright/`) or one of the packages above. The development harness under + * `sdk/dev/` is a host, not part of the library, and is exempt. + */ +const rule = ESLintUtils.RuleCreator( + () => "https://github.com/element-hq/element-call", +)({ + name: "sdk-import-boundary", + meta: { + type: "problem", + docs: { + description: + "Disallow the SDK importing Element Call (anything outside sdk/) or host-supplied packages.", + }, + messages: { + outsideSdk: + "The SDK must not import '{{specifier}}': it resolves outside sdk/. Copy or move what it needs into sdk/src.", + bannedPackage: + "The SDK must not import '{{specifier}}': a host supplies that, the SDK has no UI.", + }, + schema: [], + }, + defaultOptions: [], + create(context) { + const filename = context.filename; + const sdkRoot = resolve(context.cwd, "sdk") + sep; + if (!filename.startsWith(sdkRoot)) return {}; + if (filename.startsWith(resolve(sdkRoot, "dev") + sep)) return {}; + + const check = (node, specifier) => { + if (typeof specifier !== "string") return; + if (specifier.startsWith(".")) { + const target = resolve(dirname(filename), specifier); + if (!target.startsWith(sdkRoot)) + context.report({ + node, + messageId: "outsideSdk", + data: { specifier }, + }); + } else if ( + BANNED_PACKAGES.some( + (pkg) => + specifier === pkg || + specifier.startsWith(pkg.endsWith("/") ? pkg : `${pkg}/`), + ) + ) { + context.report({ + node, + messageId: "bannedPackage", + data: { specifier }, + }); + } + }; + + return { + ImportDeclaration(node) { + check(node, node.source.value); + }, + ExportNamedDeclaration(node) { + if (node.source) check(node, node.source.value); + }, + ExportAllDeclaration(node) { + check(node, node.source.value); + }, + ImportExpression(node) { + if (node.source.type === "Literal") check(node, node.source.value); + }, + }; + }, +}); + +export default rule; diff --git a/eslint/index.js b/eslint/index.js index 1f2e7629e..dfb7fb5e2 100644 --- a/eslint/index.js +++ b/eslint/index.js @@ -4,5 +4,6 @@ module.exports = { "no-observablescope-leak": require("./NoObservableScopeLeak").default, "no-top-level-logger-get-child": require("./NoTopLevelLoggerGetChild") .default, + "sdk-import-boundary": require("./SdkImportBoundary").default, }, }; diff --git a/sdk/README.md b/sdk/README.md index a01bba509..49d85ee07 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -9,9 +9,8 @@ in [`SdkMigration.md`](./SdkMigration.md). **Status:** first implementation. `createMatrixRTCClient` joins the session, connects to the transport, publishes the local media and exposes every member's media; the -development harness and its e2e tests in `playwright/sdk` drive it. The -implementation still imports the building blocks it shares with Element Call from -`src/` (connections, memberships, key provider); moving them here is the next slice. +development harness and its e2e tests in `playwright/sdk` drive it. It depends on +nothing in Element Call's `src/`; Element Call does not consume it yet. ## Using it diff --git a/sdk/SdkArchitecture.md b/sdk/SdkArchitecture.md index 7c4a61c41..c05a84ef8 100644 --- a/sdk/SdkArchitecture.md +++ b/sdk/SdkArchitecture.md @@ -508,18 +508,23 @@ Layout: sdk/ index.ts the entry point: re-exports the API and the primitives src/api.ts the public types above - src/session/ MatrixRTCClient, the local member, publisher, connections, transports, join + src/errors.ts MatrixRTCError and its codes + src/config.ts MatrixRTCMode, session timings, media quality + src/encryption.ts E2eeType, EncryptionSystem + src/reactive/ Behavior, ObservableScope, the observable operators + src/session/ MatrixRTCClient, the local member, publisher, connections, + memberships, transports, discovery, JWT, join src/media/ the LiveKit adapter: MemberMedia and MediaTrack - src/utils/ LazyBehavior, mapScoped, error wrapping + src/utils/ LazyBehavior, mapScoped, network retry, display names, test helpers dev/ the harness (below) SdkArchitecture.md this document SdkMigration.md how Element Call gets from CallViewModel to the client ``` -An import boundary is still to be added to the local lint plugin: nothing under -`sdk/` may import `src/`, `component/`, `playwright/`, React, i18n, Compound or the -settings. Until the shared modules have moved (see the migration document) the -SDK imports them from `src/`, so the rule cannot be turned on yet. +The lint rule `element-call/sdk-import-boundary` keeps it that way: nothing under +`sdk/` may import a file outside `sdk/` (that is `src/`, `component/`, +`playwright/`) or React, i18n or Compound. The harness under `sdk/dev/` is a host +and exempt. ## Dev harness: `sdk/dev/` diff --git a/sdk/SdkMigration.md b/sdk/SdkMigration.md index 6ad68b83a..72f5a7816 100644 --- a/sdk/SdkMigration.md +++ b/sdk/SdkMigration.md @@ -8,42 +8,48 @@ symbols, so it ages with the code; the architecture document should not. ## Status -| Slice | State | -| -------------------------------------------------------------------------------------------------------------- | -------------------- | -| 1. Scaffold: `sdk/` packaging, the interface, the harness, the smoke test | done, October 2026 | -| 2. First implementation of `createMatrixRTCClient` under `sdk/src`, the media adapter, the e2e suite | done, 1 October 2026 | -| 3. Move the shared modules out of `src/` into `sdk/src/`, leave re-exports behind, turn on the import boundary | next | -| 4. `createCallViewModel$` consumes the client; media view models take `media$` | after 3 | -| 5. Remove the re-exports, `sdk-target-based-on-call-view-model/` and the old `build:sdk` output | last | +| Slice | State | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------- | +| 1. Scaffold: `sdk/` packaging, the interface, the harness, the smoke test | done, October 2026 | +| 2. First implementation of `createMatrixRTCClient` under `sdk/src`, the media adapter, the e2e suite | done, 1 October 2026 | +| 3. Copy the shared modules into `sdk/src/` with their tests, drop every `src/` import, turn on the import boundary | done, 2 October 2026 | +| 4. `createCallViewModel$` consumes the client; media view models take `media$`; Element Call imports the primitives and enums from the SDK and deletes its own copies | next | +| 5. Remove `sdk-target-based-on-call-view-model/` and the old `build:sdk` output | last | -Slice 2 was built without touching `src/`, so the SDK imports its building blocks -from there for now. Nothing in Element Call consumes the SDK yet. +`src/` was not touched in slices 2 and 3: the SDK has its own copies of what it +needs, and the lint rule `element-call/sdk-import-boundary` fails any import from +`sdk/` that resolves outside `sdk/` or names React, i18n or Compound. Until slice 4 +the two copies drift independently, which is why slice 4 comes next. Nothing in +Element Call consumes the SDK yet. -## What the SDK still imports from `src/` +## What was copied from `src/`, and what changed on the way -These move into `sdk/src/` in slice 3, each with its tests, leaving a one-line -re-export in `src` until Element Call's importers are updated. +Each module came with its tests (`sdk/src/utils/test.ts` and `test-fixtures.ts` are +the subset of `src/utils/test.ts` and `test-fixtures.ts` they need). In slice 4 +Element Call switches to the SDK's copy and deletes its own. -| Module | Used for | Notes for the move | -| ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | -| `state/Behavior.ts`, `state/ObservableScope.ts`, `utils/observable.ts` | the reactive primitives | exported from `sdk/index.ts` already; 37 importers in `src` and `component` keep working through the re-export | -| `state/SessionBehaviors.ts` | `createMemberships$`, `membershipsAndTransports$`, `createKeyRotationSuppressed$` | as is | -| `state/CallViewModel/remoteMembers/Connection.ts` | `Connection`, `ConnectionState` | the SDK subclasses it as `ResolvedConnection` to keep the JWT answer; fold that in | -| `state/CallViewModel/remoteMembers/ConnectionFactory.ts` | the `ConnectionFactory` interface only | `ECConnectionFactory` stays behind: it reads settings and `MediaDevices`; the SDK has `LivekitConnectionFactory` | -| `state/CallViewModel/remoteMembers/ConnectionManager.ts` | `createConnectionManager$` | as is | -| `state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts` | `createRemoteMatrixLivekitMembers$`, the membership-to-participant match | the `TaggedParticipant` type goes; the SDK maps the result to `RTCMember` | -| `state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts` | `createRoomMembers$`, `createMatrixMemberMetadata$` | depends on `utils/displayname.ts`, which moves with it | -| `state/CallViewModel/localMember/HomeserverConnected.ts` | `createHomeserverConnected$` | reads `Config.get()` unless the grace period is passed; the SDK passes it | -| `state/CallViewModel/localMember/LocalTransport.ts` | the `LocalTransport` type only | `getLocalTransport` reads config and settings; the SDK has its own in `session/LocalTransport.ts` | -| `state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts` | discovery through `rtc/transports` | takes a resolved config for the fallback url; the SDK passes `DEFAULT_CONFIG`, which has none | -| `state/CallViewModel/localMember/LocalMember.ts` | `observeSharingScreen$` only | the SDK has its own local member; the rest stays until slice 4 | -| `livekit/openIDSFU.ts` | `getSFUConfigWithOpenID`, `SFUConfig` | reads `Config.get()` only on the delegated delayed leave path, which the SDK does not use | -| `livekit/options.ts` | `buildLiveKitOptions` | takes the media quality config as a parameter; `getLiveKitOptions` stays behind | -| `e2ee/matrixKeyProvider.ts`, `e2ee/e2eeType.ts`, `e2ee/sharedKeyManagement.ts` | the key provider, `E2eeType`, the `EncryptionSystem` type | `sharedKeyManagement.ts` also holds React hooks; split the type out | -| `config/ConfigOptions.ts` | `MatrixRTCMode`, `DEFAULT_CONFIG` for the timings | the enum moves; the timings become an option (see the architecture document's open decisions) | -| `utils/errors.ts` | the error classes the modules above throw | they carry translation keys, so they stay; the SDK wraps them as `MatrixRTCError` with `cause` | +| In `src/` | In `sdk/src/` | What changed | +| ----------------------------------------------------------------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `state/Behavior.ts`, `state/ObservableScope.ts`, `utils/observable.ts` | `reactive/` | as is | +| `state/SessionBehaviors.ts` | `session/SessionBehaviors.ts` | as is | +| `state/CallViewModel/remoteMembers/Connection.ts` | `session/Connection.ts` | throws `MatrixRTCError`s; the SDK still subclasses it as `ResolvedConnection` to keep the JWT answer, to be folded in | +| `state/CallViewModel/remoteMembers/ConnectionFactory.ts` | `session/ConnectionFactory.ts` | the interface only; `ECConnectionFactory` reads settings and `MediaDevices`, the SDK has `LivekitConnectionFactory` | +| `state/CallViewModel/remoteMembers/ConnectionManager.ts` | `session/ConnectionManager.ts` | as is | +| `state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts` | `session/MatrixLivekitMembers.ts` | as is; the SDK maps its result to `RTCMember`, `TaggedParticipant` is internal | +| `state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts`, `utils/displayname.ts` | `session/MatrixMemberMetadata.ts`, `utils/displayname.ts` | as is | +| `state/CallViewModel/localMember/HomeserverConnected.ts` | `session/HomeserverConnected.ts` | the grace period defaults to `defaultSessionTimings` instead of `Config.get()` | +| `state/CallViewModel/localMember/LocalTransport.ts` | `session/LocalTransport.ts` | the type and `isLocalTransport`; discovery without the custom-url setting | +| `state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts` | `session/RtcTransportAutoDiscovery.ts` | takes `fallbackTransportUrl` instead of a resolved config | +| `state/CallViewModel/localMember/LocalMember.ts` | `session/LocalMember.ts` | rewritten; only `observeSharingScreen$` was copied | +| `livekit/openIDSFU.ts` | `session/openIDSFU.ts` | the delegated leave timeout is a parameter (`delayTimeoutMs`) instead of `Config.get()` | +| `livekit/options.ts` | `session/livekitOptions.ts` | `buildLiveKitOptions(mediaQuality?)` only; `getLiveKitOptions` and the `Config` singleton are gone | +| `e2ee/matrixKeyProvider.ts` | `session/MatrixKeyProvider.ts` | as is | +| `e2ee/e2eeType.ts`, the `EncryptionSystem` type from `e2ee/sharedKeyManagement.ts` | `encryption.ts` | the React hooks stayed behind | +| `MatrixRTCMode`, the timings and media quality types from `config/ConfigOptions.ts` | `config.ts` | `defaultSessionTimings` and `defaultMediaQuality` replace `DEFAULT_CONFIG` | +| `utils/errors.ts` | `errors.ts` | `MatrixRTCError` with `code` and `category` replaces `ElementCallError`; messages are plain English, the host translates by code | +| `doNetworkOperationWithRetry` from `utils/matrix.ts` | `utils/network.ts` | as is | -Rewritten in the SDK rather than moved, because the originals read settings, +Rewritten in the SDK rather than copied, because the originals read settings, config, analytics or the host bridge: the local member (`session/LocalMember.ts`), the publisher (`session/Publisher.ts`), the connection factory and LiveKit room options (`session/ConnectionFactory.ts`), transport discovery diff --git a/sdk/index.ts b/sdk/index.ts index c8eec0eee..fbf9fedd1 100644 --- a/sdk/index.ts +++ b/sdk/index.ts @@ -17,11 +17,11 @@ Please see LICENSE in the repository root for full details. // Shared with Element Call. They live in `src` until the view model consumes // the SDK, at which point they move here; a consumer gets them from this // module either way. -export { type Behavior, constant } from "../src/state/Behavior"; -export { ObservableScope } from "../src/state/ObservableScope"; -export { E2eeType } from "../src/e2ee/e2eeType"; -export { type EncryptionSystem } from "../src/e2ee/sharedKeyManagement"; -export { MatrixRTCMode } from "../src/config/ConfigOptions"; +export { type Behavior, constant } from "./src/reactive/Behavior"; +export { ObservableScope } from "./src/reactive/ObservableScope"; +export { E2eeType } from "./src/encryption"; +export { type EncryptionSystem } from "./src/encryption"; +export { MatrixRTCMode } from "./src/config"; export * from "./src/api"; export { createMatrixRTCClient } from "./src/session/MatrixRTCClient"; diff --git a/sdk/src/api.ts b/sdk/src/api.ts index 3ab128680..55549e6fe 100644 --- a/sdk/src/api.ts +++ b/sdk/src/api.ts @@ -18,9 +18,9 @@ import { import { type Track, type TrackProcessor } from "livekit-client"; import { type Observable } from "rxjs"; -import { type Behavior } from "../../src/state/Behavior"; -import { type EncryptionSystem } from "../../src/e2ee/sharedKeyManagement"; -import { type MatrixRTCMode } from "../../src/config/ConfigOptions"; +import { type Behavior } from "./reactive/Behavior"; +import { type EncryptionSystem } from "./encryption"; +import { type MatrixRTCMode } from "./config"; // --------------------------------------------------------------------------- // Session @@ -64,16 +64,9 @@ export type ConnectionStatus = | "reconnecting" | "disconnected"; -/** - * An error raised by the session. `cause` holds the underlying error, which - * lets a host that knows the backend tell the failures apart. - */ -export class MatrixRTCError extends Error { - public constructor(message: string, options?: ErrorOptions) { - super(message, options); - this.name = "MatrixRTCError"; - } -} +import { type MatrixRTCError } from "./errors"; + +export { MatrixRTCError } from "./errors"; /** * A session in one room. The room has to be the one the client's sync loop diff --git a/sdk/src/config.ts b/sdk/src/config.ts new file mode 100644 index 000000000..c3fdb7a4d --- /dev/null +++ b/sdk/src/config.ts @@ -0,0 +1,94 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * The MatrixRTC mode determines how the client interacts with the MatrixRTC + * backend and other participants. Resolved by the host. + */ +export enum MatrixRTCMode { + /** Multi-SFU transport, legacy JWT endpoint, state events. */ + Compatibility = "compatibility", + /** + * Multi-SFU transport with sticky events, a hashed RTC backend identity and + * the new JWT endpoint for the local membership. Remote memberships always + * try the new endpoint first, then the legacy one. + */ + Matrix_2_0 = "matrix_2_0", +} + +export interface DelayedLeaveTimings { + /** How long (in milliseconds) the homeserver waits before sending the delayed leave. */ + delay_ms: number; + /** How often (in milliseconds) the client restarts the delayed leave. */ + restart_ms: number; + /** How long (in milliseconds) a restart may take before it counts as failed. */ + restart_timeout_ms?: number; +} + +/** + * The session's timings. A host that has them in its configuration passes + * them in; these are the defaults a MatrixRTC backend is expected to accept. + */ +export interface SessionTimings { + /** How long the sync may be down before the client counts as disconnected. */ + syncDisconnectGracePeriodMs: number; + /** How long to wait before retrying after a network error on any request. */ + networkErrorRetryMs: number; + /** How long to wait before rotating the media key when someone leaves. */ + waitForKeyRotationMs?: number; + /** When the membership expires if the client stops renewing it. */ + membershipEventExpiryMs?: number; + /** The session size at which the media key stops being rotated. */ + keyRotationParticipantLimit?: number; + delayedLeave: DelayedLeaveTimings; +} + +export const defaultSessionTimings: SessionTimings = { + syncDisconnectGracePeriodMs: 10_000, + networkErrorRetryMs: 1_000, + delayedLeave: { delay_ms: 18_000, restart_ms: 4_000 }, +}; + +/** Media quality settings a host may pass; everything is optional. */ +export interface MediaQuality { + /** Video codec preference. The server must also have the codec enabled. Default `vp8`. */ + video_codec?: "vp8" | "vp9" | "h264" | "av1"; + video?: { + /** Max resolution height in pixels. Default 720. */ + max_resolution?: number; + /** Max bitrate in bits per second. Default 1700000. */ + max_bitrate?: number; + /** Max framerate. Default 30. */ + max_framerate?: number; + /** Simulcast layers, lowest quality first. Default 180p and 360p. */ + simulcast_layers?: Array<{ height: number; bitrate: number }>; + }; + screen_share?: { + /** Max resolution height in pixels. Default 1080. */ + max_resolution?: number; + /** Max bitrate in bits per second. Default 5000000. */ + max_bitrate?: number; + /** Max framerate. Default 30. */ + max_framerate?: number; + /** Simulcast layers, lowest quality first. Without them LiveKit adds one layer at half resolution. */ + simulcast_layers?: Array<{ + height: number; + bitrate: number; + framerate?: number; + }>; + }; +} + +export const defaultMediaQuality = { + video_codec: "vp8", + video: { max_resolution: 720, max_bitrate: 1_700_000, max_framerate: 30 }, + screen_share: { + max_resolution: 1080, + max_bitrate: 5_000_000, + max_framerate: 30, + }, +} as const satisfies MediaQuality; diff --git a/sdk/src/encryption.ts b/sdk/src/encryption.ts new file mode 100644 index 000000000..0d60e3af0 --- /dev/null +++ b/sdk/src/encryption.ts @@ -0,0 +1,27 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +export enum E2eeType { + NONE = 0, + PER_PARTICIPANT = 1, + SHARED_KEY = 2, +} + +export interface Unencrypted { + kind: E2eeType.NONE; +} + +export interface SharedSecret { + kind: E2eeType.SHARED_KEY; + secret: string; +} + +export interface PerParticipantE2EE { + kind: E2eeType.PER_PARTICIPANT; +} + +export type EncryptionSystem = Unencrypted | SharedSecret | PerParticipantE2EE; diff --git a/sdk/src/errors.ts b/sdk/src/errors.ts new file mode 100644 index 000000000..0d3d03198 --- /dev/null +++ b/sdk/src/errors.ts @@ -0,0 +1,169 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type ConnectionError } from "livekit-client"; + +export enum ErrorCode { + /** No MatrixRTC transport provided by the homeserver and no fallback configured. */ + MISSING_MATRIX_RTC_TRANSPORT = "MISSING_MATRIX_RTC_TRANSPORT", + CONNECTION_LOST_ERROR = "CONNECTION_LOST_ERROR", + INTERNAL_MEMBERSHIP_MANAGER = "INTERNAL_MEMBERSHIP_MANAGER", + FAILED_TO_START_LIVEKIT = "FAILED_TO_START_LIVEKIT", + /** LiveKit indicates that the server has hit its track limits. */ + INSUFFICIENT_CAPACITY_ERROR = "INSUFFICIENT_CAPACITY_ERROR", + OPEN_ID_ERROR = "OPEN_ID_ERROR", + NO_MATRIX_2_AUTHORIZATION_SERVICE = "NO_MATRIX_2_0_AUTHORIZATION_SERVICE", + SFU_ERROR = "SFU_ERROR", + UNKNOWN_ERROR = "UNKNOWN_ERROR", +} + +export enum ErrorCategory { + /** Calling is not supported, the server is misconfigured (JWT service missing, no MSC support). */ + CONFIGURATION_ISSUE = "CONFIGURATION_ISSUE", + NETWORK_CONNECTIVITY = "NETWORK_CONNECTIVITY", + CLIENT_CONFIGURATION = "CLIENT_CONFIGURATION", + UNKNOWN = "UNKNOWN", + SYSTEM_FAILURE = "SYSTEM_FAILURE", +} + +/** + * An error raised by the client. `code` says what went wrong in a form a host + * can translate; `cause` holds the underlying error where there is one. + */ +export class MatrixRTCError extends Error { + public constructor( + message: string, + public readonly code: ErrorCode = ErrorCode.UNKNOWN_ERROR, + public readonly category: ErrorCategory = ErrorCategory.UNKNOWN, + options?: ErrorOptions, + ) { + super(message, options); + this.name = "MatrixRTCError"; + } +} + +/** Wraps whatever a backend threw into the SDK's error type, keeping it as the cause. */ +export function toMatrixRTCError(error: unknown): MatrixRTCError { + if (error instanceof MatrixRTCError) return error; + if (error instanceof Error) + return new MatrixRTCError( + error.message || error.name, + ErrorCode.UNKNOWN_ERROR, + ErrorCategory.UNKNOWN, + { cause: error }, + ); + return new MatrixRTCError(String(error)); +} + +export class MatrixRTCTransportMissingError extends MatrixRTCError { + public constructor(public readonly domain: string) { + super( + `The homeserver ${domain} advertises no MatrixRTC transport`, + ErrorCode.MISSING_MATRIX_RTC_TRANSPORT, + ErrorCategory.CONFIGURATION_ISSUE, + ); + } +} + +export class MembershipManagerError extends MatrixRTCError { + public constructor(cause: Error) { + super( + `The membership manager stopped: ${cause.message}`, + ErrorCode.INTERNAL_MEMBERSHIP_MANAGER, + ErrorCategory.SYSTEM_FAILURE, + { cause }, + ); + } +} + +export class UnknownCallError extends MatrixRTCError { + public constructor(cause: Error) { + super( + cause.message || "Unknown error", + ErrorCode.UNKNOWN_ERROR, + ErrorCategory.UNKNOWN, + { + cause, + }, + ); + } +} + +export class FailToGetOpenIdToken extends MatrixRTCError { + public constructor(cause: Error) { + super( + cause instanceof MatrixRTCError + ? cause.message + : `Could not authenticate with the transport: ${cause.message}`, + ErrorCode.OPEN_ID_ERROR, + ErrorCategory.CONFIGURATION_ISSUE, + { cause }, + ); + } +} + +export class NoMatrix2AuthorizationService extends MatrixRTCError { + public constructor(cause: Error) { + super( + "The transport's authorization service does not support Matrix 2.0", + ErrorCode.NO_MATRIX_2_AUTHORIZATION_SERVICE, + ErrorCategory.CONFIGURATION_ISSUE, + { cause }, + ); + } +} + +export class FailToStartLivekitConnection extends MatrixRTCError { + public constructor(detail?: string) { + super( + `Failed to start the media connection${detail ? `: ${detail}` : ""}`, + ErrorCode.FAILED_TO_START_LIVEKIT, + ErrorCategory.NETWORK_CONNECTIVITY, + ); + } +} + +export class InsufficientCapacityError extends MatrixRTCError { + public constructor() { + super( + "The media server has no capacity left", + ErrorCode.INSUFFICIENT_CAPACITY_ERROR, + ErrorCategory.UNKNOWN, + ); + } +} + +export class SFURoomCreationRestrictedError extends MatrixRTCError { + public constructor() { + super( + "The media server did not let this client create the room", + ErrorCode.SFU_ERROR, + ErrorCategory.CONFIGURATION_ISSUE, + ); + } +} + +export class PeerConnectionTimeoutError extends MatrixRTCError { + public constructor() { + super( + "Timed out establishing the media connection", + ErrorCode.SFU_ERROR, + ErrorCategory.NETWORK_CONNECTIVITY, + ); + } +} + +export class LivekitConnectionError extends MatrixRTCError { + public constructor(cause: ConnectionError) { + super( + `Could not connect to the media server: ${cause.message}`, + ErrorCode.SFU_ERROR, + ErrorCategory.NETWORK_CONNECTIVITY, + { cause }, + ); + } +} diff --git a/sdk/src/media/LivekitMediaTrack.test.ts b/sdk/src/media/LivekitMediaTrack.test.ts index 1d907a9c2..f6adda3ed 100644 --- a/sdk/src/media/LivekitMediaTrack.test.ts +++ b/sdk/src/media/LivekitMediaTrack.test.ts @@ -16,7 +16,7 @@ import { } from "livekit-client"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { ObservableScope } from "../../../src/state/ObservableScope"; +import { ObservableScope } from "../reactive/ObservableScope"; import { type AudioMediaTrack } from "../api"; import { createLivekitMediaTrack } from "./LivekitMediaTrack"; diff --git a/sdk/src/media/LivekitMediaTrack.ts b/sdk/src/media/LivekitMediaTrack.ts index 8c4d13b33..b4b065d74 100644 --- a/sdk/src/media/LivekitMediaTrack.ts +++ b/sdk/src/media/LivekitMediaTrack.ts @@ -35,8 +35,8 @@ import { switchMap, } from "rxjs"; -import { type Behavior } from "../../../src/state/Behavior"; -import { type ObservableScope } from "../../../src/state/ObservableScope"; +import { type Behavior } from "../reactive/Behavior"; +import { type ObservableScope } from "../reactive/ObservableScope"; import { type AudioMediaTrack, type MediaSource, diff --git a/sdk/src/media/LivekitMemberMedia.ts b/sdk/src/media/LivekitMemberMedia.ts index ebc99127d..315acaf94 100644 --- a/sdk/src/media/LivekitMemberMedia.ts +++ b/sdk/src/media/LivekitMemberMedia.ts @@ -20,10 +20,10 @@ import { } from "livekit-client"; import { distinctUntilChanged, filter, map, type Observable } from "rxjs"; -import { type Behavior } from "../../../src/state/Behavior"; -import { type ObservableScope } from "../../../src/state/ObservableScope"; -import { E2eeType } from "../../../src/e2ee/e2eeType"; -import { type EncryptionSystem } from "../../../src/e2ee/sharedKeyManagement"; +import { type Behavior } from "../reactive/Behavior"; +import { type ObservableScope } from "../reactive/ObservableScope"; +import { E2eeType } from "../encryption"; +import { type EncryptionSystem } from "../encryption"; import { type AudioMediaTrack, type EncryptionError, diff --git a/sdk/src/reactive/Behavior.ts b/sdk/src/reactive/Behavior.ts new file mode 100644 index 000000000..53a826e15 --- /dev/null +++ b/sdk/src/reactive/Behavior.ts @@ -0,0 +1,29 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { BehaviorSubject } from "rxjs"; + +/** + * A stateful, read-only reactive value. As an Observable, it is "hot" and + * always replays the current value upon subscription. + * + * A Behavior is to BehaviorSubject what Observable is to Subject; it does not + * provide a way to imperatively set new values. For more info on the + * distinction between Behaviors and Observables, see + * https://monoid.dk/post/behaviors-and-streams-why-both/. + */ +export type Behavior = Omit< + BehaviorSubject, + "next" | "observers" | "error" +>; + +/** + * Creates a Behavior which never changes in value. + */ +export function constant(value: T): Behavior { + return new BehaviorSubject(value); +} diff --git a/sdk/src/reactive/ObservableScope.test.ts b/sdk/src/reactive/ObservableScope.test.ts new file mode 100644 index 000000000..25bb14324 --- /dev/null +++ b/sdk/src/reactive/ObservableScope.test.ts @@ -0,0 +1,293 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { BehaviorSubject, combineLatest, Subject } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { sleep } from "matrix-js-sdk/lib/utils"; + +import { + Epoch, + mapEpoch, + ObservableScope, + trackEpoch, +} from "./ObservableScope"; +import { testScope, withTestScheduler } from "../utils/test"; + +describe("Epoch", () => { + it("should map the value correctly", () => { + const epoch = new Epoch(1); + const mappedEpoch = epoch.mapInner((v) => v + 1); + expect(mappedEpoch.value).toBe(2); + expect(mappedEpoch.epoch).toBe(0); + }); + + it("should be tracked from an observable", () => { + withTestScheduler(({ expectObservable, behavior }) => { + const observable$ = behavior("abc", { + a: 1, + b: 2, + c: 3, + }); + const epochObservable$ = observable$.pipe(trackEpoch()); + expectObservable(epochObservable$).toBe("abc", { + a: expect.toSatisfy((e) => e.epoch === 0 && e.value === 1), + b: expect.toSatisfy((e) => e.epoch === 1 && e.value === 2), + c: expect.toSatisfy((e) => e.epoch === 2 && e.value === 3), + }); + }); + }); + + it("can be mapped without loosing epoch information", () => { + withTestScheduler(({ expectObservable, behavior }) => { + const observable$ = behavior("abc", { + a: "A", + b: "B", + c: "C", + }); + const epochObservable$ = observable$.pipe(trackEpoch()); + const derivedEpoch$ = epochObservable$.pipe( + mapEpoch((e) => e + "-mapped"), + ); + + expectObservable(derivedEpoch$).toBe("abc", { + a: new Epoch("A-mapped", 0), + b: new Epoch("B-mapped", 1), + c: new Epoch("C-mapped", 2), + }); + }); + }); + + it("diamonds emits in a predictable order", () => { + const sb$ = new BehaviorSubject("initial"); + const root$ = sb$.pipe(trackEpoch()); + const derivedA$ = root$.pipe(mapEpoch((e) => e + "-A")); + const derivedB$ = root$.pipe(mapEpoch((e) => e + "-B")); + combineLatest([root$, derivedB$, derivedA$]).subscribe( + ([root, derivedA, derivedB]) => { + logger.log( + "combined" + + root.epoch + + root.value + + "\n" + + derivedA.epoch + + derivedA.value + + "\n" + + derivedB.epoch + + derivedB.value, + ); + }, + ); + sb$.next("updated"); + sb$.next("ANOTERUPDATE"); + }); + + it("behavior test", () => { + const scope = testScope(); + const s$ = new Subject(); + const behavior$ = scope.behavior(s$, 0); + behavior$.subscribe((value) => { + logger.log(`Received value: ${value}`); + }); + s$.next(1); + s$.next(2); + s$.next(3); + s$.next(3); + s$.next(3); + s$.next(3); + s$.next(3); + s$.complete(); + }); +}); + +describe("Reconcile", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("should wait clean up before processing next", async () => { + vi.useFakeTimers(); + const scope = testScope(); + const behavior$ = new BehaviorSubject(0); + + const setup = vi.fn().mockImplementation(async () => await sleep(100)); + const cleanup = vi + .fn() + .mockImplementation(async (n: number) => await sleep(100)); + scope.reconcile(behavior$, async (value) => { + await setup(); + return async (): Promise => { + await cleanup(value); + }; + }); + // Let the initial setup process + await vi.advanceTimersByTimeAsync(120); + expect(setup).toHaveBeenCalledTimes(1); + expect(cleanup).toHaveBeenCalledTimes(0); + + // Send next value + behavior$.next(1); + await vi.advanceTimersByTimeAsync(50); + // Should not have started setup for 1 yet + expect(setup).toHaveBeenCalledTimes(1); + expect(cleanup).toHaveBeenCalledTimes(1); + expect(cleanup).toHaveBeenCalledWith(0); + + // Let cleanup finish + await vi.advanceTimersByTimeAsync(50); + // Now setup for 1 should have started + expect(setup).toHaveBeenCalledTimes(2); + }); + + it("should skip intermediates values that are not setup", async () => { + vi.useFakeTimers(); + const scope = testScope(); + const behavior$ = new BehaviorSubject(0); + + const setup = vi + .fn() + .mockImplementation(async (n: number) => await sleep(100)); + + const cleanupLock = Promise.withResolvers(); + const cleanup = vi + .fn() + .mockImplementation(async (n: number) => await cleanupLock.promise); + + scope.reconcile(behavior$, async (value) => { + await setup(value); + return async (): Promise => { + await cleanup(value); + }; + }); + // Let the initial setup process (0) + await vi.advanceTimersByTimeAsync(120); + + // Send 4 next values quickly + behavior$.next(1); + behavior$.next(2); + behavior$.next(3); + behavior$.next(4); + + await vi.advanceTimersByTimeAsync(3000); + // should have only called cleanup for 0 + expect(cleanup).toHaveBeenCalledTimes(1); + expect(cleanup).toHaveBeenCalledWith(0); + // Let cleanup finish + cleanupLock.resolve(undefined); + await vi.advanceTimersByTimeAsync(120); + + // Now setup for 4 should have started, skipping 1,2,3 + expect(setup).toHaveBeenCalledTimes(2); + expect(setup).toHaveBeenCalledWith(4); + expect(setup).not.toHaveBeenCalledWith(1); + expect(setup).not.toHaveBeenCalledWith(2); + expect(setup).not.toHaveBeenCalledWith(3); + }); + + it("should wait for setup to complete before starting cleanup", async () => { + vi.useFakeTimers(); + const scope = testScope(); + const behavior$ = new BehaviorSubject(0); + + const setup = vi + .fn() + .mockImplementation(async (n: number) => await sleep(3000)); + + const cleanupLock = Promise.withResolvers(); + const cleanup = vi + .fn() + .mockImplementation(async (n: number) => await cleanupLock.promise); + + scope.reconcile(behavior$, async (value) => { + await setup(value); + return async (): Promise => { + await cleanup(value); + }; + }); + + await vi.advanceTimersByTimeAsync(500); + // Setup for 0 should be in progress + expect(setup).toHaveBeenCalledTimes(1); + + behavior$.next(1); + await vi.advanceTimersByTimeAsync(500); + + // Should not have started setup for 1 yet + expect(setup).not.toHaveBeenCalledWith(1); + // Should not have called cleanup yet, because the setup for 0 is not done + expect(cleanup).toHaveBeenCalledTimes(0); + + // Let setup for 0 finish + await vi.advanceTimersByTimeAsync(2500 + 100); + // Now cleanup for 0 should have started + expect(cleanup).toHaveBeenCalledTimes(1); + expect(cleanup).toHaveBeenCalledWith(0); + + cleanupLock.resolve(undefined); + await vi.advanceTimersByTimeAsync(100); + // Now setup for 1 should have started + expect(setup).toHaveBeenCalledWith(1); + }); +}); + +describe("behavior", () => { + it("warns with the tag and nesting depth, logging each new depth once", () => { + const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); + const scope = testScope(); + const source$ = new Subject(); + const behavior$ = scope.behavior(source$, 0, "tagged"); + // Re-enter once on value 1 (depth 2), then a nested re-entry on value 2 + // (depth 3), which is the shape needed to strand a middle subscriber. + behavior$.subscribe((v) => { + if (v === 1) source$.next(2); + }); + behavior$.subscribe((v) => { + if (v === 2) source$.next(3); + }); + const seen: number[] = []; + behavior$.subscribe((v) => seen.push(v)); + + source$.next(1); + + const messages = warn.mock.calls.map((c) => c[0] as string); + expect(messages).toEqual([ + expect.stringContaining("Behavior (tagged) re-entered at depth 2"), + expect.stringContaining("Behavior (tagged) re-entered at depth 3"), + ]); + warn.mock.calls.forEach((c) => expect(c[1]).toEqual(expect.any(String))); + // The behavior settles on the newest value while the last subscriber is + // left stranded on the oldest. + expect(behavior$.value).toBe(3); + expect(seen.at(-1)).toBe(1); + }); + + it("warns when the scope is ended while a behavior is mid-delivery", () => { + const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); + const scope = new ObservableScope(); + const source$ = new Subject(); + const behavior$ = scope.behavior(source$, 0); + behavior$.subscribe((v) => { + if (v === 1) scope.end(); + }); + // A derived behavior is bound to the scope, so ending the scope + // mid-delivery unsubscribes it before the value reaches it. + const derived$ = scope.behavior(behavior$); + + source$.next(1); + + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("Scope ended while 1 of its behaviors"), + expect.any(String), + ); + expect(behavior$.value).toBe(1); + expect(derived$.value).toBe(0); + }); +}); diff --git a/sdk/src/reactive/ObservableScope.ts b/sdk/src/reactive/ObservableScope.ts new file mode 100644 index 000000000..4f657b96a --- /dev/null +++ b/sdk/src/reactive/ObservableScope.ts @@ -0,0 +1,329 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + BehaviorSubject, + catchError, + distinctUntilChanged, + EMPTY, + endWith, + filter, + map, + type Observable, + type OperatorFunction, + share, + take, + takeUntil, +} from "rxjs"; + +import { logger } from "matrix-js-sdk/lib/logger"; + +import { type Behavior } from "./Behavior"; + +type MonoTypeOperator = (o: Observable) => Observable; + +type SplitBehavior = keyof T extends string | number + ? { [K in keyof T as `${K}$`]: Behavior } + : never; + +const nothing = Symbol("nothing"); + +/** + * A scope which limits the execution lifetime of its bound Observables. + */ +export class ObservableScope { + /** + * Number of this scope's behaviors currently mid-delivery. Used to detect a + * scope being torn down synchronously from within one of its own emissions. + */ + private delivering = 0; + + private readonly ended$ = new BehaviorSubject(false); + + private readonly bindImpl: MonoTypeOperator = takeUntil( + this.ended$.pipe(filter((ended) => ended)), + ); + + /** + * Binds an Observable to this scope, so that it completes when the scope + * ends. + */ + public bind(): MonoTypeOperator { + return this.bindImpl; + } + + private readonly shareImpl: MonoTypeOperator = share({ + resetOnError: false, + resetOnComplete: false, + resetOnRefCountZero: false, + }); + /** + * Shares (multicasts) the Observable as a hot Observable. + */ + public readonly share: MonoTypeOperator = (input$) => + input$.pipe(this.bindImpl, this.shareImpl); + + /** + * Converts an Observable to a Behavior. If no initial value is specified, the + * Observable must synchronously emit an initial value. + */ + public behavior( + setValue$: Observable, + initialValue: T | typeof nothing = nothing, + // An optional label used only in the re-entrancy warning below, so that a + // torn behavior can be identified from a rageshake. + tag?: string, + ): Behavior { + const subject$ = new BehaviorSubject(initialValue); + // Push values from the Observable into the BehaviorSubject. + // BehaviorSubjects have an undesirable feature where if you call 'complete', + // they will no longer re-emit their current value upon subscription. We want + // to support Observables that complete (for example `of({})`), so we have to + // take care to not propagate the completion event. + // If a subscriber synchronously causes this same behavior to emit again, + // rxjs delivers the nested value to every subscriber first and then + // resumes delivering the outer (older) value to the remaining subscribers, + // leaving them permanently out of sync with the others. A single such + // re-entry can only strand a contiguous run of subscribers; stranding a + // subscriber in the middle of the list needs a nested (deeper) re-entry, so + // we report the depth and log each new depth (with a stack trace) rather + // than only the first occurrence, to make a nested re-entry visible. + let depth = 0; + let maxReportedDepth = 1; + setValue$.pipe(this.bind(), distinctUntilChanged()).subscribe({ + next: (value) => { + if (depth > 0 && depth + 1 > maxReportedDepth) { + maxReportedDepth = depth + 1; + logger.warn( + `Behavior${tag ? ` (${tag})` : ""} re-entered at depth ${ + depth + 1 + } while delivering a value; later subscribers will be left with a stale value`, + new Error().stack, + ); + } + depth++; + this.delivering++; + try { + subject$.next(value); + } finally { + depth--; + this.delivering--; + } + }, + error: (err: unknown) => { + subject$.error(err); + }, + }); + if (subject$.value === nothing) + throw new Error("Behavior failed to synchronously emit an initial value"); + return subject$ as Behavior; + } + + /** + * Ends the scope, causing any bound Observables to complete. + */ + public end(): void { + if (this.delivering > 0) + logger.warn( + `Scope ended while ${this.delivering} of its behaviors were still delivering a value; later subscribers will be left with a stale value`, + new Error().stack, + ); + this.ended$.next(true); + } + + /** + * Register a callback to be executed when the scope is ended. + */ + public onEnd(callback: () => void): void { + this.ended$ + .pipe( + filter((ended) => ended), + take(1), + ) + .subscribe(callback); + } + + /** + * For the duration of the scope, sync some external state with the value of + * the provided Behavior by way of an async function which attempts to update + * (reconcile) the external state. The reconciliation function may return a + * clean-up callback which will be called and awaited before the next change + * in value (or the end of the scope). + * + * All calls to the function and its clean-up callbacks are serialized. If the + * value changes faster than the handlers can keep up with, intermediate + * values may be skipped. + * + * Basically, this is like React's useEffect but async and for Behaviors. + */ + public reconcile( + value$: Behavior, + callback: (value: T) => Promise<(() => Promise) | void>, + ): void { + let latestValue: T | typeof nothing = nothing; + let reconcilePromise: Promise | undefined = undefined; + let cleanUp: (() => Promise) | void = undefined; + let prevVal: T | typeof nothing = nothing; + + // While this loop runs it will process the latest from `value$` until it caught up with the updates. + // It might skip updates from `value$` and only process the newest value after callback has resolved. + const reconcileLoop = async (): Promise => { + while (latestValue !== prevVal) { + await cleanUp?.(); // Call the previous value's clean-up handler + prevVal = latestValue; + + if (latestValue !== nothing) cleanUp = await callback(latestValue); // Sync current value... + // `latestValue` might have gotten updated during the `await callback`. That is why we loop here + } + }; + + value$ + .pipe( + catchError(() => EMPTY), // Ignore errors + this.bind(), // Limit to the duration of the scope + endWith(nothing), // Clean up when the scope ends + ) + .subscribe((value) => { + // Always track the latest value! The `reconcileLoop` will run until it "processed" the "last" `latestValue`. + latestValue = value; + // There's already an instance of the below 'reconcileLoop' loop running + // concurrently. So lets let the loop handle it. NEVER instanciate two `reconcileLoop`s. + if (reconcilePromise) return; + + reconcilePromise = reconcileLoop().finally(() => { + reconcilePromise = undefined; + }); + }); + } + + /** + * Splits a Behavior of objects with static properties into an object with + * Behavior properties. + * + * For example, splitting a `Behavior<{ name: string; age: number }>` results + * in an object of type `{ name$: Behavior; age$: Behavior }`. + */ + public splitBehavior( + input$: Behavior, + ): SplitBehavior { + return Object.fromEntries( + Object.keys(input$.value).map((key) => [ + `${key}$`, + this.behavior( + input$.pipe(map((input) => input[key as keyof T])), + nothing, + key, + ), + ]), + ) as SplitBehavior; + } +} + +/** + * `Epoch`'s can be used to create `Behavior`s and `Observable`s which derivitives can be merged + * with `combinedLatest` without duplicated emissions. + * + * This is useful in the following example: + * ``` + * const rootObs$ = of("red","green","blue"); + * const derivedObs$ = rootObs$.pipe( + * map((v)=> {red:"fire", green:"grass", blue:"water"}[v]) + * ); + * const otherDerivedObs$ = rootObs$.pipe( + * map((v)=> {red:"tomatoes", green:"leaves", blue:"sky"}[v]) + * ); + * const mergedObs$ = combineLatest([rootObs$, derivedObs$, otherDerivedObs$]).pipe( + * map(([color, a,b]) => color + " like " + a + " and " + b) + * ); + * + * ``` + * will result in 6 emissions with mismatching items like "red like fire and leaves" + * + * # Use Epoch + * ``` + * const ancestorObs$ = of(1,2,3).pipe(trackEpoch()); + * const derivedObs$ = ancestorObs$.pipe( + * mapEpoch((v)=> "this number: " + v) + * ); + * const otherDerivedObs$ = ancestorObs$.pipe( + * mapEpoch((v)=> "multiplied by: " + v) + * ); + * const mergedObs$ = combineLatest([derivedObs$, otherDerivedObs$]).pipe( + * filter((values) => values.every((v) => v.epoch === values[0].v)), + * map(([color, a, b]) => color + " like " + a + " and " + b) + * ); + * + * ``` + * will result in 3 emissions all matching (e.g. "blue like water and sky") + */ +export class Epoch { + public readonly epoch: number; + public readonly value: T; + + public constructor(value: T, epoch?: number) { + this.value = value; + this.epoch = epoch ?? 0; + } + /** + * Maps the value inside the epoch to a new value while keeping the epoch number. + * # usage + * ``` + * const myEpoch$ = myObservable$.pipe( + * map(trackEpoch()), + * // this is the preferred way using mapEpoch + * mapEpoch((v)=> v+1) + * // This is how inner map can be used: + * map((epoch) => epoch.innerMap((v)=> v+1)) + * // It is equivalent to: + * map((epoch) => new Epoch(epoch.value + 1, epoch.epoch)) + * ) + * ``` + * See also `Epoch` + */ + public mapInner(map: (value: T) => U): Epoch { + return new Epoch(map(this.value), this.epoch); + } +} + +/** + * A `pipe` compatible map oparator that keeps the epoch in tact but allows mapping the value. + * # usage + * ``` + * const myEpoch$ = myObservable$.pipe( + * map(trackEpoch()), + * // this is the preferred way using mapEpoch + * mapEpoch((v)=> v+1) + * // This is how inner map can be used: + * map((epoch) => epoch.innerMap((v)=> v+1)) + * // It is equivalent to: + * map((epoch) => new Epoch(epoch.value + 1, epoch.epoch)) + * ) + * ``` + * See also `Epoch` + */ +export function mapEpoch( + mapFn: (value: T) => U, +): OperatorFunction, Epoch> { + return map((e) => e.mapInner(mapFn)); +} + +/** + * # usage + * ``` + * const myEpoch$ = myObservable$.pipe( + * map(trackEpoch()), + * map((epoch) => epoch.innerMap((v)=> v+1)) + * ) + * const derived = myEpoch$.pipe( + * mapEpoch((v)=>v^2) + * ) + * ``` + * See also `Epoch` + */ +export function trackEpoch(): OperatorFunction> { + return map>((value, number) => new Epoch(value, number)); +} diff --git a/sdk/src/reactive/observable.test.ts b/sdk/src/reactive/observable.test.ts new file mode 100644 index 000000000..1f613c3ec --- /dev/null +++ b/sdk/src/reactive/observable.test.ts @@ -0,0 +1,104 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "vitest"; +import { type Observable, of, Subject, switchMap } from "rxjs"; + +import { withTestScheduler } from "../utils/test"; +import { filterBehavior, generateItems, pauseWhen } from "./observable"; +import { type Behavior } from "./Behavior"; + +test("pauseWhen", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const inputMarbles = " abcdefgh-i-jk-"; + const pauseMarbles = " n-y--n-yn-y--n"; + const outputMarbles = "abc--fgh-i---k"; + expectObservable( + behavior(inputMarbles).pipe( + pauseWhen(behavior(pauseMarbles, { y: true, n: false })), + ), + ).toBe(outputMarbles); + }); +}); + +test("generateItems", () => { + const scope1$ = new Subject(); + const scope2$ = new Subject(); + const scope3$ = new Subject(); + const scope4$ = new Subject(); + const scopeSubjects = [scope1$, scope2$, scope3$, scope4$]; + + withTestScheduler(({ hot, expectObservable }) => { + // Each scope should start when the input number reaches or surpasses their + // number and end when the input number drops back below their number. + // At the very end, unsubscribing should end all remaining scopes. + const inputMarbles = " 123242"; + const outputMarbles = " abcbdb"; + const subscriptionMarbles = "^-----!"; + const scope1Marbles = " y-----n"; + const scope2Marbles = " -y----n"; + const scope3Marbles = " --ynyn"; + const scope4Marbles = " ----yn"; + + expectObservable( + hot(inputMarbles).pipe( + generateItems( + "test items", + function* (input) { + for (let i = 1; i <= +input; i++) { + yield { keys: [i], data: undefined }; + } + }, + (scope, data$, i) => { + scopeSubjects[i - 1].next("y"); + scope.onEnd(() => scopeSubjects[i - 1].next("n")); + return i.toString(); + }, + ), + ), + subscriptionMarbles, + ).toBe(outputMarbles, { + a: ["1"], + b: ["1", "2"], + c: ["1", "2", "3"], + d: ["1", "2", "3", "4"], + }); + + expectObservable(scope1$).toBe(scope1Marbles); + expectObservable(scope2$).toBe(scope2Marbles); + expectObservable(scope3$).toBe(scope3Marbles); + expectObservable(scope4$).toBe(scope4Marbles); + }); +}); + +test("filterBehavior", () => { + withTestScheduler(({ behavior, expectObservable }) => { + // Filtering the input should segment it into 2 modes of non-null behavior. + const inputMarbles = " abcxabx"; + const filteredMarbles = "a--xa-x"; + + const input$ = behavior(inputMarbles, { + a: "a", + b: "b", + c: "c", + x: null, + }); + const filtered$: Observable | null> = input$.pipe( + filterBehavior((value) => typeof value === "string"), + ); + + expectObservable(filtered$).toBe(filteredMarbles, { + a: expect.any(Object), + x: null, + }); + expectObservable( + filtered$.pipe( + switchMap((value$) => (value$ === null ? of(null) : value$)), + ), + ).toBe(inputMarbles, { a: "a", b: "b", c: "c", x: null }); + }); +}); diff --git a/sdk/src/reactive/observable.ts b/sdk/src/reactive/observable.ts new file mode 100644 index 000000000..b4d464f9b --- /dev/null +++ b/sdk/src/reactive/observable.ts @@ -0,0 +1,264 @@ +/* +Copyright 2023, 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type Observable, + audit, + defer, + filter, + finalize, + map, + of, + scan, + tap, + withLatestFrom, + BehaviorSubject, + type OperatorFunction, + distinctUntilChanged, +} from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { type Behavior } from "./Behavior"; +import { Epoch, ObservableScope } from "./ObservableScope"; + +const nothing = Symbol("nothing"); + +/** + * RxJS operator that invokes a callback when the Observable is finalized, + * passing the most recently emitted value. If no value was emitted, the + * callback will not be invoked. + */ +export function finalizeValue(callback: (finalValue: T) => void) { + return (source$: Observable): Observable => + defer(() => { + let finalValue: T | typeof nothing = nothing; + return source$.pipe( + tap((value) => (finalValue = value)), + finalize(() => { + if (finalValue !== nothing) callback(finalValue); + }), + ); + }); +} + +/** + * RxJS operator that pauses all changes in the input value whenever a Behavior + * is true. When the Behavior returns to being false, the most recently + * suppressed change is emitted as the most recent value. + */ +export function pauseWhen(pause$: Behavior) { + return (value$: Observable): Observable => + value$.pipe( + withLatestFrom(pause$), + audit(([, pause]) => + pause ? pause$.pipe(filter((pause) => !pause)) : of(null), + ), + map(([value]) => value), + ); +} + +interface ItemHandle { + scope: ObservableScope; + keys: readonly [...Keys]; + data$: BehaviorSubject; + item: Item; +} + +/** + * Maps a changing input value to a collection of items that each capture some + * dynamic data and are tied to a key. Items will be automatically created when + * their key is requested for the first time, reused when the same key is + * requested at a later time, and destroyed (have their scope ended) when the + * key is no longer requested. + * + * @param name A name for this collection to use in debug logs. + * @param generator A generator function yielding a tuple of keys and the + * currently associated data for each item that it wants to exist. + * @param factory A function constructing an individual item, given the item's key, + * dynamic data, and an automatically managed ObservableScope for the item. + */ +export function generateItems< + Input, + Keys extends [unknown, ...unknown[]], + Data, + Item, +>( + name: string, + generator: ( + input: Input, + ) => Iterable<{ keys: readonly [...Keys]; data: Data }, void, void>, + factory: ( + scope: ObservableScope, + data$: Behavior, + ...keys: Keys + ) => Item, +): OperatorFunction { + return generateItemsInternal(name, generator, factory, (items) => items); +} + +/** + * Same as generateItems, but preserves epoch data. + */ +export function generateItemsWithEpoch< + Input, + Keys extends [unknown, ...unknown[]], + Data, + Item, +>( + name: string, + generator: ( + input: Input, + ) => Iterable<{ keys: readonly [...Keys]; data: Data }, void, void>, + factory: ( + scope: ObservableScope, + data$: Behavior, + ...keys: Keys + ) => Item, +): OperatorFunction, Epoch> { + return generateItemsInternal( + name, + function* (input) { + yield* generator(input.value); + }, + factory, + (items, input) => new Epoch(items, input.epoch), + ); +} + +/** + * Segments a behavior into periods during which its value matches the filter + * (outputting a behavior with a narrowed type) and periods during which it does + * not match (outputting null). + */ +export function filterBehavior( + predicate: (value: T) => value is S, +): OperatorFunction | null> { + return (input$) => + input$.pipe( + scan | null>((acc$, input) => { + if (predicate(input)) { + const output$ = acc$ ?? new BehaviorSubject(input); + output$.next(input); + return output$; + } + return null; + }, null), + distinctUntilChanged(), + ); +} + +function generateItemsInternal< + Input, + Keys extends [unknown, ...unknown[]], + Data, + Item, + Output, +>( + name: string, + generator: ( + input: Input, + ) => Iterable<{ keys: readonly [...Keys]; data: Data }, void, void>, + factory: ( + scope: ObservableScope, + data$: Behavior, + ...keys: Keys + ) => Item, + project: (items: Item[], input: Input) => Output, +): OperatorFunction { + /* eslint-disable @typescript-eslint/no-explicit-any */ + return (input$) => + input$.pipe( + // Keep track of the existing items over time, so they can persist + scan< + Input, + { + map: Map; + items: Set>; + input: Input; + }, + { map: Map; items: Set> } + >( + ({ map: prevMap, items: prevItems }, input) => { + const nextMap = new Map(); + const nextItems = new Set>(); + + for (const { keys, data } of generator(input)) { + // Disable type checks for a second to grab the item out of a nested map + let i: any = prevMap; + for (const key of keys) i = i?.get(key); + let item = i as ItemHandle | undefined; + + if (item === undefined) { + // First time requesting the key; create the item + const scope = new ObservableScope(); + const data$ = new BehaviorSubject(data); + logger.debug( + `[${name}] Creating item with keys ${keys.join(", ")}`, + ); + item = { + scope, + keys, + data$, + item: factory(scope, data$, ...keys), + }; + } else { + item.data$.next(data); + } + + // Likewise, disable type checks to insert the item in the nested map + let m: Map = nextMap; + for (let i = 0; i < keys.length - 1; i++) { + let inner = m.get(keys[i]); + if (inner === undefined) { + inner = new Map(); + m.set(keys[i], inner); + } + m = inner; + } + const finalKey = keys[keys.length - 1]; + if (m.has(finalKey)) + throw new Error( + `Keys must be unique (tried to generate multiple items for key ${keys.join(", ")})`, + ); + m.set(keys[keys.length - 1], item); + nextItems.add(item); + } + + // Destroy all items that are no longer being requested + for (const item of prevItems) + if (!nextItems.has(item)) { + logger.debug( + `[${name}] Destroying item with keys ${item.keys.join(", ")}`, + ); + item.scope.end(); + } + + return { map: nextMap, items: nextItems, input }; + }, + { map: new Map(), items: new Set() }, + ), + finalizeValue(({ items }) => { + // Destroy all remaining items when no longer subscribed + logger.debug( + `[${name}] End of scope, destroying all ${items.size} items…`, + ); + for (const item of items) { + logger.debug( + `[${name}] Destroying item with keys ${item.keys.join(", ")}`, + ); + item.scope.end(); + } + }), + map(({ items, input }) => + project( + [...items].map(({ item }) => item), + input, + ), + ), + ); + /* eslint-enable @typescript-eslint/no-explicit-any */ +} diff --git a/sdk/src/session/Connection.test.ts b/sdk/src/session/Connection.test.ts new file mode 100644 index 000000000..ca644cf6d --- /dev/null +++ b/sdk/src/session/Connection.test.ts @@ -0,0 +1,733 @@ +/* +Copyright 2025 Element Creations Ltd. +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + afterEach, + beforeEach, + describe, + expect, + it, + type MockedObject, + onTestFinished, + vi, +} from "vitest"; +import { + type LocalParticipant, + type RemoteParticipant, + type RemoteTrack, + type RemoteTrackPublication, + type Room as LivekitRoom, + RoomEvent, + Track, + ConnectionState as LivekitConnectionState, +} from "livekit-client"; +import fetchMock from "fetch-mock"; +import EventEmitter from "events"; +import { type IOpenIDToken } from "matrix-js-sdk"; +import { logger, type Logger } from "matrix-js-sdk/lib/logger"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; + +import { + Connection, + ConnectionState, + type ConnectionOpts, +} from "./Connection.ts"; +import { ObservableScope } from "../reactive/ObservableScope"; +import { type OpenIDClientParts } from "./openIDSFU"; +import { MatrixRTCError, FailToGetOpenIdToken } from "../errors"; +import { testJWTToken } from "../utils/test-fixtures"; +import { mockRemoteParticipant, ownMemberMock } from "../utils/test"; + +let testScope: ObservableScope; + +let client: MockedObject; + +let fakeLivekitRoom: MockedObject; + +let localParticipantEventEmiter: EventEmitter; +let fakeLocalParticipant: MockedObject; + +const ROOM_ID = "!roomID:example.org"; + +const livekitFocus: LivekitTransport = { + livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", + type: "livekit", +}; + +function setupTest(): void { + testScope = new ObservableScope(); + client = vi.mocked({ + getOpenIdToken: vi.fn().mockResolvedValue({ + access_token: "rYsmGUEwNjKgJYyeNUkZseJN", + token_type: "Bearer", + matrix_server_name: "example.org", + expires_in: 3600, + }), + getDeviceId: vi.fn().mockReturnValue("ABCDEF"), + } as unknown as OpenIDClientParts); + + localParticipantEventEmiter = new EventEmitter(); + + fakeLocalParticipant = vi.mocked({ + identity: "@me:example.org", + isMicrophoneEnabled: vi.fn().mockReturnValue(true), + getTrackPublication: vi.fn().mockReturnValue(undefined), + on: localParticipantEventEmiter.on.bind(localParticipantEventEmiter), + off: localParticipantEventEmiter.off.bind(localParticipantEventEmiter), + addListener: localParticipantEventEmiter.addListener.bind( + localParticipantEventEmiter, + ), + removeListener: localParticipantEventEmiter.removeListener.bind( + localParticipantEventEmiter, + ), + removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind( + localParticipantEventEmiter, + ), + } as unknown as LocalParticipant); + + const fakeRoomEventEmitter = new EventEmitter(); + fakeLivekitRoom = vi.mocked({ + connect: vi.fn(), + disconnect: vi.fn(), + remoteParticipants: new Map(), + localParticipant: fakeLocalParticipant, + state: LivekitConnectionState.Disconnected, + on: fakeRoomEventEmitter.on.bind(fakeRoomEventEmitter), + off: fakeRoomEventEmitter.off.bind(fakeRoomEventEmitter), + addListener: fakeRoomEventEmitter.addListener.bind(fakeRoomEventEmitter), + removeListener: + fakeRoomEventEmitter.removeListener.bind(fakeRoomEventEmitter), + removeAllListeners: + fakeRoomEventEmitter.removeAllListeners.bind(fakeRoomEventEmitter), + setE2EEEnabled: vi.fn().mockResolvedValue(undefined), + emit: (eventName: string | symbol, ...args: unknown[]) => { + fakeRoomEventEmitter.emit(eventName, ...args); + }, + } as unknown as LivekitRoom); +} + +function setupRemoteConnection(): Connection { + const opts: ConnectionOpts = { + client: client, + roomId: ROOM_ID, + transport: livekitFocus, + scope: testScope, + ownMembershipIdentity: ownMemberMock, + livekitRoomFactory: () => fakeLivekitRoom, + }; + + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { + return { + status: 200, + body: { + url: "wss://matrix-rtc.m.localhost/livekit/sfu", + jwt: testJWTToken, + }, + }; + }); + + fakeLivekitRoom.connect.mockImplementation(async (): Promise => { + const changeEv = RoomEvent.ConnectionStateChanged; + + fakeLivekitRoom.state = LivekitConnectionState.Connecting; + fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state); + fakeLivekitRoom.state = LivekitConnectionState.Connected; + fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state); + + return Promise.resolve(); + }); + + return new Connection(opts, logger); +} + +afterEach(() => { + vi.useRealTimers(); + vi.clearAllMocks(); + fetchMock.reset(); +}); + +describe("Start connection states", () => { + beforeEach(() => { + fetchMock.post( + `https://matrix-rtc.example.org/livekit/jwt/get_token`, + () => { + return { + // Return a non-retryable error, if not, the retry logic will + // wait and fail the test with a timeout. + status: 404, + }; + }, + ); + }); + + it("start in initialized state", () => { + setupTest(); + + const opts: ConnectionOpts = { + client: client, + roomId: ROOM_ID, + transport: livekitFocus, + scope: testScope, + ownMembershipIdentity: ownMemberMock, + livekitRoomFactory: () => fakeLivekitRoom, + }; + const connection = new Connection(opts, logger); + + expect(connection.state$.getValue()).toEqual("Initialized"); + }); + + it("fail to getOpenId token then error state", async () => { + setupTest(); + vi.useFakeTimers(); + + const opts: ConnectionOpts = { + client: client, + roomId: ROOM_ID, + transport: livekitFocus, + scope: testScope, + ownMembershipIdentity: ownMemberMock, + livekitRoomFactory: () => fakeLivekitRoom, + }; + + const connection = new Connection(opts, logger); + + const capturedStates: (ConnectionState | Error)[] = []; + const s = connection.state$.subscribe((value) => { + capturedStates.push(value); + }); + onTestFinished(() => s.unsubscribe()); + + const deferred = Promise.withResolvers(); + + client.getOpenIdToken.mockImplementation( + async (): Promise => { + return await deferred.promise; + }, + ); + + connection.start().catch(() => { + // expected to throw + }); + + let capturedState = capturedStates.pop(); + expect(capturedState).toBeDefined(); + expect(capturedState!).toEqual("FetchingConfig"); + + deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); + + await vi.runAllTimersAsync(); + + capturedState = capturedStates.pop(); + if (capturedState instanceof Error) { + expect(capturedState).toBeInstanceOf(FailToGetOpenIdToken); + expect(connection.transport).toEqual(livekitFocus); + } else { + expect.fail("Expected FailedToStart state but got " + capturedState); + } + }); + + it("fail to get JWT token and error state", async () => { + setupTest(); + vi.useFakeTimers(); + + const opts: ConnectionOpts = { + client: client, + roomId: ROOM_ID, + transport: livekitFocus, + scope: testScope, + ownMembershipIdentity: ownMemberMock, + livekitRoomFactory: () => fakeLivekitRoom, + }; + + const connection = new Connection(opts, logger); + + const capturedStates: (ConnectionState | Error)[] = []; + const s = connection.state$.subscribe((value) => { + capturedStates.push(value); + }); + onTestFinished(() => s.unsubscribe()); + + const deferredSFU = Promise.withResolvers(); + // mock the /sfu/get call + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => { + await deferredSFU.promise; + return { + status: 500, + body: { + errcode: "M_LOOKUP_FAILED", + error: "Failed to look up user info from homeserver", + }, + }; + }); + + connection.start().catch(() => { + // expected to throw + }); + + let capturedState = capturedStates.pop(); + expect(capturedState).toBeDefined(); + expect(capturedState).toEqual(ConnectionState.FetchingConfig); + + deferredSFU.resolve(); + await vi.runAllTimersAsync(); + + capturedState = capturedStates.pop(); + + if ( + capturedState instanceof MatrixRTCError && + capturedState.cause instanceof Error + ) { + expect(capturedState.cause.message).toContain( + "Failed to look up user info from homeserver", + ); + expect(connection.transport).toEqual(livekitFocus); + } else { + expect.fail("Expected FailedToStart state but got " + capturedState); + } + }); + + it("fail to connect to livekit error state", async () => { + setupTest(); + vi.useFakeTimers(); + + const opts: ConnectionOpts = { + client: client, + roomId: ROOM_ID, + transport: livekitFocus, + scope: testScope, + ownMembershipIdentity: ownMemberMock, + livekitRoomFactory: () => fakeLivekitRoom, + }; + + const connection = new Connection(opts, logger); + + const capturedStates: (ConnectionState | Error)[] = []; + const s = connection.state$.subscribe((value) => { + capturedStates.push(value); + }); + onTestFinished(() => s.unsubscribe()); + + const deferredSFU = Promise.withResolvers(); + // mock the /sfu/get call + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { + return { + status: 200, + body: { + url: "wss://matrix-rtc.m.localhost/livekit/sfu", + jwt: testJWTToken, + }, + }; + }); + + fakeLivekitRoom.connect.mockImplementation(async () => { + await deferredSFU.promise; + throw new Error("Failed to connect to livekit"); + }); + + connection.start().catch(() => { + // expected to throw + }); + + let capturedState = capturedStates.pop(); + expect(capturedState).toBeDefined(); + + expect(capturedState).toEqual(ConnectionState.FetchingConfig); + + deferredSFU.resolve(); + await vi.runAllTimersAsync(); + + capturedState = capturedStates.pop(); + + if ( + capturedState instanceof MatrixRTCError && + capturedState.cause instanceof Error + ) { + expect(capturedState.cause.message).toContain( + "Failed to connect to livekit", + ); + expect(connection.transport).toEqual(livekitFocus); + } else { + expect.fail( + "Expected FailedToStart state but got " + JSON.stringify(capturedState), + ); + } + }); + + it("connection states happy path", async () => { + vi.useFakeTimers(); + setupTest(); + + const connection = setupRemoteConnection(); + + const capturedStates: (ConnectionState | Error)[] = []; + const s = connection.state$.subscribe((value) => { + capturedStates.push(value); + }); + onTestFinished(() => s.unsubscribe()); + + await connection.start(); + await vi.runAllTimersAsync(); + + const initialState = capturedStates.shift(); + expect(initialState).toEqual(ConnectionState.Initialized); + const fetchingState = capturedStates.shift(); + expect(fetchingState).toEqual(ConnectionState.FetchingConfig); + const disconnectedState = capturedStates.shift(); + expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected); + const connectingState = capturedStates.shift(); + expect(connectingState).toEqual(ConnectionState.LivekitConnecting); + const connectedState = capturedStates.shift(); + expect(connectedState).toEqual(ConnectionState.LivekitConnected); + }); + + it("stopping while connecting does not report an error", async () => { + setupTest(); + const connection = setupRemoteConnection(); + + const capturedStates: (ConnectionState | Error)[] = []; + const s = connection.state$.subscribe((value) => { + capturedStates.push(value); + }); + onTestFinished(() => s.unsubscribe()); + + // livekit-client rejects a pending connect() when disconnect() is called. + const pendingConnect = Promise.withResolvers(); + fakeLivekitRoom.connect.mockReturnValue(pendingConnect.promise); + fakeLivekitRoom.disconnect.mockResolvedValue(undefined); + + const started = connection.start(); + await vi.waitFor(() => expect(fakeLivekitRoom.connect).toHaveBeenCalled()); + const stopping = connection.stop(); + pendingConnect.reject(new Error("Client initiated disconnect")); + await stopping; + + // start() resolves rather than rejecting (it is not awaited by the + // ConnectionManager, so a rejection would be unhandled). + await expect(started).resolves.toBeUndefined(); + expect(capturedStates.at(-1)).toEqual(ConnectionState.Stopped); + expect(capturedStates.some((st) => st instanceof Error)).toBe(false); + }); + + it("shutting down the scope should stop the connection", async () => { + setupTest(); + vi.useFakeTimers(); + + const connection = setupRemoteConnection(); + await connection.start(); + + const stopSpy = vi.spyOn(connection, "stop"); + testScope.end(); + + expect(stopSpy).toHaveBeenCalled(); + expect(fakeLivekitRoom.disconnect).toHaveBeenCalled(); + }); +}); + +describe("remote participants", () => { + it("emits the list of remote participants", () => { + setupTest(); + + const connection = setupRemoteConnection(); + + const observedParticipants: RemoteParticipant[][] = []; + const s = connection.remoteParticipants$.subscribe((participants) => { + observedParticipants.push(participants); + }); + onTestFinished(() => s.unsubscribe()); + // The remoteParticipants$ observable is derived from the current members of the + // livekitRoom and the rtc membership in order to publish the members that are publishing + // on this connection. + + let participants: RemoteParticipant[] = [ + mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }), + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), + mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }), + // Mock Dan to have no published tracks. We want him to still show show up + // in the participants list. + mockRemoteParticipant({ + identity: "@dan:example.org:DEV333", + getTrackPublication: () => undefined, + getTrackPublications: () => [], + }), + ]; + + // Let's simulate 3 members on the livekitRoom + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation( + () => new Map(participants.map((p) => [p.identity, p])), + ); + + participants.forEach((p) => + fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p), + ); + + // At this point there should be ~~no~~ publishers + // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks) + // The filtering we do is just based on the matrixRTC member events. + expect(observedParticipants.pop()!.length).toEqual(4); + + participants = [ + mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }), + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), + mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }), + mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }), + ]; + participants.forEach((p) => + fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p), + ); + + // At this point there should be no publishers + expect(observedParticipants.pop()!.length).toEqual(4); + }); + + it("should be scoped to parent scope", (): void => { + setupTest(); + + const connection = setupRemoteConnection(); + + let observedParticipants: RemoteParticipant[][] = []; + const s = connection.remoteParticipants$.subscribe((participants) => { + observedParticipants.push(participants); + }); + onTestFinished(() => s.unsubscribe()); + + let participants: RemoteParticipant[] = [ + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), + ]; + + // Let's simulate 3 members on the livekitRoom + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation( + () => new Map(participants.map((p) => [p.identity, p])), + ); + + for (const participant of participants) { + fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, participant); + } + + // We should have bob as a participant now + const ps = observedParticipants.pop(); + expect(ps?.length).toEqual(1); + expect(ps?.[0]?.identity).toEqual("@bob:example.org:DEV111"); + + // end the parent scope + testScope.end(); + observedParticipants = []; + + // SHOULD NOT emit any more participants as the scope is ended + participants = participants.filter( + (p) => p.identity !== "@bob:example.org:DEV111", + ); + + fakeLivekitRoom.emit( + RoomEvent.ParticipantDisconnected, + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), + ); + + expect(observedParticipants.length).toEqual(0); + }); +}); + +describe("remote track logging", () => { + it("logs remote participant and track events on the connection logger", () => { + setupTest(); + const info = vi.fn(); + const warn = vi.fn(); + const testLogger = { + getChild: (): unknown => testLogger, + info, + debug: vi.fn(), + warn, + error: vi.fn(), + } as unknown as Logger; + new Connection( + { + client, + roomId: ROOM_ID, + transport: livekitFocus, + scope: testScope, + ownMembershipIdentity: ownMemberMock, + livekitRoomFactory: () => fakeLivekitRoom, + }, + testLogger, + ); + info.mockClear(); // drop the constructor log line + + const bob = mockRemoteParticipant({ + identity: "@bob:example.org:DEV111", + sid: "PA_bob", + }); + const pub = { + kind: "audio", + source: "microphone", + trackSid: "TR_mic", + isMuted: false, + isEncrypted: true, + } as unknown as RemoteTrackPublication; + const messages = (): string[] => info.mock.calls.map((c) => c[0] as string); + + fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, bob); + fakeLivekitRoom.emit( + RoomEvent.TrackSubscribed, + {} as RemoteTrack, + pub, + bob, + ); + fakeLivekitRoom.emit(RoomEvent.TrackMuted, pub, bob); + fakeLivekitRoom.emit( + RoomEvent.TrackStreamStateChanged, + pub, + Track.StreamState.Paused, + bob, + ); + expect(messages()).toEqual([ + "Participant connected: @bob:example.org:DEV111 (PA_bob)", + "Subscribed: audio microphone TR_mic of @bob:example.org:DEV111 encrypted=true muted=false", + "Muted: audio microphone TR_mic of @bob:example.org:DEV111 encrypted=true", + "Stream paused: audio microphone TR_mic of @bob:example.org:DEV111 encrypted=true", + ]); + + // Encryption status changes are logged; cryptor errors are warnings + fakeLivekitRoom.emit( + RoomEvent.ParticipantEncryptionStatusChanged, + false, + bob, + ); + expect(messages().at(-1)).toBe( + "Encryption status of @bob:example.org:DEV111: encrypted=false", + ); + const cryptorError = new Error("missing key at index 3"); + fakeLivekitRoom.emit(RoomEvent.EncryptionError, cryptorError, bob); + expect(warn).toHaveBeenCalledWith( + "Encryption error for @bob:example.org:DEV111:", + cryptorError, + ); + + // Local mute events are already logged by the Publisher + fakeLivekitRoom.emit(RoomEvent.TrackMuted, pub, { + ...fakeLocalParticipant, + isLocal: true, + } as unknown as LocalParticipant); + expect(messages().filter((m) => m.startsWith("Muted"))).toHaveLength(1); + + // Listeners are removed when the scope ends + testScope.end(); + fakeLivekitRoom.emit(RoomEvent.ParticipantDisconnected, bob); + expect( + messages().filter((m) => m.startsWith("Participant disconnected")), + ).toHaveLength(0); + }); +}); + +// +// NOT USED ANYMORE ? +// +// This setup look like sth for the Publisher. Not a connection. + +// describe("PublishConnection", () => { +// // let fakeBlurProcessor: ProcessorWrapper; +// let roomFactoryMock: Mock<() => LivekitRoom>; +// let muteStates: MockedObject; + +// function setUpPublishConnection(): void { +// setupTest(); + +// roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); + +// muteStates = mockMuteStates(); + +// // fakeBlurProcessor = vi.mocked>({ +// // name: "BackgroundBlur", +// // restart: vi.fn().mockResolvedValue(undefined), +// // setOptions: vi.fn().mockResolvedValue(undefined), +// // getOptions: vi.fn().mockReturnValue({ strength: 0.5 }), +// // isRunning: vi.fn().mockReturnValue(false) +// // }); +// } + +// describe("Livekit room creation", () => { +// function createSetup(): void { +// setUpPublishConnection(); + +// const fakeTrackProcessorSubject$ = new BehaviorSubject({ +// supported: true, +// processor: undefined, +// }); + +// const opts: ConnectionOpts = { +// client: client, +// transport: livekitFocus, +// scope: testScope, +// livekitRoomFactory: roomFactoryMock, +// }; + +// const audioInput = { +// available$: of(new Map([["mic1", { id: "mic1" }]])), +// selected$: new BehaviorSubject({ id: "mic1" }), +// select(): void {}, +// }; + +// const videoInput = { +// available$: of(new Map([["cam1", { id: "cam1" }]])), +// selected$: new BehaviorSubject({ id: "cam1" }), +// select(): void {}, +// }; + +// const audioOutput = { +// available$: of(new Map([["speaker", { id: "speaker" }]])), +// selected$: new BehaviorSubject({ id: "speaker" }), +// select(): void {}, +// }; + +// // TODO understand what is wrong with our mocking that requires ts-expect-error +// const fakeDevices = mockMediaDevices({ +// // @ts-expect-error Mocking only +// audioInput, +// // @ts-expect-error Mocking only +// videoInput, +// // @ts-expect-error Mocking only +// audioOutput, +// }); + +// new Connection( +// opts, +// fakeDevices, +// muteStates, +// undefined, +// fakeTrackProcessorSubject$, +// ); +// } + +// it("should create room with proper initial audio and video settings", () => { +// createSetup(); + +// expect(roomFactoryMock).toHaveBeenCalled(); + +// const lastCallArgs = +// roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; + +// const roomOptions = lastCallArgs.pop() as unknown as RoomOptions; +// expect(roomOptions).toBeDefined(); + +// expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1"); +// expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1"); +// expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker"); +// }); + +// it("respect controlledAudioDevices", () => { +// // TODO: Refactor the code to make it testable. +// // The UrlParams module is a singleton has a cache and is very hard to test. +// // This breaks other tests as well if not handled properly. +// // vi.mock(import("./../UrlParams"), () => { +// // return { +// // getUrlParams: vi.fn().mockReturnValue({ +// // controlledAudioDevices: true +// // }) +// // }; +// // }); +// }); +// }); +// }); diff --git a/sdk/src/session/Connection.ts b/sdk/src/session/Connection.ts new file mode 100644 index 000000000..cb02dd079 --- /dev/null +++ b/sdk/src/session/Connection.ts @@ -0,0 +1,426 @@ +/* +Copyright 2025 Element Creations Ltd. +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + connectedParticipantsObserver, + connectionStateObserver, +} from "@livekit/components-core"; +import { + ConnectionError, + ConnectionErrorReason, + type Participant, + type RemoteParticipant, + type RemoteTrackPublication, + type Room as LivekitRoom, + RoomEvent, + type SubscriptionError, + type Track, + type TrackPublication, +} from "livekit-client"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject, map } from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { + getSFUConfigWithOpenID, + type OpenIDClientParts, + type SFUConfig, +} from "./openIDSFU"; +import { type Behavior } from "../reactive/Behavior"; +import { type ObservableScope } from "../reactive/ObservableScope"; +import { + MatrixRTCError, + InsufficientCapacityError, + LivekitConnectionError, + PeerConnectionTimeoutError, + SFURoomCreationRestrictedError, + UnknownCallError, +} from "../errors"; + +export interface ConnectionOpts { + /** + * For the local transport we already do know the jwt token and url. We can reuse it. + * On top the local transport will send additional data to the jwt server to use delayed event delegation. + */ + existingSFUConfig?: SFUConfig; + /** The identity parts to use on this connection */ + ownMembershipIdentity: CallMembershipIdentityParts; + /** The media transport to connect to. */ + transport: LivekitTransport; + /** The Matrix client to use for OpenID and SFU config requests. */ + client: OpenIDClientParts; + /** The room ID this connection is associated with. */ + roomId: string; + /** The observable scope to use for this connection. */ + scope: ObservableScope; + + /** Optional factory to create the LiveKit room, mainly for testing purposes. */ + livekitRoomFactory: () => LivekitRoom; +} + +export enum ConnectionState { + /** The start state of a connection. It has been created but nothing has loaded yet. */ + Initialized = "Initialized", + /** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room */ + FetchingConfig = "FetchingConfig", + Stopped = "Stopped", + /** The same as ConnectionState.Disconnected from `livekit-client` */ + LivekitDisconnected = "disconnected", + /** The same as ConnectionState.Connecting from `livekit-client` */ + LivekitConnecting = "connecting", + /** The same as ConnectionState.Connected from `livekit-client` */ + LivekitConnected = "connected", + /** The same as ConnectionState.Reconnecting from `livekit-client` */ + LivekitReconnecting = "reconnecting", + /** The same as ConnectionState.SignalReconnecting from `livekit-client` */ + LivekitSignalReconnecting = "signalReconnecting", +} + +/** + * A connection to a Matrix RTC LiveKit backend. + * + * Expose observables for participants and connection state. + */ +export class Connection { + // Private Behavior + private readonly _state$ = new BehaviorSubject< + ConnectionState | MatrixRTCError + >(ConnectionState.Initialized); + + /** + * The current state of the connection to the media transport. + */ + public readonly state$: Behavior = this._state$; + + /** + * The media transport to connect to. + */ + public readonly transport: LivekitTransport; + + public readonly livekitRoom: LivekitRoom; + + private scope: ObservableScope; + + /** + * The remote LiveKit participants that are visible on this connection. + * + * Note that this may include participants that are connected only to + * subscribe, or publishers that are otherwise unattested in MatrixRTC state. + * It is therefore more low-level than what should be presented to the user. + */ + public readonly remoteParticipants$: Behavior; + + /** + * The alias of the LiveKit room. + */ + public get livekitAlias(): string | undefined { + return this._livekitAlias; + } + private _livekitAlias?: string; + + /** + * Whether the connection has been stopped. + * @see Connection.stop + * */ + protected stopped = false; + + // TODO: can we just keep the ConnectionOpts object instead of spreading? + private readonly client: OpenIDClientParts; + private readonly roomId: string; + private readonly logger: Logger; + private readonly ownMembershipIdentity: CallMembershipIdentityParts; + private readonly existingSFUConfig?: SFUConfig; + /** + * Creates a new connection to a matrix RTC LiveKit backend. + * + * @param opts - Connection options {@link ConnectionOpts}. + * + * @param logger - The logger to use. + */ + public constructor(opts: ConnectionOpts, logger: Logger) { + this.ownMembershipIdentity = opts.ownMembershipIdentity; + this.existingSFUConfig = opts.existingSFUConfig; + this.roomId = opts.roomId; + this.logger = logger.getChild( + "[Connection " + opts.transport.livekit_service_url + "]", + ); + this.logger.info( + `constructor: ${opts.transport.livekit_service_url} roomId: ${this.roomId} withSfuConfig?: ${opts.existingSFUConfig ? JSON.stringify(opts.existingSFUConfig) : "undefined"}`, + ); + const { transport, client, scope } = opts; + + this.scope = scope; + this.livekitRoom = opts.livekitRoomFactory(); + this.transport = transport; + this.client = client; + + this.remoteParticipants$ = scope.behavior( + // Only tracks remote participants + connectedParticipantsObserver(this.livekitRoom), + ); + this.logRemoteTrackEvents(); + + scope.onEnd(() => { + this.logger.info(`Connection scope ended, stopping connection`); + void this.stop(); + }); + } + + /** + * Logs the lifecycle of remote participants and their tracks as seen by + * livekit-client, so that rageshakes show what a tile should have been + * rendering (published, subscribed, muted, paused) for each remote member. + */ + private logRemoteTrackEvents(): void { + const room = this.livekitRoom; + const log = this.logger.getChild("[RemoteTracks]"); + // The encryption flag matters: if the publisher encrypts but this client + // believes the track is unencrypted, the cryptor is bypassed and raw + // ciphertext reaches the decoder (audible as loud noise bursts). + const track = (pub: TrackPublication, p: Participant): string => + `${pub.kind} ${pub.source} ${pub.trackSid} of ${p.identity} encrypted=${pub.isEncrypted}`; + + const onParticipantConnected = (p: RemoteParticipant): void => + log.info(`Participant connected: ${p.identity} (${p.sid})`); + const onParticipantDisconnected = (p: RemoteParticipant): void => + log.info(`Participant disconnected: ${p.identity} (${p.sid})`); + const onTrackPublished = ( + pub: RemoteTrackPublication, + p: RemoteParticipant, + ): void => log.info(`Published: ${track(pub, p)} muted=${pub.isMuted}`); + const onTrackUnpublished = ( + pub: RemoteTrackPublication, + p: RemoteParticipant, + ): void => log.info(`Unpublished: ${track(pub, p)}`); + const onTrackSubscribed = ( + _t: Track, + pub: RemoteTrackPublication, + p: RemoteParticipant, + ): void => log.info(`Subscribed: ${track(pub, p)} muted=${pub.isMuted}`); + const onTrackUnsubscribed = ( + _t: Track, + pub: RemoteTrackPublication, + p: RemoteParticipant, + ): void => log.info(`Unsubscribed: ${track(pub, p)}`); + const onTrackSubscriptionFailed = ( + trackSid: string, + p: RemoteParticipant, + reason?: SubscriptionError, + ): void => + log.warn( + `Subscription failed: ${trackSid} of ${p.identity} reason=${reason}`, + ); + // Mute events also fire for the local participant, which the Publisher + // already logs. + const onTrackMuted = (pub: TrackPublication, p: Participant): void => { + if (!p.isLocal) log.info(`Muted: ${track(pub, p)}`); + }; + const onTrackUnmuted = (pub: TrackPublication, p: Participant): void => { + if (!p.isLocal) log.info(`Unmuted: ${track(pub, p)}`); + }; + const onTrackStreamStateChanged = ( + pub: RemoteTrackPublication, + state: Track.StreamState, + p: RemoteParticipant, + ): void => log.info(`Stream ${state}: ${track(pub, p)}`); + const onEncryptionStatusChanged = ( + encrypted: boolean, + p?: Participant, + ): void => + log.info( + `Encryption status of ${p?.identity ?? "unknown participant"}: encrypted=${encrypted}`, + ); + // livekit-client throttles these per cryptor; they indicate frames being + // dropped (missing/invalid key). + const onEncryptionError = (error: Error, p?: Participant): void => + log.warn( + `Encryption error for ${p?.identity ?? "unknown participant"}:`, + error, + ); + + room + .on(RoomEvent.ParticipantConnected, onParticipantConnected) + .on(RoomEvent.ParticipantDisconnected, onParticipantDisconnected) + .on(RoomEvent.TrackPublished, onTrackPublished) + .on(RoomEvent.TrackUnpublished, onTrackUnpublished) + .on(RoomEvent.TrackSubscribed, onTrackSubscribed) + .on(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed) + .on(RoomEvent.TrackSubscriptionFailed, onTrackSubscriptionFailed) + .on(RoomEvent.TrackMuted, onTrackMuted) + .on(RoomEvent.TrackUnmuted, onTrackUnmuted) + .on(RoomEvent.TrackStreamStateChanged, onTrackStreamStateChanged) + .on( + RoomEvent.ParticipantEncryptionStatusChanged, + onEncryptionStatusChanged, + ) + .on(RoomEvent.EncryptionError, onEncryptionError); + this.scope.onEnd(() => { + room + .off(RoomEvent.ParticipantConnected, onParticipantConnected) + .off(RoomEvent.ParticipantDisconnected, onParticipantDisconnected) + .off(RoomEvent.TrackPublished, onTrackPublished) + .off(RoomEvent.TrackUnpublished, onTrackUnpublished) + .off(RoomEvent.TrackSubscribed, onTrackSubscribed) + .off(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed) + .off(RoomEvent.TrackSubscriptionFailed, onTrackSubscriptionFailed) + .off(RoomEvent.TrackMuted, onTrackMuted) + .off(RoomEvent.TrackUnmuted, onTrackUnmuted) + .off(RoomEvent.TrackStreamStateChanged, onTrackStreamStateChanged) + .off( + RoomEvent.ParticipantEncryptionStatusChanged, + onEncryptionStatusChanged, + ) + .off(RoomEvent.EncryptionError, onEncryptionError); + }); + } + + /** + * Starts the connection. + * + * This will: + * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) + * 2. Use this token to request the SFU config to the MatrixRtc authentication service. + * 3. Connect to the configured LiveKit room. + * + * The errors are also represented as a state in the `state$` observable. + * It is safe to ignore those errors and handle them accordingly via the `state$` observable. + * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. + * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. + */ + // TODO consider an autostart pattern... + public async start(): Promise { + this.logger.debug("Starting Connection"); + this.stopped = false; + try { + this._state$.next(ConnectionState.FetchingConfig); + // We should already have this information after creating the localTransport. + // only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined) + const { url, jwt, livekitAlias } = + this.existingSFUConfig ?? + (await this.getSFUConfigForRemoteConnection()); + this.logger.debug( + "Starting Connection to: ", + this.transport.livekit_service_url, + "jwt: ", + jwt, + "wss: ", + url, + "livekitAlias: ", + livekitAlias, + ); + this._livekitAlias = livekitAlias; + // If we were stopped while fetching the config, don't proceed to connect + if (this.stopped) return; + + // Setup observer once we are done with getSFUConfigWithOpenID + connectionStateObserver(this.livekitRoom) + .pipe( + this.scope.bind(), + map((s) => s as unknown as ConnectionState), + ) + .subscribe((lkState) => { + // It is save to cast lkState to ConnectionState as they are fully overlapping. + this._state$.next(lkState); + }); + + try { + this.logger.info(`livekitRoom.connect ${url}`); + await this.livekitRoom.connect(url, jwt); + this.logger.info(`livekitRoom.connect SUCCESS ${url}`); + } catch (e) { + this.logger.info(`livekitRoom.connect FAILED ${url}`, e); + // LiveKit uses 503 to indicate that the server has hit its track limits. + // https://github.com/livekit/livekit/blob/fcb05e97c5a31812ecf0ca6f7efa57c485cea9fb/pkg/service/rtcservice.go#L171 + // It also errors with a status code of 200 (yes, really) for room + // participant limits. + // LiveKit Cloud uses 429 for connection limits. + // Either way, all these errors can be explained as "insufficient capacity". + if (e instanceof ConnectionError) { + if (e.status === 503 || e.status === 200 || e.status === 429) { + throw new InsufficientCapacityError(); + } + if (e.status === 404) { + // error msg is "Failed to create call" + // error description is "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists." + // The room does not exist. There are two different modes of operation for the SFU: + // - the room is created on the fly when connecting (livekit `auto_create` option) + // - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service) + // In the first case there will not be a 404, so we are in the second case. + throw new SFURoomCreationRestrictedError(); + } + + if (e.reason === ConnectionErrorReason.Timeout) { + // Unabled to establish peer connection within the timeout + throw new PeerConnectionTimeoutError(); + } + + throw new LivekitConnectionError(e); + } + throw e; + } + // If we were stopped while connecting, don't proceed to update state. + if (this.stopped) return; + } catch (error) { + if (this.stopped) { + // stop() was called while we were connecting, which makes the pending + // connect reject. That is the abort we asked for, not a failure, so + // don't record an error state on a stopped connection or rethrow it + // (start() is not awaited by the ConnectionManager, so a throw here + // becomes an unhandled promise rejection). + this.logger.debug(`Connect aborted because the connection was stopped`); + return; + } + this.logger.debug(`Failed to connect to LiveKit room: ${error}`); + this._state$.next( + error instanceof MatrixRTCError + ? error + : error instanceof Error + ? new UnknownCallError(error) + : new UnknownCallError(new Error(`${error}`)), + ); + // Its okay to ignore the throw. The error is part of the state. + throw error; + } + } + + protected async getSFUConfigForRemoteConnection(): Promise { + // This will only be called for sfu's where we do not publish ourselves. + // For the local connection we will use the existingJwtTokenData + return await getSFUConfigWithOpenID( + this.client, + this.ownMembershipIdentity, + this.transport.livekit_service_url, + this.roomId, + // dont pass any custom opts for the subscribe only connections + {}, + this.logger, + ); + } + + /** + * Stops the connection. + * + * This will disconnect from the LiveKit room. + * If the connection is already stopped, this is a no-op. + */ + public async stop(): Promise { + this.logger.debug( + `stop: disconnecing from lk room ${this.transport.livekit_service_url}`, + ); + if (this.stopped) return; + // Mark as stopped before disconnecting so that a connect() aborted by the + // disconnect sees the flag and does not report the abort as an error. + this.stopped = true; + await this.livekitRoom.disconnect(); + this._state$.next(ConnectionState.Stopped); + this.logger.debug( + `stop: DONE disconnecing from lk room ${this.transport.livekit_service_url}`, + ); + } +} diff --git a/sdk/src/session/ConnectionFactory.ts b/sdk/src/session/ConnectionFactory.ts index 16d7f1a07..5864cb4c5 100644 --- a/sdk/src/session/ConnectionFactory.ts +++ b/sdk/src/session/ConnectionFactory.ts @@ -18,21 +18,23 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { BehaviorSubject, combineLatest, map } from "rxjs"; -import { buildLiveKitOptions } from "../../../src/livekit/options"; -import { - type OpenIDClientParts, - type SFUConfig, -} from "../../../src/livekit/openIDSFU"; -import { type Behavior } from "../../../src/state/Behavior"; -import { - Connection, - type ConnectionOpts, - ConnectionState, -} from "../../../src/state/CallViewModel/remoteMembers/Connection"; -import { type ConnectionFactory } from "../../../src/state/CallViewModel/remoteMembers/ConnectionFactory"; -import { type ObservableScope } from "../../../src/state/ObservableScope"; +import { buildLiveKitOptions } from "./livekitOptions"; +import { type OpenIDClientParts, type SFUConfig } from "./openIDSFU"; +import { type Behavior } from "../reactive/Behavior"; +import { Connection, type ConnectionOpts, ConnectionState } from "./Connection"; +import { type ObservableScope } from "../reactive/ObservableScope"; import { type LocalMediaInputs, type ResolvedTransport } from "../api"; +export interface ConnectionFactory { + createConnection( + scope: ObservableScope, + transport: LivekitTransport, + ownMembershipIdentity: CallMembershipIdentityParts, + logger: Logger, + sfuConfig?: SFUConfig, + ): Connection; +} + /** * A connection that keeps what it fetched from the JWT service, so that the * transport can report it as `resolved$`. Nothing while the config is still diff --git a/sdk/src/session/ConnectionManager.test.ts b/sdk/src/session/ConnectionManager.test.ts new file mode 100644 index 000000000..7f15da96b --- /dev/null +++ b/sdk/src/session/ConnectionManager.test.ts @@ -0,0 +1,358 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { BehaviorSubject, NEVER } from "rxjs"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type RemoteParticipant } from "livekit-client"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { Epoch, mapEpoch, ObservableScope } from "../reactive/ObservableScope"; +import { + createConnectionManager$, + ConnectionManagerData, +} from "./ConnectionManager.ts"; +import { type ConnectionFactory } from "./ConnectionFactory.ts"; +import { type Connection } from "./Connection.ts"; +import { ownMemberMock, withTestScheduler } from "../utils/test"; +import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; +import { type Behavior } from "../reactive/Behavior"; + +// Some test constants + +const TRANSPORT_1: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", +}; + +const TRANSPORT_2: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.sample.com", +}; + +let fakeConnectionFactory: ConnectionFactory; +let testScope: ObservableScope; + +// Can be useful to track all created connections in tests, even the disposed ones +let allCreatedConnections: Connection[]; + +beforeEach(() => { + testScope = new ObservableScope(); + allCreatedConnections = []; + fakeConnectionFactory = {} as unknown as ConnectionFactory; + vi.mocked(fakeConnectionFactory).createConnection = vi + .fn() + .mockImplementation( + (scope: ObservableScope, transport: LivekitTransport) => { + const mockConnection = { + transport, + remoteParticipants$: new BehaviorSubject([]), + } as unknown as Connection; + vi.mocked(mockConnection).start = vi.fn(); + vi.mocked(mockConnection).stop = vi.fn(); + // Tie the connection's lifecycle to the scope to test scope lifecycle management + scope.onEnd(() => { + void mockConnection.stop(); + }); + allCreatedConnections.push(mockConnection); + return mockConnection; + }, + ); +}); + +afterEach(() => { + testScope.end(); +}); + +describe("connections$ stream", () => { + test("Should create and start new connections for each transports", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { connectionManagerData$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport$: NEVER, + remoteTransports$: behavior("a", { + a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable( + connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + ).toBe("a", { + a: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + expect(connections.length).toBe(2); + + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledTimes(2); + + const conn1 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_1), + ); + expect(conn1).toBeDefined(); + expect(conn1!.start).toHaveBeenCalled(); + + const conn2 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_2), + ); + expect(conn2).toBeDefined(); + expect(conn2!.start).toHaveBeenCalled(); + return true; + }), + }); + }); + }); + + test("Should start connection only once", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { connectionManagerData$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport$: NEVER, + remoteTransports$: behavior("abcdef", { + a: new Epoch([TRANSPORT_1], 0), + b: new Epoch([TRANSPORT_1], 1), + c: new Epoch([TRANSPORT_1], 2), + d: new Epoch([TRANSPORT_1], 3), + e: new Epoch([TRANSPORT_1], 4), + f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable( + connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + ).toBe("xxxxxa", { + x: expect.anything(), + a: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + + expect(connections.length).toBe(2); + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledTimes(2); + + const conn2 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_2), + ); + expect(conn2).toBeDefined(); + + const conn1 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_1), + ); + expect(conn1).toBeDefined(); + expect(conn1!.start).toHaveBeenCalledOnce(); + + return true; + }), + }); + }); + }); + + test("Should cleanup connections when not needed anymore", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { connectionManagerData$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport$: NEVER, + remoteTransports$: behavior("abc", { + a: new Epoch([TRANSPORT_1], 0), + b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), + c: new Epoch([TRANSPORT_1], 2), + }), + logger: logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable( + connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())), + ).toBe("xab", { + x: expect.anything(), + a: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + expect(connections.length).toBe(2); + return true; + }), + b: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + + expect(connections.length).toBe(1); + // The second connection should have been stopped has it is no longer needed. + const connection2 = allCreatedConnections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_2), + ); + expect(connection2).toBeDefined(); + expect(connection2!.stop).toHaveBeenCalled(); + + // The first connection should still be active + const conn1 = connections[0]; + expect(conn1.stop).not.toHaveBeenCalledOnce(); + + return true; + }), + }); + }); + }); +}); + +describe("ConnectionManagerData", () => { + test("warns when a second connection to the same URL is merged", () => { + const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); + const data = new ConnectionManagerData(logger); + const connection = { transport: TRANSPORT_1 } as unknown as Connection; + const p = (identity: string): RemoteParticipant => + ({ identity }) as unknown as RemoteParticipant; + data.add(connection, [p("a")]); + data.add({ ...connection } as Connection, [p("b")]); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining( + "second connection to https://lk.example.org: existing [a], adding [b]", + ), + ); + expect(data.getParticipantsForTransport(TRANSPORT_1)).toHaveLength(2); + }); +}); + +describe("connectionManagerData$ stream", () => { + // Used in test to control fake connections' remoteParticipants$ streams + let fakeRemoteParticipantsStreams: Map>; + + function keyForTransport(transport: LivekitTransport): string { + return `${transport.livekit_service_url}`; + } + + beforeEach(() => { + fakeRemoteParticipantsStreams = new Map(); + + function getRemoteParticipantsFor( + transport: LivekitTransport, + ): Behavior { + return ( + fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ?? + new BehaviorSubject([]) + ); + } + + // need a more advanced fake connection factory + vi.mocked(fakeConnectionFactory).createConnection = vi + .fn() + .mockImplementation( + (scope: ObservableScope, transport: LivekitTransport) => { + const fakeRemoteParticipants$ = new BehaviorSubject< + RemoteParticipant[] + >([]); + const mockConnection = { + transport, + remoteParticipants$: getRemoteParticipantsFor(transport), + } as unknown as Connection; + vi.mocked(mockConnection).start = vi.fn(); + vi.mocked(mockConnection).stop = vi.fn(); + // Tie the connection's lifecycle to the scope to test scope lifecycle management + scope.onEnd(() => { + void mockConnection.stop(); + }); + + fakeRemoteParticipantsStreams.set( + keyForTransport(transport), + fakeRemoteParticipants$, + ); + return mockConnection; + }, + ); + }); + + test("Should report connections with the remote participants", () => { + withTestScheduler(({ expectObservable, schedule, behavior }) => { + // Setup the fake participants streams behavior + // ============================== + fakeRemoteParticipantsStreams.set( + keyForTransport(TRANSPORT_1), + behavior("oa-b", { + o: [], + a: [{ identity: "user1A" } as RemoteParticipant], + b: [ + { identity: "user1A" } as RemoteParticipant, + { identity: "user1B" } as RemoteParticipant, + ], + }), + ); + + fakeRemoteParticipantsStreams.set( + keyForTransport(TRANSPORT_2), + behavior("o-a", { + o: [], + a: [{ identity: "user2A" } as RemoteParticipant], + }), + ); + // ============================== + + const { connectionManagerData$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + localTransport$: NEVER, + remoteTransports$: behavior("a", { + a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), + }), + logger, + ownMembershipIdentity: ownMemberMock, + }); + + expectObservable(connectionManagerData$).toBe("abcd", { + a: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); + return true; + }), + b: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[0].identity, + ).toBe("user1A"); + return true; + }), + c: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[0].identity, + ).toBe("user1A"); + expect( + data.getParticipantsForTransport(TRANSPORT_2)[0].identity, + ).toBe("user2A"); + return true; + }), + d: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2); + expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[0].identity, + ).toBe("user1A"); + expect( + data.getParticipantsForTransport(TRANSPORT_1)[1].identity, + ).toBe("user1B"); + expect( + data.getParticipantsForTransport(TRANSPORT_2)[0].identity, + ).toBe("user2A"); + return true; + }), + }); + }); + }); +}); diff --git a/sdk/src/session/ConnectionManager.ts b/sdk/src/session/ConnectionManager.ts new file mode 100644 index 000000000..0ef72cb59 --- /dev/null +++ b/sdk/src/session/ConnectionManager.ts @@ -0,0 +1,279 @@ +/* +Copyright 2025 Element Creations Ltd. +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { + combineLatest, + map, + type Observable, + of, + switchMap, + startWith, + catchError, + NEVER, +} from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type RemoteParticipant } from "livekit-client"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +import { type Behavior } from "../reactive/Behavior"; +import { type Connection } from "./Connection.ts"; +import { Epoch, type ObservableScope } from "../reactive/ObservableScope"; +import { generateItemsWithEpoch } from "../reactive/observable"; +import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; +import { type ConnectionFactory } from "./ConnectionFactory.ts"; +import { isLocalTransport, type LocalTransport } from "./LocalTransport"; +import { type SFUConfig } from "./openIDSFU"; + +export class ConnectionManagerData { + private readonly store: Map< + string, + { connection: Connection; participants: RemoteParticipant[] } + > = new Map(); + + public constructor(private readonly logger?: Logger) {} + + public add(connection: Connection, participants: RemoteParticipant[]): void { + const key = this.getKey(connection.transport); + const existing = this.store.get(key); + if (!existing) { + this.store.set(key, { connection, participants }); + } else { + // Transports are deduplicated by URL upstream, so this should never + // happen; if it does, members may be matched against the wrong room. + this.logger?.warn( + `Merging participants from a second connection to ${key}: existing [${existing.participants.map((p) => p.identity).join(", ")}], adding [${participants.map((p) => p.identity).join(", ")}]`, + ); + existing.participants.push(...participants); + } + } + + private getKey(transport: LivekitTransport): string { + // This is enough as a key because the ConnectionManager is already scoped by room. + // We also do not need to consider the slotId at this point since each `MatrixRTCSession` is already scoped by `slotDescription: {id, application}`. + return transport.livekit_service_url; + } + + public getConnections(): Connection[] { + return Array.from(this.store.values()).map(({ connection }) => connection); + } + + public getConnectionForTransport( + transport: LivekitTransport, + ): Connection | null { + return this.store.get(this.getKey(transport))?.connection ?? null; + } + + public getParticipantsForTransport( + transport: LivekitTransport, + ): RemoteParticipant[] { + const key = this.getKey(transport); + const existing = this.store.get(key); + if (existing) { + return existing.participants; + } + return []; + } +} + +interface Props { + scope: ObservableScope; + connectionFactory: ConnectionFactory; + localTransport$: Observable; + remoteTransports$: Behavior>; + + logger: Logger; + ownMembershipIdentity: CallMembershipIdentityParts; +} + +// TODO - write test for scopes (do we really need to bind scope) +export interface IConnectionManager { + connectionManagerData$: Behavior>; +} + +/** + * Crete a `ConnectionManager` + * @param props - Configuration object + * @param props.scope - The observable scope used by this object + * @param props.connectionFactory - Used to create new connections + * @param props.localTransport$ - The transport to publish local media on. (deduplicated with remoteTransports$) + * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$) + * @param props.ownMembershipIdentity - The own membership identity to use. + * @param props.logger - The logger to use. + + * + * Each of these behaviors can be interpreted as subscribed list of transports. + * + * Using `registerTransports` independent external modules can control what connections + * are created by the ConnectionManager. + * + * The connection manager will remove all duplicate transports in each subscibed list. + * + * See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe. + */ +export function createConnectionManager$({ + scope, + connectionFactory, + localTransport$, + remoteTransports$, + logger: parentLogger, + ownMembershipIdentity, +}: Props): IConnectionManager { + const logger = parentLogger.getChild("[ConnectionManager]"); + // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing + + const localTransportAsArray$ = localTransport$.pipe( + // LocalMember already surfaces local transport errors properly in the UI, + // here we can just swallow them + catchError(() => NEVER), + map((transport) => [transport]), + startWith([]), + ); + + /** + * All transports currently managed by the ConnectionManager. + * + * This list does not include duplicate transports. + * + * It is build based on the list of subscribed transports (`transportsSubscriptions$`). + * externally this is modified via `registerTransports()`. + */ + const localAndRemoteTransports$: Behavior< + Epoch<(LivekitTransport | LocalTransport)[]> + > = scope.behavior( + combineLatest([localTransportAsArray$, remoteTransports$]).pipe( + // Combine local and remote transports into one transport array + // and set the forceOldJwtEndpoint property on the local transport + map(([localTransportAsArray, remoteTransports]) => { + const dedupedRemote = removeDuplicateTransports(remoteTransports.value); + const remoteWithoutLocal = dedupedRemote.filter( + (transport) => + !localTransportAsArray.find((l) => + areLivekitTransportsEqual(l.transport, transport), + ), + ); + logger.debug( + "remoteWithoutLocal", + remoteWithoutLocal, + "localTransportAsArray", + localTransportAsArray, + ); + return new Epoch( + [...localTransportAsArray, ...remoteWithoutLocal], + remoteTransports.epoch, + ); + }), + ), + ); + + /** + * Connections for each transport in use by one or more session members. + */ + const connections$ = scope.behavior( + localAndRemoteTransports$.pipe( + generateItemsWithEpoch( + "ConnectionManager connections$", + function* (transports) { + for (const transport of transports) { + if (isLocalTransport(transport)) { + // This is the local transport; only the `LocalTransport` + // interface has a `sfuConfig` field. + yield { + keys: [ + transport.transport.livekit_service_url, + transport.sfuConfig, + ], + data: undefined, + }; + } else { + yield { + keys: [ + transport.livekit_service_url, + undefined as SFUConfig | undefined, + ], + data: undefined, + }; + } + } + }, + (scope, _data$, serviceUrl, sfuConfig) => { + const connection = connectionFactory.createConnection( + scope, + { + type: "livekit", + livekit_service_url: serviceUrl, + }, + ownMembershipIdentity, + logger, + // TODO: This whole optional SFUConfig parameter is not particularly elegant. + // I would like it if connections always fetched the SFUConfig by themselves. + sfuConfig, + ); + // Start the connection immediately + // Use connection state to track connection progress + void connection.start(); + // TODO subscribe to connection state to retry or log issues? + return connection; + }, + ), + ), + ); + + const connectionManagerData$ = scope.behavior( + connections$.pipe( + switchMap((connections) => { + const epoch = connections.epoch; + + // Map the connections to list of {connection, participants}[] + const listOfConnectionsWithRemoteParticipants = connections.value.map( + (connection) => { + return connection.remoteParticipants$.pipe( + map((participants) => ({ + connection, + participants, + })), + ); + }, + ); + + // probably not required + + if (listOfConnectionsWithRemoteParticipants.length === 0) { + return of(new Epoch(new ConnectionManagerData(), epoch)); + } + + // combineLatest the several streams into a single stream with the ConnectionManagerData + return combineLatest(listOfConnectionsWithRemoteParticipants).pipe( + map( + (lists) => + new Epoch( + lists.reduce((data, { connection, participants }) => { + data.add(connection, participants); + return data; + }, new ConnectionManagerData(logger)), + epoch, + ), + ), + ); + }), + ), + new Epoch(new ConnectionManagerData(), -1), + ); + + return { connectionManagerData$ }; +} + +function removeDuplicateTransports( + transports: T[], +): T[] { + return transports.reduce((acc, transport) => { + if (!acc.some((t) => areLivekitTransportsEqual(t, transport))) + acc.push(transport); + return acc; + }, [] as T[]); +} diff --git a/sdk/src/session/HomeserverConnected.test.ts b/sdk/src/session/HomeserverConnected.test.ts new file mode 100644 index 000000000..85daf53ed --- /dev/null +++ b/sdk/src/session/HomeserverConnected.test.ts @@ -0,0 +1,336 @@ +/* +Copyright 2025 Element Creations Ltd. +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { EventEmitter } from "events"; +import { ClientEvent, SyncState } from "matrix-js-sdk"; +import { MembershipManagerEvent, Status } from "matrix-js-sdk/lib/matrixrtc"; + +import { ObservableScope } from "../reactive/ObservableScope"; +import { createHomeserverConnected$ } from "./HomeserverConnected"; +import { testScope, withTestScheduler } from "../utils/test"; + +/** + * Minimal stub of a Matrix client sufficient for our tests: + ``` + createHomeserverConnected$( + scope: ObservableScope, + client: NodeStyleEventEmitter & Pick, + matrixRTCSession: NodeStyleEventEmitter & + Pick, + ) + ``` + */ +class MockMatrixClient extends EventEmitter { + private syncState: SyncState; + public constructor(initial: SyncState) { + super(); + this.syncState = initial; + } + public setSyncState(state: SyncState): void { + this.syncState = state; + // Matrix's Sync event in createHomeserverConnected$ expects [SyncState] + this.emit(ClientEvent.Sync, [state]); + } + public getSyncState(): SyncState { + return this.syncState; + } +} + +/** + * Minimal stub of MatrixRTCSession (membership manager): + ``` + createHomeserverConnected$( + scope: ObservableScope, + client: NodeStyleEventEmitter & Pick, + matrixRTCSession: NodeStyleEventEmitter & + Pick, + ) + ``` + */ +class MockMatrixRTCSession extends EventEmitter { + public membershipStatus: Status; + public probablyLeft: boolean; + + public constructor(props: { + membershipStatus: Status; + probablyLeft: boolean; + }) { + super(); + this.membershipStatus = props.membershipStatus; + this.probablyLeft = props.probablyLeft; + } + + public setMembershipStatus(status: Status): void { + this.membershipStatus = status; + this.emit(MembershipManagerEvent.StatusChanged); + } + + public setProbablyLeft(flag: boolean): void { + this.probablyLeft = flag; + this.emit(MembershipManagerEvent.ProbablyLeft); + } +} + +describe("createHomeserverConnected$", () => { + let scope: ObservableScope; + let client: MockMatrixClient; + let session: MockMatrixRTCSession; + + beforeEach(() => { + scope = new ObservableScope(); + client = new MockMatrixClient(SyncState.Error); // start disconnected + session = new MockMatrixRTCSession({ + membershipStatus: Status.Disconnected, + probablyLeft: false, + }); + }); + + afterEach(() => { + scope.end(); + }); + + // LLM generated test cases. They are a bit overkill but I improved the mocking so it is + // easy enough to read them so I think they can stay. + // Note: gracePeriodMs is set to 0 to avoid debouncing delays in tests + it("reports syncing reason when sync state is not Syncing", () => { + const hsConnected = createHomeserverConnected$(scope, client, session, 0); + expect(hsConnected.combined$.value).toEqual([false, "sync"]); + }); + + it("reports membership reason when sync is Syncing but membership is not Connected", () => { + const hsConnected = createHomeserverConnected$(scope, client, session, 0); + client.setSyncState(SyncState.Syncing); + expect(hsConnected.combined$.value).toEqual([false, "membership"]); + }); + + it("reports probablyLeft reason when membership transitions to Connected but ProbablyLeft is true", () => { + const hsConnected = createHomeserverConnected$(scope, client, session, 0); + // Make sync loop OK + client.setSyncState(SyncState.Syncing); + // Indicate probable leave before connection + session.setProbablyLeft(true); + session.setMembershipStatus(Status.Connected); + expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]); + }); + + it("becomes null (connected) only when all three conditions are satisfied", () => { + const hsConnected = createHomeserverConnected$(scope, client, session, 0); + // 1. Sync loop connected + client.setSyncState(SyncState.Syncing); + expect(hsConnected.combined$.value).toEqual([false, "membership"]); // not yet membership connected + // 2. Membership connected + session.setMembershipStatus(Status.Connected); + expect(hsConnected.combined$.value).toEqual([true, null]); // probablyLeft is false + }); + + it("returns syncing reason when sync loop leaves Syncing", () => { + const hsConnected = createHomeserverConnected$(scope, client, session, 0); + // Reach connected state + client.setSyncState(SyncState.Syncing); + session.setMembershipStatus(Status.Connected); + expect(hsConnected.combined$.value).toEqual([true, null]); + + // Sync loop error => should report syncing reason + client.setSyncState(SyncState.Error); + expect(hsConnected.combined$.value).toEqual([false, "sync"]); + }); + + it("returns membershipConnected reason when membership status becomes disconnected", () => { + const hsConnected = createHomeserverConnected$(scope, client, session, 0); + client.setSyncState(SyncState.Syncing); + session.setMembershipStatus(Status.Connected); + expect(hsConnected.combined$.value).toEqual([true, null]); + + session.setMembershipStatus(Status.Disconnected); + expect(hsConnected.combined$.value).toEqual([false, "membership"]); + }); + + it("returns certainlyConnected reason when ProbablyLeft is emitted", () => { + const hsConnected = createHomeserverConnected$(scope, client, session, 0); + client.setSyncState(SyncState.Syncing); + session.setMembershipStatus(Status.Connected); + expect(hsConnected.combined$.value).toEqual([true, null]); + + session.setProbablyLeft(true); + expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]); + }); + + it("recovers to null (connected) if ProbablyLeft becomes false again while other conditions remain true", () => { + const hsConnected = createHomeserverConnected$(scope, client, session, 0); + client.setSyncState(SyncState.Syncing); + session.setMembershipStatus(Status.Connected); + expect(hsConnected.combined$.value).toEqual([true, null]); + + session.setProbablyLeft(true); + expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]); + + // Simulate clearing the flag (in realistic scenario membership manager would update) + session.setProbablyLeft(false); + expect(hsConnected.combined$.value).toEqual([true, null]); + }); + + it("composite sequence reflects each individual failure reason", () => { + const hsConnected = createHomeserverConnected$(scope, client, session, 0); + + // Initially: sync error + membership disconnected → syncing wins (highest priority) + expect(hsConnected.combined$.value).toEqual([false, "sync"]); + + // Fix sync only → membershipConnected is now the blocker + client.setSyncState(SyncState.Syncing); + expect(hsConnected.combined$.value).toEqual([false, "membership"]); + + // Fix membership → all conditions satisfied + session.setMembershipStatus(Status.Connected); + expect(hsConnected.combined$.value).toEqual([true, null]); + + // Introduce probablyLeft → certainlyConnected + session.setProbablyLeft(true); + expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]); + + // Restore notProbablyLeft → connected again + session.setProbablyLeft(false); + expect(hsConnected.combined$.value).toEqual([true, null]); + + // Drop sync → syncing reason + client.setSyncState(SyncState.Error); + expect(hsConnected.combined$.value).toEqual([false, "sync"]); + }); +}); + +describe("createHomeserverConnected$ - combined$ reason values", () => { + let scope: ObservableScope; + let client: MockMatrixClient; + let session: MockMatrixRTCSession; + + beforeEach(() => { + scope = new ObservableScope(); + // Start with sync failing and membership disconnected + client = new MockMatrixClient(SyncState.Error); + session = new MockMatrixRTCSession({ + membershipStatus: Status.Disconnected, + probablyLeft: false, + }); + }); + + afterEach(() => { + scope.end(); + }); + + it("is [true, null] when all three conditions are satisfied", () => { + const { combined$ } = createHomeserverConnected$(scope, client, session, 0); + client.setSyncState(SyncState.Syncing); + session.setMembershipStatus(Status.Connected); + expect(combined$.value).toEqual([true, null]); + }); + + it("reports syncing when sync loop is not Syncing", () => { + const { combined$ } = createHomeserverConnected$(scope, client, session, 0); + // client starts with SyncState.Error, membership also disconnected + expect(combined$.value).toEqual([false, "sync"]); + }); + + it("reports membershipConnected when sync is fine but membership is not Connected", () => { + const { combined$ } = createHomeserverConnected$(scope, client, session, 0); + client.setSyncState(SyncState.Syncing); + // session still Status.Disconnected + expect(combined$.value).toEqual([false, "membership"]); + }); + + it("reports certainlyConnected when probablyLeft is true", () => { + const { combined$ } = createHomeserverConnected$(scope, client, session, 0); + client.setSyncState(SyncState.Syncing); + session.setMembershipStatus(Status.Connected); + session.setProbablyLeft(true); + expect(combined$.value).toEqual([false, "probablyLeft"]); + }); + + it("prioritises syncing over membershipConnected when both fail", () => { + const { combined$ } = createHomeserverConnected$(scope, client, session, 0); + // Both sync (Error) and membership (Disconnected) are failing + expect(combined$.value).toEqual([false, "sync"]); + }); + + it("updates reason as conditions change", () => { + const { combined$ } = createHomeserverConnected$(scope, client, session, 0); + // Initially: syncing fails + expect(combined$.value).toEqual([false, "sync"]); + + // Fix sync → membershipConnected is now the blocker + client.setSyncState(SyncState.Syncing); + expect(combined$.value).toEqual([false, "membership"]); + + // Fix membership → probablyLeft makes certainlyConnected fail + session.setProbablyLeft(true); + session.setMembershipStatus(Status.Connected); + expect(combined$.value).toEqual([false, "probablyLeft"]); + + // Clear probablyLeft → all conditions satisfied + session.setProbablyLeft(false); + expect(combined$.value).toEqual([true, null]); + }); +}); + +describe("createHomeserverConnected$ - Grace Period", () => { + const GRACE_PERIOD = 5; + + function marbleTest( + syncStateMarbles: string, + expectedConnectedMarbles: string, + ): void { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const syncState$ = behavior(syncStateMarbles, { + s: SyncState.Syncing, + e: SyncState.Error, + }); + const client = new MockMatrixClient(syncState$.value); + schedule(syncStateMarbles, { + s: () => client.setSyncState(SyncState.Syncing), + e: () => client.setSyncState(SyncState.Error), + }); + const session = new MockMatrixRTCSession({ + membershipStatus: Status.Connected, + probablyLeft: false, + }); + const hsConnected = createHomeserverConnected$( + testScope(), + client, + session, + GRACE_PERIOD, + ); + expectObservable(hsConnected.combined$).toBe(expectedConnectedMarbles, { + y: [true, null], + n: [false, "sync"], + }); + }); + } + + it("respects gracePeriodMs: stays true during grace period and flips false after", () => { + // - Initial state: Everything is connected + // - Sync error occurs -> should remain connected due to grace period + // - After grace period, not connected + marbleTest("se", "y-----n"); + // If the sync error takes longer to occur, it should take equally long for + // the connection state to change + marbleTest("s--e", "y-------n"); + }); + + it("recovers immediately if sync returns during grace period", () => { + // - Initial state: Connected + // - Sync error occurs + // - Sync recovers BEFORE the grace period expires + // - Connection state remains constant + marbleTest("se--s", "y"); + }); + + it("flips to true IMMEDIATELY even if a grace period was pending", () => { + // - Initial error: connection eventually flips to false + // - Back to Syncing -> Must be connected immediately (synchronously) + marbleTest("e-----s", "y----ny"); + }); +}); diff --git a/sdk/src/session/HomeserverConnected.ts b/sdk/src/session/HomeserverConnected.ts new file mode 100644 index 000000000..efcf4cc7d --- /dev/null +++ b/sdk/src/session/HomeserverConnected.ts @@ -0,0 +1,143 @@ +/* +Copyright 2025 Element Creations Ltd. +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + MembershipManagerEvent, + Status, + type MatrixRTCSession, +} from "matrix-js-sdk/lib/matrixrtc"; +import { ClientEvent, type MatrixClient, SyncState } from "matrix-js-sdk"; +import { + fromEvent, + startWith, + map, + tap, + type Observable, + distinctUntilChanged, + switchMap, + of, + delay, + combineLatest, +} from "rxjs"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; + +import { type ObservableScope } from "../reactive/ObservableScope"; +import { type Behavior } from "../reactive/Behavior"; +import { defaultSessionTimings } from "../config"; + +type NodeEventHandler = (...args: unknown[]) => void; + +interface NodeStyleEventEmitter { + addListener(eventName: string | symbol, handler: NodeEventHandler): this; + removeListener(eventName: string | symbol, handler: NodeEventHandler): this; +} + +export type HomeserverDisconnectReason = "sync" | "membership" | "probablyLeft"; + +export interface HomeserverConnected { + /** + * Emits `[true, null]` when the homeserver connection is healthy, or + * `[false, reason]` when one of the three sub-conditions fails. + */ + combined$: Behavior<[boolean, HomeserverDisconnectReason | null]>; + rtsSession$: Behavior; +} + +/** + * Behavior representing whether we consider ourselves connected to the Matrix homeserver + * for the purposes of a MatrixRTC session. + * + * `combined$` emits `null` when all conditions are satisfied, or the first failing + * reason (priority: syncing > membershipConnected > certainlyConnected): + * 1. Sync loop is not in SyncState.Syncing (after grace period) → "sync" + * 2. membershipStatus !== Status.Connected → "membership" + * 3. probablyLeft === true → "probablyLeft" + * + * @param scope - The observable scope for lifecycle management. + * @param client - The Matrix client to monitor sync state. + * @param matrixRTCSession - The RTC session to monitor membership. + * @param gracePeriodMs - Grace period in milliseconds to wait before reporting sync disconnect. + * If not provided, uses the config value (default 10000ms). + */ +export function createHomeserverConnected$( + scope: ObservableScope, + client: NodeStyleEventEmitter & Pick, + matrixRTCSession: NodeStyleEventEmitter & + Pick, + gracePeriodMs?: number, +): HomeserverConnected { + const logger = rootLogger.getChild("[HomeserverConnected]"); + // Get grace period from parameter or config (default 10000ms) + const graceMs = + gracePeriodMs ?? defaultSessionTimings.syncDisconnectGracePeriodMs; + + const syncing$ = ( + fromEvent(client, ClientEvent.Sync) as Observable<[SyncState]> + ).pipe( + startWith([client.getSyncState()]), + map(([state]) => state === SyncState.Syncing), + distinctUntilChanged(), + switchMap((isSyncing) => { + if (isSyncing || graceMs <= 0) { + return of(isSyncing); + } + return of(false).pipe(delay(graceMs), startWith(true)); + }), + distinctUntilChanged(), + ); + + const rtsSession$ = scope.behavior( + fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe( + map(() => matrixRTCSession.membershipStatus ?? Status.Unknown), + ), + matrixRTCSession.membershipStatus ?? Status.Unknown, + ); + + const membershipConnected$ = rtsSession$.pipe( + map((status) => status === Status.Connected), + ); + + // This is basically notProbablyLeft$ + // + // probablyLeft is computed by a local timer that mimics the server delayed event. + // If we locally predict our server event timed out. We consider ourselves as probablyLeft + // even though we might not yet have received the delayed event leave. + // + // If that is not the case we certainly still have a valid membership on the matrix network + // independet if the sync currently works. + const certainlyConnected$ = fromEvent( + matrixRTCSession, + MembershipManagerEvent.ProbablyLeft, + ).pipe( + startWith(null), + map(() => matrixRTCSession.probablyLeft !== true), + ); + + const combined$ = scope.behavior( + combineLatest([syncing$, membershipConnected$, certainlyConnected$]).pipe( + map( + ([syncing, membership, certainly]): [ + boolean, + HomeserverDisconnectReason | null, + ] => { + if (!syncing) return [false, "sync"]; + if (!membership) return [false, "membership"]; + if (!certainly) return [false, "probablyLeft"]; + return [true, null]; + }, + ), + tap(([connected, reason]) => { + logger.info( + `Homeserver connected update: ${connected ? "connected" : reason}`, + ); + }), + ), + ); + + return { combined$, rtsSession$ }; +} diff --git a/sdk/src/session/KeyProvider.ts b/sdk/src/session/KeyProvider.ts index daeaa595d..faead16a8 100644 --- a/sdk/src/session/KeyProvider.ts +++ b/sdk/src/session/KeyProvider.ts @@ -9,9 +9,9 @@ import { type BaseKeyProvider, ExternalE2EEKeyProvider } from "livekit-client"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type MatrixRTCSession as JsSdkRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { E2eeType } from "../../../src/e2ee/e2eeType"; -import { MatrixKeyProvider } from "../../../src/e2ee/matrixKeyProvider"; -import { type EncryptionSystem } from "../../../src/e2ee/sharedKeyManagement"; +import { E2eeType } from "../encryption"; +import { MatrixKeyProvider } from "./MatrixKeyProvider"; +import { type EncryptionSystem } from "../encryption"; /** The LiveKit key provider for the encryption the host asked for, or none. */ export function createKeyProvider( diff --git a/sdk/src/session/LocalMember.ts b/sdk/src/session/LocalMember.ts index ee4385dd9..23dd8b604 100644 --- a/sdk/src/session/LocalMember.ts +++ b/sdk/src/session/LocalMember.ts @@ -5,8 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +import { observeParticipantEvents } from "@livekit/components-core"; import { type LocalParticipant, + type Participant, + ParticipantEvent, type ScreenShareCaptureOptions, } from "livekit-client"; import { type Logger } from "matrix-js-sdk/lib/logger"; @@ -32,17 +35,17 @@ import { switchMap, } from "rxjs"; -import { type Behavior } from "../../../src/state/Behavior"; -import { type ObservableScope } from "../../../src/state/ObservableScope"; -import { type HomeserverConnected } from "../../../src/state/CallViewModel/localMember/HomeserverConnected"; -import { observeSharingScreen$ } from "../../../src/state/CallViewModel/localMember/LocalMember"; +import { type Behavior } from "../reactive/Behavior"; +import { type ObservableScope } from "../reactive/ObservableScope"; +import { type HomeserverConnected } from "./HomeserverConnected"; +import { type Connection, ConnectionState } from "./Connection"; +import { type IConnectionManager } from "./ConnectionManager"; import { - type Connection, - ConnectionState, -} from "../../../src/state/CallViewModel/remoteMembers/Connection"; -import { type IConnectionManager } from "../../../src/state/CallViewModel/remoteMembers/ConnectionManager"; -import { type MatrixRTCError } from "../api"; -import { toMatrixRTCError } from "../utils/errors"; + FailToStartLivekitConnection, + type MatrixRTCError, + MembershipManagerError, + toMatrixRTCError, +} from "../errors"; import { type LocalTransport } from "./LocalTransport"; import { type Publisher } from "./Publisher"; @@ -171,7 +174,11 @@ export function createLocalMembership$({ } else if (publisher.shouldPublish) await publisher.stopPublishing(); } catch (e) { if (publishError$.value === null) - publishError$.next(toMatrixRTCError(e)); + publishError$.next( + new FailToStartLivekitConnection( + e instanceof Error ? e.message : String(e), + ), + ); else logger.error("Another publish error", e); } }, @@ -199,7 +206,12 @@ export function createLocalMembership$({ membershipManagerError$.pipe(scope.bind()).subscribe((e) => { logger.error("The membership manager stopped", e); - if (matrixError$.value === null) matrixError$.next(toMatrixRTCError(e)); + if (matrixError$.value === null) + matrixError$.next( + new MembershipManagerError( + e instanceof Error ? e : new Error(String(e)), + ), + ); }); cameraEnabled$.pipe(scope.bind()).subscribe((videoEnabled) => { @@ -356,3 +368,13 @@ const screenShareCaptureOptions: ScreenShareCaptureOptions = { surfaceSwitching: "include", systemAudio: "include", }; + +function observeSharingScreen$(participant: Participant): Observable { + return observeParticipantEvents( + participant, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled)); +} diff --git a/sdk/src/session/LocalTransport.ts b/sdk/src/session/LocalTransport.ts index 668e65c15..4d0c4f077 100644 --- a/sdk/src/session/LocalTransport.ts +++ b/sdk/src/session/LocalTransport.ts @@ -7,18 +7,25 @@ Please see LICENSE in the repository root for full details. import { type MatrixClient } from "matrix-js-sdk"; import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { - DEFAULT_CONFIG, - type MatrixRTCMode, -} from "../../../src/config/ConfigOptions"; -import { getSFUConfigWithOpenID } from "../../../src/livekit/openIDSFU"; -import { type LocalTransport } from "../../../src/state/CallViewModel/localMember/LocalTransport"; -import { RtcTransportAutoDiscovery } from "../../../src/state/CallViewModel/localMember/RtcTransportAutoDiscovery"; -import { MatrixRTCTransportMissingError } from "../../../src/utils/errors"; +import { type MatrixRTCMode } from "../config"; +import { getSFUConfigWithOpenID, type SFUConfig } from "./openIDSFU"; +import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery"; +import { MatrixRTCTransportMissingError } from "../errors"; -export type { LocalTransport }; +/** The transport the local member publishes on, authenticated with. */ +export interface LocalTransport { + transport: LivekitTransport; + sfuConfig: SFUConfig; +} + +export function isLocalTransport( + obj: LivekitTransport | LocalTransport, +): obj is LocalTransport { + return "transport" in obj && "sfuConfig" in obj; +} interface Props { client: Pick< @@ -49,7 +56,6 @@ export async function getLocalTransport({ }: Props): Promise { const transport = await new RtcTransportAutoDiscovery({ client, - resolvedConfig: DEFAULT_CONFIG, logger, }).discoverPreferredTransport(); if (transport === null) diff --git a/sdk/src/session/MatrixKeyProvider.test.ts b/sdk/src/session/MatrixKeyProvider.test.ts new file mode 100644 index 000000000..ea7f9ea1b --- /dev/null +++ b/sdk/src/session/MatrixKeyProvider.test.ts @@ -0,0 +1,72 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, test, vi } from "vitest"; +import { + type MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/lib/matrixrtc"; +import { KeyProviderEvent } from "livekit-client"; + +import { MatrixKeyProvider } from "./MatrixKeyProvider"; + +function mockRTCSession(): MatrixRTCSession { + return { + on: vi.fn(), + off: vi.fn(), + reemitEncryptionKeys: vi.fn(), + } as unknown as MatrixRTCSession; +} + +describe("matrixKeyProvider", () => { + test("initializes", () => { + const keyProvider = new MatrixKeyProvider(); + expect(keyProvider).toBeTruthy(); + }); + + test("listens for key requests and emits existing keys", () => { + const keyProvider = new MatrixKeyProvider(); + + const session = mockRTCSession(); + + keyProvider.setRTCSession(session); + + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + expect.any(Function), + ); + expect(session.off).not.toHaveBeenCalled(); + }); + + test("stops listening when session changes", () => { + const keyProvider = new MatrixKeyProvider(); + + const session1 = mockRTCSession(); + const session2 = mockRTCSession(); + + keyProvider.setRTCSession(session1); + expect(session1.off).not.toHaveBeenCalled(); + + keyProvider.setRTCSession(session2); + expect(session1.off).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + expect.any(Function), + ); + }); + + test("emits existing keys", () => { + const keyProvider = new MatrixKeyProvider(); + const setKeyListener = vi.fn(); + keyProvider.on(KeyProviderEvent.SetKey, setKeyListener); + + const session = mockRTCSession(); + + keyProvider.setRTCSession(session); + + expect(session.reemitEncryptionKeys).toHaveBeenCalled(); + }); +}); diff --git a/sdk/src/session/MatrixKeyProvider.ts b/sdk/src/session/MatrixKeyProvider.ts new file mode 100644 index 000000000..f1a66a28a --- /dev/null +++ b/sdk/src/session/MatrixKeyProvider.ts @@ -0,0 +1,75 @@ +/* +Copyright 2023, 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { BaseKeyProvider } from "livekit-client"; +import { + type MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/lib/matrixrtc"; +import { logger as rootLogger, type Logger } from "matrix-js-sdk/lib/logger"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; + +export class MatrixKeyProvider extends BaseKeyProvider { + private rtcSession?: MatrixRTCSession; + private logger: Logger; + public constructor() { + super({ ratchetWindowSize: 10, keyringSize: 256 }); + this.logger = rootLogger.getChild("[MatrixKeyProvider]"); + } + + public setRTCSession(rtcSession: MatrixRTCSession): void { + if (this.rtcSession) { + this.rtcSession.off( + MatrixRTCSessionEvent.EncryptionKeyChanged, + this.onEncryptionKeyChanged, + ); + } + + this.rtcSession = rtcSession; + + this.rtcSession.on( + MatrixRTCSessionEvent.EncryptionKeyChanged, + this.onEncryptionKeyChanged, + ); + + // The new session could be aware of keys of which the old session wasn't, + // so emit key changed events + this.rtcSession.reemitEncryptionKeys(); + } + + private onEncryptionKeyChanged = ( + encryptionKey: Uint8Array, + encryptionKeyIndex: number, + membershipParts: CallMembershipIdentityParts, + rtcBackendIdentity: string, + ): void => { + crypto.subtle + .importKey("raw", encryptionKey, "HKDF", false, [ + "deriveBits", + "deriveKey", + ]) + .then( + (keyMaterial) => { + this.onSetEncryptionKey( + keyMaterial, + rtcBackendIdentity, + encryptionKeyIndex, + ); + + this.logger.debug( + `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}:${membershipParts.deviceId}) encryptionKeyIndex=${encryptionKeyIndex}`, + ); + }, + (e) => { + this.logger.error( + `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipParts.userId}:${membershipParts.deviceId} encryptionKeyIndex=${encryptionKeyIndex}`, + e, + ); + }, + ); + }; +} diff --git a/sdk/src/session/MatrixLivekitMembers.test.ts b/sdk/src/session/MatrixLivekitMembers.test.ts new file mode 100644 index 000000000..4a0779119 --- /dev/null +++ b/sdk/src/session/MatrixLivekitMembers.test.ts @@ -0,0 +1,333 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, test, expect, beforeEach, afterEach, vi } from "vitest"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { + type CallMembership, + type LivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs"; + +import { type IConnectionManager } from "./ConnectionManager.ts"; +import { + type RemoteMatrixLivekitMember, + createRemoteMatrixLivekitMembers$, +} from "./MatrixLivekitMembers.ts"; +import { + Epoch, + mapEpoch, + ObservableScope, + trackEpoch, +} from "../reactive/ObservableScope"; +import { ConnectionManagerData } from "./ConnectionManager.ts"; +import { + flushPromises, + mockRtcMembership, + mockRemoteParticipant, +} from "../utils/test"; +import { type Connection } from "./Connection.ts"; +import { constant } from "../reactive/Behavior"; +import { localRtcMember } from "../utils/test-fixtures"; + +let testScope: ObservableScope; + +const fallbackMemberId = (userId: string, deviceId: string): string => + `${userId}:${deviceId}`; + +const transportA: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", +}; + +const transportB: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.sample.com", +}; + +const bobMembership = mockRtcMembership("@bob:example.org", "DEV000", { + fociPreferred: [transportA], +}); +const carlMembership = mockRtcMembership("@carl:sample.com", "DEV111", { + fociPreferred: [transportB], +}); + +beforeEach(() => { + testScope = new ObservableScope(); +}); + +afterEach(() => { + testScope.end(); +}); + +function epochMeWith$( + source$: Observable>, + me$: Observable, +): Observable> { + return combineLatest([source$, me$]).pipe( + map(([ep, cd]) => { + return new Epoch(cd, ep.epoch); + }), + ); +} + +test("should signal participant not yet connected to livekit", async () => { + const info = vi.spyOn(logger, "info"); + const mockedMemberships$ = new BehaviorSubject([bobMembership]); + const mockConnectionManagerData$ = new BehaviorSubject( + new ConnectionManagerData(), + ); + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(mockedMemberships$); + + const connectionManagerData$ = epochMeWith$( + memberships$, + mockConnectionManagerData$, + ); + + const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + localUser: localRtcMember, + }); + + await flushPromises(); + expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].participant.value$.value).toBe(null); + expect(data[0].connection$.value).toBe(null); + return true; + }, + ); + expect(info).toHaveBeenCalledWith( + "[RemoteMatrixLivekitMembers] @bob:example.org:DEV000: LiveKit participant missing on no connection", + ); +}); + +// Helper to create epoch'ed memberships$ and membershipsWithTransport$ from memberships observable. +function createEpochedMemberships$(m$: Observable): { + memberships$: Observable>; + membershipsWithTransport$: Observable< + Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + >; +} { + const memberships$ = m$.pipe(trackEpoch()); + const membershipsWithTransport$ = memberships$.pipe( + mapEpoch((members) => { + return members.map((m) => { + const tr = m.getTransport(); + return { + membership: m, + transport: + tr?.type === "livekit" ? (tr as LivekitTransport) : undefined, + }; + }); + }), + ); + return { + memberships$, + membershipsWithTransport$, + }; +} + +test("should signal participant on a connection that is publishing", async () => { + const bobParticipantId = fallbackMemberId( + bobMembership.userId, + bobMembership.deviceId, + ); + + const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( + constant([bobMembership]), + ); + + const connection = { + transport: bobMembership.getTransport(), + } as unknown as Connection; + const dataWithPublisher = new ConnectionManagerData(); + dataWithPublisher.add(connection, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + + const connectionManagerData$ = epochMeWith$( + memberships$, + constant(dataWithPublisher), + ); + + const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + localUser: localRtcMember, + }); + + await flushPromises(); + expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].participant.value$.value).toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }); + expect(data[0].connection$.value).toBe(connection); + return true; + }, + ); +}); + +test("should signal participant on a connection that is not publishing", async () => { + const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$( + constant([bobMembership]), + ); + + const connection = { + transport: bobMembership.getTransport(), + } as unknown as Connection; + const dataWithPublisher = new ConnectionManagerData(); + dataWithPublisher.add(connection, []); + + const connectionManagerData$ = epochMeWith$( + memberships$, + constant(dataWithPublisher), + ); + + const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + localUser: localRtcMember, + }); + await flushPromises(); + expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].participant.value$.value).toBe(null); + expect(data[0].connection$.value).toBe(connection); + return true; + }, + ); +}); + +describe("Publication edge case", () => { + test("bob is publishing in several connections", async () => { + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(constant([bobMembership, carlMembership])); + + const connectionWithPublisher = new ConnectionManagerData(); + const bobParticipantId = fallbackMemberId( + bobMembership.userId, + bobMembership.deviceId, + ); + const connectionA = { + transport: transportA, + } as unknown as Connection; + const connectionB = { + transport: transportB, + } as unknown as Connection; + + connectionWithPublisher.add(connectionA, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + connectionWithPublisher.add(connectionB, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + + const connectionManagerData$ = epochMeWith$( + memberships$, + constant(connectionWithPublisher), + ); + + const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + localUser: localRtcMember, + }); + await flushPromises(); + expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(2); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].connection$.value).toBe(connectionA); + expect(data[0].participant.value$.value).toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }); + + return true; + }, + ); + }); +}); + +test("bob is publishing in the wrong connection", async () => { + const mockedMemberships$ = new BehaviorSubject([ + bobMembership, + carlMembership, + ]); + + const { memberships$, membershipsWithTransport$ } = + createEpochedMemberships$(mockedMemberships$); + + const connectionWithPublisher = new ConnectionManagerData(); + + const bobParticipantId = fallbackMemberId( + bobMembership.userId, + bobMembership.deviceId, + ); + const connectionA = { transport: transportA } as unknown as Connection; + const connectionB = { transport: transportB } as unknown as Connection; + + // Bob is not publishing on A + connectionWithPublisher.add(connectionA, []); + // Bob is publishing on B but his membership says A + connectionWithPublisher.add(connectionB, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + + const connectionsWithPublisher$ = new BehaviorSubject( + connectionWithPublisher, + ); + const connectionManagerData$ = epochMeWith$( + memberships$, + connectionsWithPublisher$, + ); + + const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + localUser: localRtcMember, + }); + + await flushPromises(); + expect(remoteMatrixLivekitMembers$.value.value).toSatisfy( + (data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(2); + expect(data[0].membership$.value).toBe(bobMembership); + expect(data[0].connection$.value).toBe(connectionA); + expect(data[0].participant.value$.value).toBe(null); + return true; + }, + ); +}); diff --git a/sdk/src/session/MatrixLivekitMembers.ts b/sdk/src/session/MatrixLivekitMembers.ts new file mode 100644 index 000000000..d3daa1dcb --- /dev/null +++ b/sdk/src/session/MatrixLivekitMembers.ts @@ -0,0 +1,160 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type RemoteParticipant } from "livekit-client"; +import { + type CallMembership, + type LivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; +import { combineLatest, filter, map } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { type Behavior } from "../reactive/Behavior"; +import { type IConnectionManager } from "./ConnectionManager"; +import { Epoch, type ObservableScope } from "../reactive/ObservableScope"; +import { type Connection } from "./Connection"; +import { generateItemsWithEpoch } from "../reactive/observable"; + +interface RemoteTaggedParticipant { + type: "remote"; + value$: Behavior; +} + +export interface MatrixLivekitMember { + membership$: Behavior; + connection$: Behavior; + // participantId: string; We do not want a participantId here since it will be generated by the jwt + // TODO decide if we can also drop the userId. Its in the matrix membership anyways. + userId: string; +} + +/** + * Represents a remote Matrix call member and their associated LiveKit participation. + * `livekitParticipant` can be null if the member is not yet connected to the livekit room + * or if it has no livekit transport at all. + */ +export interface RemoteMatrixLivekitMember extends MatrixLivekitMember { + participant: RemoteTaggedParticipant; +} + +interface Props { + scope: ObservableScope; + membershipsWithTransport$: Behavior< + Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + >; + connectionManager: IConnectionManager; + localUser: { deviceId: string; userId: string }; +} + +/** + * Combines MatrixRTC and Livekit worlds. + * + * It has a small public interface: + * - in (via constructor): + * - an observable of CallMembership[] to track the call members (The matrix side) + * - a `ConnectionManager` for the lk rooms (The livekit side) + * - out (via public Observable): + * - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data. + */ +export function createRemoteMatrixLivekitMembers$({ + scope, + membershipsWithTransport$, + connectionManager, + localUser, +}: Props): Behavior> { + /** + * Behavior of all the remote call members and their associated livekit data (if available). + */ + return scope.behavior( + combineLatest([ + membershipsWithTransport$, + connectionManager.connectionManagerData$, + ]).pipe( + filter((values) => + values.every((value) => value.epoch === values[0].epoch), + ), + map(([ms, data]) => new Epoch([ms.value, data.value] as const, ms.epoch)), + generateItemsWithEpoch( + "RemoteMatrixLivekitMembers", + // Generator function. + // creates an array of `{key, data}[]` + // Each change in the keys (new key) will result in a call to the factory function. + function* ([membershipsWithTransport, managerData]) { + for (const { membership, transport } of membershipsWithTransport) { + // Exclude the local membership + if ( + membership.userId === localUser.userId && + membership.deviceId === localUser.deviceId + ) + continue; + + const participants = transport + ? managerData.getParticipantsForTransport(transport) + : []; + const matches = participants.filter( + (p) => p.identity == membership.rtcBackendIdentity, + ); + const participant = matches[0] ?? null; + const connection = transport + ? managerData.getConnectionForTransport(transport) + : null; + if (matches.length > 1) + logger.warn( + `[RemoteMatrixLivekitMembers] ${membership.rtcBackendIdentity}: ${matches.length} LiveKit participants match (sids ${matches.map((p) => p.sid).join(", ")}), using ${participant?.sid}`, + ); + + yield { + // This could just be the backend identity without the other keys. + // The user ID, device ID, and member ID are included however so + // they show up in debug logs. + keys: [ + membership.userId, + membership.deviceId, + membership.memberId, + membership.rtcBackendIdentity, + ], + data: { membership, participant, connection }, + }; + } + }, + // Each update where the key of the generator array do not change will result in updates to the `data$` behavior. + (scope, data$, userId, _deviceId, _memberId, rtcBackendIdentity) => { + const { participant$, ...rest } = scope.splitBehavior(data$); + // Log whether the member could be matched to a LiveKit participant, + // since a tile shows "waiting for media" for as long as it cannot. + participant$.pipe(scope.bind()).subscribe((p) => { + const url = data$.value.connection?.transport.livekit_service_url; + logger.info( + `[RemoteMatrixLivekitMembers] ${rtcBackendIdentity}: LiveKit participant ${p ? `matched (${p.sid})` : "missing"} on ${url ?? "no connection"}`, + ); + }); + // will only get called once per backend identity. + // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. + return { + userId, + participant: { type: "remote" as const, value$: participant$ }, + ...rest, + }; + }, + ), + ), + new Epoch([], -1), + ); +} + +// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) + +// TODO add this to the JS-SDK +export function areLivekitTransportsEqual( + t1: T | null, + t2: T | null, +): boolean { + if (t1 && t2) { + return t1.livekit_service_url === t2.livekit_service_url; + } + return !t1 && !t2; +} diff --git a/sdk/src/session/MatrixMemberMetadata.test.ts b/sdk/src/session/MatrixMemberMetadata.test.ts new file mode 100644 index 000000000..8b1731ecd --- /dev/null +++ b/sdk/src/session/MatrixMemberMetadata.test.ts @@ -0,0 +1,632 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, vi } from "vitest"; +import { + type MatrixEvent, + type RoomMember, + type RoomState, + RoomStateEvent, +} from "matrix-js-sdk"; +import EventEmitter from "events"; +import { it } from "vitest"; + +import { ObservableScope } from "../reactive/ObservableScope"; +import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; +import { + mockRtcMembership, + mockMatrixRoomMember, + withTestScheduler, +} from "../utils/test"; +import { + createMatrixMemberMetadata$, + createRoomMembers$, +} from "./MatrixMemberMetadata.ts"; +let testScope: ObservableScope; +let mockMatrixRoom: MatrixRoom; + +describe("MatrixMemberMetadata", () => { + /* + * To be populated in the test setup. + * Maps userId to a partial/mock RoomMember object. + */ + let fakeMembersMap: Map>; + + beforeEach(() => { + testScope = new ObservableScope(); + fakeMembersMap = new Map>(); + + const roomEmitter = new EventEmitter(); + mockMatrixRoom = { + on: roomEmitter.on.bind(roomEmitter), + off: roomEmitter.off.bind(roomEmitter), + emit: roomEmitter.emit.bind(roomEmitter), + // addListener: roomEmitter.addListener.bind(roomEmitter), + // removeListener: roomEmitter.removeListener.bind(roomEmitter), + getMember: vi.fn().mockImplementation((userId: string) => { + const member = fakeMembersMap.get(userId); + if (member) { + return member as RoomMember; + } + return null; + }), + getMembers: vi.fn().mockImplementation(() => { + const members = Array.from(fakeMembersMap.values()); + return members; + }), + getMembersWithMembership: vi.fn().mockImplementation(() => { + const members = Array.from(fakeMembersMap.values()); + return members; + }), + } as unknown as MatrixRoom; + }); + + function fakeMemberWith(data: Partial): void { + const userId = data.userId || "@alice:example.com"; + const member: Partial = { + userId: userId, + rawDisplayName: data.rawDisplayName ?? userId, + getMxcAvatarUrl: + data.getMxcAvatarUrl || + vi.fn().mockImplementation(() => { + return `mxc://example.com/${userId}`; + }), + ...data, + } as unknown as RoomMember; + fakeMembersMap.set(userId, member); + } + + afterEach(() => { + fakeMembersMap.clear(); + }); + + describe("displayname", () => { + function updateDisplayName( + userId: `@${string}:${string}`, + newDisplayName: string, + ): void { + const member = fakeMembersMap.get(userId); + if (member) { + member.rawDisplayName = newDisplayName; + // Emit the event to notify listeners + mockMatrixRoom.emit( + RoomStateEvent.Members, + {} as unknown as MatrixEvent, + {} as unknown as RoomState, + member as RoomMember, + ); + } else { + throw new Error(`No member found with userId: ${userId}`); + } + } + + it("should show our own user if present in rtc session and room", () => { + withTestScheduler(({ scope, behavior, expectObservable }) => { + fakeMemberWith({ + userId: "@local:example.com", + rawDisplayName: "it's a me", + }); + const memberships$ = behavior("a", { + a: [mockRtcMembership("@local:example.com", "DEVICE1")], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + const dn$ = metadataStore.createDisplayNameBehavior$( + scope, + "@local:example.com", + ); + + expectObservable(dn$).toBe("a", { + a: "it's a me", + }); + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([["@local:example.com", "it's a me"]]), + }); + }); + }); + + function setUpBasicRoom(): void { + fakeMemberWith({ + userId: "@local:example.com", + rawDisplayName: "it's a me", + }); + fakeMemberWith({ userId: "@alice:example.com", rawDisplayName: "Alice" }); + fakeMemberWith({ userId: "@bob:example.com", rawDisplayName: "Bob" }); + fakeMemberWith({ userId: "@carl:example.com", rawDisplayName: "Carl" }); + fakeMemberWith({ userId: "@evil:example.com", rawDisplayName: "Carl" }); + fakeMemberWith({ userId: "@bob:foo.bar", rawDisplayName: "Bob" }); + fakeMemberWith({ userId: "@no-name:foo.bar" }); + } + + it("should get displayName for users", () => { + setUpBasicRoom(); + + withTestScheduler(({ scope, behavior, expectObservable }) => { + const memberships$ = behavior("a", { + a: [ + mockRtcMembership("@alice:example.com", "DEVICE1"), + mockRtcMembership("@bob:example.com", "DEVICE1"), + ], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + const aliceDispName$ = metadataStore.createDisplayNameBehavior$( + scope, + "@alice:example.com", + ); + + expectObservable(aliceDispName$).toBe("a", { + a: "Alice", + }); + + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([ + ["@alice:example.com", "Alice"], + ["@bob:example.com", "Bob"], + ]), + }); + }); + }); + + it("should use userId if no display name", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("a", { + a: [mockRtcMembership("@no-name:foo.bar", "D000")], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([ + ["@no-name:foo.bar", "@no-name:foo.bar"], + ]), + }); + }); + }); + + it("should disambiguate users with same display name", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("a", { + a: [ + mockRtcMembership("@bob:example.com", "DEVICE1"), + mockRtcMembership("@bob:example.com", "DEVICE2"), + mockRtcMembership("@bob:foo.bar", "BOB000"), + mockRtcMembership("@carl:example.com", "C000"), + mockRtcMembership("@evil:example.com", "E000"), + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([ + // ["@local:example.com", "it's a me"], + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], + ["@carl:example.com", "Carl (@carl:example.com)"], + ["@evil:example.com", "Carl (@evil:example.com)"], + ]), + }); + }); + }); + + it("should start to disambiguate reactivly when needed", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("ab", { + a: [mockRtcMembership("@bob:example.com", "DEVICE1")], + b: [ + mockRtcMembership("@bob:example.com", "DEVICE1"), + mockRtcMembership("@bob:foo.bar", "BOB000"), + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + a: new Map([["@bob:example.com", "Bob"]]), + b: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], + ]), + }); + }); + }); + + it("should keep disambiguated name when other leave", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("ab", { + a: [ + mockRtcMembership("@bob:example.com", "DEVICE1"), + mockRtcMembership("@bob:foo.bar", "BOB000"), + ], + b: [mockRtcMembership("@bob:example.com", "DEVICE1")], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + a: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], + ]), + b: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ]), + }); + }); + }); + + it("should disambiguate on name change", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("a", { + a: [ + mockRtcMembership("@bob:example.com", "B000"), + mockRtcMembership("@carl:example.com", "C000"), + ], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + schedule("-a", { + a: () => { + updateDisplayName("@carl:example.com", "Bob"); + }, + }); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + a: new Map([ + ["@bob:example.com", "Bob"], + ["@carl:example.com", "Carl"], + ]), + b: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@carl:example.com", "Bob (@carl:example.com)"], + ]), + }); + }); + }); + + it("should track individual member id with createDisplayNameBehavior", () => { + withTestScheduler(({ scope, behavior, schedule, expectObservable }) => { + setUpBasicRoom(); + const BOB = "@bob:example.com"; + const CARL = "@carl:example.com"; + // for this test we build a mock environment that does all possible changes: + // - memberships join/leave + // - room join/leave + // - disambiguate + const memberships$ = behavior("ab-d", { + a: [mockRtcMembership(CARL, "C000")], + b: [ + mockRtcMembership(CARL, "C000"), + // bob joins + mockRtcMembership(BOB, "B000"), + ], + // c carl gets renamed to BOB + d: [ + // carl leaves + mockRtcMembership(BOB, "B000"), + ], + }); + schedule("--a-", { + a: () => { + // carl renames + updateDisplayName(CARL, "Bob"); + }, + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + const bob$ = metadataStore.createDisplayNameBehavior$(scope, BOB); + const carl$ = metadataStore.createDisplayNameBehavior$(scope, CARL); + + expectObservable(bob$).toBe("abc-", { + a: undefined, + b: "Bob", + c: "Bob (@bob:example.com)", + // bob stays disambiguate even though carl left + // d: "Bob (@bob:example.com)", + }); + + expectObservable(carl$).toBe("a-cd", { + a: "Carl", + // b: "Carl", + // carl gets renamed and disambiguate + c: "Bob (@carl:example.com)", + d: undefined, + }); + }); + }); + + it("should disambiguate users with invisible characters", () => { + withTestScheduler(({ scope, behavior, expectObservable }) => { + const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); + const bobZeroWidthSpaceRtcMember = mockRtcMembership( + "@bob2:example.org", + "BBBB", + ); + const bob = mockMatrixRoomMember(bobRtcMember, { + rawDisplayName: "Bob", + }); + const bobZeroWidthSpace = mockMatrixRoomMember( + bobZeroWidthSpaceRtcMember, + { + rawDisplayName: "Bo\u200bb", + }, + ); + fakeMemberWith(bob); + fakeMemberWith(bobZeroWidthSpace); + fakeMemberWith({ userId: "@carol:example.org" }); + const memberships$ = behavior("ab", { + a: [mockRtcMembership("@carol:example.org", "1111"), bobRtcMember], + b: [ + mockRtcMembership("@carol:example.org", "1111"), + bobRtcMember, + bobZeroWidthSpaceRtcMember, + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + const bob$ = metadataStore.createDisplayNameBehavior$( + scope, + "@bob:example.org", + ); + const bob2$ = metadataStore.createDisplayNameBehavior$( + scope, + "@bob2:example.org", + ); + const carol$ = metadataStore.createDisplayNameBehavior$( + scope, + "@carol:example.org", + ); + expectObservable(bob$).toBe("ab", { + a: "Bob", + b: "Bob (@bob:example.org)", + }); + expectObservable(bob2$).toBe("ab", { + a: undefined, + b: "Bo\u200bb (@bob2:example.org)", + }); + expectObservable(carol$).toBe("a-", { + a: "@carol:example.org", + }); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + // Carol has no displayname - So userId is used. + a: new Map([ + ["@carol:example.org", "@carol:example.org"], + ["@bob:example.org", "Bob"], + ]), + // Other Bob joins, and should handle zero width hacks. + b: new Map([ + ["@carol:example.org", "@carol:example.org"], + [bobRtcMember.userId, `Bob (@bob:example.org)`], + [ + bobZeroWidthSpace.userId, + `${bobZeroWidthSpace.rawDisplayName} (${bobZeroWidthSpace.userId})`, + ], + ]), + }); + }); + }); + + it("should strip RTL characters from displayname", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); + const daveRTLRtcMember = mockRtcMembership( + "@dave2:example.org", + "DDDD", + ); + const dave = mockMatrixRoomMember(daveRtcMember, { + rawDisplayName: "Dave", + }); + const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { + rawDisplayName: "\u202eevaD", + }); + + fakeMemberWith({ userId: "@carol:example.org" }); + fakeMemberWith(daveRTL); + fakeMemberWith(dave); + const memberships$ = behavior("ab", { + a: [mockRtcMembership("@carol:example.org", "DDDD")], + b: [ + mockRtcMembership("@carol:example.org", "DDDD"), + daveRtcMember, + daveRTLRtcMember, + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + // Carol has no displayname - So userId is used. + a: new Map([["@carol:example.org", "@carol:example.org"]]), + // Both Dave's join. Since after stripping + b: new Map([ + ["@carol:example.org", "@carol:example.org"], + // Not disambiguated + ["@dave:example.org", "Dave"], + // This one is, since it's using RTL. + ["@dave2:example.org", "evaD (@dave2:example.org)"], + ]), + }); + }); + }); + }); + + describe("avatarUrl", () => { + function updateAvatarUrl( + userId: `@${string}:${string}`, + avatarUrl: string, + ): void { + const member = fakeMembersMap.get(userId); + if (member) { + member.getMxcAvatarUrl = vi.fn().mockReturnValue(avatarUrl); + // Emit the event to notify listeners + mockMatrixRoom.emit( + RoomStateEvent.Members, + {} as unknown as MatrixEvent, + {} as unknown as RoomState, + member as RoomMember, + ); + } else { + throw new Error(`No member found with userId: ${userId}`); + } + } + + it("should use avatar url from room members", () => { + withTestScheduler(({ scope, behavior, expectObservable }) => { + fakeMemberWith({ + userId: "@local:example.com", + }); + fakeMemberWith({ + userId: "@alice:example.com", + getMxcAvatarUrl: vi.fn().mockReturnValue("mxc://custom.url/avatar"), + }); + const memberships$ = behavior("a", { + a: [ + mockRtcMembership("@local:example.com", "DEVICE1"), + mockRtcMembership("@alice:example.com", "DEVICE1"), + ], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + const local$ = metadataStore.createAvatarUrlBehavior$( + scope, + "@local:example.com", + ); + + const alice$ = metadataStore.createAvatarUrlBehavior$( + scope, + "@alice:example.com", + ); + + expectObservable(local$).toBe("a", { + a: "mxc://example.com/@local:example.com", + }); + expectObservable(alice$).toBe("a", { + a: "mxc://custom.url/avatar", + }); + expectObservable(metadataStore.avatarMap$).toBe("a", { + a: new Map([ + ["@local:example.com", "mxc://example.com/@local:example.com"], + ["@alice:example.com", "mxc://custom.url/avatar"], + ]), + }); + }); + }); + + it("should update on avatar change and user join/leave", () => { + withTestScheduler(({ scope, behavior, schedule, expectObservable }) => { + fakeMemberWith({ userId: "@carl:example.com" }); + fakeMemberWith({ userId: "@bob:example.com" }); + const memberships$ = behavior("ab-d", { + a: [mockRtcMembership("@bob:example.com", "B000")], + b: [ + mockRtcMembership("@bob:example.com", "B000"), + mockRtcMembership("@carl:example.com", "C000"), + ], + d: [mockRtcMembership("@carl:example.com", "C000")], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + schedule("--c-", { + c: () => { + updateAvatarUrl( + "@carl:example.com", + "mxc://updated.me/updatedAvatar", + ); + }, + }); + + const bob$ = metadataStore.createAvatarUrlBehavior$( + scope, + "@bob:example.com", + ); + const carl$ = metadataStore.createAvatarUrlBehavior$( + scope, + "@carl:example.com", + ); + expectObservable(bob$).toBe("a---", { + a: "mxc://example.com/@bob:example.com", + }); + expectObservable(carl$).toBe("a-c-", { + a: "mxc://example.com/@carl:example.com", + + c: "mxc://updated.me/updatedAvatar", + }); + expectObservable(metadataStore.avatarMap$).toBe("a-c-", { + a: new Map([ + ["@bob:example.com", "mxc://example.com/@bob:example.com"], + ["@carl:example.com", "mxc://example.com/@carl:example.com"], + ]), + // expect an update once we update the avatar URL + c: new Map([ + ["@bob:example.com", "mxc://example.com/@bob:example.com"], + ["@carl:example.com", "mxc://updated.me/updatedAvatar"], + ]), + }); + }); + }); + }); +}); diff --git a/sdk/src/session/MatrixMemberMetadata.ts b/sdk/src/session/MatrixMemberMetadata.ts new file mode 100644 index 000000000..f0e155e66 --- /dev/null +++ b/sdk/src/session/MatrixMemberMetadata.ts @@ -0,0 +1,157 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type RoomMember, RoomStateEvent } from "matrix-js-sdk"; +import { combineLatest, fromEvent, map } from "rxjs"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { + KnownMembership, + type Room as MatrixRoom, +} from "matrix-js-sdk/lib/matrix"; +// eslint-disable-next-line rxjs/no-internal + +import { type ObservableScope } from "../reactive/ObservableScope"; +import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; +import { type Behavior } from "../reactive/Behavior"; + +export type RoomMemberMap = Map< + string, + Pick +>; +export function roomToMembersMap(matrixRoom: MatrixRoom): RoomMemberMap { + const members = matrixRoom + .getMembersWithMembership(KnownMembership.Join) + .concat(matrixRoom.getMembersWithMembership(KnownMembership.Invite)); + return members.reduce((acc, member) => { + acc.set(member.userId, { + userId: member.userId, + getMxcAvatarUrl: member.getMxcAvatarUrl.bind(member), + rawDisplayName: member.rawDisplayName, + }); + return acc; + }, new Map()); +} + +export function createRoomMembers$( + scope: ObservableScope, + matrixRoom: MatrixRoom, +): Behavior { + return scope.behavior( + fromEvent(matrixRoom, RoomStateEvent.Members).pipe( + map(() => roomToMembersMap(matrixRoom)), + ), + roomToMembersMap(matrixRoom), + ); +} + +/** + * Displayname for each member of the call. This will disambiguate + * any displayname that clashes with another member. Only members + * joined to the call are considered here. + * + * @returns Map uses the Matrix user ID as the key. + */ +// don't do this work more times than we need to. This is achieved by converting to a behavior: +export const memberDisplaynames$ = ( + scope: ObservableScope, + memberships$: Behavior[]>, + roomMembers$: Behavior, +): Behavior> => { + const logger = rootLogger.getChild("[MatrixMemberMetadata]"); + // This map tracks userIds that at some point needed disambiguation. + // This is a memory leak bound to the number of participants. + // A call application will always increase the memory if there have been more members in a call. + // Its capped by room member participants. + const shouldDisambiguateTrackerMap = new Set(); + return scope.behavior( + combineLatest([ + // Handle call membership changes + memberships$, + // Additionally handle display name changes (implicitly reacting to them) + roomMembers$, + // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), + ]).pipe( + map(([memberships, roomMembers]) => { + const displaynameMap = new Map(); + // We only consider RTC members for disambiguation as they are the only visible members. + for (const rtcMember of memberships) { + const member = roomMembers.get(rtcMember.userId); + if (!member) { + logger.error(`Could not find member for user ${rtcMember.userId}`); + continue; + } + const disambiguateComputed = shouldDisambiguate( + member, + memberships, + roomMembers, + ); + + const disambiguate = + shouldDisambiguateTrackerMap.has(rtcMember.userId) || + disambiguateComputed; + if (disambiguate) shouldDisambiguateTrackerMap.add(rtcMember.userId); + displaynameMap.set( + rtcMember.userId, + calculateDisplayName(member, disambiguate), + ); + } + return displaynameMap; + }), + ), + ); +}; + +export const createMatrixMemberMetadata$ = ( + scope: ObservableScope, + memberships$: Behavior[]>, + roomMembers$: Behavior, +): { + createDisplayNameBehavior$: ( + scope: ObservableScope, + userId: string, + ) => Behavior; + createAvatarUrlBehavior$: ( + scope: ObservableScope, + userId: string, + ) => Behavior; + displaynameMap$: Behavior>; + avatarMap$: Behavior>; +} => { + const displaynameMap$ = memberDisplaynames$( + scope, + memberships$, + roomMembers$, + ); + const avatarMap$ = scope.behavior( + roomMembers$.pipe( + map((roomMembers) => + Array.from(roomMembers.keys()).reduce((acc, key) => { + acc.set(key, roomMembers.get(key)?.getMxcAvatarUrl()); + return acc; + }, new Map()), + ), + ), + ); + return { + createDisplayNameBehavior$: (scope: ObservableScope, userId: string) => + scope.behavior( + displaynameMap$.pipe( + map((displaynameMap) => displaynameMap.get(userId)), + ), + ), + createAvatarUrlBehavior$: (scope: ObservableScope, userId: string) => + scope.behavior( + roomMembers$.pipe( + map((roomMembers) => roomMembers.get(userId)?.getMxcAvatarUrl()), + ), + ), + // mostly for testing purposes + displaynameMap$, + avatarMap$, + }; +}; diff --git a/sdk/src/session/MatrixRTCClient.ts b/sdk/src/session/MatrixRTCClient.ts index 096031531..8bd6d5f4c 100644 --- a/sdk/src/session/MatrixRTCClient.ts +++ b/sdk/src/session/MatrixRTCClient.ts @@ -15,21 +15,21 @@ import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/En import { v4 as uuidv4 } from "uuid"; import { combineLatest, from, fromEvent, map } from "rxjs"; -import { MatrixRTCMode } from "../../../src/config/ConfigOptions"; -import { type ObservableScope } from "../../../src/state/ObservableScope"; +import { MatrixRTCMode } from "../config"; +import { type ObservableScope } from "../reactive/ObservableScope"; import { createKeyRotationSuppressed$, createMemberships$, membershipsAndTransports$, -} from "../../../src/state/SessionBehaviors"; -import { createHomeserverConnected$ } from "../../../src/state/CallViewModel/localMember/HomeserverConnected"; -import { createConnectionManager$ } from "../../../src/state/CallViewModel/remoteMembers/ConnectionManager"; -import { createRemoteMatrixLivekitMembers$ } from "../../../src/state/CallViewModel/remoteMembers/MatrixLivekitMembers"; +} from "./SessionBehaviors"; +import { createHomeserverConnected$ } from "./HomeserverConnected"; +import { createConnectionManager$ } from "./ConnectionManager"; +import { createRemoteMatrixLivekitMembers$ } from "./MatrixLivekitMembers"; import { createMatrixMemberMetadata$, createRoomMembers$, -} from "../../../src/state/CallViewModel/remoteMembers/MatrixMemberMetadata"; -import { filterBehavior, generateItems } from "../../../src/utils/observable"; +} from "./MatrixMemberMetadata"; +import { filterBehavior, generateItems } from "../reactive/observable"; import { type LocalMediaInputs, type LocalRTCMember, diff --git a/sdk/src/session/Members.ts b/sdk/src/session/Members.ts index 362ee8629..7b295e9aa 100644 --- a/sdk/src/session/Members.ts +++ b/sdk/src/session/Members.ts @@ -13,12 +13,12 @@ import { import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, distinctUntilChanged, map } from "rxjs"; -import { type Behavior } from "../../../src/state/Behavior"; -import { type ObservableScope } from "../../../src/state/ObservableScope"; -import { type Connection } from "../../../src/state/CallViewModel/remoteMembers/Connection"; -import { type createMatrixMemberMetadata$ } from "../../../src/state/CallViewModel/remoteMembers/MatrixMemberMetadata"; -import { type RemoteMatrixLivekitMember } from "../../../src/state/CallViewModel/remoteMembers/MatrixLivekitMembers"; -import { type EncryptionSystem } from "../../../src/e2ee/sharedKeyManagement"; +import { type Behavior } from "../reactive/Behavior"; +import { type ObservableScope } from "../reactive/ObservableScope"; +import { type Connection } from "./Connection"; +import { type createMatrixMemberMetadata$ } from "./MatrixMemberMetadata"; +import { type RemoteMatrixLivekitMember } from "./MatrixLivekitMembers"; +import { type EncryptionSystem } from "../encryption"; import { type LocalRTCMember, type MemberMedia, diff --git a/sdk/src/session/Publisher.ts b/sdk/src/session/Publisher.ts index 0dc449afa..adcba8b8d 100644 --- a/sdk/src/session/Publisher.ts +++ b/sdk/src/session/Publisher.ts @@ -17,8 +17,8 @@ import { import { type Logger } from "matrix-js-sdk/lib/logger"; import { combineLatest, distinctUntilChanged, map, skip } from "rxjs"; -import { type Behavior } from "../../../src/state/Behavior"; -import { ObservableScope } from "../../../src/state/ObservableScope"; +import { type Behavior } from "../reactive/Behavior"; +import { ObservableScope } from "../reactive/ObservableScope"; import { type LocalMediaInputs } from "../api"; /** diff --git a/sdk/src/session/RtcTransportAutoDiscovery.test.ts b/sdk/src/session/RtcTransportAutoDiscovery.test.ts new file mode 100644 index 000000000..76cbf2169 --- /dev/null +++ b/sdk/src/session/RtcTransportAutoDiscovery.test.ts @@ -0,0 +1,142 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + beforeEach, + describe, + expect, + it, + type MockedObject, + vi, +} from "vitest"; +import { MatrixError } from "matrix-js-sdk"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { + type LivekitTransport, + type Transport, +} from "matrix-js-sdk/lib/matrixrtc"; + +import { + RtcTransportAutoDiscovery, + type RtcTransportAutoDiscoveryProps, +} from "./RtcTransportAutoDiscovery.ts"; + +type DiscoveryClient = RtcTransportAutoDiscoveryProps["client"]; + +const backendTransport: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://backend.example.org", +}; + +const configTransport: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://config.example.org", +}; + +function makeClient(): MockedObject { + return { + getDomain: vi.fn().mockReturnValue("example.org"), + baseUrl: "https://matrix.example.org", + _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), + getOpenIdToken: vi.fn(), + getDeviceId: vi.fn(), + } as unknown as MockedObject; +} + +describe("RtcTransportAutoDiscovery", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + const VALID_TEST_CASES: Array<{ transports: Transport[] }> = [ + { transports: [backendTransport] }, + // will pick the first livekit transport in the list, even if there are other non-livekit transports + { transports: [{ type: "not_livekit" }, backendTransport] }, + ]; + it.each(VALID_TEST_CASES)( + "prefers backend transport other app config $transports", + async ({ transports }) => { + const client = makeClient(); + client._unstable_getRTCTransports.mockResolvedValue(transports); + + const discovery = new RtcTransportAutoDiscovery({ + client, + fallbackTransportUrl: configTransport.livekit_service_url, + logger: rootLogger, + }); + + const discoveredTransport = await discovery.discoverPreferredTransport(); + + expect(discoveredTransport).toStrictEqual(backendTransport); + expect(discoveredTransport).not.toStrictEqual(configTransport); + + expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(1); + }, + ); + + it("Retries limit_exceeded backend transport", async () => { + const client = makeClient(); + client._unstable_getRTCTransports + .mockRejectedValueOnce( + new MatrixError( + { + errcode: "M_LIMIT_EXCEEDED", + error: "Too many requests", + retry_after_ms: 100, + }, + 429, + ), + ) + .mockResolvedValue([backendTransport]); + + const discovery = new RtcTransportAutoDiscovery({ + client, + fallbackTransportUrl: "https://config.example.org", + logger: rootLogger, + }); + + await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual( + backendTransport, + ); + + expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(2); + }); + + const INVALID_TEST_CASES: Array<{ transports: Transport[] }> = [ + { transports: [] }, + { transports: [{ type: "not_livekit" }] }, + ]; + it.each(INVALID_TEST_CASES)( + "falls back to config when backend has no (valid) livekit transports $transports", + async ({ transports }) => { + const client = makeClient(); + client._unstable_getRTCTransports.mockResolvedValue(transports); + + const discovery = new RtcTransportAutoDiscovery({ + client, + fallbackTransportUrl: configTransport.livekit_service_url, + logger: rootLogger, + }); + + const discoveredTransport = await discovery.discoverPreferredTransport(); + expect(discoveredTransport).not.toStrictEqual(backendTransport); + expect(discoveredTransport).toStrictEqual(configTransport); + }, + ); + + it("returns null when backend and config are all unavailable", async () => { + const client = makeClient(); + client._unstable_getRTCTransports.mockResolvedValue([]); + + const discovery = new RtcTransportAutoDiscovery({ + client, + fallbackTransportUrl: undefined, + logger: rootLogger, + }); + + await expect(discovery.discoverPreferredTransport()).resolves.toBeNull(); + }); +}); diff --git a/sdk/src/session/RtcTransportAutoDiscovery.ts b/sdk/src/session/RtcTransportAutoDiscovery.ts new file mode 100644 index 000000000..45503e3a6 --- /dev/null +++ b/sdk/src/session/RtcTransportAutoDiscovery.ts @@ -0,0 +1,103 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ +import { + isLivekitTransport, + type LivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; +import { type MatrixClient } from "matrix-js-sdk"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { doNetworkOperationWithRetry } from "../utils/network"; + +type TransportDiscoveryClient = Pick< + MatrixClient, + "getDomain" | "_unstable_getRTCTransports" +>; + +export interface RtcTransportAutoDiscoveryProps { + client: TransportDiscoveryClient; + /** A transport to use when the homeserver advertises none. */ + fallbackTransportUrl?: string; + logger: Logger; +} + +export class RtcTransportAutoDiscovery { + private readonly client: TransportDiscoveryClient; + private readonly fallbackTransportUrl: string | undefined; + private readonly logger: Logger; + + public constructor({ + client, + fallbackTransportUrl, + logger, + }: RtcTransportAutoDiscoveryProps) { + this.client = client; + this.fallbackTransportUrl = fallbackTransportUrl; + this.logger = logger.getChild("[RtcTransportAutoDiscovery]"); + } + + public async discoverPreferredTransport(): Promise { + // 1) backend transports + const backendTransport = await this.tryBackendTransports(); + if (backendTransport) { + this.logger.info( + `Found backend transport: ${backendTransport.livekit_service_url}`, + ); + return backendTransport; + } + + // 2) app config URL + const configTransport = this.tryConfigTransport(); + if (configTransport) { + this.logger.info( + `Found app config transport: ${configTransport.livekit_service_url}`, + ); + return configTransport; + } + + return null; + } + + /** + * Fetches the first rtc_foci from the backend. + * This will not throw errors, but instead just log them and return null if the expected config is not found or malformed. + * @private + */ + private async tryBackendTransports(): Promise { + const client = this.client; + // MSC4143: Attempt to fetch transports from backend. + this.logger.info("First try to use getRTCTransports end point ..."); + try { + const transportList = await doNetworkOperationWithRetry(async () => + client._unstable_getRTCTransports(), + ); + const first = transportList.find(isLivekitTransport); + if (first) { + return first; + } else { + this.logger.info( + `No livekit transport found in getRTCTransports end point`, + transportList, + ); + } + } catch (ex) { + this.logger.info(`Failed to use getRTCTransports end point: ${ex}`); + } + return null; + } + + private tryConfigTransport(): LivekitTransport | null { + const url = this.fallbackTransportUrl; + if (url) { + return { + type: "livekit", + livekit_service_url: url, + }; + } + return null; + } +} diff --git a/sdk/src/session/SessionBehaviors.test.ts b/sdk/src/session/SessionBehaviors.test.ts new file mode 100644 index 000000000..fca24767c --- /dev/null +++ b/sdk/src/session/SessionBehaviors.test.ts @@ -0,0 +1,41 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, it, vi } from "vitest"; +import { MatrixRTCSessionEvent } from "matrix-js-sdk/lib/matrixrtc"; +import { EventEmitter } from "events"; + +import { createKeyRotationSuppressed$ } from "./SessionBehaviors"; +import { testScope } from "../utils/test"; + +describe("SessionBehaviors", () => { + describe("createKeyRotationSuppressed$", () => { + it("emits initial value from isKeyRotationSuppressed and updates when KeyRotationSuppressedChanged event is emitted", () => { + const scope = testScope(); + const emitter = new EventEmitter(); + const mockSession = Object.assign(emitter, { + on: vi.fn(), + off: vi.fn(), + isKeyRotationSuppressed: false, + }); + + const keyRotationSuppressed$ = createKeyRotationSuppressed$( + scope, + mockSession as any, + ); + + expect(keyRotationSuppressed$.value).toBe(false); + emitter.emit(MatrixRTCSessionEvent.KeyRotationSuppressedChanged, true); + + expect(keyRotationSuppressed$.value).toBe(true); + + emitter.emit(MatrixRTCSessionEvent.KeyRotationSuppressedChanged, false); + + expect(keyRotationSuppressed$.value).toBe(false); + }); + }); +}); diff --git a/sdk/src/session/SessionBehaviors.ts b/sdk/src/session/SessionBehaviors.ts new file mode 100644 index 000000000..385a5349f --- /dev/null +++ b/sdk/src/session/SessionBehaviors.ts @@ -0,0 +1,102 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type CallMembership, + type LivekitTransport, + type MatrixRTCSession, + MatrixRTCSessionEvent, + isLivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; +import { fromEvent } from "rxjs"; + +import { + Epoch, + mapEpoch, + trackEpoch, + type ObservableScope, +} from "../reactive/ObservableScope"; +import { type Behavior } from "../reactive/Behavior"; + +/** + * Tracks the transports used by ourselves, plus all other MatrixRTC session + * members. + */ +export const membershipsAndTransports$ = ( + scope: ObservableScope, + memberships$: Behavior>, +): { + membershipsWithTransport$: Behavior< + Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + >; + transports$: Behavior>; +} => { + const membershipsWithTransport$: Behavior< + Epoch< + { + membership: CallMembership; + transport: LivekitTransport | undefined; + }[] + > + > = scope.behavior( + memberships$.pipe( + mapEpoch((memberships) => { + return memberships.map((membership) => { + const transport = membership.getTransport(); + return { + membership, + transport: isLivekitTransport(transport) ? transport : undefined, + }; + }); + }), + ), + ); + + const transports$: Behavior> = scope.behavior( + membershipsWithTransport$.pipe( + mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), + ), + ); + + return { + membershipsWithTransport$, + transports$, + }; +}; + +export const createMemberships$ = ( + scope: ObservableScope, + matrixRTCSession: MatrixRTCSession, +): Behavior> => { + return scope.behavior( + fromEvent( + matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + (_, memberships: CallMembership[]) => memberships, + ).pipe(trackEpoch()), + new Epoch(matrixRTCSession.memberships), + ); +}; + +/** + * Whether the session has grown large enough that MatrixRTC has stopped rotating the media + * encryption key. While this is true the key in use is still shared with new joiners, but no new + * key is generated when someone joins or leaves. + */ +export const createKeyRotationSuppressed$ = ( + scope: ObservableScope, + matrixRTCSession: MatrixRTCSession, +): Behavior => { + return scope.behavior( + fromEvent( + matrixRTCSession, + MatrixRTCSessionEvent.KeyRotationSuppressedChanged, + (suppressed: boolean) => suppressed, + ), + matrixRTCSession.isKeyRotationSuppressed, + ); +}; diff --git a/sdk/src/session/Transports.ts b/sdk/src/session/Transports.ts index a7bd57a93..a5064e579 100644 --- a/sdk/src/session/Transports.ts +++ b/sdk/src/session/Transports.ts @@ -11,10 +11,10 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, distinctUntilChanged, map, of, switchMap } from "rxjs"; -import { constant, type Behavior } from "../../../src/state/Behavior"; -import { type ObservableScope } from "../../../src/state/ObservableScope"; -import { ConnectionState } from "../../../src/state/CallViewModel/remoteMembers/Connection"; -import { type IConnectionManager } from "../../../src/state/CallViewModel/remoteMembers/ConnectionManager"; +import { constant, type Behavior } from "../reactive/Behavior"; +import { type ObservableScope } from "../reactive/ObservableScope"; +import { ConnectionState } from "./Connection"; +import { type IConnectionManager } from "./ConnectionManager"; import { type TransportMetadata } from "../api"; import { ResolvedConnection } from "./ConnectionFactory"; diff --git a/sdk/src/session/joinJsSdkSession.ts b/sdk/src/session/joinJsSdkSession.ts index 9b1fe1e5d..d176ddd7e 100644 --- a/sdk/src/session/joinJsSdkSession.ts +++ b/sdk/src/session/joinJsSdkSession.ts @@ -13,10 +13,7 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { - DEFAULT_CONFIG, - MatrixRTCMode, -} from "../../../src/config/ConfigOptions"; +import { defaultSessionTimings, MatrixRTCMode } from "../config"; interface Options { encryptMedia: boolean; @@ -25,11 +22,8 @@ interface Options { applicationData?: Record; } -/** The session's timing defaults; a later slice makes them an input. */ -export const sessionTimings = { - syncDisconnectGracePeriodMs: DEFAULT_CONFIG.sync_disconnect_grace_period_ms, - ...DEFAULT_CONFIG.matrix_rtc_session, -}; +/** The session's timings; a later slice makes them an input. */ +export const sessionTimings = defaultSessionTimings; /** * Sends the membership and starts the membership manager, which keeps it @@ -51,7 +45,7 @@ export function joinJsSdkSession( const intent = applicationData?.["m.call.intent"]; const callIntent = typeof intent === "string" ? (intent as RTCCallIntent) : undefined; - const timings = sessionTimings.delayed_leave; + const timings = sessionTimings.delayedLeave; // Give up on the network as soon as either the sync has been down for the // grace period or the delayed leave has probably been sent, whichever is // sooner @@ -66,12 +60,12 @@ export function joinJsSdkSession( delayedLeaveEventRestartMs: timings.restart_ms, delayedLeaveEventDelayMs: timings.delay_ms, delayedLeaveEventRestartLocalTimeoutMs: timings.restart_timeout_ms, - networkErrorRetryMs: sessionTimings.network_error_retry_ms, - makeKeyDelay: sessionTimings.wait_for_key_rotation_ms, - membershipEventExpiryMs: sessionTimings.membership_event_expiry_ms, - keyRotationParticipantLimit: sessionTimings.key_rotation_participant_limit, + networkErrorRetryMs: sessionTimings.networkErrorRetryMs, + makeKeyDelay: sessionTimings.waitForKeyRotationMs, + membershipEventExpiryMs: sessionTimings.membershipEventExpiryMs, + keyRotationParticipantLimit: sessionTimings.keyRotationParticipantLimit, unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0, maximumNetworkErrorRetryCount: - Math.ceil(maxWaitMs / sessionTimings.network_error_retry_ms) + 1, + Math.ceil(maxWaitMs / sessionTimings.networkErrorRetryMs) + 1, }); } diff --git a/sdk/src/session/livekitOptions.test.ts b/sdk/src/session/livekitOptions.test.ts new file mode 100644 index 000000000..d9424ac50 --- /dev/null +++ b/sdk/src/session/livekitOptions.test.ts @@ -0,0 +1,189 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, it } from "vitest"; +import { VideoPresets, type VideoPreset } from "livekit-client"; + +import { buildLiveKitOptions } from "./livekitOptions"; + +describe("buildLiveKitOptions", () => { + it("returns sensible defaults with no config", () => { + const opts = buildLiveKitOptions(); + expect(opts.adaptiveStream).toBe(true); + expect(opts.dynacast).toBe(true); + expect(opts.videoCaptureDefaults?.resolution).toEqual( + VideoPresets.h720.resolution, + ); + expect(opts.publishDefaults?.videoCodec).toBe("vp8"); + expect(opts.publishDefaults?.videoEncoding).toEqual({ + maxBitrate: 1_700_000, + maxFramerate: 30, + }); + expect(opts.publishDefaults?.screenShareEncoding).toEqual({ + maxBitrate: 5_000_000, + maxFramerate: 30, + }); + expect(opts.publishDefaults?.videoSimulcastLayers).toEqual([ + VideoPresets.h180, + VideoPresets.h360, + ]); + }); + + it("applies video codec from config", () => { + const opts = buildLiveKitOptions({ video_codec: "vp9" }); + expect(opts.publishDefaults?.videoCodec).toBe("vp9"); + }); + + it("applies video resolution and encoding from config", () => { + const baseVideoConfig = { + max_resolution: 1080, + max_bitrate: 3_000_000, + max_framerate: 60, + }; + const opts1080 = buildLiveKitOptions({ + video: baseVideoConfig, + }); + const opts1440 = buildLiveKitOptions({ + video: { ...baseVideoConfig, max_resolution: 1440 }, + }); + const opts2160 = buildLiveKitOptions({ + video: { ...baseVideoConfig, max_resolution: 2160 }, + }); + expect(opts1080.videoCaptureDefaults?.resolution).toEqual( + VideoPresets.h1080.resolution, + ); + expect(opts1440.videoCaptureDefaults?.resolution).toEqual( + VideoPresets.h1440.resolution, + ); + expect(opts2160.videoCaptureDefaults?.resolution).toEqual( + VideoPresets.h2160.resolution, + ); + expect(opts1080.publishDefaults?.videoEncoding).toEqual({ + maxBitrate: 3_000_000, + maxFramerate: 60, + }); + }); + + it("applies screen share encoding from config", () => { + const opts = buildLiveKitOptions({ + screen_share: { + max_bitrate: 8_000_000, + max_framerate: 15, + }, + }); + expect(opts.publishDefaults?.screenShareEncoding).toEqual({ + maxBitrate: 8_000_000, + maxFramerate: 15, + }); + }); + + it("uses DEFAULT_CONFIG defaults when only resolution is set", () => { + const opts = buildLiveKitOptions({ + screen_share: { + max_resolution: 720, + }, + }); + // Bitrate and framerate fall back to DEFAULT_CONFIG, not the preset + expect(opts.publishDefaults?.screenShareEncoding).toEqual({ + maxBitrate: 5_000_000, + maxFramerate: 30, + }); + }); + + it("maps low resolutions to the closest preset, rounding up", () => { + const expectations: [number, VideoPreset][] = [ + [180, VideoPresets.h180], + [360, VideoPresets.h360], + [480, VideoPresets.h540], + [540, VideoPresets.h540], + [720, VideoPresets.h720], + ]; + for (const [height, preset] of expectations) { + const opts = buildLiveKitOptions({ video: { max_resolution: height } }); + expect(opts.videoCaptureDefaults?.resolution).toEqual(preset.resolution); + } + }); + + it("screen share layers fall back to max_framerate, then 30", () => { + const fromMax = buildLiveKitOptions({ + screen_share: { + max_framerate: 15, + simulcast_layers: [{ height: 540, bitrate: 1_000_000 }], + }, + }); + expect( + fromMax.publishDefaults?.screenShareSimulcastLayers?.[0], + ).toMatchObject({ encoding: { maxFramerate: 15 } }); + + const fromDefault = buildLiveKitOptions({ + screen_share: { + simulcast_layers: [{ height: 540, bitrate: 1_000_000 }], + }, + }); + expect( + fromDefault.publishDefaults?.screenShareSimulcastLayers?.[0], + ).toMatchObject({ encoding: { maxFramerate: 30 } }); + }); + + it("applies custom video simulcast layers", () => { + const opts = buildLiveKitOptions({ + video: { + simulcast_layers: [ + { height: 180, bitrate: 100_000 }, + { height: 360, bitrate: 300_000 }, + { height: 540, bitrate: 600_000 }, + ], + max_framerate: 24, + }, + }); + const layers = opts.publishDefaults?.videoSimulcastLayers; + expect(layers).toHaveLength(3); + expect(layers?.[0]).toMatchObject({ + width: 320, + height: 180, + encoding: { maxBitrate: 100_000, maxFramerate: 24 }, + }); + expect(layers?.[2]).toMatchObject({ + width: 960, + height: 540, + encoding: { maxBitrate: 600_000, maxFramerate: 24 }, + }); + }); + + it("applies custom screen share simulcast layers", () => { + const opts = buildLiveKitOptions({ + screen_share: { + simulcast_layers: [{ height: 540, bitrate: 1_000_000, framerate: 5 }], + }, + }); + const layers = opts.publishDefaults?.screenShareSimulcastLayers; + expect(layers).toHaveLength(1); + expect(layers?.[0]).toMatchObject({ + width: 960, + height: 540, + encoding: { maxBitrate: 1_000_000, maxFramerate: 5 }, + }); + }); + + it("does not include screenShareSimulcastLayers when not configured", () => { + const opts = buildLiveKitOptions(); + expect(opts.publishDefaults?.screenShareSimulcastLayers).toBeUndefined(); + }); + + it("backupCodec always uses stock VP8 720p encoding", () => { + const opts = buildLiveKitOptions({ + video_codec: "av1", + video: { max_bitrate: 10_000_000, max_framerate: 60 }, + }); + const backup = opts.publishDefaults?.backupCodec as { + codec: string; + encoding: { maxBitrate: number; maxFramerate: number }; + }; + expect(backup.codec).toBe("vp8"); + expect(backup.encoding).toEqual(VideoPresets.h720.encoding); + }); +}); diff --git a/sdk/src/session/livekitOptions.ts b/sdk/src/session/livekitOptions.ts new file mode 100644 index 000000000..f871bc17a --- /dev/null +++ b/sdk/src/session/livekitOptions.ts @@ -0,0 +1,140 @@ +/* +Copyright 2023, 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + AudioPresets, + DefaultReconnectPolicy, + type RoomOptions, + type TrackPublishDefaults, + type VideoPreset, + VideoPresets, + VideoPreset as VideoPresetClass, +} from "livekit-client"; + +import { defaultMediaQuality, type MediaQuality } from "../config"; + +/** + * Find the closest matching VideoPreset for a given height. + */ +function videoPresetForHeight(height: number): VideoPreset { + if (height <= 180) return VideoPresets.h180; + if (height <= 360) return VideoPresets.h360; + if (height <= 540) return VideoPresets.h540; + if (height <= 720) return VideoPresets.h720; + if (height <= 1080) return VideoPresets.h1080; + if (height <= 1440) return VideoPresets.h1440; + return VideoPresets.h2160; +} + +/** + * Build LiveKit publish options from config, falling back to sensible defaults. + */ +function buildPublishOptions( + mediaQuality: MediaQuality | undefined, +): TrackPublishDefaults { + const defaults = defaultMediaQuality; + const videoConf = mediaQuality?.video; + const screenConf = mediaQuality?.screen_share; + const codec = mediaQuality?.video_codec ?? defaults.video_codec; + + // Camera video encoding + const videoEncoding = { + maxBitrate: videoConf?.max_bitrate ?? defaults.video.max_bitrate, + maxFramerate: videoConf?.max_framerate ?? defaults.video.max_framerate, + }; + + // Camera simulcast layers + let videoSimulcastLayers: VideoPreset[]; + if (videoConf?.simulcast_layers) { + videoSimulcastLayers = videoConf.simulcast_layers.map( + (layer) => + new VideoPresetClass( + Math.round((layer.height * 16) / 9), + layer.height, + layer.bitrate, + videoConf?.max_framerate ?? defaults.video.max_framerate, + ), + ); + } else { + videoSimulcastLayers = [VideoPresets.h180, VideoPresets.h360]; + } + + // Screen share encoding + const screenShareEncoding = { + maxBitrate: screenConf?.max_bitrate ?? defaults.screen_share.max_bitrate, + maxFramerate: + screenConf?.max_framerate ?? defaults.screen_share.max_framerate, + }; + + // Screen share simulcast layers + let screenShareSimulcastLayers: VideoPreset[] | undefined; + if (screenConf?.simulcast_layers) { + screenShareSimulcastLayers = screenConf.simulcast_layers.map( + (layer) => + new VideoPresetClass( + Math.round((layer.height * 16) / 9), + layer.height, + layer.bitrate, + layer.framerate ?? screenConf?.max_framerate ?? 30, + ), + ); + } + + return { + audioPreset: AudioPresets.music, + dtx: true, + // disable red because the livekit server strips out red packets for clients + // that don't support it (firefox) but of course that doesn't work with e2ee. + red: false, + forceStereo: false, + simulcast: true, + videoSimulcastLayers: videoSimulcastLayers as VideoPreset[], + screenShareEncoding, + ...(screenShareSimulcastLayers && { + screenShareSimulcastLayers: screenShareSimulcastLayers as VideoPreset[], + }), + stopMicTrackOnMute: false, + videoCodec: codec, + videoEncoding, + backupCodec: { + codec: "vp8", + encoding: VideoPresets.h720.encoding, + }, + } as TrackPublishDefaults; +} + +/** + * Build LiveKit RoomOptions from the media quality a host asks for. + */ +export function buildLiveKitOptions(mediaQuality?: MediaQuality): RoomOptions { + const videoHeight = + mediaQuality?.video?.max_resolution ?? + defaultMediaQuality.video.max_resolution; + const basePreset = videoPresetForHeight(videoHeight); + + return { + // automatically manage subscribed video quality + adaptiveStream: true, + + // optimize publishing bandwidth and CPU for published tracks + dynacast: true, + + // capture settings + videoCaptureDefaults: { + resolution: basePreset.resolution, + }, + + // publish settings + publishDefaults: buildPublishOptions(mediaQuality), + + // default LiveKit options that seem to be sane + stopLocalTrackOnUnpublish: true, + reconnectPolicy: new DefaultReconnectPolicy(), + disconnectOnPageLeave: true, + webAudioMix: false, + }; +} diff --git a/sdk/src/session/openIDSFU.test.ts b/sdk/src/session/openIDSFU.test.ts new file mode 100644 index 000000000..6464e80d2 --- /dev/null +++ b/sdk/src/session/openIDSFU.test.ts @@ -0,0 +1,333 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + beforeEach, + afterEach, + describe, + expect, + it, + type MockedObject, + vitest, +} from "vitest"; +import fetchMock from "fetch-mock"; +import { MatrixError } from "matrix-js-sdk"; + +import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU"; +import { testJWTToken } from "../utils/test-fixtures"; +import { ownMemberMock } from "../utils/test"; +import { FailToGetOpenIdToken } from "../errors"; + +const sfuUrl = "https://sfu.example.org"; + +describe("getSFUConfigWithOpenID", () => { + let matrixClient: MockedObject; + beforeEach(() => { + fetchMock.catch(404); + matrixClient = { + getOpenIdToken: vitest.fn(), + getDeviceId: vitest.fn(), + }; + }); + afterEach(() => { + vitest.clearAllMocks(); + fetchMock.reset(); + }); + + it("should handle fetching a token", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: sfuUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfigWithOpenID( + matrixClient, + ownMemberMock, + "https://sfu.example.org", + "!example_room_id", + ); + expect(config).toEqual({ + jwt: testJWTToken, + url: sfuUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + void (await fetchMock.flush()); + }); + + it("should fail if the SFU errors", async () => { + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { + errcode: "M_LOOKUP_FAILED", + error: "Failed to look up user info from homeserver", + }, + }; + }); + try { + await getSFUConfigWithOpenID( + matrixClient, + ownMemberMock, + "https://sfu.example.org", + "!example_room_id", + ); + } catch (ex: unknown) { + expect(ex).toBeInstanceOf(FailToGetOpenIdToken); + expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); + const mxError = (ex as Error).cause as MatrixError; + expect(mxError.message).toEqual( + "MatrixError: [500] Failed to look up user info from homeserver", + ); + + void (await fetchMock.flush()); + return; + } + expect.fail("Expected test to throw;"); + }); + + it("should retry without delay params if the JWT service legacy endpoint returns M_BAD_JSON 400", async () => { + let callCount = 0; + + fetchMock.post( + "https://sfu.example.org/sfu/get", + (url, opts) => { + callCount++; + const body = JSON.parse(opts.body as string); + + // First call: check if it has delay parts and return 400 + if (callCount === 1) { + expect(body).toHaveProperty("delay_id", "mock_delay_id"); + return { + status: 400, + body: { errcode: "M_BAD_JSON", error: "Unsupported parameters" }, + }; + } + + // Second call: check if delay parts were stripped and return success + expect(body).not.toHaveProperty("delay_id"); + expect(body).not.toHaveProperty("delay_timeout"); + expect(body).not.toHaveProperty("delay_cs_api_url"); + + return { + status: 200, + body: { url: sfuUrl, jwt: testJWTToken }, + }; + }, + { overwriteRoutes: true }, + ); + + // Note: Assuming getSFUConfigWithOpenID eventually calls getLiveKitJWT + const config = await getSFUConfigWithOpenID( + matrixClient, + ownMemberMock, + "https://sfu.example.org", + "!example_room_id", + { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + delayTimeoutMs: 3_600_000, + }, + ); + + expect(config.jwt).toBe(testJWTToken); + expect(callCount).toBe(2); + void (await fetchMock.flush()); + }); + + it("should successfully send delay parameters to the JWT service legacy endpoint", async () => { + fetchMock.post( + "https://sfu.example.org/sfu/get", + (url, opts) => { + const body = JSON.parse(opts.body as string); + + // Verify, that the request contains the expected delay parameters + if ( + body.delay_id === "mock_delay_id" && + body.delay_timeout === 3600000 && + body.delay_cs_api_url === "https://homeserverserver.org/cs_api" + ) { + return { + status: 200, + body: { url: sfuUrl, jwt: testJWTToken }, + }; + } + return { + status: 400, + body: { error: "Missing expected delay params" }, + }; + }, + { overwriteRoutes: true }, + ); + + const config = await getSFUConfigWithOpenID( + matrixClient, + ownMemberMock, + "https://sfu.example.org", + "!example_room_id", + { + delayEndpointBaseUrl: "https://homeserverserver.org/cs_api", + delayId: "mock_delay_id", + delayTimeoutMs: 3_600_000, + }, + ); + + // Prüfe das Ergebnis + expect(config).toMatchObject({ + jwt: testJWTToken, + url: sfuUrl, + }); + + void (await fetchMock.flush()); + }); + + it("should try legacy and then new endpoint with delay delegation", async () => { + fetchMock.post("https://sfu.example.org/get_token", () => { + return { + status: 500, + body: { + errcode: "M_LOOKUP_FAILED", + error: "Failed to look up user info from homeserver", + }, + }; + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { + errcode: "M_LOOKUP_FAILED", + error: "Failed to look up user info from homeserver", + }, + }; + }); + try { + await getSFUConfigWithOpenID( + matrixClient, + ownMemberMock, + "https://sfu.example.org", + "!example_room_id", + { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + delayTimeoutMs: 3_600_000, + }, + ); + } catch (ex) { + expect(ex).toBeInstanceOf(FailToGetOpenIdToken); + expect((ex as FailToGetOpenIdToken).cause).toBeInstanceOf(MatrixError); + const mxError = (ex as Error).cause as MatrixError; + expect(mxError.message).toEqual( + "MatrixError: [500] Failed to look up user info from homeserver", + ); + void (await fetchMock.flush()); + } + const calls = fetchMock.calls(); + expect(calls.length).toBe(2); + + expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); + expect(calls[0][1]).toStrictEqual({ + // check if it uses correct delayID! + body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', + method: "POST", + headers: { + "Content-Type": "application/json", + }, + }); + + expect(calls[1][0]).toStrictEqual("https://sfu.example.org/sfu/get"); + + expect(calls[1][1]).toStrictEqual({ + body: '{"room":"!example_room_id","device_id":"DEVICE","delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', + headers: { + "Content-Type": "application/json", + }, + method: "POST", + }); + }); + + it("dont try legacy if endpoint with delay delegation is sucessful", async () => { + fetchMock.post("https://sfu.example.org/get_token", () => { + return { + status: 200, + body: { url: sfuUrl, jwt: testJWTToken }, + }; + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 500, + body: { error: "Test failure" }, + }; + }); + try { + await getSFUConfigWithOpenID( + matrixClient, + ownMemberMock, + "https://sfu.example.org", + "!example_room_id", + { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + delayTimeoutMs: 3_600_000, + }, + ); + } catch (ex) { + expect(ex).toBeInstanceOf(FailToGetOpenIdToken); + expect((ex as FailToGetOpenIdToken).cause).toEqual( + new Error("SFU Config fetch failed with status code 500"), + ); + void (await fetchMock.flush()); + } + const calls = fetchMock.calls(); + expect(calls.length).toBe(1); + + expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token"); + expect(calls[0][1]).toStrictEqual({ + // check if it uses correct delayID! + body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":3600000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}', + method: "POST", + headers: { + "Content-Type": "application/json", + }, + }); + }); + + it("should retry fetching the openid token", async () => { + let count = 0; + matrixClient.getOpenIdToken.mockImplementation(async () => { + count++; + if (count < 2) { + throw Error("Test failure"); + } + return Promise.resolve({ + token_type: "Bearer", + access_token: "foobar", + matrix_server_name: "example.org", + expires_in: 30, + }); + }); + fetchMock.post("https://sfu.example.org/sfu/get", () => { + return { + status: 200, + body: { url: sfuUrl, jwt: testJWTToken }, + }; + }); + const config = await getSFUConfigWithOpenID( + matrixClient, + ownMemberMock, + "https://sfu.example.org", + "!example_room_id", + ); + expect(config).toEqual({ + jwt: testJWTToken, + url: sfuUrl, + livekitIdentity: "@me:example.org:ABCDEF", + livekitAlias: "!example_room_id", + }); + void (await fetchMock.flush()); + }); +}); diff --git a/sdk/src/session/openIDSFU.ts b/sdk/src/session/openIDSFU.ts new file mode 100644 index 000000000..190bf4fe4 --- /dev/null +++ b/sdk/src/session/openIDSFU.ts @@ -0,0 +1,315 @@ +/* +Copyright 2023, 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type IOpenIDToken, + type MatrixClient, + parseErrorResponse, +} from "matrix-js-sdk"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { FailToGetOpenIdToken, NoMatrix2AuthorizationService } from "../errors"; +import { doNetworkOperationWithRetry } from "../utils/network"; +import { MatrixRTCMode } from "../config"; + +/** + * Configuration and access tokens provided by the SFU on successful authentication. + */ +export interface SFUConfig { + url: string; + jwt: string; + livekitAlias: string; + // NOTE: Currently unused. + livekitIdentity: string; +} + +/** + * Decoded details from the JWT. + */ +interface SFUJWTPayload { + /** + * Expiration time for the JWT. + * Note: This value is in seconds since Unix epoch. + */ + exp: number; + /** + * Name of the instance which authored the JWT + */ + iss: string; + /** + * Time at which the JWT can start to be used. + * Note: This value is in seconds since Unix epoch. + */ + nbf: number; + /** + * Subject. The Livekit alias in this context. + */ + sub: string; + /** + * The set of permissions for the user. + */ + video: { + canPublish: boolean; + canSubscribe: boolean; + room: string; + roomJoin: boolean; + }; +} + +// The bits we need from MatrixClient +export type OpenIDClientParts = Pick< + MatrixClient, + "getOpenIdToken" | "getDeviceId" +>; + +/** + * Gets a bearer token from the homeserver and then use it to authenticate + * to the matrix RTC backend in order to get acces to the SFU. + * It has built-in retry for calls to the homeserver with a backoff policy. + * @param client The Matrix client + * @param membership Our own membership identity parts used to send to jwt service. + * @param serviceUrl The URL of the livekit SFU service + * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. + * @param opts Additional options to modify which endpoint with which data will be used to acquire the jwt token. + * @param opts.matrixRTCMode Determines which version of the JWT endpoint to use, which affects whether the + * RTC backend identity is based on string concatenation (legacy) or a hash (Matrix 2.0). + * This function by default uses whatever is possible with the current jwt service installed next to the SFU. + * For remote connections this does not matter, since we will not publish there we can rely on the newest option. + * @param opts.delayEndpointBaseUrl The URL of the matrix homeserver. + * @param opts.delayId The delay id used for the jwt service to manage. + * @param logger optional logger. + * @returns Object containing the token information + * @throws FailToGetOpenIdToken + */ +export async function getSFUConfigWithOpenID( + client: OpenIDClientParts, + membership: CallMembershipIdentityParts, + serviceUrl: string, + roomId: string, + opts?: { + matrixRTCMode?: MatrixRTCMode; + delayEndpointBaseUrl?: string; + delayId?: string; + /** How long the SFU waits before sending the delegated leave. Needed with `delayId`. */ + delayTimeoutMs?: number; + }, + logger?: Logger, +): Promise { + let openIdToken: IOpenIDToken; + try { + openIdToken = await doNetworkOperationWithRetry(async () => + client.getOpenIdToken(), + ); + } catch (error) { + throw new FailToGetOpenIdToken( + error instanceof Error ? error : new Error("Unknown error"), + ); + } + logger?.debug("Got openID token", openIdToken); + let sfuConfig: { url: string; jwt: string } | undefined; + + const tryBothJwtEndpoints = opts?.matrixRTCMode === undefined; // This is for SFUs where we do not publish. + + const forceMatrix2Jwt = opts?.matrixRTCMode === MatrixRTCMode.Matrix_2_0; + + // We want to start using the new endpoint (with optional delay delegation) + // if we can use both or if we are forced to use the new one. + if (tryBothJwtEndpoints || forceMatrix2Jwt) { + try { + logger?.info( + `Trying to get JWT via default endpoint for focus ${serviceUrl}...`, + ); + const sfuConfig = await getLiveKitJWT( + membership, + serviceUrl, + roomId, + openIdToken, + opts?.delayEndpointBaseUrl, + opts?.delayId, + opts?.delayTimeoutMs, + ); + + return extractFullConfigFromToken(sfuConfig); + } catch (e) { + logger?.debug(`Failed fetching jwt with matrix 2.0 endpoint:`, e); + // Make this throw a hard error in case we force the matrix2.0 endpoint. + if (forceMatrix2Jwt) { + throw new NoMatrix2AuthorizationService(e as Error); + } + } + } + + // DEPRECATED + // here we either have a sfuConfig or we already exited because of `if (forceMatrix2) throw ...` + // The only case we can get into this condition is, if `forceMatrix2` is `false` + try { + logger?.info( + `Trying to get JWT with legacy endpoint for focus ${serviceUrl}...`, + ); + sfuConfig = await getLiveKitJWTLegacy( + membership.deviceId, + serviceUrl, + roomId, + openIdToken, + opts?.delayEndpointBaseUrl, + opts?.delayId, + opts?.delayTimeoutMs, + ); + logger?.info(`Got JWT from call's active focus URL.`); + return extractFullConfigFromToken(sfuConfig); + } catch (ex) { + throw new FailToGetOpenIdToken( + ex instanceof Error ? ex : new Error(`Unknown error ${ex}`), + ); + } +} + +function extractFullConfigFromToken(sfuConfig: { + url: string; + jwt: string; +}): SFUConfig { + const [, payloadStr] = sfuConfig.jwt.split("."); + const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload; + return { + jwt: sfuConfig.jwt, + url: sfuConfig.url, + livekitAlias: payload.video.room, + // NOTE: Currently unused. + // Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager. + // The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys) + livekitIdentity: payload.sub, + }; +} + +async function getLiveKitJWTLegacy( + deviceId: string, + livekitServiceURL: string, + matrixRoomId: string, + openIDToken: IOpenIDToken, + delayEndpointBaseUrl?: string, + delayId?: string, + delayTimeoutMs?: number, +): Promise<{ url: string; jwt: string }> { + interface IDelayParams { + delay_id?: string; + delay_timeout?: number; + delay_cs_api_url?: string; + } + let bodyDalayParts: IDelayParams = {}; + // Also check for empty string + if (delayId && delayEndpointBaseUrl) { + bodyDalayParts = { + delay_id: delayId, + delay_timeout: delayTimeoutMs, + delay_cs_api_url: delayEndpointBaseUrl, + }; + } + + const makeRequest = async (delayParts: IDelayParams): Promise => { + return await fetch(livekitServiceURL + "/sfu/get", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + // The legacy JWT endpoint uses only the matrix room id to calculate the livekit room alias. + // However, the livekit room alias is provided as part of the JWT payload. + room: matrixRoomId, + openid_token: openIDToken, + device_id: deviceId, + ...delayParts, + }), + }); + }; + + const res = await doNetworkOperationWithRetry(async () => { + let response = await makeRequest(bodyDalayParts); + + // Old service compatibility check + const oldServiceDoesNotSupportDelayParts = + response.status === 400 && Object.keys(bodyDalayParts).length > 0; + // If http status 400 with M_BAD_JSON and we sent delay parts, retry without them + if (oldServiceDoesNotSupportDelayParts) { + try { + const errorBody = await response.json(); + if (errorBody.errcode === "M_BAD_JSON") { + response = await makeRequest({}); + } + } catch { + // If we can't parse the error, treat as real error + } + } + + return response; + }); + + if (!res.ok) { + throw parseErrorResponse(res, await res.text()); + } + return await res.json(); +} + +class NotSupportedError extends Error { + public constructor(message: string) { + super(message); + this.name = "NotSupported"; + } +} + +export async function getLiveKitJWT( + membership: CallMembershipIdentityParts, + livekitServiceURL: string, + matrixRoomId: string, + openIDToken: IOpenIDToken, + delayEndpointBaseUrl?: string, + delayId?: string, + delayTimeoutMs?: number, +): Promise<{ url: string; jwt: string }> { + const { userId, deviceId, memberId } = membership; + + const body = { + room_id: matrixRoomId, + slot_id: "m.call#ROOM", + openid_token: openIDToken, + member: { + id: memberId, + claimed_user_id: userId, + claimed_device_id: deviceId, + }, + }; + + let bodyDalayParts = {}; + // Also check for empty string + if (delayId && delayEndpointBaseUrl) { + bodyDalayParts = { + delay_id: delayId, + delay_timeout: delayTimeoutMs, + delay_cs_api_url: delayEndpointBaseUrl, + }; + } + + const res = await doNetworkOperationWithRetry(async () => { + return await fetch(livekitServiceURL + "/get_token", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ ...body, ...bodyDalayParts }), + }); + }); + + if (!res.ok) { + const msg = "SFU Config fetch failed with status code " + res.status; + if (res.status === 404) { + throw new NotSupportedError(msg); + } else { + throw parseErrorResponse(res, await res.text()); + } + } + return await res.json(); +} diff --git a/sdk/src/session/status.test.ts b/sdk/src/session/status.test.ts index 7152ad8a8..8e385a2a8 100644 --- a/sdk/src/session/status.test.ts +++ b/sdk/src/session/status.test.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { describe, expect, it } from "vitest"; import { Status } from "matrix-js-sdk/lib/matrixrtc"; -import { ConnectionState } from "../../../src/state/CallViewModel/remoteMembers/Connection"; +import { ConnectionState } from "./Connection"; import { MatrixRTCError } from "../api"; import { type LocalMemberState, diff --git a/sdk/src/utils/displayname.test.ts b/sdk/src/utils/displayname.test.ts new file mode 100644 index 000000000..4d6653d8f --- /dev/null +++ b/sdk/src/utils/displayname.test.ts @@ -0,0 +1,133 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, test } from "vitest"; + +import { calculateDisplayName, shouldDisambiguate } from "./displayname"; +import { + alice, + aliceDoppelganger, + aliceDoppelgangerRtcMember, + aliceRtcMember, + bob, + bobRtcMember, + bobZeroWidthSpace, + bobZeroWidthSpaceRtcMember, + daveRTL, +} from "./test-fixtures"; +import { mockMatrixRoom } from "./test"; +import { roomToMembersMap } from "../session/MatrixMemberMetadata"; + +describe("shouldDisambiguate", () => { + test("should not disambiguate a solo member", () => { + const room = mockMatrixRoom({ + getMembersWithMembership: () => [], + }); + expect(shouldDisambiguate(alice, [], roomToMembersMap(room))).toEqual( + false, + ); + }); + test("should not disambiguate a member with an empty displayname", () => { + const room = mockMatrixRoom({ + getMembersWithMembership: () => [alice, aliceDoppelganger], + }); + expect( + shouldDisambiguate( + { rawDisplayName: "", userId: alice.userId }, + [aliceRtcMember, aliceDoppelgangerRtcMember], + roomToMembersMap(room), + ), + ).toEqual(false); + }); + test("should disambiguate a member with RTL characters", () => { + const room = mockMatrixRoom({ getMembersWithMembership: () => [] }); + expect(shouldDisambiguate(daveRTL, [], roomToMembersMap(room))).toEqual( + true, + ); + }); + test("should disambiguate a member with a matching displayname", () => { + const room = mockMatrixRoom({ + getMembersWithMembership: () => [alice, aliceDoppelganger], + }); + expect( + shouldDisambiguate( + alice, + [aliceRtcMember, aliceDoppelgangerRtcMember], + roomToMembersMap(room), + ), + ).toEqual(true); + expect( + shouldDisambiguate( + aliceDoppelganger, + [aliceRtcMember, aliceDoppelgangerRtcMember], + roomToMembersMap(room), + ), + ).toEqual(true); + }); + test("should disambiguate a member with a matching displayname with hidden spaces", () => { + const room = mockMatrixRoom({ + getMembersWithMembership: () => [bob, bobZeroWidthSpace], + }); + expect( + shouldDisambiguate( + bob, + [bobRtcMember, bobZeroWidthSpaceRtcMember], + roomToMembersMap(room), + ), + ).toEqual(true); + expect( + shouldDisambiguate( + bobZeroWidthSpace, + [bobRtcMember, bobZeroWidthSpaceRtcMember], + roomToMembersMap(room), + ), + ).toEqual(true); + }); + test.for(["Alice @foo:bar", "@foo:b", "A@foo:lice", "A @f oo: ba r"])( + "should disambiguate a member with a displayname containing a mxid-like string '%s'", + (rawDisplayName) => { + const room = mockMatrixRoom({ + getMembersWithMembership: () => [alice, aliceDoppelganger], + }); + expect( + shouldDisambiguate( + { rawDisplayName, userId: alice.userId }, + [], + roomToMembersMap(room), + ), + ).toEqual(true); + }, + ); +}); + +describe("calculateDisplayName", () => { + test.for<[{ rawDisplayName?: string; userId: string }, boolean, string]>([ + [alice, false, alice.rawDisplayName], + [alice, true, `${alice.rawDisplayName} (${alice.userId})`], + // Empty strings and zero width strings that are effectively empty are resolved as userIds + [{ rawDisplayName: "", userId: alice.userId }, false, alice.userId], + [ + { rawDisplayName: "\u200b\u200b\u200b", userId: alice.userId }, + false, + alice.userId, + ], + [ + { rawDisplayName: alice.userId, userId: alice.userId }, + false, + alice.userId, + ], + // Zero width strings are kept intact + [bobZeroWidthSpace, false, bobZeroWidthSpace.rawDisplayName], + // Directional characters are stripped. + [daveRTL, false, daveRTL.rawDisplayName.slice(1)], + [daveRTL, true, `${daveRTL.rawDisplayName.slice(1)} (${daveRTL.userId})`], + // Ensure we do NOT unhomoglyth + [{ ...alice, rawDisplayName: "alice m" }, false, "alice m"], + ])("correctly calculates displayname", ([member, disambiguate, result]) => + expect(calculateDisplayName(member, disambiguate)).toEqual(result), + ); +}); diff --git a/sdk/src/utils/displayname.ts b/sdk/src/utils/displayname.ts new file mode 100644 index 000000000..bc49b29ec --- /dev/null +++ b/sdk/src/utils/displayname.ts @@ -0,0 +1,114 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + removeDirectionOverrideChars, + removeHiddenChars as removeHiddenCharsUncached, +} from "matrix-js-sdk/lib/utils"; + +import type { RoomMember } from "matrix-js-sdk"; +import type { CallMembership } from "matrix-js-sdk/lib/matrixrtc"; + +// Calling removeHiddenChars() can be slow on Safari, so we cache the results. +// To illustrate a simple benchmark: +// Chrome: 10,000 calls took 2.599ms +// Safari: 10,000 calls took 242ms +// See: https://github.com/element-hq/element-call/issues/3065 + +const removeHiddenCharsCache = new Map(); + +/** + * Calls removeHiddenCharsUncached and caches the result + */ +function removeHiddenChars(str: string): string { + if (removeHiddenCharsCache.has(str)) { + return removeHiddenCharsCache.get(str)!; + } + const result = removeHiddenCharsUncached(str); + // this is naive but should be good enough for our purposes + if (removeHiddenCharsCache.size > 500) { + removeHiddenCharsCache.clear(); + } + removeHiddenCharsCache.set(str, result); + return result; +} + +// Borrowed from https://github.com/matrix-org/matrix-js-sdk/blob/f10deb5ef2e8f061ff005af0476034382ea128ca/src/models/room-member.ts#L409 +export function shouldDisambiguate( + member: { rawDisplayName?: string; userId: string }, + memberships: Pick[], + roomMembers: Map>, +): boolean { + const { rawDisplayName: displayName, userId } = member; + if (!displayName || displayName === userId) return false; + + // First check if the displayname is something we consider truthy + // after stripping it of zero width characters and padding spaces + const strippedDisplayName = removeHiddenChars(displayName); + if (!strippedDisplayName) return false; + + // Next check if the name contains something that look like a mxid + // If it does, it may be someone trying to impersonate someone else + // Show full mxid in this case + if (/@.+:.+/.test(displayName)) return true; + + // Also show mxid if the display name contains any LTR/RTL characters as these + // make it very difficult for us to find similar *looking* display names + // E.g "Mark" could be cloned by writing "kraM" but in RTL. + if (/[\u200E\u200F\u202A-\u202F]/.test(displayName)) return true; + + // Also show mxid if there are other people with the same or similar + // displayname, after hidden character removal. + return ( + memberships + .map((m) => m.userId && roomMembers.get(m.userId)) + // NOTE: We *should* have a room member for everyone. + .filter((m) => !!m) + .filter((m) => m.userId !== userId) + .some( + (m) => + removeHiddenChars(calculateDisplayName(m, false)) === + strippedDisplayName, + ) + ); +} + +/** + * Calculates a display name for a member, optionally disambiguating it. + * @param member - The member to calculate the display name for. + * @param member.rawDisplayName - The raw display name of the member + * @param member.userId - The user ID of the member + * @param disambiguate - Whether to disambiguate the display name. + */ +export function calculateDisplayName( + member: { rawDisplayName?: string; userId: string }, + disambiguate: boolean, +): string { + const { rawDisplayName: displayName, userId } = member; + if (!displayName || displayName === userId) return userId; + + const resultDisplayname = removeDirectionOverrideChars(displayName); + + if (disambiguate) return resultDisplayname + " (" + userId + ")"; + + // First check if the displayname is something we consider truthy + // after stripping it of zero width characters and padding spaces + if (!removeHiddenChars(displayName)) return userId; + + // We always strip the direction override characters (LRO and RLO). + // These override the text direction for all subsequent characters + // in the paragraph so if display names contained these, they'd + // need to be wrapped in something to prevent this from leaking out + // (which we can do in HTML but not text) or we'd need to add + // control characters to the string to reset any overrides (eg. + // adding PDF characters at the end). As far as we can see, + // there should be no reason these would be necessary - rtl display + // names should flip into the correct direction automatically based on + // the characters, and you can still embed rtl in ltr or vice versa + // with the embed chars or marker chars. + return resultDisplayname; +} diff --git a/sdk/src/utils/errors.test.ts b/sdk/src/utils/errors.test.ts deleted file mode 100644 index a96fa352b..000000000 --- a/sdk/src/utils/errors.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { describe, expect, it } from "vitest"; - -import { MatrixRTCError } from "../api"; -import { toMatrixRTCError } from "./errors"; - -describe("toMatrixRTCError", () => { - it("keeps an MatrixRTCError as it is", () => { - const error = new MatrixRTCError("gone"); - expect(toMatrixRTCError(error)).toBe(error); - }); - - it("wraps another error as the cause", () => { - const cause = new Error("boom"); - const error = toMatrixRTCError(cause); - expect(error).toBeInstanceOf(MatrixRTCError); - expect(error.message).toBe("boom"); - expect(error.cause).toBe(cause); - }); - - it("falls back to the code of an error without a message", () => { - const cause = Object.assign(new Error(""), { code: "SFU_ERROR" }); - expect(toMatrixRTCError(cause).message).toBe("SFU_ERROR"); - }); - - it("stringifies anything else", () => { - expect(toMatrixRTCError("nope").message).toBe("nope"); - }); -}); diff --git a/sdk/src/utils/errors.ts b/sdk/src/utils/errors.ts deleted file mode 100644 index d461d2f07..000000000 --- a/sdk/src/utils/errors.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { MatrixRTCError } from "../api"; - -/** Wraps whatever a backend threw into the SDK's error type, keeping it as the cause. */ -export function toMatrixRTCError(error: unknown): MatrixRTCError { - if (error instanceof MatrixRTCError) return error; - if (error instanceof Error) - return new MatrixRTCError(messageOf(error), { cause: error }); - return new MatrixRTCError(String(error)); -} - -/** - * Element Call's errors carry a translated title as their message, which is - * empty where no translations are loaded; their code says what went wrong - * regardless. - */ -function messageOf(error: Error): string { - if (error.message !== "") return error.message; - if ("code" in error && typeof error.code === "string") return error.code; - return error.name; -} diff --git a/sdk/src/utils/mapScoped.test.ts b/sdk/src/utils/mapScoped.test.ts index 5f3231d48..19148abb6 100644 --- a/sdk/src/utils/mapScoped.test.ts +++ b/sdk/src/utils/mapScoped.test.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { describe, expect, it, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; -import { ObservableScope } from "../../../src/state/ObservableScope"; +import { ObservableScope } from "../reactive/ObservableScope"; import { mapScoped } from "./mapScoped"; describe("mapScoped", () => { diff --git a/sdk/src/utils/mapScoped.ts b/sdk/src/utils/mapScoped.ts index 15af0f714..c9663313f 100644 --- a/sdk/src/utils/mapScoped.ts +++ b/sdk/src/utils/mapScoped.ts @@ -7,8 +7,8 @@ Please see LICENSE in the repository root for full details. import { Observable, of, switchMap } from "rxjs"; -import { type Behavior } from "../../../src/state/Behavior"; -import { ObservableScope } from "../../../src/state/ObservableScope"; +import { type Behavior } from "../reactive/Behavior"; +import { ObservableScope } from "../reactive/ObservableScope"; /** * Builds one item per present value of a behavior, each in a scope of its own diff --git a/sdk/src/utils/network.ts b/sdk/src/utils/network.ts new file mode 100644 index 000000000..ea8def05d --- /dev/null +++ b/sdk/src/utils/network.ts @@ -0,0 +1,31 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { calculateRetryBackoff } from "matrix-js-sdk"; +import { sleep } from "matrix-js-sdk/lib/utils"; + +/** + * Performs a network operation with retries on connection errors, honouring + * the homeserver's rate limits. Rethrows once the error is not retryable or + * the retries are used up. + */ +export async function doNetworkOperationWithRetry( + operation: () => Promise, +): Promise { + let currentRetryCount = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + try { + return await operation(); + } catch (e) { + currentRetryCount++; + const backoff = calculateRetryBackoff(e, currentRetryCount, true); + if (backoff < 0) throw e; + await sleep(backoff); + } + } +} diff --git a/sdk/src/utils/test-fixtures.ts b/sdk/src/utils/test-fixtures.ts new file mode 100644 index 000000000..62e70f5bc --- /dev/null +++ b/sdk/src/utils/test-fixtures.ts @@ -0,0 +1,65 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { mockRtcMembership, mockMatrixRoomMember } from "./test"; + +export const localRtcMember = mockRtcMembership("@local:example.org", "1111"); + +export const aliceDeviceId = "AAAA"; +export const aliceUserId = "@alice:example.org"; +export const aliceRtcMember = mockRtcMembership(aliceUserId, aliceDeviceId); +export const alice = mockMatrixRoomMember(aliceRtcMember, { + rawDisplayName: "Alice", +}); + +export const aliceDoppelgangerRtcMember = mockRtcMembership( + "@alice2:example.org", + "AAAA", +); +export const aliceDoppelganger = mockMatrixRoomMember( + aliceDoppelgangerRtcMember, + { + rawDisplayName: "Alice", + }, +); + +export const bobDeviceId = "BBBB"; +export const bobUserId = "@bob:example.org"; +export const bobRtcMember = mockRtcMembership(bobUserId, bobDeviceId); +export const bob = mockMatrixRoomMember(bobRtcMember, { + rawDisplayName: "Bob", +}); + +export const bobZeroWidthSpaceRtcMember = mockRtcMembership( + "@bob2:example.org", + "BBBB", +); +export const bobZeroWidthSpace = mockMatrixRoomMember( + bobZeroWidthSpaceRtcMember, + { + rawDisplayName: "Bo\u200bb", + }, +); + +export const daveRTLRtcMember = mockRtcMembership("@dave2:example.org", "DDDD"); +export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { + rawDisplayName: "\u202eevaD", +}); + +export const testJWTToken = [ + {}, // header + { + // payload + sub: "@me:example.org:ABCDEF", + video: { + room: "!example_room_id", + }, + }, + {}, // signature +] + .map((d) => global.btoa(JSON.stringify(d))) + .join("."); diff --git a/sdk/src/utils/test.ts b/sdk/src/utils/test.ts new file mode 100644 index 000000000..1d913c139 --- /dev/null +++ b/sdk/src/utils/test.ts @@ -0,0 +1,342 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** Test helpers shared by the SDK's unit tests. Not part of the package. */ + +import { map, type SchedulerLike } from "rxjs"; +import { type RunHelpers, TestScheduler } from "rxjs/testing"; +import { expect, type MockedObject, onTestFinished, vi, vitest } from "vitest"; +import { + EventType, + MatrixEvent, + type Room, + type Room as MatrixRoom, + type RoomMember, +} from "matrix-js-sdk"; +import { + CallMembership, + type LivekitFocusSelection, + type LivekitTransport, + type MatrixRTCSession, + MatrixRTCSessionEvent, + type MatrixRTCSessionEventHandlerMap, + MembershipManagerEvent, + type SessionMembershipData, + Status, + type Transport, +} from "matrix-js-sdk/lib/matrixrtc"; +import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; +import { + type KeyTransportEvents, + type KeyTransportEventsHandlerMap, +} from "matrix-js-sdk/lib/matrixrtc/IKeyTransport"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { TypedEventEmitter } from "matrix-js-sdk/lib/models/typed-event-emitter"; +import { + type RemoteParticipant, + type RemoteTrackPublication, +} from "livekit-client"; +import { randomUUID } from "crypto"; +import EventEmitter from "events"; + +import { type Behavior } from "../reactive/Behavior"; +import { ObservableScope } from "../reactive/ObservableScope"; + +export async function flushPromises(): Promise { + await new Promise((resolve) => window.setTimeout(resolve)); +} + +export interface OurRunHelpers extends RunHelpers { + /** + * Schedules a sequence of actions to happen, as described by a marble + * diagram. + */ + schedule: (marbles: string, actions: Record void>) => void; + behavior: ( + marbles: string, + values?: { [marble: string]: T }, + error?: unknown, + ) => Behavior; + scope: ObservableScope; +} + +interface TestRunnerGlobal { + rxjsTestScheduler?: SchedulerLike; +} + +/** + * Create a new ObservableScope which ends when the current test ends. + */ +export function testScope(): ObservableScope { + const scope = new ObservableScope(); + onTestFinished(() => scope.end()); + return scope; +} + +/** + * Run Observables with a scheduler that virtualizes time, for testing purposes. + */ +export function withTestScheduler( + continuation: (helpers: OurRunHelpers) => void, +): void { + const scheduler = new TestScheduler((actual, expected) => { + expect(actual).toStrictEqual(expected); + }); + const scope = new ObservableScope(); + // we set the test scheduler as a global so that you can watch it in a debugger + // and get the frame number. e.g. `rxjsTestScheduler?.now()` + (global as unknown as TestRunnerGlobal).rxjsTestScheduler = scheduler; + scheduler.run((helpers) => + continuation({ + ...helpers, + scope, + schedule(marbles, actions) { + const actionsObservable$ = helpers + .cold(marbles) + .pipe(map((value) => actions[value]())); + const results = Object.fromEntries( + Object.keys(actions).map((value) => [value, undefined] as const), + ); + // Run the actions and verify that none of them error + helpers.expectObservable(actionsObservable$).toBe(marbles, results); + }, + behavior( + marbles: string, + values?: { [marble: string]: T }, + error?: unknown, + ) { + // Generate a hot Observable with helpers.hot and use it as a Behavior. + // To do this, we need to ensure that the initial value emits + // synchronously upon subscription. The issue is that helpers.hot emits + // frame 0 of the marble diagram *asynchronously*, only once we return + // from the continuation, so we need to splice out the initial marble + // and turn it into a proper initial value. + const initialMarbleIndex = marbles.search(/[^ ]/); + if (initialMarbleIndex === -1) + throw new Error("Behavior must have an initial value"); + const initialMarble = marbles[initialMarbleIndex]; + const initialValue = + values === undefined ? (initialMarble as T) : values[initialMarble]; + // The remainder of the marble diagram should start on frame 1 + return scope.behavior( + helpers.hot( + `-${marbles.slice(initialMarbleIndex + 1)}`, + values, + error, + ), + initialValue, + ); + }, + }), + ); + scope.end(); +} + +interface EmitterMock { + on: (...args: unknown[]) => T; + off: (...args: unknown[]) => T; + addListener: (...args: unknown[]) => T; + removeListener: (...args: unknown[]) => T; + emit: (event: string | symbol, ...args: unknown[]) => boolean; +} + +export function mockEmitter(): EmitterMock { + const ee = new EventEmitter(); + return { + on: ee.on.bind(ee) as unknown as (...args: unknown[]) => T, + off: ee.off.bind(ee) as unknown as (...args: unknown[]) => T, + addListener: ee.addListener.bind(ee) as unknown as ( + ...args: unknown[] + ) => T, + removeListener: ee.removeListener.bind(ee) as unknown as ( + ...args: unknown[] + ) => T, + emit: ee.emit.bind(ee), + }; +} + +export const exampleTransport: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", +}; + +export function mockRtcMembership( + user: string | RoomMember, + deviceId: string, + customOverwrites?: { + rtcBackendIdentity?: string; + callId?: string; + fociPreferred?: Transport[]; + focusActive?: LivekitFocusSelection; + membership?: Partial; + }, +): CallMembership { + // setup defaults based on overwrites and fallback values. + const { rtcBackendIdentity, callId, fociPreferred, focusActive, membership } = + { + fociPreferred: [exampleTransport], + focusActive: { + type: "livekit" as const, + focus_selection: "multi_sfu" as const, + }, + callId: "", + membership: {}, + ...customOverwrites, + }; + + const data: SessionMembershipData = { + application: "m.call", + call_id: callId, + device_id: deviceId, + foci_preferred: fociPreferred, + focus_active: focusActive, + ...membership, + }; + const userId = typeof user === "string" ? user : user.userId; + const event = new MatrixEvent({ + sender: userId, + event_id: `$-ev-${randomUUID()}:example.org`, + type: EventType.GroupCallMemberPrefix, + content: data, + }); + + const membershipData = CallMembership.membershipDataFromMatrixEvent(event); + const cms = new CallMembership( + event, + membershipData, + rtcBackendIdentity ?? `${userId}:${deviceId}`, + ); + vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); + + return cms; +} + +export const ownMemberMock: CallMembershipIdentityParts = { + userId: "@alice:example.org", + deviceId: "DEVICE", + memberId: "@alice:example.org:DEVICE", +}; +// Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are +// rather simple, but if one util to mock a member is good enough for us, maybe +// it's useful for matrix-js-sdk consumers in general. +export function mockMatrixRoomMember( + rtcMembership: CallMembership, + member: Partial = {}, +): RoomMember { + return { + ...mockEmitter(), + userId: rtcMembership.userId, + getMxcAvatarUrl(): string | undefined { + return undefined; + }, + rawDisplayName: rtcMembership.userId, + ...member, + } as RoomMember; +} + +export function mockMatrixRoom(room: Partial): MatrixRoom { + return { ...mockEmitter(), ...room } as Partial as MatrixRoom; +} + +export function mockRemoteParticipant( + participant: Partial, +): RemoteParticipant { + return { + isLocal: false, + setVolume() {}, + getTrackPublication: () => + ({}) as Partial as RemoteTrackPublication, + // this will only get used for `getTrackPublications().length` + getTrackPublications: () => [0], + ...mockEmitter(), + ...participant, + } as RemoteParticipant; +} + +export class MockRTCSession extends TypedEventEmitter< + MatrixRTCSessionEvent | MembershipManagerEvent | KeyTransportEvents, + KeyTransportEventsHandlerMap & + MatrixRTCSessionEventHandlerMap & + MembershipManagerEventHandlerMap +> { + public asMockedSession(): MockedObject { + const session = this as unknown as MockedObject; + + session.reemitEncryptionKeys = vi + .fn<() => void>() + .mockReturnValue(undefined); + + return session; + } + + public readonly statistics = { + counters: {}, + }; + + public leaveRoomSession: ReturnType = vitest + .fn() + .mockResolvedValue(undefined); + + public constructor( + public readonly room: Room, + public memberships: CallMembership[] = [], + ) { + super(); + } + + public joined = true; + public isJoined(): boolean { + return this.joined; + } + + public isKeyRotationSuppressed = false; + + public withMemberships( + rtcMembers$: Behavior[]>, + ): MockRTCSession { + rtcMembers$.subscribe((m) => { + const old = this.memberships; + this.memberships = m as CallMembership[]; + this.emit( + MatrixRTCSessionEvent.MembershipsChanged, + old, + this.memberships, + ); + }); + + return this; + } + + public updateCallIntent: ReturnType = vitest + .fn() + .mockImplementation(async () => Promise.resolve()); + + private _membershipStatus = Status.Connected; + public get membershipStatus(): Status { + return this._membershipStatus; + } + public set membershipStatus(value: Status) { + const prev = this._membershipStatus; + this._membershipStatus = value; + if (value !== prev) + this.emit(MembershipManagerEvent.StatusChanged, prev, value); + } + + private _probablyLeft = false; + public get probablyLeft(): boolean { + return this._probablyLeft; + } + public set probablyLeft(value: boolean) { + const prev = this._probablyLeft; + this._probablyLeft = value; + if (value !== prev) this.emit(MembershipManagerEvent.ProbablyLeft, value); + } + + public async joinRTCSession(): Promise { + return Promise.resolve(); + } +}