diff --git a/eslint/SdkImportBoundary.js b/eslint/SdkImportBoundary.js index 7dd8a7790..a86ab6196 100644 --- a/eslint/SdkImportBoundary.js +++ b/eslint/SdkImportBoundary.js @@ -20,11 +20,21 @@ const BANNED_PACKAGES = [ "@vector-im/", ]; +/** + * Packages only a media backend may import. The session speaks MatrixRTC and + * sees media through `sdk/src/backend/api.ts`; a backend under + * `sdk/src/backend//` is the one place its protocol's library appears. + */ +const BACKEND_PACKAGES = ["livekit-client", "@livekit/"]; + /** * Keeps the SDK free of Element Call: nothing under `sdk/` may import a file * outside `sdk/` by relative path (that is `src/`, `component/`, * `playwright/`) or one of the packages above. The development harness under - * `sdk/dev/` is a host, not part of the library, and is exempt. + * `sdk/dev/` is a host, not part of the library, and is exempt. Keeps the + * session free of any one media backend, too: outside `sdk/src/backend/`, + * a backend's packages may only be imported for their types, and only in + * the public API, the errors and the test helpers that have to name them. */ const rule = ESLintUtils.RuleCreator( () => "https://github.com/element-hq/element-call", @@ -41,6 +51,8 @@ const rule = ESLintUtils.RuleCreator( "The SDK must not import '{{specifier}}': it resolves outside sdk/. Copy or move what it needs into sdk/src.", bannedPackage: "The SDK must not import '{{specifier}}': a host supplies that, the SDK has no UI.", + backendPackage: + "Only a media backend under sdk/src/backend/ may import '{{specifier}}'; the session sees media through backend/api.ts.", }, schema: [], }, @@ -50,9 +62,28 @@ const rule = ESLintUtils.RuleCreator( const sdkRoot = resolve(context.cwd, "sdk") + sep; if (!filename.startsWith(sdkRoot)) return {}; if (filename.startsWith(resolve(sdkRoot, "dev") + sep)) return {}; + const inBackend = filename.startsWith( + resolve(sdkRoot, "src", "backend") + sep, + ); - const check = (node, specifier) => { + const matchesPackage = (specifier, pkg) => + specifier === pkg || + specifier.startsWith(pkg.endsWith("/") ? pkg : `${pkg}/`); + + const check = (node, specifier, typeOnly = false) => { if (typeof specifier !== "string") return; + if ( + !inBackend && + !typeOnly && + BACKEND_PACKAGES.some((pkg) => matchesPackage(specifier, pkg)) + ) { + context.report({ + node, + messageId: "backendPackage", + data: { specifier }, + }); + return; + } if (specifier.startsWith(".")) { const target = resolve(dirname(filename), specifier); if (!target.startsWith(sdkRoot)) @@ -62,11 +93,7 @@ const rule = ESLintUtils.RuleCreator( data: { specifier }, }); } else if ( - BANNED_PACKAGES.some( - (pkg) => - specifier === pkg || - specifier.startsWith(pkg.endsWith("/") ? pkg : `${pkg}/`), - ) + BANNED_PACKAGES.some((pkg) => matchesPackage(specifier, pkg)) ) { context.report({ node, @@ -78,7 +105,12 @@ const rule = ESLintUtils.RuleCreator( return { ImportDeclaration(node) { - check(node, node.source.value); + check( + node, + node.source.value, + node.importKind === "type" || + node.specifiers.every((s) => s.importKind === "type"), + ); }, ExportNamedDeclaration(node) { if (node.source) check(node, node.source.value); diff --git a/sdk/MediaBackendPlan.md b/sdk/MediaBackendPlan.md index 1a6a32e56..222aa1557 100644 --- a/sdk/MediaBackendPlan.md +++ b/sdk/MediaBackendPlan.md @@ -1,9 +1,16 @@ # Media backend: trapping LiveKit behind one seam +> **Status: implemented, 4 October 2026.** All six steps landed in one change; +> the interface is in `sdk/src/backend/api.ts`, the LiveKit backend in +> `sdk/src/backend/livekit/`, and the architecture document has the section +> "Media backend". The text below is the plan as it was written, kept for the +> reasoning behind the shape. + The SDK's public API in `sdk/src/api.ts` names LiveKit in three places: the video -processor type, the video codec in the capture settings, and the LiveKit-flavoured -client options (`mediaQuality`, `capture`, `transportUrl`, `fallbackTransportUrl`). -The implementation is a different story. LiveKit participants, rooms, connections, +processor type in `PublishRequest` and `LocalVideoMediaTrack.setProcessor`, the +video codec in `VideoCaptureSettings`, and the LiveKit-flavoured client options +(`mediaQuality`, `transportUrl`, `fallbackTransportUrl`). The implementation is a +different story. LiveKit participants, rooms, connections, tokens and data packets run through most of `sdk/src/session/`. Only the two adapters in `sdk/src/media/` are the clean boundary they were meant to be. @@ -33,23 +40,23 @@ slice 11 (the error renames) get easier once the folder exists. Everything under `sdk/src/session/` that imports `livekit-client`, `@livekit/components-core`, or speaks the LiveKit token protocol: -| Module | What it is | -| --- | --- | -| `media/LivekitMediaTrack.ts`, `media/LivekitMemberMedia.ts` | Participant to `MemberMedia`. Clean already. | -| `session/Connection.ts`, `ConnectionFactory.ts`, `ConnectionManager.ts` | One LiveKit room per transport URL, the multi-SFU policy, the token fetch per connection. | -| `session/MatrixLivekitMembers.ts` | Matches a participant identity to a membership, epoch-paired. | -| `session/Publisher.ts` | Local tracks, upstream pausing, device switching, the processor. | -| `session/KeyProvider.ts`, `MatrixKeyProvider.ts` | LiveKit's key provider on the js-sdk key stream. | -| `session/DataChannel.ts` | LiveKit data packets. | -| `session/openIDSFU.ts`, `livekitOptions.ts` | The JWT service protocol, including delegation, and room options. | -| `session/LocalTransport.ts` | The pre-join token fetch. Discovery itself is a homeserver endpoint and stays; see "Discovery stays in the session". | -| `session/RtcTransportAutoDiscovery.ts` | Generic apart from its `isLivekitTransport` filter and the fallback URL. Stays. | -| `session/LocalMember.ts` | `participant$`, `connection$`, upstream pausing, screen share, the delegation probe and handover, `Track.Source` in `setEnabled`. Roughly 150 of 530 lines. | -| `session/Members.ts` | `mediaFor` takes a participant and a room. | -| `session/Transports.ts` | Transport id, the connected check and `resolved$` assume LiveKit. | -| `session/SessionBehaviors.ts` | `membershipsAndTransports$` filters to LiveKit transports. Deleted in step 3, not moved. | -| `session/MatrixRTCClient.ts` | Wires all of the above. | -| `utils/test.ts` | `mockRemoteParticipant`, `exampleTransport`. | +| Module | What it is | +| ----------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `media/LivekitMediaTrack.ts`, `media/LivekitMemberMedia.ts` | Participant to `MemberMedia`. Clean already. | +| `session/Connection.ts`, `ConnectionFactory.ts`, `ConnectionManager.ts` | One LiveKit room per transport URL, the multi-SFU policy, the token fetch per connection. | +| `session/MatrixLivekitMembers.ts` | Matches a participant identity to a membership, epoch-paired. | +| `session/Publisher.ts`, `publishOptions.ts` | Publish requests onto the local participant, upstream pausing, audio output switching, the capture and publish option builders. | +| `session/KeyProvider.ts`, `MatrixKeyProvider.ts` | LiveKit's key provider on the js-sdk key stream. | +| `session/DataChannel.ts` | LiveKit data packets. | +| `session/openIDSFU.ts`, `livekitOptions.ts` | The JWT service protocol, including delegation, and room options. | +| `session/LocalTransport.ts` | The pre-join token fetch. Discovery itself is a homeserver endpoint and stays; see "Discovery stays in the session". | +| `session/RtcTransportAutoDiscovery.ts` | Generic apart from its `isLivekitTransport` filter and the fallback URL. Stays. | +| `session/LocalMember.ts` | `participant$`, `connection$`, upstream pausing, the delegation probe and handover, and `publish` resolving with a `LocalTrackPublication`. Roughly 120 of 490 lines. | +| `session/Members.ts` | `mediaFor` takes a participant and a room; `publish` maps a `trackSid` back to a track. | +| `session/Transports.ts` | Transport id, the connected check and `resolved$` assume LiveKit. | +| `session/SessionBehaviors.ts` | `membershipsAndTransports$` filters to LiveKit transports. Deleted in step 3, not moved. | +| `session/MatrixRTCClient.ts` | Wires all of the above. | +| `utils/test.ts` | `mockRemoteParticipant`, `exampleTransport`. | `errors.ts` has a type-only import for `LivekitConnectionError`; that is slice 11's business and stays. `index.ts` exports `getSFUConfigWithOpenID` as @@ -61,8 +68,10 @@ the backend folder. All of it lives in `sdk/src/backend/api.ts`. Types not defined here come from `sdk/src/api.ts` (`MemberMedia`, `LocalMemberMedia`, `LocalMediaInputs`, `ResolvedTransport`), `sdk/src/config.ts` (`SessionTimings`), -`sdk/src/encryption.ts` and matrix-js-sdk (`Transport`, `CallMembership`, -`CallMembershipIdentityParts`). The js-sdk session itself never reaches a +`sdk/src/encryption.ts` and matrix-js-sdk (`Transport`, +`CallMembershipIdentityParts`). Memberships reach a backend as the SDK's +`RTCMembership`, whose pick covers the two things a backend reads, +`rtcBackendIdentity` and `getTransport()`. The js-sdk session itself never reaches a backend: `MatrixKeyProvider` reads three things from it today, the `EncryptionKeyChanged` event, one `reemitEncryptionKeys()` call to replay the keys it already holds, and the room id for a log line, and `mediaKeys$` covers @@ -138,22 +147,33 @@ export type MediaBackendFactory = ( ``` `LocalMediaInputs` is unchanged from `sdk/src/api.ts` and is the only input the -backend reads about the local devices: +backend reads about the local media. Since the `MemberMediaPlan.md` slices, the +devices and the processor are no longer behaviors the SDK watches: they are +fields of the initial requests, and later changes go to the track's own +`setDevice` and `setProcessor`. ```ts export interface LocalMediaInputs { - /** Initial state; `setMicrophoneEnabled` and `setCameraEnabled` change it from then on. */ - microphoneEnabled: boolean; - cameraEnabled: boolean; - audioInputDeviceId$: Behavior; - videoInputDeviceId$: Behavior; + /** Published at the join; `publish` on the local member adds to it from then on. */ + publish: PublishRequest[]; /** Undefined where the host routes audio itself, or to leave the browser's choice. */ audioOutputDeviceId$: Behavior; - /** Background blur and the like. The LiveKit type on the public API; slice 7. */ - videoProcessor$: Behavior | undefined>; } + +export type PublishRequest = + | { source: "microphone"; deviceId?: string; capture?: AudioCaptureSettings } + | { + source: "camera"; + deviceId?: string; + processor?: TrackProcessor; + capture?: VideoCaptureSettings; + } + | { source: "screenShare"; audio?: boolean; capture?: VideoCaptureSettings }; ``` +The processor type in the camera request is the LiveKit type on the public API; +slice 7 of the migration. + ### The backend ```ts @@ -220,7 +240,7 @@ export interface MediaBackend { */ mediaFor$( scope: ObservableScope, - membership$: Behavior, + membership$: Behavior, ): Behavior; /** @@ -278,22 +298,28 @@ export interface LocalMediaBackend { readonly publishError$: Behavior; /** - * Enables or disables a source and resolves with what resulted, which - * differs from the request where the device could not be used. Remembered - * across connections, so a request before the transport is up is applied - * when the tracks are created. Today's `DesiredMedia` lives behind this. + * Publishes a source and resolves with its track once it is in `media$`'s + * `tracks$`. Remembered across connections, so a request before the + * transport is up is applied when the connection comes, and republished + * after a reconnection. Rejects where the device could not be used, + * including the user closing the screen picker. Today's `DesiredMedia` + * map lives behind this. */ - setEnabled(source: "microphone" | "camera", enabled: boolean): Promise; - - readonly sharingScreen$: Behavior; - /** - * Rejects with the browser's `DOMException` when the user closes the - * picker; the session filters that out before showing an error. - */ - setScreenShareEnabled(enabled: boolean): Promise; + publish( + request: PublishRequest, + ): Promise; + /** Removes one of our tracks by id; a screen share takes its audio with it. */ + unpublish(id: string): Promise; } ``` +Muting, device switching and the processor are not on the backend: they are +methods of `LocalAudioMediaTrack` and `LocalVideoMediaTrack`, which the backend +builds in `media$`. The `setEnabled` callback that `createLocalLivekitMemberMedia` +takes today, so that a track's mute keeps the desired map in step, is wired +inside the backend. Screen sharing has no members of its own any more: it is a +`publish` request, and "sharing" is `tracks$` holding a `screenShare` track. + There is one publish switch, not two. Today `LocalMember.ts` pauses the upstream from two places: `stopPublishing()` on leave, and a separate subscription on the homeserver connectivity that pauses every publication while @@ -310,12 +336,9 @@ There is no publisher on the interface. Today's `Publisher` wraps one LiveKit room's local participant and so lives and dies with the local connection, and `LocalMember.ts` re-applies the join intent to each new one. That lifecycle is a LiveKit detail: the backend keeps the publish intent, as it keeps the desired -media for `setEnabled`, and re-applies both to every connection it opens. A +requests for `publish`, and re-applies both to every connection it opens. A mesh backend has no publisher object at all. -Whether the platform can share a screen at all stays in the session: it is a -`getDisplayMedia` check, the same for every backend. - ### Delayed leave delegation How it works today, all in `LocalMember.ts` and `openIDSFU.ts`: @@ -362,23 +385,31 @@ const prepared$ = from( ), ); // errors become fatalTransportError$ -scope.reconcile(combineLatest([prepared$, joinRequested$]), ([prepared, join]) => { - if (prepared === null || !join) return; - joinMatrixRTC( - prepared.transport, - prepared.canDelegateDelayedLeave ? timings.delegatedDelayedLeave : timings.delayedLeave, - ); - return leaveOnCleanup; -}); +scope.reconcile( + combineLatest([prepared$, joinRequested$]), + ([prepared, join]) => { + if (prepared === null || !join) return; + joinMatrixRTC( + prepared.transport, + prepared.canDelegateDelayedLeave + ? timings.delegatedDelayedLeave + : timings.delayedLeave, + ); + return leaveOnCleanup; + }, +); -scope.reconcile(combineLatest([prepared$, delayId$]), async ([prepared, delayId]) => { - if (!prepared?.canDelegateDelayedLeave || delayId === null) return; - try { - await backend.delegateDelayedLeave(delayId); - } catch (e) { - logger.error("Failed to delegate the leave", e); - } -}); +scope.reconcile( + combineLatest([prepared$, delayId$]), + async ([prepared, delayId]) => { + if (!prepared?.canDelegateDelayedLeave || delayId === null) return; + try { + await backend.delegateDelayedLeave(delayId); + } catch (e) { + logger.error("Failed to delegate the leave", e); + } + }, +); ``` A mesh backend answers `canDelegateDelayedLeave: false` and is never asked to @@ -420,19 +451,19 @@ nobody has a tile for, and that member's transport would never connect. ### Where the LiveKit backend gets each member from -| Interface member | Today | Moves from | -| --- | --- | --- | -| `prepareLocalTransport` | OpenID and JWT fetch kept as `existingSFUConfig`, delegation probe | `LocalTransport.ts`, `LocalMember.ts` | -| `delegateDelayedLeave` | `getSFUConfigWithOpenID` with `delayId` | `LocalMember.ts` | -| `local.connectionState$` | `connection$.state$` for the local transport | `LocalMember.ts` | -| `local.media$` | `mediaFor(participant$, connection$)` with `createLocalLivekitMemberMedia` | `Members.ts` | -| `local.setPublishing`, `publishError$` | `scope.reconcile(connection$, createPublisher)`, the `[publisher$, joinRequested$]` reconcile with its `publishError$`, and the homeserver-connectivity loop over `participant.trackPublications` | `LocalMember.ts`, `MatrixRTCClient.ts` | -| `local.setEnabled` | `DesiredMedia` plus `Publisher.setEnabled(Track.Source, …)` | `LocalMember.ts`, `Publisher.ts` | -| `local.sharingScreen$`, `setScreenShareEnabled` | `observeSharingScreen$`, `setScreenShareEnabled` with capture and publish options | `LocalMember.ts` | -| `mediaFor$` | `createRemoteMatrixLivekitMembers$` matching plus `mediaFor` | `Members.ts`, `MatrixLivekitMembers.ts` | -| `connections$` | `connectionManagerData$` joined with each connection's `state$` and `ResolvedConnection.resolved$` | `Transports.ts`, `ConnectionFactory.ts` | -| `sendData`, `data$` | `createDataChannel$` minus the member lookup | `DataChannel.ts` | -| internal | key provider on `mediaKeys$` instead of `setRTCSession`, room options, E2EE worker, connection manager fed by the transports registered through `mediaFor$` | `KeyProvider.ts`, `MatrixKeyProvider.ts`, `livekitOptions.ts`, `ConnectionFactory.ts`, `ConnectionManager.ts` | +| Interface member | Today | Moves from | +| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `prepareLocalTransport` | OpenID and JWT fetch kept as `existingSFUConfig`, delegation probe | `LocalTransport.ts`, `LocalMember.ts` | +| `delegateDelayedLeave` | `getSFUConfigWithOpenID` with `delayId` | `LocalMember.ts` | +| `local.connectionState$` | `connection$.state$` for the local transport | `LocalMember.ts` | +| `local.media$` | `mediaFor(participant$, connection$)` with `createLocalLivekitMemberMedia` | `Members.ts` | +| `local.setPublishing`, `publishError$` | `scope.reconcile(connection$, createPublisher)`, the `[publisher$, joinRequested$]` reconcile with its `publishError$`, and the homeserver-connectivity loop over `participant.trackPublications` | `LocalMember.ts`, `MatrixRTCClient.ts` | +| `local.publish`, `unpublish` | `publish` and `unpublish` with the `DesiredMedia` map and `publication$`, and the `trackSid` to track lookup | `LocalMember.ts`, `Members.ts`, `Publisher.ts` | +| internal | the `setEnabled(source, enabled)` callback into `createLocalLivekitMemberMedia` | `LocalMember.ts`, `Members.ts` | +| `mediaFor$` | `createRemoteMatrixLivekitMembers$` matching plus `mediaFor` | `Members.ts`, `MatrixLivekitMembers.ts` | +| `connections$` | `connectionManagerData$` joined with each connection's `state$` and `ResolvedConnection.resolved$` | `Transports.ts`, `ConnectionFactory.ts` | +| `sendData`, `data$` | `createDataChannel$` minus the member lookup | `DataChannel.ts` | +| internal | key provider on `mediaKeys$` instead of `setRTCSession`, room options, E2EE worker, connection manager fed by the transports registered through `mediaFor$` | `KeyProvider.ts`, `MatrixKeyProvider.ts`, `livekitOptions.ts`, `ConnectionFactory.ts`, `ConnectionManager.ts` | ### Discovery stays in the session @@ -446,9 +477,12 @@ only asked to prepare what the session picked: export async function discoverLocalTransport( client: Pick, transportType: string, - options: Pick, + options: Pick< + MatrixRTCClientOptions, + "transportUrl" | "fallbackTransportUrl" + >, logger: Logger, -): Promise; // throws MatrixRTCTransportMissingError +): Promise; // throws MatrixRTCTransportMissingError ``` `RtcTransportAutoDiscovery` does the list fetch today and stays where it is, @@ -487,7 +521,6 @@ export interface LivekitBackendOptions { */ client: Pick; mediaQuality?: MediaQuality; - capture?: CaptureSettings; /** * Which token service endpoint to use for the local transport: the MSC4195 * one, or the legacy one with the old identity format. Remote transports @@ -520,8 +553,9 @@ Pure move, no code change beyond import paths. - `git mv` into `sdk/src/backend/livekit/`: `media/*`, `Connection`, `ConnectionFactory`, `ConnectionManager`, `DataChannel`, `KeyProvider`, - `MatrixKeyProvider`, `MatrixLivekitMembers`, `Publisher`, `livekitOptions`, - `openIDSFU`, each with its test. `RtcTransportAutoDiscovery` and + `MatrixKeyProvider`, `MatrixLivekitMembers`, `Publisher`, `publishOptions`, + `livekitOptions`, `openIDSFU`, each with its test. `utils/tracks.ts` is + neutral and stays. `RtcTransportAutoDiscovery` and `LocalTransport` stay in `session/`. - Split `utils/test.ts`: `mockRemoteParticipant` and `exampleTransport` go to `backend/livekit/test.ts`. @@ -595,24 +629,28 @@ with the second's. transport's connection mapped onto `MediaConnectionState`, `media$` through `createLocalLivekitMemberMedia`, `setPublishing` and `publishError$` by reconciling a `Publisher` per connection against the remembered intent, - which is the two reconciles from `LocalMember.ts` moved as they are, - `setEnabled` over a backend-owned `DesiredMedia` and the current publisher, - `sharingScreen$` and `setScreenShareEnabled` with today's capture and publish - options. The homeserver-connectivity loop is not moved; `setPublishing` - covers it. + which is the two reconciles from `LocalMember.ts` moved as they are, and + `publish` and `unpublish` over a backend-owned `DesiredMedia` map and the + current publisher, with the `trackSid` to track lookup that `Members.ts` + does today done inside. The `setEnabled` callback into + `createLocalLivekitMemberMedia` is wired here too. The + homeserver-connectivity loop is not moved; `setPublishing` covers it. - `LocalMember.ts` takes `backend.local` instead of `connectionManager`, `createPublisher` and `desired`. Its publisher reconcile and the pause and resume loop become one subscription, `joinRequested$ && homeserverConnected.combined$` into `backend.local.setPublishing`, and `backend.local.publishError$` feeds - `state$` where the local subject did. It - loses `participant$`, `connection$`, the `livekitRoom` access, the pause and - resume loop, `Track.Source`, `ScreenShareCaptureOptions` and - `TrackPublishOptions`, and compares against `MediaConnectionState.Connected` - in `mediaState$` and `disconnectReason$`. `LocalMembership` no longer exposes - `participant$` or `connection$`. + `state$` where the local subject did. It loses `participant$`, + `connection$`, the `livekitRoom` access, the pause and resume loop, + `publish`, `unpublish`, `setEnabled` and `publication$`, and compares + against `MediaConnectionState.Connected` in `mediaState$` and + `disconnectReason$`. `LocalMembership` shrinks to the join state machine + and no longer names a LiveKit type. +- `Members.ts`: `createLocalRTCMember` forwards `publish` and `unpublish` to + `backend.local` and drops its own `trackSid` lookup. - `MatrixRTCClient.ts` stops passing `createPublisher`, `desired` and the - connection manager. The transitional field goes. + connection manager, and stops building the `DesiredMedia` map. The + transitional field goes. - `status.test.ts` moves to `MediaConnectionState`. New unit test: `LocalMember.test.ts` against a fake `LocalMediaBackend`. This is @@ -660,8 +698,7 @@ and that a new delay id triggers a handover. - Extend `eslint/SdkImportBoundary.js`: under `sdk/src/session/`, in `sdk/src/api.ts` and in `sdk/index.ts`, `livekit-client` and `@livekit/*` are banned except as `import type`. From here on the seam is enforced, not - documented. `oxlint` currently reports no files under `sdk/`, so that has to - be fixed first for the rule to bite at all. + documented. - `SdkArchitecture.md` gets a "Media backend" section built from this plan, the README's description of what imports LiveKit is updated, and `SdkMigration.md` records the slice as done. diff --git a/sdk/README.md b/sdk/README.md index 7c30f9950..dffdb7fe4 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -1,11 +1,13 @@ # MatrixRTC SDK (EXPERIMENTAL) `@element-hq/matrixrtc-sdk` is the call model under Element Call, on its own: -MatrixRTC memberships and transports, LiveKit connections, publishing, E2EE keys and +MatrixRTC memberships and transports, media connections, publishing, E2EE keys and the media of every member, as observables. It has no UI. Element Call's own `CallViewModel` is meant to become one consumer of it. The design is in [`SdkArchitecture.md`](./SdkArchitecture.md), the migration from the view model -in [`SdkMigration.md`](./SdkMigration.md). +in [`SdkMigration.md`](./SdkMigration.md). Media goes through one media backend +behind `MediaBackend`; LiveKit, under `src/backend/livekit/`, is the only one +today and the only place that imports `livekit-client`. **Status:** first implementation. `createMatrixRTCClient` joins the session, connects to the transport, publishes the local media and exposes every member's media; the diff --git a/sdk/SdkArchitecture.md b/sdk/SdkArchitecture.md index 6f471b746..b2ff14db1 100644 --- a/sdk/SdkArchitecture.md +++ b/sdk/SdkArchitecture.md @@ -25,7 +25,7 @@ therefore `Element Call → sdk`, never the reverse. The migration from today's │ │ │ public API, all Behaviors: │ │ join() leave() status$ connected$ reconnecting$ fatalError$ │ -│ localMember$ remoteMembers$ memberCount$ │ +│ localMember$ remoteMembers$ memberCount$ sendData() data$ │ │ keyRotationSuppressed$ connectedTransports$ │ │ │ │ per member RTCMember { id userId deviceId membership$ displayName$ │ @@ -33,33 +33,42 @@ therefore `Element Call → sdk`, never the reverse. The migration from today's │ per media MemberMedia { tracks$ encryptionError$ } │ │ per track MediaTrack { source kind muted$ encrypted$ stats$ attach() }│ │ │ -│ ┌──────────────────────────┐ memberships ┌───────────────────────────┐ │ -│ │ MatrixRTC core │ ───────────────▶ │ transports │ │ -│ │ (js-sdk MatrixRTCSession,│ transports │ │ │ -│ │ later the rust-rtc │ │ ConnectionManager │ │ -│ │ crate behind a bridge) │ │ one Connection per │ │ -│ │ │ │ transport url │ │ -│ │ memberships from room │ │ Connection │ │ -│ │ state and sticky events│ │ OpenID → JWT → LiveKit │ │ -│ │ own membership, delayed │ │ room, participants, │ │ -│ │ leave, retries │ media keys │ tracks │ │ -│ │ media keys over │ ───────────────▶ │ Publisher │ │ -│ │ to-device messages │ (key provider) │ local tracks, upstream │ │ -│ └──────────────────────────┘ └───────────────────────────┘ │ -│ │ │ │ -│ └──── RTCMember = membership + participant ─┘ │ -│ matched on rtcBackendIdentity = LiveKit identity │ +│ ┌───────────────────────────┐ MediaBackend ┌───────────────────────────┐ │ +│ │ session: MatrixRTC │ backend/api.ts │ media backend │ │ +│ │ │ │ backend/livekit/ today, │ │ +│ │ js-sdk MatrixRTCSession, │ │ or options.backend │ │ +│ │ later the rust-rtc │ prepareLocal │ │ │ +│ │ crate behind a bridge │ Transport ───▶│ token service: │ │ +│ │ memberships from room │ delegateDelayed │ OpenID → JWT, │ │ +│ │ state and sticky events │ Leave ───────▶│ delegation probe │ │ +│ │ own membership, delayed │ │ ConnectionManager: │ │ +│ │ leave, retries │ mediaKeys$ ────▶│ one LiveKit room per │ │ +│ │ discovery of the local │ │ transport url, local │ │ +│ │ transport (homeserver) │ mediaFor$ ─────▶│ published, remote │ │ +│ │ join state machine │ ◀────── media$ │ subscribed │ │ +│ │ (LocalMember) │ local.publish, │ Publisher: local tracks, │ │ +│ │ transport registry │ setPublishing▶│ upstream paused until │ │ +│ │ (TransportMetadata) │ ◀─ connections$ │ joined │ │ +│ │ data packet → member │ ◀─ data$ │ LivekitMemberMedia: │ │ +│ │ │ │ participant → tracks │ │ +│ └───────────────────────────┘ └───────────────────────────┘ │ +│ │ +│ RTCMember = membership + mediaFor$(membership): the backend matches the │ +│ member's rtcBackendIdentity to the identity it sees on the transport │ └──────────────────────────────────────────────────────────────────────────────┘ │ │ ▼ ▼ MatrixClient (sync, room state, to-device, OpenID) LiveKit SFU ``` -The left box is MatrixRTC: who is in the session, with which transport, and the -keys. The right box is media: a connection to every transport any member -advertises, and our own publication on the one we advertise. The client is the -join of the two: a member exists once its membership does, and gets its media once -a participant with the matching identity shows up on the member's transport. +The left box is MatrixRTC: who is in the session, with which transport, the +keys, the delayed leave, and the join state machine. The right box is a media +backend: whatever carries the media, behind the `MediaBackend` interface in +`sdk/src/backend/api.ts`. The client is the join of the two: a member exists +once its membership does, and gets its media once the backend has something +for that membership on its transport. LiveKit is the one backend today; a +cascading SFU or full mesh is another folder under `sdk/src/backend/`, handed +in through `options.backend`, and nothing in the left box changes. ### How Element Call uses it @@ -88,26 +97,33 @@ LiveKit participants. ### Joining, in order ``` - host MatrixRTCClient homeserver / JWT service LiveKit SFU - │ create ───▶│ │ │ - │ │ GET rtc/transports ──────────────▶│ │ - │ │◀── preferred transport ───────────│ │ - │ │ OpenID token, then /sfu/get ─────▶│ │ - │ │◀── sfu url, jwt, alias, identity ─│ = resolved$ of the │ - │ │ │ local transport │ - │ │ connect(url, jwt) ───────────────────────────────────────▶│ - │ join() ───▶│ │ │ - │ │ membership state event + delayed leave ──▶│ │ - │ │ create tracks, resume upstream ──────────────────────────▶│ - │ │ media key to every other member (to-device) ──▶│ │ - │ │◀── other memberships (sync) ──────│◀── participants ────────│ - │◀─ status$ "connected", localMember$, remoteMembers$ with media$ ──────────│ + host MatrixRTCClient / backend homeserver / JWT service LiveKit SFU + │ create ───▶│ │ │ + │ │ GET rtc/transports (session) ────────▶│ │ + │ │◀── first transport of the backend's type │ + │ │ prepareLocalTransport (backend): │ │ + │ │ OpenID token, then /sfu/get ───────▶│ │ + │ │◀── sfu url, jwt, alias, identity ─────│ = resolved$ of the │ + │ │ probe delegate_delayed_leave ──────▶│ local transport │ + │ │ connect(url, jwt) ───────────────────────────────────────────▶│ + │ join() ───▶│ │ │ + │ │ membership state event + delayed leave ──▶│ │ + │ │ delegateDelayedLeave(delayId) (backend) ─▶│ │ + │ │ create tracks, resume upstream ──────────────────────────────▶│ + │ │ media key to every other member (to-device) ──▶│ │ + │ │◀── other memberships (sync) ──────────│◀── participants ────────│ + │◀─ status$ "connected", localMember$, remoteMembers$ with media$ ──────────────│ ``` -The transport is resolved and connected before `join()`, so that the membership -can name it and the first frames go out as soon as the state event is sent. Until -`join()` the tracks exist but their upstream is paused: a host can show a preview -without anyone hearing it. `leave()` pauses the upstream again and sends the leave. +The session discovers the transport from the homeserver and the backend prepares +it, both before `join()`, so that the membership can name it and the first frames +go out as soon as the state event is sent. Preparing is the backend's own +business: for LiveKit the token exchange, and the probe that decides whether the +SFU can take over the delayed leave, which picks the delayed leave timings the +join uses. Until `join()` the tracks exist but their upstream is paused: a host +can show a preview without anyone hearing it. The upstream flows exactly while the +member is joined and the homeserver is reachable; `leave()` or a sync outage pauses +it, and `leave()` also sends the leave. ## Vocabulary @@ -123,6 +139,7 @@ without anyone hearing it. `leave()` pauses the upstream again and sends the lea | active | `isActive$` on an audio track: the track carries sound, as the backend measures it. "Speaking" is a call's reading of it on the microphone track | LiveKit "speaking", `isSpeaking` | | attach, detach | handing a `