mirror of
https://github.com/vector-im/element-call.git
synced 2026-10-16 23:45:50 +00:00
renames + rewriting docs
This commit is contained in:
Timo K.
+10
-9
@@ -3,10 +3,11 @@
|
||||
`@element-hq/matrixrtc-sdk` is the call model under Element Call, on its own:
|
||||
MatrixRTC memberships and transports, LiveKit connections, publishing, E2EE keys and
|
||||
the media of every member, as observables. It has no UI. Element Call's own
|
||||
`CallViewModel` is meant to become one consumer of it; the design is in
|
||||
[`sdk-plan.md`](../sdk-plan.md).
|
||||
`CallViewModel` is meant to become one consumer of it. The design is in
|
||||
[`SdkArchitecture.md`](./SdkArchitecture.md), the migration from the view model
|
||||
in [`SdkMigration.md`](./SdkMigration.md).
|
||||
|
||||
**Status:** first implementation. `createRtcSession` joins the session, connects to
|
||||
**Status:** first implementation. `createMatrixRTCClient` joins the session, connects to
|
||||
the transport, publishes the local media and exposes every member's media; the
|
||||
development harness and its e2e tests in `playwright/sdk` drive it. The
|
||||
implementation still imports the building blocks it shares with Element Call from
|
||||
@@ -17,17 +18,17 @@ implementation still imports the building blocks it shares with Element Call fro
|
||||
```ts
|
||||
import {
|
||||
constant,
|
||||
createRtcSession,
|
||||
createMatrixRTCClient,
|
||||
E2eeType,
|
||||
MatrixRTCMode,
|
||||
ObservableScope,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
const scope = new ObservableScope();
|
||||
const session = createRtcSession(
|
||||
const rtcClient = createMatrixRTCClient(
|
||||
scope,
|
||||
client, // a matrix-js-sdk MatrixClient, logged in and syncing
|
||||
room, // the matrix-js-sdk Room to hold the session in, from client.getRoom()
|
||||
room, // the matrix-js-sdk Room, from client.getRoom() once the join has synced
|
||||
{
|
||||
microphoneEnabled$: constant(true),
|
||||
cameraEnabled$: constant(true),
|
||||
@@ -40,15 +41,15 @@ const session = createRtcSession(
|
||||
matrixRTCMode: MatrixRTCMode.Compatibility,
|
||||
},
|
||||
);
|
||||
session.join();
|
||||
rtcClient.join();
|
||||
|
||||
session.remoteMembers$.subscribe((members) => {
|
||||
rtcClient.remoteMembers$.subscribe((members) => {
|
||||
// each member has displayName$, media$ and more; a media track is rendered
|
||||
// by handing it a <video> or <audio> element: track.attach(element)
|
||||
});
|
||||
|
||||
// later
|
||||
session.leave();
|
||||
rtcClient.leave();
|
||||
scope.end();
|
||||
```
|
||||
|
||||
|
||||
@@ -0,0 +1,570 @@
|
||||
# MatrixRTC SDK architecture
|
||||
|
||||
`@element-hq/matrixrtc-sdk` is the model under Element Call, on its own: one
|
||||
`MatrixRTCClient` per room that joins and leaves the MatrixRTC session, discovers
|
||||
and connects to the media transports, publishes the local media, distributes the
|
||||
encryption keys, and hands out every member with its media as observables. It has
|
||||
no UI, reads no configuration and no settings, and never names LiveKit in its
|
||||
interface.
|
||||
|
||||
Element Call becomes one consumer of it: `CallViewModel` keeps what makes a session
|
||||
a _call_ (ringing, auto-leave, layout, sounds, reactions, settings, the host
|
||||
bridge) and takes everything else from the client. The dependency direction is
|
||||
therefore `Element Call → sdk`, never the reverse. The migration from today's
|
||||
`CallViewModel` to that shape is in [`SdkMigration.md`](./SdkMigration.md).
|
||||
|
||||
## The architecture
|
||||
|
||||
```
|
||||
host application (Element Call, a whiteboard, watch-together, …)
|
||||
│
|
||||
│ createMatrixRTCClient(scope, matrixClient, room, localMedia, options)
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────────────────┐
|
||||
│ MatrixRTCClient │
|
||||
│ │
|
||||
│ public API, all Behaviors: │
|
||||
│ join() leave() status$ connected$ reconnecting$ fatalError$ │
|
||||
│ localMember$ remoteMembers$ memberCount$ │
|
||||
│ keyRotationSuppressed$ connectedTransports$ │
|
||||
│ │
|
||||
│ per member RTCMember { id userId deviceId membership$ displayName$ │
|
||||
│ avatarUrl$ transport$ media$ } │
|
||||
│ per media MemberMedia { speaking$ microphone$ camera$ screenShare$ … } │
|
||||
│ per track MediaTrack { muted$ encrypted$ stats$ attach() detach() } │
|
||||
│ │
|
||||
│ ┌──────────────────────────┐ memberships ┌───────────────────────────┐ │
|
||||
│ │ MatrixRTC core │ ───────────────▶ │ transports │ │
|
||||
│ │ (js-sdk MatrixRTCSession,│ transports │ │ │
|
||||
│ │ later the rust-rtc │ │ ConnectionManager │ │
|
||||
│ │ crate behind a bridge) │ │ one Connection per │ │
|
||||
│ │ │ │ transport url │ │
|
||||
│ │ memberships from room │ │ Connection │ │
|
||||
│ │ state and sticky events│ │ OpenID → JWT → LiveKit │ │
|
||||
│ │ own membership, delayed │ │ room, participants, │ │
|
||||
│ │ leave, retries │ media keys │ tracks │ │
|
||||
│ │ media keys over │ ───────────────▶ │ Publisher │ │
|
||||
│ │ to-device messages │ (key provider) │ local tracks, upstream │ │
|
||||
│ └──────────────────────────┘ └───────────────────────────┘ │
|
||||
│ │ │ │
|
||||
│ └──── RTCMember = membership + participant ─┘ │
|
||||
│ matched on rtcBackendIdentity = LiveKit identity │
|
||||
└──────────────────────────────────────────────────────────────────────────────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
MatrixClient (sync, room state, to-device, OpenID) LiveKit SFU
|
||||
```
|
||||
|
||||
The left box is MatrixRTC: who is in the session, with which transport, and the
|
||||
keys. The right box is media: a connection to every transport any member
|
||||
advertises, and our own publication on the one we advertise. The client is the
|
||||
join of the two: a member exists once its membership does, and gets its media once
|
||||
a participant with the matching identity shows up on the member's transport.
|
||||
|
||||
### How Element Call uses it
|
||||
|
||||
```
|
||||
Element Call, as app, widget or component
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ views: InCallView, tiles, header, footer, settings │
|
||||
│ ▲ Behaviors │
|
||||
│ CallViewModel │
|
||||
│ layout, ringing and notifications, auto-leave, sounds, │
|
||||
│ reactions, hand raise, settings, audio routing, host bridge │
|
||||
│ ▲ status$, localMember$, remoteMembers$, MemberMedia │
|
||||
│ MatrixRTCClient @element-hq/matrixrtc-sdk │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
│ MatrixClient + Room │ livekit-client
|
||||
▼ ▼
|
||||
homeserver SFU
|
||||
```
|
||||
|
||||
All three modes already hold a `MatrixClient` and a `Room` by the time a call
|
||||
starts: standalone after `useLoadGroupCall`, the component from the host's `client`
|
||||
prop, the widget from `createRoomWidgetClient`. The view model passes them on and
|
||||
builds its media view models from `RTCMember` and `MemberMedia` instead of from
|
||||
LiveKit participants.
|
||||
|
||||
### Joining, in order
|
||||
|
||||
```
|
||||
host MatrixRTCClient homeserver / JWT service LiveKit SFU
|
||||
│ create ───▶│ │ │
|
||||
│ │ GET rtc/transports ──────────────▶│ │
|
||||
│ │◀── preferred transport ───────────│ │
|
||||
│ │ OpenID token, then /sfu/get ─────▶│ │
|
||||
│ │◀── sfu url, jwt, alias, identity ─│ = resolved$ of the │
|
||||
│ │ │ local transport │
|
||||
│ │ connect(url, jwt) ───────────────────────────────────────▶│
|
||||
│ join() ───▶│ │ │
|
||||
│ │ membership state event + delayed leave ──▶│ │
|
||||
│ │ create tracks, resume upstream ──────────────────────────▶│
|
||||
│ │ media key to every other member (to-device) ──▶│ │
|
||||
│ │◀── other memberships (sync) ──────│◀── participants ────────│
|
||||
│◀─ status$ "connected", localMember$, remoteMembers$ with media$ ──────────│
|
||||
```
|
||||
|
||||
The transport is resolved and connected before `join()`, so that the membership
|
||||
can name it and the first frames go out as soon as the state event is sent. Until
|
||||
`join()` the tracks exist but their upstream is paused: a host can show a preview
|
||||
without anyone hearing it. `leave()` pauses the upstream again and sends the leave.
|
||||
|
||||
## Vocabulary
|
||||
|
||||
| Word | Meaning here | Elsewhere |
|
||||
| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ |
|
||||
| client | a `MatrixRTCClient`: one room, one slot, one local member | `MatrixClient` is always written out |
|
||||
| session | the MatrixRTC session in the room: the set of memberships. Only used for the js-sdk object, which stays inside the SDK as `JsSdkRTCSession` | MSC4143 |
|
||||
| membership | one device's MatrixRTC state event, the js-sdk `CallMembership`. The js-sdk name is kept until it renames it | |
|
||||
| member | an `RTCMember`: a membership plus what the SDK derives from it (display name, transport, media) | Element Call's "matrixLivekitMember" |
|
||||
| participant | a LiveKit participant, the media side of a member. Never leaves the SDK | |
|
||||
| transport | where media is exchanged: a LiveKit service url today, described by `TransportMetadata` | MSC4143 "focus" |
|
||||
| application, slot | what the session is for (`m.call`) and which one of them in the room (`ROOM`) | MSC4143 |
|
||||
|
||||
Names use `RTC` in capitals, as the js-sdk does: `MatrixRTCClient`, `RTCMember`,
|
||||
`matrixRTCMode`.
|
||||
|
||||
## Reactive primitives
|
||||
|
||||
The whole public API is made of `Behavior`s: an rxjs observable that always holds a
|
||||
current value (`.value`) and replays it on subscription. A consumer subscribes to
|
||||
what it renders and reads `.value` where it needs the present state. Everything is
|
||||
bound to an `ObservableScope` the host creates; ending the scope ends the client,
|
||||
its connections and its tracks. Both come from the SDK (`Behavior`, `constant`,
|
||||
`ObservableScope`); the host needs rxjs and nothing else.
|
||||
|
||||
## What a host must provide
|
||||
|
||||
Found the hard way while building the harness; each one fails in a way that looks
|
||||
like an SDK bug.
|
||||
|
||||
- **The room the sync loop maintains.** `client.getRoom(roomId)` once the join has
|
||||
synced. `joinRoom` returns a detached copy for a room joined just now, and a
|
||||
client built on it never sees a member.
|
||||
- **Power to send the membership event.** Every member needs to write
|
||||
`org.matrix.msc3401.call.member` (and the sticky event in Matrix 2.0 mode). A
|
||||
plain room only lets its creator send state; Element Call's rooms set the level
|
||||
to 0.
|
||||
- **An encrypted room for per-participant keys.** Media keys travel in encrypted
|
||||
to-device messages, which the crypto only delivers to devices it tracks, which
|
||||
are the members of encrypted rooms. In an unencrypted room the key never arrives
|
||||
and every remote track decodes nothing.
|
||||
- **A logged-in, syncing client with crypto set up**, and the transport advertised
|
||||
by the homeserver (`rtc/transports`). The SDK has no fallback url of its own.
|
||||
|
||||
## The client: `MatrixRTCClient`
|
||||
|
||||
Rule of thumb for what belongs here: if it would be the same for a different
|
||||
application on top of the same MatrixRTC session, it is in the client. If it is
|
||||
specific to calls, or depends on the window, a setting, a platform, or a React
|
||||
context, it is in the view model.
|
||||
|
||||
### Creating one
|
||||
|
||||
```ts
|
||||
import { type MatrixClient, type Room } from "matrix-js-sdk";
|
||||
import { type RTCNotificationType } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
export interface MatrixRTCClientOptions {
|
||||
encryptionSystem: EncryptionSystem;
|
||||
/** Resolved by the host; the SDK reads neither config.json nor settings. */
|
||||
matrixRTCMode: MatrixRTCMode;
|
||||
/**
|
||||
* MSC4075 notification sent with the join. A parameter of the MatrixRTC
|
||||
* join itself, so it is here even though it is named after calls; reacting
|
||||
* to a notification (ringing, timeouts, declines) is the application's job.
|
||||
*/
|
||||
sendNotificationType?: RTCNotificationType;
|
||||
/** The application the session belongs to, as named in the membership. Default `m.call`. */
|
||||
application?: string;
|
||||
/** The application's slot in the room. Default `ROOM`. */
|
||||
slot?: string;
|
||||
/**
|
||||
* Whatever the application wants to say about itself in the membership.
|
||||
* Opaque to the SDK. Today the js-sdk carries one key, `m.call.intent`
|
||||
* (`"audio"` or `"video"`); anything else is dropped until it can.
|
||||
*/
|
||||
applicationData?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* What the local member publishes. The SDK publishes what it is told; device
|
||||
* enumeration, permission prompts and the lobby preview stay with the host.
|
||||
*/
|
||||
export interface LocalMediaInputs {
|
||||
microphoneEnabled$: Behavior<boolean>;
|
||||
cameraEnabled$: Behavior<boolean>;
|
||||
audioInputDeviceId$: Behavior<string | undefined>;
|
||||
videoInputDeviceId$: Behavior<string | undefined>;
|
||||
/** Background blur and the like. */
|
||||
videoProcessor$: Behavior<TrackProcessor<Track.Kind.Video> | undefined>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes the whole MatrixClient, not a narrow pick, and finds the MatrixRTC
|
||||
* session itself. The js-sdk is the MatrixRTC implementation today; when the
|
||||
* rust-rtc crate replaces it, the SDK has to bridge the crate to the js-sdk
|
||||
* client (events, state, to-device, delayed events, OpenID), and only the SDK
|
||||
* knows what that bridge needs. Holding the client keeps that change inside
|
||||
* the SDK. The js-sdk session is not part of the public API.
|
||||
*
|
||||
* The room has to be the one the client's sync loop maintains; see "What a
|
||||
* host must provide".
|
||||
*/
|
||||
export function createMatrixRTCClient(
|
||||
scope: ObservableScope,
|
||||
client: MatrixClient,
|
||||
room: Room,
|
||||
localMedia: LocalMediaInputs,
|
||||
options: MatrixRTCClientOptions,
|
||||
): MatrixRTCClient;
|
||||
```
|
||||
|
||||
### The client
|
||||
|
||||
```ts
|
||||
export type ConnectionStatus =
|
||||
| "waitingForTransport"
|
||||
| "connecting"
|
||||
| "connected"
|
||||
| "reconnecting"
|
||||
| "disconnected";
|
||||
|
||||
/** An error raised by the client. `cause` holds the backend's error, for a host that knows the backend. */
|
||||
export class MatrixRTCError extends Error {}
|
||||
|
||||
export interface MatrixRTCClient {
|
||||
join(): void;
|
||||
leave(): void;
|
||||
/** The local member's state machine, collapsed to what a host shows. */
|
||||
status$: Behavior<ConnectionStatus>;
|
||||
/** Connected to the homeserver, the session and the local transport, all three. */
|
||||
connected$: Behavior<boolean>;
|
||||
/** Connected once, and currently not. */
|
||||
reconnecting$: Behavior<boolean>;
|
||||
/**
|
||||
* A transport, Matrix or connection error that stops the session. Null
|
||||
* while fine. A failed publication is not fatal: the member can still
|
||||
* receive, so it stays on the local member's media.
|
||||
*/
|
||||
fatalError$: Behavior<MatrixRTCError | null>;
|
||||
|
||||
/** Null until our own membership has been seen in the room. */
|
||||
localMember$: Behavior<LocalRTCMember | null>;
|
||||
remoteMembers$: Behavior<RemoteRTCMember[]>;
|
||||
/** `remoteMembers.length`, plus one for the local member once it exists. One per membership, so one user may count several times. */
|
||||
memberCount$: Behavior<number>;
|
||||
|
||||
/**
|
||||
* Whether the session has grown large enough that MatrixRTC has stopped
|
||||
* rotating the media encryption key.
|
||||
*/
|
||||
keyRotationSuppressed$: Behavior<boolean>;
|
||||
|
||||
/**
|
||||
* Transports the client currently holds a live connection to. The
|
||||
* connections themselves stay internal; this is what a developer panel
|
||||
* needs.
|
||||
*/
|
||||
connectedTransports$: Behavior<TransportMetadata[]>;
|
||||
}
|
||||
```
|
||||
|
||||
### Members
|
||||
|
||||
```ts
|
||||
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
export interface RTCMember {
|
||||
local: boolean;
|
||||
/**
|
||||
* The identity the media backend knows this member by: the membership's
|
||||
* `rtcBackendIdentity`, which is `${userId}:${deviceId}` before sticky
|
||||
* events and a hash of the member id in Matrix 2.0 mode.
|
||||
*/
|
||||
id: string;
|
||||
userId: string;
|
||||
deviceId: string;
|
||||
membership$: Behavior<CallMembership>;
|
||||
/**
|
||||
* Matrix room state rather than MatrixRTC, but the SDK already holds the
|
||||
* room and every consumer wants them next to the membership. Disambiguated
|
||||
* among the members of the session.
|
||||
*/
|
||||
displayName$: Behavior<string>;
|
||||
avatarUrl$: Behavior<string | undefined>;
|
||||
/** Which transport this member is on; undefined when the membership names none. */
|
||||
transport$: Behavior<TransportMetadata | undefined>;
|
||||
/**
|
||||
* Null while the member has a transport but no participant has shown up on
|
||||
* it yet ("waiting for media"). Hand raise and reactions are room events
|
||||
* keyed by member and stay with the host for now; they are the obvious next
|
||||
* fields here.
|
||||
*/
|
||||
media$: Behavior<MemberMedia | null>;
|
||||
}
|
||||
|
||||
export interface RemoteRTCMember extends RTCMember {
|
||||
local: false;
|
||||
}
|
||||
|
||||
export interface LocalRTCMember extends RTCMember {
|
||||
local: true;
|
||||
media$: Behavior<LocalMemberMedia | null>;
|
||||
sharingScreen$: Behavior<boolean>;
|
||||
/** Null when the platform cannot share a screen (no getDisplayMedia). Hiding the button is the host's policy. */
|
||||
toggleScreenSharing: (() => void) | null;
|
||||
/** The last error from toggling, until dismissed. The user closing the picker is not one. */
|
||||
screenShareError$: Behavior<Error | null>;
|
||||
dismissScreenShareError(): void;
|
||||
}
|
||||
```
|
||||
|
||||
### Transports
|
||||
|
||||
```ts
|
||||
import { type Transport } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
/**
|
||||
* One transport advertised in a membership. Transport independent: `type` and
|
||||
* `id` are all the SDK needs; `raw` and `resolved$` are there for a
|
||||
* backend-specific developer panel and for connection diagnostics. One object
|
||||
* per transport, however many memberships name it.
|
||||
*/
|
||||
export interface TransportMetadata {
|
||||
/** `"livekit"` today. */
|
||||
type: string;
|
||||
/** Stable key, unique per transport in the session. For LiveKit, the service url. */
|
||||
id: string;
|
||||
/** The transport object as it appears in the membership, e.g. `{ type, livekit_service_url }`. */
|
||||
raw: Transport;
|
||||
/**
|
||||
* What the backend had to fetch before it could connect. Undefined until
|
||||
* the connection has resolved it, and again after the connection stops or
|
||||
* fails. The local transport resolves before the join, since the membership
|
||||
* needs it; remote ones when their connection starts.
|
||||
*/
|
||||
resolved$: Behavior<ResolvedTransport | undefined>;
|
||||
}
|
||||
|
||||
export type ResolvedTransport =
|
||||
| {
|
||||
type: "livekit";
|
||||
/** The SFU websocket url, as opposed to the JWT service url in `raw`. */
|
||||
url: string;
|
||||
/** A secret: fit for a developer panel, never for a log line. */
|
||||
token: string;
|
||||
roomAlias: string;
|
||||
identity: string;
|
||||
}
|
||||
| { type: string; [key: string]: unknown };
|
||||
```
|
||||
|
||||
### Call-specific things kept out of the client
|
||||
|
||||
Checked against `CallViewModel` and its options. These stay in Element Call because
|
||||
they only make sense for a call:
|
||||
|
||||
| Today | Why it is call-specific | Where it lives instead |
|
||||
| ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- |
|
||||
| `autoLeave$` / `AutoLeaveReason` (`"allOthersLeft" \| "timeout" \| "decline"`) | hanging up because the other side declined or everyone left is call etiquette, not session semantics | `CallViewModel.leave$`, built from `remoteMembers$` and the room timeline |
|
||||
| `ringAttempts$`, `createCallNotificationLifecycle$`, `createSentCallNotification$`, `createReceivedDecline$` | ringing, pickup timeouts and decline events are MSC4075 call-notify UX | `src/state/CallViewModel/CallNotificationLifecycle.ts`, fed by `remoteMembers$` and the Matrix room |
|
||||
| `waitForCallPickup`, `autoLeaveWhenOthersLeft` options | inputs to the above | `CallViewModelOptions` |
|
||||
| `hideScreensharing` option | a UI policy; the session can always share a screen if the platform can | `CallViewModel` nulls `toggleScreenSharing` |
|
||||
| `ringingVm$`, `ringingStatusLocation` | view models | `CallViewModel` |
|
||||
| `hostBridge.hangUp$`, `hangup()`, `userHangup$` | widget / host integration | `CallViewModel` |
|
||||
| `ElementCallError` | carries Element Call translation keys | the view model wraps `MatrixRTCError`, reading `cause` |
|
||||
| `participantCount$` sound limits, join/leave sounds | call UX | `CallViewModel` |
|
||||
| `callIntent` | what a membership says about the call | `applicationData["m.call.intent"]`, set by the host |
|
||||
|
||||
Kept despite the name: `sendNotificationType`, because the js-sdk join sends the
|
||||
notification as part of entering the session. Only _reacting_ to it is
|
||||
call-specific.
|
||||
|
||||
## Media: `MemberMedia`
|
||||
|
||||
Every member has media. Today, in Element Call, that media _is_ a `livekit-client`
|
||||
participant, handed straight through to the view models. In the SDK the participant
|
||||
is wrapped in a `MemberMedia`, so that nothing outside the SDK imports
|
||||
`Participant`, `TrackPublication`, `Track` or `TrackReference`. The adapter in
|
||||
`sdk/src/media/` is the only place that reads a participant.
|
||||
|
||||
```ts
|
||||
export type MediaSource =
|
||||
"microphone" | "camera" | "screenShare" | "screenShareAudio";
|
||||
|
||||
export type MediaStreamStats =
|
||||
RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined;
|
||||
|
||||
/** One published track of a member. */
|
||||
export interface MediaTrack {
|
||||
source: MediaSource;
|
||||
kind: "audio" | "video";
|
||||
/** Stable for the life of the track. */
|
||||
id: string;
|
||||
muted$: Behavior<boolean>;
|
||||
/** False when the SFU reports the track as unencrypted. */
|
||||
encrypted$: Behavior<boolean>;
|
||||
/**
|
||||
* Inbound statistics for a remote track, outbound for a local one. Polled
|
||||
* once a second, but only while someone is subscribed, so a statistics
|
||||
* panel costs nothing while closed.
|
||||
*/
|
||||
stats$: Behavior<MediaStreamStats>;
|
||||
/**
|
||||
* Rendering. The view hands its <video> or <audio> element over; the SDK
|
||||
* sets its stream and, for video, registers the size and on-screen
|
||||
* observers that pick a simulcast layer and pause the subscription while
|
||||
* the element is hidden. attach is idempotent per element; detach has to be
|
||||
* called before the element leaves the DOM so those observers are released.
|
||||
*/
|
||||
attach(element: HTMLMediaElement): void;
|
||||
detach(element: HTMLMediaElement): void;
|
||||
}
|
||||
|
||||
export interface AudioMediaTrack extends MediaTrack {
|
||||
kind: "audio";
|
||||
/** Route playback through Web Audio, for earpiece pan and gain. Undefined resets. */
|
||||
setAudioContext(ctx: AudioContext | undefined, plugins?: AudioNode[]): void;
|
||||
/** No-op for our own audio, which is never played back. */
|
||||
setVolume(volume: number): void;
|
||||
}
|
||||
|
||||
export interface VideoMediaTrack extends MediaTrack {
|
||||
kind: "video";
|
||||
/** For mirroring and switching cameras. Undefined for remote tracks. */
|
||||
facingMode$?: Behavior<"user" | "environment" | undefined>;
|
||||
}
|
||||
|
||||
export type EncryptionError = "MissingKey" | "InvalidKey";
|
||||
|
||||
/**
|
||||
* The media of one member, backed by a LiveKit participant inside the SDK.
|
||||
* There is no identity field: the LiveKit identity is the member's `id`, and
|
||||
* the SDK does the matching before a MemberMedia exists.
|
||||
*/
|
||||
export interface MemberMedia {
|
||||
local: boolean;
|
||||
speaking$: Behavior<boolean>;
|
||||
screenShareEnabled$: Behavior<boolean>;
|
||||
|
||||
microphone$: Behavior<AudioMediaTrack | undefined>;
|
||||
camera$: Behavior<VideoMediaTrack | undefined>;
|
||||
screenShare$: Behavior<VideoMediaTrack | undefined>;
|
||||
screenShareAudio$: Behavior<AudioMediaTrack | undefined>;
|
||||
|
||||
/** Emits when the SFU reports a key problem for this member. */
|
||||
encryptionError$: Observable<EncryptionError>;
|
||||
}
|
||||
|
||||
export interface LocalMemberMedia extends MemberMedia {
|
||||
local: true;
|
||||
}
|
||||
```
|
||||
|
||||
### Rendering: `attach` and `detach`
|
||||
|
||||
The view passes its DOM element into the SDK and the SDK does the rest; there is no
|
||||
`MediaStreamTrack` on the interface. The reason is adaptive stream: Element Call
|
||||
runs with `adaptiveStream: true`, which means LiveKit decides the simulcast layer
|
||||
from the size of the attached video element and pauses the subscription at the
|
||||
SFU while no attached element is on screen. Those size and visibility observers are
|
||||
set up inside LiveKit's `attach`. Handing the element over is the only way to keep
|
||||
that without reimplementing the observers outside the SDK. The e2e tests check both
|
||||
effects: a larger tile receives a higher resolution, a hidden tile stops receiving
|
||||
frames.
|
||||
|
||||
What `attach` covers for free, so the host does not have to: `srcObject` and
|
||||
`autoplay` setup with the Safari and Firefox black-video workarounds; `play()` and
|
||||
the autoplay-blocked signal; following the track through mute, reconnect and
|
||||
re-subscription into every attached element; the Web Audio graph for earpiece pan
|
||||
and gain; the adaptive stream observers.
|
||||
|
||||
The interface is web-only. A non-DOM host would need a different media surface; if
|
||||
one shows up, a `MediaStreamTrack` accessor is added next to `attach`, not instead
|
||||
of it.
|
||||
|
||||
## Packaging
|
||||
|
||||
`sdk/` is packaged exactly as `component/` is: a `package.json` with peer
|
||||
dependencies (`matrix-js-sdk`, `livekit-client`, `rxjs`), `exports` with types,
|
||||
type emission through `sdk/tsconfig.build.json`, an externals list in
|
||||
`vite-sdk.config.ts` that `pnpm lint:externals` enforces, and installation from the
|
||||
repository as a git dependency with a `path:` suffix. `pnpm build:sdk` produces
|
||||
`sdk/dist`. It is not a pnpm workspace package yet; the root is not a workspace, and
|
||||
moving to one later carries the package name, entry point and externals over
|
||||
unchanged.
|
||||
|
||||
Element Call consumes the SDK by its package name from day one, so the swap to a
|
||||
published dependency is one line: `tsconfig.json` `paths` and the `resolve.alias`
|
||||
in `vitePluginsConfig` both map `@element-hq/matrixrtc-sdk` to `sdk/index.ts`. The
|
||||
alias has to live in `vitePluginsConfig`, which every config is built on. Put only
|
||||
into the app's own config, the harness resolves the name through `sdk/package.json`
|
||||
to whatever stale `sdk/dist` is lying around.
|
||||
|
||||
Layout:
|
||||
|
||||
```
|
||||
sdk/
|
||||
index.ts the entry point: re-exports the API and the primitives
|
||||
src/api.ts the public types above
|
||||
src/session/ MatrixRTCClient, the local member, publisher, connections, transports, join
|
||||
src/media/ the LiveKit adapter: MemberMedia and MediaTrack
|
||||
src/utils/ LazyBehavior, mapScoped, error wrapping
|
||||
dev/ the harness (below)
|
||||
SdkArchitecture.md this document
|
||||
SdkMigration.md how Element Call gets from CallViewModel to the client
|
||||
```
|
||||
|
||||
An import boundary is still to be added to the local lint plugin: nothing under
|
||||
`sdk/` may import `src/`, `component/`, `playwright/`, React, i18n, Compound or the
|
||||
settings. Until the shared modules have moved (see the migration document) the
|
||||
SDK imports them from `src/`, so the rule cannot be turned on yet.
|
||||
|
||||
## Dev harness: `sdk/dev/`
|
||||
|
||||
A web app that uses the SDK and nothing else: plain DOM, no React, no Compound, no
|
||||
i18n, imports only `@element-hq/matrixrtc-sdk`, `matrix-js-sdk` and `rxjs`. It is
|
||||
where the SDK is run during development and what the e2e tests in `playwright/sdk`
|
||||
drive. If something cannot be done in the harness without reaching into `src/`, the
|
||||
SDK API is missing something.
|
||||
|
||||
What it does: log in with a password, or with an access token the URL hands it (the
|
||||
tests register users through the admin API and skip `/login`, which the dev
|
||||
homeserver rate-limits); join the room and wait for the sync to hold it; create the
|
||||
client with microphone and camera enabled; show one tile per member with the display
|
||||
name, a `<video>` attached from `media$.camera$` and, for remote members, an
|
||||
`<audio>` from `media$.microphone$`; toggle the microphone and camera; leave.
|
||||
|
||||
Every media element carries the track's state as `data-*` attributes (`muted`,
|
||||
`encrypted`, `frameWidth`, `frames`), which is what the tests read to check mute
|
||||
propagation, encryption, adaptive resolution and paused subscriptions. That is more
|
||||
than the smallest consumer needs, and the price of testing the SDK through its
|
||||
public API only.
|
||||
|
||||
## Open decisions
|
||||
|
||||
- **Timings.** Delayed-leave timings, the sync grace period, the key rotation
|
||||
delay and participant limit come from `DEFAULT_CONFIG` today. The SDK must not
|
||||
read `config.json`, so they become an option (`timings`, with these defaults).
|
||||
Delegating the delayed leave to the SFU is not implemented, since it needs the
|
||||
same timings; the js-sdk restarts the delayed event itself meanwhile.
|
||||
- **`LocalMediaInputs` as behaviors or methods.** Behaviors match how the mute
|
||||
states drive the publisher and keep a lobby's pre-join state in one place;
|
||||
methods are the more conventional SDK shape. Two things are missing either way:
|
||||
creating the tracks before `join()` for a preview (today they are created at the
|
||||
join), and screen share capture options, which are fixed.
|
||||
- **`applicationData` after the join.** The option is read once. The implementation
|
||||
still updates `m.call.intent` from the camera input, as Element Call does; that
|
||||
moves to the host once the data can be updated, as a behavior or a setter.
|
||||
- **Backend overrides.** Tests of the client itself need to inject the transport,
|
||||
the connection factory and the LiveKit room. They were planned as
|
||||
`options.backend`, grouped so that no LiveKit type appears on the options; not
|
||||
yet added, so `createMatrixRTCClient` is covered by the e2e tests only.
|
||||
- **Raw local state.** The local member's state machine (`LocalMemberState`) stays
|
||||
internal; `status$` and `fatalError$` are its public view. A developer panel may
|
||||
want it as `LocalRTCMember.state$`.
|
||||
- **Developer panel.** `resolved$.identity` covers the local identity and
|
||||
`remoteMembers$` filtered by `transport$.id` the remote list. Anything beyond
|
||||
that (LiveKit room state, connection quality) needs an opaque `debug$` per
|
||||
transport.
|
||||
@@ -0,0 +1,328 @@
|
||||
# From `CallViewModel` to `MatrixRTCClient`
|
||||
|
||||
How Element Call gets from today's `CallViewModel`, which does everything, to a
|
||||
view model that consumes the SDK. The target shape is in
|
||||
[`SdkArchitecture.md`](./SdkArchitecture.md); this document is the inventory of
|
||||
what moves, what stays, and how far along it is. It refers to source files and
|
||||
symbols, so it ages with the code; the architecture document should not.
|
||||
|
||||
## Status
|
||||
|
||||
| Slice | State |
|
||||
| -------------------------------------------------------------------------------------------------------------- | -------------------- |
|
||||
| 1. Scaffold: `sdk/` packaging, the interface, the harness, the smoke test | done, October 2026 |
|
||||
| 2. First implementation of `createMatrixRTCClient` under `sdk/src`, the media adapter, the e2e suite | done, 1 October 2026 |
|
||||
| 3. Move the shared modules out of `src/` into `sdk/src/`, leave re-exports behind, turn on the import boundary | next |
|
||||
| 4. `createCallViewModel$` consumes the client; media view models take `media$` | after 3 |
|
||||
| 5. Remove the re-exports, `sdk-target-based-on-call-view-model/` and the old `build:sdk` output | last |
|
||||
|
||||
Slice 2 was built without touching `src/`, so the SDK imports its building blocks
|
||||
from there for now. Nothing in Element Call consumes the SDK yet.
|
||||
|
||||
## What the SDK still imports from `src/`
|
||||
|
||||
These move into `sdk/src/` in slice 3, each with its tests, leaving a one-line
|
||||
re-export in `src` until Element Call's importers are updated.
|
||||
|
||||
| Module | Used for | Notes for the move |
|
||||
| ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
|
||||
| `state/Behavior.ts`, `state/ObservableScope.ts`, `utils/observable.ts` | the reactive primitives | exported from `sdk/index.ts` already; 37 importers in `src` and `component` keep working through the re-export |
|
||||
| `state/SessionBehaviors.ts` | `createMemberships$`, `membershipsAndTransports$`, `createKeyRotationSuppressed$` | as is |
|
||||
| `state/CallViewModel/remoteMembers/Connection.ts` | `Connection`, `ConnectionState` | the SDK subclasses it as `ResolvedConnection` to keep the JWT answer; fold that in |
|
||||
| `state/CallViewModel/remoteMembers/ConnectionFactory.ts` | the `ConnectionFactory` interface only | `ECConnectionFactory` stays behind: it reads settings and `MediaDevices`; the SDK has `LivekitConnectionFactory` |
|
||||
| `state/CallViewModel/remoteMembers/ConnectionManager.ts` | `createConnectionManager$` | as is |
|
||||
| `state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts` | `createRemoteMatrixLivekitMembers$`, the membership-to-participant match | the `TaggedParticipant` type goes; the SDK maps the result to `RTCMember` |
|
||||
| `state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts` | `createRoomMembers$`, `createMatrixMemberMetadata$` | depends on `utils/displayname.ts`, which moves with it |
|
||||
| `state/CallViewModel/localMember/HomeserverConnected.ts` | `createHomeserverConnected$` | reads `Config.get()` unless the grace period is passed; the SDK passes it |
|
||||
| `state/CallViewModel/localMember/LocalTransport.ts` | the `LocalTransport` type only | `getLocalTransport` reads config and settings; the SDK has its own in `session/LocalTransport.ts` |
|
||||
| `state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts` | discovery through `rtc/transports` | takes a resolved config for the fallback url; the SDK passes `DEFAULT_CONFIG`, which has none |
|
||||
| `state/CallViewModel/localMember/LocalMember.ts` | `observeSharingScreen$` only | the SDK has its own local member; the rest stays until slice 4 |
|
||||
| `livekit/openIDSFU.ts` | `getSFUConfigWithOpenID`, `SFUConfig` | reads `Config.get()` only on the delegated delayed leave path, which the SDK does not use |
|
||||
| `livekit/options.ts` | `buildLiveKitOptions` | takes the media quality config as a parameter; `getLiveKitOptions` stays behind |
|
||||
| `e2ee/matrixKeyProvider.ts`, `e2ee/e2eeType.ts`, `e2ee/sharedKeyManagement.ts` | the key provider, `E2eeType`, the `EncryptionSystem` type | `sharedKeyManagement.ts` also holds React hooks; split the type out |
|
||||
| `config/ConfigOptions.ts` | `MatrixRTCMode`, `DEFAULT_CONFIG` for the timings | the enum moves; the timings become an option (see the architecture document's open decisions) |
|
||||
| `utils/errors.ts` | the error classes the modules above throw | they carry translation keys, so they stay; the SDK wraps them as `MatrixRTCError` with `cause` |
|
||||
|
||||
Rewritten in the SDK rather than moved, because the originals read settings,
|
||||
config, analytics or the host bridge: the local member (`session/LocalMember.ts`),
|
||||
the publisher (`session/Publisher.ts`), the connection factory and LiveKit room
|
||||
options (`session/ConnectionFactory.ts`), transport discovery
|
||||
(`session/LocalTransport.ts`), the join (`session/joinJsSdkSession.ts`) and the
|
||||
key provider selection (`session/KeyProvider.ts`).
|
||||
|
||||
## What becomes what in `createCallViewModel$`
|
||||
|
||||
Everything below `createCallViewModel$` that is not presentation and not in the
|
||||
call-specific table of the architecture document. Grouped by what it is today in
|
||||
`src/state/CallViewModel/CallViewModel.ts`.
|
||||
|
||||
| Binding today | In the SDK |
|
||||
| ---------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- |
|
||||
| `livekitKeyProvider` via `getE2eeKeyProvider` | internal, keyed off `options.encryptionSystem` |
|
||||
| `matrixRTCMode` resolution from `Config.get()` | an input; resolution stays in the caller |
|
||||
| `matrixRTCSession` parameter | internal: the session manager for the slot, `getRoomSession(room)` |
|
||||
| `memberships$`, `membershipsAndTransports` | internal; surfaces as `RTCMember.transport$` |
|
||||
| `ownMembershipIdentity` | internal; surfaces as `RTCMember.id` |
|
||||
| `localTransport$` via `getLocalTransport` | internal |
|
||||
| `connectionFactory`, `connectionManager` | internal; surfaces as `connectedTransports$` |
|
||||
| `remoteMatrixLivekitMembers$` | `remoteMembers$` |
|
||||
| `localMembership` via `createLocalMembership$`, `enterRTCSession`, `Publisher`, `delayId$` | internal; surfaces as `join`, `leave`, `status$`, `connected$`, `reconnecting$`, the screen share fields |
|
||||
| `localRtcMembership$`, `localMatrixLivekitMember$` | `localMember$` |
|
||||
| `matrixLivekitMembers$`, `participantCount$` | `memberCount$` |
|
||||
| `matrixRoomMembers$`, `matrixMemberMetadataStore` | `RTCMember.displayName$` / `avatarUrl$` |
|
||||
| `allConnections$` | `connectedTransports$` |
|
||||
| `livekitRoomItems$` | dropped; audio renders from `remoteMembers$[].media$` |
|
||||
| `keyRotationSuppressed$` | same |
|
||||
| `sharingScreen$`, `toggleScreenSharing` | `localMember$.sharingScreen$`, `.toggleScreenSharing` |
|
||||
| `errors$` and the `fatalError$` mapping | `fatalError$`, as `MatrixRTCError` |
|
||||
| `join`, `leave`, `connected$`, `reconnecting$`, `screenShareError$`, `dismissScreenShareError` | same names on `MatrixRTCClient` / `LocalRTCMember` |
|
||||
| `callIntent` option, `updateCallIntent` from the camera | `applicationData["m.call.intent"]`; the camera sync is still in the SDK, see the open decisions |
|
||||
|
||||
Changes to the call sites that the rename brings: `localMember$` and
|
||||
`remoteMembers$` replace `localMatrixLivekitMember$` and
|
||||
`remoteMatrixLivekitMembers$`; `connection$` on a member goes away, since the view
|
||||
model only ever used `livekitRoom` (for `waitingForMedia$` and encryption errors,
|
||||
both on `MemberMedia` now) and `transport.livekit_service_url` (now
|
||||
`transport$.id`); `connectedTransports$` replaces `allConnections$`, since the
|
||||
developer panel only needed which transports exist and whether they are up.
|
||||
|
||||
## What stays in `CallViewModel`
|
||||
|
||||
Everything in the call-specific table, plus everything that takes `userMedia$` or a
|
||||
UI input and turns it into something to draw:
|
||||
|
||||
- Media view models (`createWrappedUserMedia` and below), built from
|
||||
`remoteMembers$` / `localMember$` instead of participants.
|
||||
- `CallNotificationLifecycle.ts` in full: `ringAttempts$`, `autoLeave$`,
|
||||
`ringingMedia$`, `ringingStatusLocation`.
|
||||
- `leave$` (= `autoLeave$` + `hangup()` + `hostBridge.hangUp$`), `hangup()`.
|
||||
- `screenShares$`.
|
||||
- Sounds: `joinSoundEffect$`, `leaveSoundEffect$`, `newHandRaised$`,
|
||||
`newScreenShare$`, `audibleReactions$`, `visibleReactions$`.
|
||||
- `handsRaised$`, `reactions$`, until they move onto `RTCMember`.
|
||||
- All layout: `spotlightSpeaker$`, `grid$`, `spotlight$`, `pip*`, `windowMode$`,
|
||||
`layoutSwitchVm`, every `*LayoutMedia$`, `layout$`, `visibleTiles$`,
|
||||
`tileStoreGeneration$`, `overflowing$`, `spotlightExpanded$`.
|
||||
- Chrome: `showHeader$`, `showFooter$`, `edgeToEdge$`, `showModals$`,
|
||||
`settingsOpen$`, `showSpotlightIndicators$`, `showSpeakingIndicators$`,
|
||||
`showNameTags$`, tap and hover subjects.
|
||||
- Audio routing: `earpieceMode$`, `audioOutputSwitcher$`, which read
|
||||
`MediaDevices`.
|
||||
- `CallViewModelOptions` keeps `hostBridge`, `controlledAudioDevices`, `header`,
|
||||
`showControls`, `hideScreensharing`, `waitForCallPickup`,
|
||||
`autoLeaveWhenOthersLeft`, `windowSize$`, `toggleScreensharing` (test override),
|
||||
and forwards `encryptionSystem`, `matrixRTCMode`, `sendNotificationType` and
|
||||
`applicationData` to `MatrixRTCClientOptions`.
|
||||
|
||||
## The media view models and the participant
|
||||
|
||||
The goal of slice 4 on the media side: nothing in Element Call imports
|
||||
`Participant`, `RemoteParticipant`, `LocalParticipant`, `TrackPublication`,
|
||||
`Track` or `TrackReference`. This is where the participant flows today, and what
|
||||
each use becomes.
|
||||
|
||||
### Where the participant flows today
|
||||
|
||||
Start: `remoteMatrixLivekitMembers$` in `CallViewModel.ts`, built by
|
||||
`createRemoteMatrixLivekitMembers$` in
|
||||
`src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts`, as
|
||||
`participant: { type: "remote"; value$: Behavior<RemoteParticipant | null> }`.
|
||||
Consumers, in order of depth:
|
||||
|
||||
1. `MatrixLivekitMembers.ts` matches `participant.identity` against
|
||||
`membership.rtcBackendIdentity`; logs `participant.sid`.
|
||||
2. `CallViewModel.ts` `livekitRoomItems$` reads `isLocal`, `identity`, hands
|
||||
`connection.livekitRoom` plus identities to `LivekitRoomAudioRenderer`.
|
||||
3. `CallViewModel.ts` `userMedia$` passes the whole `TaggedParticipant` into
|
||||
`createWrappedUserMedia`.
|
||||
4. `src/state/media/WrappedUserMediaViewModel.ts`: `observeParticipantEvents(p,
|
||||
TrackPublished, TrackUnpublished, LocalTrackPublished, LocalTrackUnpublished)`
|
||||
then `p.isScreenShareEnabled` to spawn screen share view models.
|
||||
5. `src/state/media/UserMediaViewModel.ts`: `observeParticipantMedia(p)` for
|
||||
`microphoneTrack?.isMuted` / `cameraTrack?.isMuted`; `IsSpeakingChanged` →
|
||||
`p.isSpeaking`; `observeRtpStreamStats$(p, source, type)`.
|
||||
6. `src/state/media/MemberMediaViewModel.ts`: `observeTrackReference$(p, source)`
|
||||
for audio and video; `publication.isEncrypted`; `p.isLocal`; `p.identity`
|
||||
matched inside `RoomEvent.EncryptionError` messages; inbound RTP frame counters.
|
||||
7. `src/state/media/RemoteUserMediaViewModel.ts`, `RemoteScreenShareViewModel.ts`:
|
||||
`p.setVolume(v)` / `p.setVolume(v, ScreenShareAudio)`; `participant === null`
|
||||
means "waiting for media".
|
||||
8. `src/state/observeTrackReference.ts`: `p.getTrackPublication(source)` wrapped as
|
||||
`{ participant, publication, source }`.
|
||||
9. `src/state/media/observeRtpStreamStats.ts`: `publication.track` must be
|
||||
`RemoteTrack | LocalTrack`, then `track.getRTCStatsReport()`.
|
||||
10. Views: `src/tile/MediaView.tsx` renders `<VideoTrack trackRef={video}>` from
|
||||
`@livekit/components-react`; `src/livekit/MatrixAudioRenderer.tsx` uses
|
||||
`useTracks` on the LiveKit room, filters on `ref.participant.isLocal` /
|
||||
`.identity`, and calls `RemoteAudioTrack.setAudioContext` /
|
||||
`setWebAudioPlugins`.
|
||||
|
||||
Local-only extras, same shape but on `LocalParticipant`
|
||||
(`localMember/LocalMember.ts`, `Publisher.ts`,
|
||||
`src/state/media/LocalUserMediaViewModel.ts`): `trackPublications`,
|
||||
`audioTrackPublications`, `setScreenShareEnabled`, `setMicrophoneEnabled`,
|
||||
`setCameraEnabled`, `enableCameraAndMicrophone`, `isMicrophoneEnabled`,
|
||||
`isCameraEnabled`, `unpublishTrack`, and on the camera `LocalVideoTrack`:
|
||||
`facingModeFromLocalTrack`, `restartTrack`, `mediaStreamTrack`,
|
||||
`TrackEvent.Restarted` / `Muted`.
|
||||
|
||||
### Participant API actually used, and what covers it
|
||||
|
||||
| LiveKit surface | Used for | Used in | Now |
|
||||
| --------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | ----------- | ------------------------------------------------ |
|
||||
| `identity` | match to `rtcBackendIdentity`; audio renderer allow-list; E2EE error matching; logs | 1, 2, 6, 10 | matched inside the SDK; `RTCMember.id` |
|
||||
| `sid` | logs only | 1 | SDK logs |
|
||||
| `isLocal` | skip local audio; encryption status short-circuit | 2, 6, 10 | `MemberMedia.local` |
|
||||
| `isSpeaking` + `IsSpeakingChanged` | `speaking$` | 5 | `MemberMedia.speaking$` |
|
||||
| `isScreenShareEnabled` + publish events | spawn screen share VM | 4 | `MemberMedia.screenShareEnabled$` |
|
||||
| `observeParticipantMedia(p)` → `microphoneTrack?.isMuted`, `cameraTrack?.isMuted` | `audioEnabled$`, `videoEnabled$` | 5 | `microphone$` / `camera$` → `muted$` |
|
||||
| `getTrackPublication(source)` for the four sources | `TrackReference` for video, encryption, stats, screen share audio presence | 6, 7, 8 | the four track behaviors |
|
||||
| `publication.isEncrypted` | `unencryptedWarning$` | 6 | `MediaTrack.encrypted$` |
|
||||
| `publication.track.getRTCStatsReport()` | `audioStreamStats$`, `videoStreamStats$`, "receiving okay" heuristic | 5, 6, 9 | `MediaTrack.stats$` |
|
||||
| `setVolume(v)`, `setVolume(v, ScreenShareAudio)` | `VolumeControls` sink | 7 | `AudioMediaTrack.setVolume` |
|
||||
| `TrackReference` → `<VideoTrack>` | render video | 10 | `MediaTrack.attach` on a `<video>` |
|
||||
| room `useTracks` + `RemoteAudioTrack.setAudioContext/setWebAudioPlugins` | render audio with earpiece pan/gain | 10 | one `<audio>` per audio track, `setAudioContext` |
|
||||
| room `RoomEvent.EncryptionError` with `identity` + `"MissingKey"`/`"InvalidKey"` | `encryptionStatus$` | 6 | `MemberMedia.encryptionError$` |
|
||||
|
||||
Everything else in the media view models (`observeSpeaker$`, `VolumeControls`,
|
||||
`SortingBin`, hand raise, reactions, display name, avatar, aspect ratio) already
|
||||
works on plain booleans and observables and does not change.
|
||||
|
||||
### How the call sites map onto `MemberMedia`
|
||||
|
||||
- `audioEnabled$` = `microphone$ → track?.muted$ === false`; `videoEnabled$`
|
||||
likewise from `camera$`.
|
||||
- `unencryptedWarning$` = any of the four tracks with `encrypted$ === false`.
|
||||
- `encryptionStatus$` = `encryptionError$` plus a "frames decoded" check on
|
||||
`camera$.stats$` / `microphone$.stats$`; same logic, no participant.
|
||||
- `screenShares$` in `WrappedUserMediaViewModel` keys off `screenShareEnabled$`.
|
||||
- `VolumeControls` sink = `microphone$.value?.setVolume` /
|
||||
`screenShareAudio$.value?.setVolume`.
|
||||
- `waitingForMedia$` = transport present and `media === null`, unchanged
|
||||
semantics; `TaggedParticipant` goes away.
|
||||
- `video$: Behavior<TrackReference | undefined>` on `BaseMemberMediaViewModel`
|
||||
becomes `Behavior<VideoMediaTrack | undefined>`; `MediaView` replaces
|
||||
`<VideoTrack trackRef>` with a `<video>` plus `attach` / `detach` in an effect,
|
||||
keeping the `onResize` callback for the aspect ratio. `VideoPreview.tsx` already
|
||||
follows this pattern.
|
||||
- `LivekitRoomAudioRenderer` stops using `useTracks` on the room and renders one
|
||||
`<audio>` per `microphone$` / `screenShareAudio$` across `remoteMembers$`. That
|
||||
removes the "no matching matrix call member" warning path, since an unmatched
|
||||
track can no longer be seen.
|
||||
- The `showConnectionStats` setting gates the subscription to `stats$` in the view
|
||||
model; the SDK polls only while subscribed.
|
||||
- Stories and tests mock `attach` by setting `srcObject` from
|
||||
`canvas.captureStream()` on the element, or by doing nothing and asserting the
|
||||
call.
|
||||
|
||||
## Packaging, as it was planned
|
||||
|
||||
`sdk/` mirrors `component/` file for file. The table that drove the scaffold, kept
|
||||
for the record:
|
||||
|
||||
| `component/` | `sdk/` | Differences |
|
||||
| ------------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `package.json` (`@element-hq/element-call-component`) | `package.json` (`@element-hq/matrixrtc-sdk`) | peers: `matrix-js-sdk`, `livekit-client`, `rxjs`. No React, no Compound. `exports`: `"."` only. No `./style.css`, no `sideEffects`. Same `prepare`. |
|
||||
| `pnpm-workspace.yaml` (marker, `autoInstallPeers: false`) | same | none |
|
||||
| `tsconfig.build.json` | same | `include: ["./index.ts"]` |
|
||||
| `index.tsx` + `api.ts` | `index.ts` + `src/api.ts` | one entry; no separate types entry until a host asks |
|
||||
| `host.ts`, `localization.ts`, `ElementCall.module.css`, `build/scopeStylesToRoot.ts` | none | no host bridge, no i18n, no styles |
|
||||
| `dev/` on port 3001 | `dev/` on port 3002 via `vite-sdk-dev.config.ts` | plain DOM |
|
||||
| `vite-component.config.ts` | `vite-sdk.config.ts` | `vitePluginsConfig({ mode, html: false })`, `outDir: "sdk/dist"`, externals = js-sdk and LiveKit entries plus `rxjs` |
|
||||
| root scripts `build:component*` | `build:sdk*` | `build:sdk:types` = `tsc -p sdk/tsconfig.build.json` |
|
||||
| root `tsconfig.json` `include` | adds `./sdk/**/*.ts` | |
|
||||
| `scripts/check-component-externals.mjs` | parameterised by config; `sdk` added to the sources, `sdk/dev` excluded | |
|
||||
| `knip.ts` | `vite-sdk.config.ts`, `vite-sdk-dev.config.ts`, `sdk/index.ts` as entry | |
|
||||
| `vitest.config.ts` unit project | adds `sdk/**/*.test.ts` | |
|
||||
| `package.json` `lint:oxlint` paths | adds `sdk` | |
|
||||
|
||||
Why not a pnpm workspace package yet: the root has no `packages:` in
|
||||
`pnpm-workspace.yaml`, so a workspace split means restructuring the app, the
|
||||
component's `prepare` trick, CI and the lockfile at the same time as moving the
|
||||
code, and dev-time HMR across workspace packages needs the same source alias
|
||||
anyway. It is the right end state if the SDK gets a release cadence of its own.
|
||||
|
||||
## Reference: the `CallViewModel` public API before the migration
|
||||
|
||||
Source: `src/state/CallViewModel/CallViewModel.ts`. Only consumer outside tests:
|
||||
`src/room/InCallView.tsx`. A snapshot, so that the split can be checked against
|
||||
what was there.
|
||||
|
||||
### Factory
|
||||
|
||||
`createCallViewModel$(scope, matrixRTCSession, matrixRoom, mediaDevices, muteStates, options, handsRaisedSubject$, reactionsSubject$, trackProcessorState$): CallViewModel`
|
||||
|
||||
- `scope: ObservableScope`: ending the scope is one of the two ways to leave.
|
||||
- `matrixRTCSession: MatrixRTCSession`, `matrixRoom: MatrixRoom`: one room per view model.
|
||||
- `mediaDevices: MediaDevices`, `muteStates: MuteStates`
|
||||
- `options: CallViewModelOptions` (below)
|
||||
- `handsRaisedSubject$: Observable<Record<string, RaisedHandInfo>>`
|
||||
- `reactionsSubject$: Observable<Record<string, ReactionInfo>>`
|
||||
- `trackProcessorState$: Behavior<ProcessorState>`
|
||||
|
||||
Helper: `callViewModelOptionsFromParams(params: UrlParams)` picks
|
||||
`controlledAudioDevices`, `header`, `showControls`, `hideScreensharing`,
|
||||
`sendNotificationType`, `callIntent` out of the URL params. `autoLeaveWhenOthersLeft`
|
||||
and `waitForCallPickup` are left to the caller.
|
||||
|
||||
### `CallViewModelOptions`
|
||||
|
||||
| Field | Type | Note |
|
||||
| ----------------------------------------------------------------------------------------------------------- | --------------------------- | --------------------------------------------------- |
|
||||
| `encryptionSystem` | `EncryptionSystem` | required |
|
||||
| `hostBridge?` | `HostBridge` | host can hang up / observe join+leave; default none |
|
||||
| `controlledAudioDevices?` | `boolean` | host controls audio output devices |
|
||||
| `header?` | `HeaderStyle` | default `Standard` |
|
||||
| `showControls?` | `boolean` | default `true` |
|
||||
| `hideScreensharing?` | `boolean` | default `false` |
|
||||
| `sendNotificationType?` | `RTCNotificationType` | notification sent on join |
|
||||
| `callIntent?` | `RTCCallIntent` | |
|
||||
| `autoLeaveWhenOthersLeft?` | `boolean` | |
|
||||
| `waitForCallPickup?` | `boolean` | telephone-style ringing UI |
|
||||
| `windowSize$` | `Behavior<{width; height}>` | required; drives `WindowMode` |
|
||||
| `matrixRTCMode?` | `MatrixRTCMode` | MatrixRTC version / compat mode |
|
||||
| `livekitRoomFactory?`, `connectionState$?`, `localTransport?`, `connectionFactory?`, `toggleScreensharing?` | | test overrides |
|
||||
|
||||
### The interface, grouped
|
||||
|
||||
**Lifecycle / connection** (to the client): `join()`, `leave()`, `hangup()`,
|
||||
`leave$: Observable<"user" | AutoLeaveReason>`, `autoLeave$`, `connected$`,
|
||||
`reconnecting$`, `allConnections$: Behavior<ConnectionManagerData>`,
|
||||
`fatalError$: Behavior<ElementCallError | null>`, `keyRotationSuppressed$`.
|
||||
|
||||
**Members / participants** (to the client): `participantCount$`,
|
||||
`localMatrixLivekitMember$`, `remoteMatrixLivekitMembers$`, `livekitRoomItems$`,
|
||||
`handsRaised$`, `reactions$` (keyed by `${userId}:${deviceId}`).
|
||||
|
||||
**Screen sharing** (capability to the client, error UI stays): `toggleScreenSharing`,
|
||||
`sharingScreen$`, `screenShareError$`, `dismissScreenShareError()`.
|
||||
|
||||
**Ringing** (stays): `ringingVm$`, `ringingStatusLocation`.
|
||||
|
||||
**Audio routing** (stays): `earpieceMode$`, `audioOutputSwitcher$`.
|
||||
|
||||
**Sounds and transient events** (stay): `joinSoundEffect$`, `leaveSoundEffect$`,
|
||||
`newHandRaised$`, `newScreenShare$`, `audibleReactions$`, `visibleReactions$`,
|
||||
`MAX_PARTICIPANT_COUNT_FOR_SOUND = 8`, `THROTTLE_SOUND_EFFECT_MS = 500`.
|
||||
|
||||
**Layout and tiles** (stay): `layout$`, `tileStoreGeneration$`,
|
||||
`showSpotlightIndicators$`, `showSpeakingIndicators$`, `showNameTags$`,
|
||||
`spotlightExpanded$`, `toggleSpotlightExpanded$`, `layoutSwitchVm$`.
|
||||
|
||||
**Chrome visibility and UI interaction** (stay): `showHeader$`, `showFooter$`,
|
||||
`edgeToEdge$`, `overflowing$`, `showModals$`, `settingsOpen$`, `setSettingsOpen$`,
|
||||
`tapScreen()`, `tapControls()`, `hoverScreen()`, `unhoverScreen()`.
|
||||
|
||||
### Pending rename noted in the source
|
||||
|
||||
A TODO above `CallViewModelOptions` proposes: member/membership → `rtcMember`,
|
||||
participant → `livekitParticipant`, matrixLivekitItem → `callMember`; and in the
|
||||
js-sdk `callMembership` → `rtcMembership`. The SDK's vocabulary (architecture
|
||||
document, "Vocabulary") settles the first three; the js-sdk rename is still open.
|
||||
|
||||
### Build targets before the migration
|
||||
|
||||
`vite-sdk.config.ts` (`pnpm build:sdk`), `vite-component.config.ts`,
|
||||
`vite-embedded.config.ts`, `vite.config.ts`. The old SDK target
|
||||
(`sdk-target-based-on-call-view-model/`) is the one whose dependency direction the
|
||||
migration flips; it goes in slice 5.
|
||||
+12
-12
@@ -21,13 +21,13 @@ import {
|
||||
} from "rxjs";
|
||||
import {
|
||||
constant,
|
||||
createRtcSession,
|
||||
createMatrixRTCClient,
|
||||
E2eeType,
|
||||
MatrixRTCMode,
|
||||
type MediaTrack,
|
||||
ObservableScope,
|
||||
type RtcMember,
|
||||
type RtcSession,
|
||||
type RTCMember,
|
||||
type MatrixRTCClient,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import { createSession, joinRoom, type Login } from "./session";
|
||||
@@ -73,7 +73,7 @@ async function start(
|
||||
const scope = new ObservableScope();
|
||||
const microphoneEnabled$ = new BehaviorSubject(true);
|
||||
const cameraEnabled$ = new BehaviorSubject(true);
|
||||
const session = createRtcSession(
|
||||
const rtcClient = createMatrixRTCClient(
|
||||
scope,
|
||||
client,
|
||||
room,
|
||||
@@ -89,20 +89,20 @@ async function start(
|
||||
matrixRTCMode: MatrixRTCMode.Compatibility,
|
||||
},
|
||||
);
|
||||
session.status$.pipe(scope.bind()).subscribe((s) => {
|
||||
rtcClient.status$.pipe(scope.bind()).subscribe((s) => {
|
||||
status.textContent = s;
|
||||
});
|
||||
session.fatalError$.pipe(scope.bind()).subscribe((error) => {
|
||||
rtcClient.fatalError$.pipe(scope.bind()).subscribe((error) => {
|
||||
if (error !== null) status.textContent = `Error: ${error.message}`;
|
||||
});
|
||||
showMembers(scope, session);
|
||||
session.join();
|
||||
showMembers(scope, rtcClient);
|
||||
rtcClient.join();
|
||||
|
||||
toggle(buttons.microphone, microphoneEnabled$);
|
||||
toggle(buttons.camera, cameraEnabled$);
|
||||
buttons.leave.hidden = false;
|
||||
buttons.leave.onclick = (): void => {
|
||||
session.leave();
|
||||
rtcClient.leave();
|
||||
scope.end();
|
||||
members.replaceChildren();
|
||||
for (const button of Object.values(buttons)) button.hidden = true;
|
||||
@@ -125,9 +125,9 @@ function toggle(
|
||||
};
|
||||
}
|
||||
|
||||
function showMembers(scope: ObservableScope, session: RtcSession): void {
|
||||
function showMembers(scope: ObservableScope, rtcClient: MatrixRTCClient): void {
|
||||
const tiles = new Map<string, HTMLElement>();
|
||||
combineLatest([session.localMember$, session.remoteMembers$])
|
||||
combineLatest([rtcClient.localMember$, rtcClient.remoteMembers$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([local, remote]) => {
|
||||
const current = local === null ? remote : [local, ...remote];
|
||||
@@ -145,7 +145,7 @@ function showMembers(scope: ObservableScope, session: RtcSession): void {
|
||||
});
|
||||
}
|
||||
|
||||
function memberTile(scope: ObservableScope, member: RtcMember): HTMLElement {
|
||||
function memberTile(scope: ObservableScope, member: RTCMember): HTMLElement {
|
||||
const tile = document.createElement("section");
|
||||
tile.dataset.testid = "member";
|
||||
tile.dataset.userId = member.userId;
|
||||
|
||||
+1
-1
@@ -24,4 +24,4 @@ export { type EncryptionSystem } from "../src/e2ee/sharedKeyManagement";
|
||||
export { MatrixRTCMode } from "../src/config/ConfigOptions";
|
||||
|
||||
export * from "./src/api";
|
||||
export { createRtcSession } from "./src/session/RtcSession";
|
||||
export { createMatrixRTCClient } from "./src/session/MatrixRTCClient";
|
||||
|
||||
+25
-17
@@ -6,13 +6,12 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The public types of the SDK. `createRtcSession` in `session/RtcSession.ts`
|
||||
* The public types of the SDK. `createMatrixRTCClient` in `session/MatrixRTCClient.ts`
|
||||
* is the only way to obtain an implementation of them.
|
||||
*/
|
||||
|
||||
import {
|
||||
type CallMembership,
|
||||
type RTCCallIntent,
|
||||
type RTCNotificationType,
|
||||
type Transport,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
@@ -26,17 +25,26 @@ import { type MatrixRTCMode } from "../../src/config/ConfigOptions";
|
||||
// ---------------------------------------------------------------------------
|
||||
// Session
|
||||
|
||||
export interface RtcSessionOptions {
|
||||
export interface MatrixRTCClientOptions {
|
||||
encryptionSystem: EncryptionSystem;
|
||||
/** Resolved by the host; the SDK reads neither config.json nor settings. */
|
||||
matrixRTCMode: MatrixRTCMode;
|
||||
/**
|
||||
* MSC4075 notification sent with the join. Parameters of the MatrixRTC join
|
||||
* itself, so they are here even though they are named after calls; reacting
|
||||
* MSC4075 notification sent with the join. A parameter of the MatrixRTC
|
||||
* join itself, so it is here even though it is named after calls; reacting
|
||||
* to a notification (ringing, timeouts, declines) is the application's job.
|
||||
*/
|
||||
sendNotificationType?: RTCNotificationType;
|
||||
callIntent?: RTCCallIntent;
|
||||
/** The application the session belongs to, as named in the membership. Default `m.call`. */
|
||||
application?: string;
|
||||
/** The application's slot in the room. Default `ROOM`. */
|
||||
slot?: string;
|
||||
/**
|
||||
* Whatever the application wants to say about itself in the membership.
|
||||
* Opaque to the SDK. Today the js-sdk carries one key, `m.call.intent`
|
||||
* (`"audio"` or `"video"`); anything else is dropped until it can.
|
||||
*/
|
||||
applicationData?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** What the local member publishes. */
|
||||
@@ -49,7 +57,7 @@ export interface LocalMediaInputs {
|
||||
videoProcessor$: Behavior<TrackProcessor<Track.Kind.Video> | undefined>;
|
||||
}
|
||||
|
||||
export type SessionConnectionStatus =
|
||||
export type ConnectionStatus =
|
||||
| "waitingForTransport"
|
||||
| "connecting"
|
||||
| "connected"
|
||||
@@ -60,10 +68,10 @@ export type SessionConnectionStatus =
|
||||
* An error raised by the session. `cause` holds the underlying error, which
|
||||
* lets a host that knows the backend tell the failures apart.
|
||||
*/
|
||||
export class RtcSessionError extends Error {
|
||||
export class MatrixRTCError extends Error {
|
||||
public constructor(message: string, options?: ErrorOptions) {
|
||||
super(message, options);
|
||||
this.name = "RtcSessionError";
|
||||
this.name = "MatrixRTCError";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,18 +81,18 @@ export class RtcSessionError extends Error {
|
||||
* detached copy `joinRoom` returns for a room joined just now never receives
|
||||
* the state the members are read from.
|
||||
*/
|
||||
export interface RtcSession {
|
||||
export interface MatrixRTCClient {
|
||||
join(): void;
|
||||
leave(): void;
|
||||
/** Collapsed view of the local member's state machine. */
|
||||
status$: Behavior<SessionConnectionStatus>;
|
||||
status$: Behavior<ConnectionStatus>;
|
||||
connected$: Behavior<boolean>;
|
||||
reconnecting$: Behavior<boolean>;
|
||||
/** A transport, Matrix or connection error that stops the session. */
|
||||
fatalError$: Behavior<RtcSessionError | null>;
|
||||
fatalError$: Behavior<MatrixRTCError | null>;
|
||||
|
||||
localMember$: Behavior<LocalRtcMember | null>;
|
||||
remoteMembers$: Behavior<RemoteRtcMember[]>;
|
||||
localMember$: Behavior<LocalRTCMember | null>;
|
||||
remoteMembers$: Behavior<RemoteRTCMember[]>;
|
||||
/** `remoteMembers.length`, plus one for the local member once it exists. */
|
||||
memberCount$: Behavior<number>;
|
||||
|
||||
@@ -135,7 +143,7 @@ export type ResolvedTransport =
|
||||
// ---------------------------------------------------------------------------
|
||||
// Members
|
||||
|
||||
export interface RtcMember {
|
||||
export interface RTCMember {
|
||||
local: boolean;
|
||||
/** The identity the media backend knows this member by. */
|
||||
id: string;
|
||||
@@ -153,11 +161,11 @@ export interface RtcMember {
|
||||
media$: Behavior<MemberMedia | null>;
|
||||
}
|
||||
|
||||
export interface RemoteRtcMember extends RtcMember {
|
||||
export interface RemoteRTCMember extends RTCMember {
|
||||
local: false;
|
||||
}
|
||||
|
||||
export interface LocalRtcMember extends RtcMember {
|
||||
export interface LocalRTCMember extends RTCMember {
|
||||
local: true;
|
||||
media$: Behavior<LocalMemberMedia | null>;
|
||||
sharingScreen$: Behavior<boolean>;
|
||||
|
||||
@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type BaseKeyProvider, ExternalE2EEKeyProvider } from "livekit-client";
|
||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
import { type MatrixRTCSession as JsSdkRtcSession } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type MatrixRTCSession as JsSdkRTCSession } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { E2eeType } from "../../../src/e2ee/e2eeType";
|
||||
import { MatrixKeyProvider } from "../../../src/e2ee/matrixKeyProvider";
|
||||
@@ -16,7 +16,7 @@ import { type EncryptionSystem } from "../../../src/e2ee/sharedKeyManagement";
|
||||
/** The LiveKit key provider for the encryption the host asked for, or none. */
|
||||
export function createKeyProvider(
|
||||
encryptionSystem: EncryptionSystem,
|
||||
session: JsSdkRtcSession,
|
||||
session: JsSdkRTCSession,
|
||||
logger: Logger,
|
||||
): BaseKeyProvider | undefined {
|
||||
switch (encryptionSystem.kind) {
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
import {
|
||||
type LivekitTransport,
|
||||
type MatrixRTCSession as JsSdkRtcSession,
|
||||
type MatrixRTCSession as JsSdkRTCSession,
|
||||
type Status as RTCSessionStatus,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { deepCompare } from "matrix-js-sdk/lib/utils";
|
||||
@@ -41,8 +41,8 @@ import {
|
||||
ConnectionState,
|
||||
} from "../../../src/state/CallViewModel/remoteMembers/Connection";
|
||||
import { type IConnectionManager } from "../../../src/state/CallViewModel/remoteMembers/ConnectionManager";
|
||||
import { type RtcSessionError } from "../api";
|
||||
import { toRtcSessionError } from "../utils/errors";
|
||||
import { type MatrixRTCError } from "../api";
|
||||
import { toMatrixRTCError } from "../utils/errors";
|
||||
import { type LocalTransport } from "./LocalTransport";
|
||||
import { type Publisher } from "./Publisher";
|
||||
|
||||
@@ -56,16 +56,16 @@ export enum PublishState {
|
||||
}
|
||||
|
||||
export type LocalMemberMediaState =
|
||||
| { connection: ConnectionState | RtcSessionError }
|
||||
| { connection: ConnectionState | MatrixRTCError }
|
||||
| PublishState
|
||||
| RtcSessionError;
|
||||
| MatrixRTCError;
|
||||
|
||||
export type LocalMemberState =
|
||||
| RtcSessionError
|
||||
| MatrixRTCError
|
||||
| TransportState.Waiting
|
||||
| {
|
||||
media: LocalMemberMediaState;
|
||||
matrix: RtcSessionError | RTCSessionStatus;
|
||||
matrix: MatrixRTCError | RTCSessionStatus;
|
||||
};
|
||||
|
||||
interface Props {
|
||||
@@ -78,7 +78,7 @@ interface Props {
|
||||
/** The membership manager giving up on keeping the membership alive. */
|
||||
membershipManagerError$: Observable<unknown>;
|
||||
matrixRTCSession: Pick<
|
||||
JsSdkRtcSession,
|
||||
JsSdkRTCSession,
|
||||
"updateCallIntent" | "leaveRoomSession"
|
||||
>;
|
||||
cameraEnabled$: Behavior<boolean>;
|
||||
@@ -121,10 +121,10 @@ export function createLocalMembership$({
|
||||
}: Props): LocalMembership {
|
||||
const logger = parentLogger.getChild("[LocalMember]");
|
||||
|
||||
const fatalTransportError$ = new Subject<RtcSessionError>();
|
||||
const fatalTransportError$ = new Subject<MatrixRTCError>();
|
||||
const localTransport$ = localTransportWithErrors$.pipe(
|
||||
catchError((e: unknown) => {
|
||||
fatalTransportError$.next(toRtcSessionError(e));
|
||||
fatalTransportError$.next(toMatrixRTCError(e));
|
||||
return NEVER;
|
||||
}),
|
||||
);
|
||||
@@ -147,8 +147,8 @@ export function createLocalMembership$({
|
||||
|
||||
const joinRequested$ = new BehaviorSubject(false);
|
||||
const publisher$ = new BehaviorSubject<Publisher | null>(null);
|
||||
const publishError$ = new BehaviorSubject<RtcSessionError | null>(null);
|
||||
const matrixError$ = new BehaviorSubject<RtcSessionError | null>(null);
|
||||
const publishError$ = new BehaviorSubject<MatrixRTCError | null>(null);
|
||||
const matrixError$ = new BehaviorSubject<MatrixRTCError | null>(null);
|
||||
|
||||
scope.reconcile(connection$, async (connection) => {
|
||||
if (connection === null) return;
|
||||
@@ -171,7 +171,7 @@ export function createLocalMembership$({
|
||||
} else if (publisher.shouldPublish) await publisher.stopPublishing();
|
||||
} catch (e) {
|
||||
if (publishError$.value === null)
|
||||
publishError$.next(toRtcSessionError(e));
|
||||
publishError$.next(toMatrixRTCError(e));
|
||||
else logger.error("Another publish error", e);
|
||||
}
|
||||
},
|
||||
@@ -185,8 +185,7 @@ export function createLocalMembership$({
|
||||
joinMatrixRTC(transport.transport);
|
||||
} catch (e) {
|
||||
logger.error("Failed to enter the session", e);
|
||||
if (matrixError$.value === null)
|
||||
matrixError$.next(toRtcSessionError(e));
|
||||
if (matrixError$.value === null) matrixError$.next(toMatrixRTCError(e));
|
||||
}
|
||||
return Promise.resolve(async (): Promise<void> => {
|
||||
try {
|
||||
@@ -200,7 +199,7 @@ export function createLocalMembership$({
|
||||
|
||||
membershipManagerError$.pipe(scope.bind()).subscribe((e) => {
|
||||
logger.error("The membership manager stopped", e);
|
||||
if (matrixError$.value === null) matrixError$.next(toRtcSessionError(e));
|
||||
if (matrixError$.value === null) matrixError$.next(toMatrixRTCError(e));
|
||||
});
|
||||
|
||||
cameraEnabled$.pipe(scope.bind()).subscribe((videoEnabled) => {
|
||||
@@ -224,7 +223,7 @@ export function createLocalMembership$({
|
||||
return {
|
||||
connection:
|
||||
connectionState instanceof Error
|
||||
? toRtcSessionError(connectionState)
|
||||
? toMatrixRTCError(connectionState)
|
||||
: (connectionState ?? ConnectionState.Initialized),
|
||||
};
|
||||
return shouldPublish
|
||||
|
||||
@@ -6,8 +6,11 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type MatrixClient, type Room } from "matrix-js-sdk";
|
||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import { MatrixRTCSessionEvent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import {
|
||||
MatrixRTCSessionEvent,
|
||||
MatrixRTCSessionManager,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { combineLatest, from, fromEvent, map } from "rxjs";
|
||||
@@ -29,20 +32,20 @@ import {
|
||||
import { filterBehavior, generateItems } from "../../../src/utils/observable";
|
||||
import {
|
||||
type LocalMediaInputs,
|
||||
type LocalRtcMember,
|
||||
type RemoteRtcMember,
|
||||
type RtcSession,
|
||||
RtcSessionError,
|
||||
type RtcSessionOptions,
|
||||
type LocalRTCMember,
|
||||
type RemoteRTCMember,
|
||||
type MatrixRTCClient,
|
||||
MatrixRTCError,
|
||||
type MatrixRTCClientOptions,
|
||||
} from "../api";
|
||||
import { LivekitConnectionFactory } from "./ConnectionFactory";
|
||||
import { joinRtcSession, sessionTimings } from "./joinRtcSession";
|
||||
import { joinJsSdkSession, sessionTimings } from "./joinJsSdkSession";
|
||||
import { createKeyProvider } from "./KeyProvider";
|
||||
import { createLocalMembership$ } from "./LocalMember";
|
||||
import { getLocalTransport } from "./LocalTransport";
|
||||
import {
|
||||
createLocalRtcMember,
|
||||
createRemoteRtcMember,
|
||||
createLocalRTCMember,
|
||||
createRemoteRTCMember,
|
||||
membershipKeys,
|
||||
} from "./Members";
|
||||
import { Publisher } from "./Publisher";
|
||||
@@ -57,21 +60,26 @@ import { createTransportRegistry } from "./Transports";
|
||||
* js-sdk client, and only the SDK knows what that bridge needs. Holding the
|
||||
* client keeps that change inside the SDK.
|
||||
*/
|
||||
export function createRtcSession(
|
||||
export function createMatrixRTCClient(
|
||||
scope: ObservableScope,
|
||||
client: MatrixClient,
|
||||
room: Room,
|
||||
localMedia: LocalMediaInputs,
|
||||
options: RtcSessionOptions,
|
||||
): RtcSession {
|
||||
const logger = rootLogger.getChild("[RtcSession]");
|
||||
options: MatrixRTCClientOptions,
|
||||
): MatrixRTCClient {
|
||||
const logger = rootLogger.getChild("[MatrixRTCClient]");
|
||||
const userId = client.getUserId();
|
||||
const deviceId = client.getDeviceId();
|
||||
if (!(userId && deviceId))
|
||||
throw new RtcSessionError("The client has to be logged in");
|
||||
throw new MatrixRTCError("The client has to be logged in");
|
||||
const { encryptionSystem, matrixRTCMode } = options;
|
||||
|
||||
const jsSdkSession = client.matrixRTC.getRoomSession(room);
|
||||
const jsSdkSession = sessionManager(
|
||||
scope,
|
||||
client,
|
||||
options,
|
||||
logger,
|
||||
).getRoomSession(room);
|
||||
const keyProvider = createKeyProvider(encryptionSystem, jsSdkSession, logger);
|
||||
|
||||
const ownMembershipIdentity: CallMembershipIdentityParts = {
|
||||
@@ -136,11 +144,11 @@ export function createRtcSession(
|
||||
),
|
||||
),
|
||||
joinMatrixRTC: (transport) =>
|
||||
joinRtcSession(jsSdkSession, ownMembershipIdentity, transport, {
|
||||
joinJsSdkSession(jsSdkSession, ownMembershipIdentity, transport, {
|
||||
encryptMedia: keyProvider !== undefined,
|
||||
matrixRTCMode,
|
||||
sendNotificationType: options.sendNotificationType,
|
||||
callIntent: options.callIntent,
|
||||
applicationData: options.applicationData,
|
||||
}),
|
||||
membershipManagerError$: fromEvent(
|
||||
jsSdkSession,
|
||||
@@ -161,7 +169,7 @@ export function createRtcSession(
|
||||
encryptionSystem,
|
||||
};
|
||||
|
||||
const remoteMembers$ = scope.behavior<RemoteRtcMember[]>(
|
||||
const remoteMembers$ = scope.behavior<RemoteRTCMember[]>(
|
||||
createRemoteMatrixLivekitMembers$({
|
||||
scope,
|
||||
membershipsWithTransport$,
|
||||
@@ -170,7 +178,7 @@ export function createRtcSession(
|
||||
}).pipe(
|
||||
map(({ value }) => value),
|
||||
generateItems(
|
||||
"RtcSession remoteMembers",
|
||||
"MatrixRTCClient remoteMembers",
|
||||
function* (members) {
|
||||
for (const member of members)
|
||||
yield {
|
||||
@@ -179,7 +187,7 @@ export function createRtcSession(
|
||||
};
|
||||
},
|
||||
(memberScope, member$) =>
|
||||
createRemoteRtcMember(memberScope, member$.value, context),
|
||||
createRemoteRTCMember(memberScope, member$.value, context),
|
||||
),
|
||||
),
|
||||
);
|
||||
@@ -196,9 +204,9 @@ export function createRtcSession(
|
||||
filterBehavior((membership) => membership !== null),
|
||||
),
|
||||
);
|
||||
const localMember$ = scope.behavior<LocalRtcMember | null>(
|
||||
const localMember$ = scope.behavior<LocalRTCMember | null>(
|
||||
mapScoped(scope, localMembership$, (memberScope, membership$) =>
|
||||
createLocalRtcMember(memberScope, membership$, localMembership, context),
|
||||
createLocalRTCMember(memberScope, membership$, localMembership, context),
|
||||
).pipe(map((member) => member ?? null)),
|
||||
);
|
||||
|
||||
@@ -233,3 +241,24 @@ export function createRtcSession(
|
||||
connectedTransports$: transports.connected$,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The js-sdk keeps one session manager per slot. The client comes with the
|
||||
* manager for the default slot; any other slot gets a manager of its own,
|
||||
* which lives as long as the scope.
|
||||
*/
|
||||
function sessionManager(
|
||||
scope: ObservableScope,
|
||||
client: MatrixClient,
|
||||
{ application = "m.call", slot = "ROOM" }: MatrixRTCClientOptions,
|
||||
logger: Logger,
|
||||
): MatrixRTCSessionManager {
|
||||
if (application === "m.call" && slot === "ROOM") return client.matrixRTC;
|
||||
const manager = new MatrixRTCSessionManager(logger, client, {
|
||||
application,
|
||||
id: slot,
|
||||
});
|
||||
manager.start();
|
||||
scope.onEnd(() => manager.stop());
|
||||
return manager;
|
||||
}
|
||||
+11
-11
@@ -20,10 +20,10 @@ import { type createMatrixMemberMetadata$ } from "../../../src/state/CallViewMod
|
||||
import { type RemoteMatrixLivekitMember } from "../../../src/state/CallViewModel/remoteMembers/MatrixLivekitMembers";
|
||||
import { type EncryptionSystem } from "../../../src/e2ee/sharedKeyManagement";
|
||||
import {
|
||||
type LocalRtcMember,
|
||||
type LocalRTCMember,
|
||||
type MemberMedia,
|
||||
type RemoteRtcMember,
|
||||
type RtcMember,
|
||||
type RemoteRTCMember,
|
||||
type RTCMember,
|
||||
} from "../api";
|
||||
import {
|
||||
createLivekitMemberMedia,
|
||||
@@ -51,13 +51,13 @@ export function membershipKeys(
|
||||
];
|
||||
}
|
||||
|
||||
export function createRemoteRtcMember(
|
||||
export function createRemoteRTCMember(
|
||||
scope: ObservableScope,
|
||||
member: RemoteMatrixLivekitMember,
|
||||
context: MemberContext,
|
||||
): RemoteRtcMember {
|
||||
): RemoteRTCMember {
|
||||
return {
|
||||
...createRtcMember(scope, member.membership$, context),
|
||||
...createRTCMember(scope, member.membership$, context),
|
||||
local: false,
|
||||
media$: mediaFor(
|
||||
scope,
|
||||
@@ -74,14 +74,14 @@ export function createRemoteRtcMember(
|
||||
};
|
||||
}
|
||||
|
||||
export function createLocalRtcMember(
|
||||
export function createLocalRTCMember(
|
||||
scope: ObservableScope,
|
||||
membership$: Behavior<CallMembership>,
|
||||
localMembership: LocalMembership,
|
||||
context: MemberContext,
|
||||
): LocalRtcMember {
|
||||
): LocalRTCMember {
|
||||
return {
|
||||
...createRtcMember(scope, membership$, context),
|
||||
...createRTCMember(scope, membership$, context),
|
||||
local: true,
|
||||
media$: mediaFor(
|
||||
scope,
|
||||
@@ -102,11 +102,11 @@ export function createLocalRtcMember(
|
||||
};
|
||||
}
|
||||
|
||||
function createRtcMember(
|
||||
function createRTCMember(
|
||||
scope: ObservableScope,
|
||||
membership$: Behavior<CallMembership>,
|
||||
{ metadata, transports }: MemberContext,
|
||||
): Omit<RtcMember, "local" | "media$"> {
|
||||
): Omit<RTCMember, "local" | "media$"> {
|
||||
const { userId, deviceId, rtcBackendIdentity } = membership$.value;
|
||||
return {
|
||||
id: rtcBackendIdentity,
|
||||
|
||||
@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import {
|
||||
type LivekitTransport,
|
||||
type MatrixRTCSession as JsSdkRtcSession,
|
||||
type MatrixRTCSession as JsSdkRTCSession,
|
||||
type RTCCallIntent,
|
||||
type RTCNotificationType,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
@@ -22,7 +22,7 @@ interface Options {
|
||||
encryptMedia: boolean;
|
||||
matrixRTCMode: MatrixRTCMode;
|
||||
sendNotificationType?: RTCNotificationType;
|
||||
callIntent?: RTCCallIntent;
|
||||
applicationData?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** The session's timing defaults; a later slice makes them an input. */
|
||||
@@ -36,12 +36,21 @@ export const sessionTimings = {
|
||||
* alive and retries on failure.
|
||||
* @throws If the join could not be started.
|
||||
*/
|
||||
export function joinRtcSession(
|
||||
session: JsSdkRtcSession,
|
||||
export function joinJsSdkSession(
|
||||
session: JsSdkRTCSession,
|
||||
ownMembershipIdentity: CallMembershipIdentityParts,
|
||||
transport: LivekitTransport,
|
||||
{ encryptMedia, matrixRTCMode, sendNotificationType, callIntent }: Options,
|
||||
{
|
||||
encryptMedia,
|
||||
matrixRTCMode,
|
||||
sendNotificationType,
|
||||
applicationData,
|
||||
}: Options,
|
||||
): void {
|
||||
// The one piece of application data the js-sdk can put in a membership
|
||||
const intent = applicationData?.["m.call.intent"];
|
||||
const callIntent =
|
||||
typeof intent === "string" ? (intent as RTCCallIntent) : undefined;
|
||||
const timings = sessionTimings.delayed_leave;
|
||||
// Give up on the network as soon as either the sync has been down for the
|
||||
// grace period or the delayed leave has probably been sent, whichever is
|
||||
@@ -9,7 +9,7 @@ import { describe, expect, it } from "vitest";
|
||||
import { Status } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { ConnectionState } from "../../../src/state/CallViewModel/remoteMembers/Connection";
|
||||
import { RtcSessionError } from "../api";
|
||||
import { MatrixRTCError } from "../api";
|
||||
import {
|
||||
type LocalMemberState,
|
||||
PublishState,
|
||||
@@ -25,7 +25,7 @@ const connecting: LocalMemberState = {
|
||||
media: { connection: ConnectionState.LivekitConnecting },
|
||||
matrix: Status.Connecting,
|
||||
};
|
||||
const error = new RtcSessionError("gone");
|
||||
const error = new MatrixRTCError("gone");
|
||||
|
||||
describe("sessionStatus", () => {
|
||||
it.each<[string, LocalMemberState, boolean, boolean, boolean, string]>([
|
||||
|
||||
@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type RtcSessionError, type SessionConnectionStatus } from "../api";
|
||||
import { type MatrixRTCError, type ConnectionStatus } from "../api";
|
||||
import { type LocalMemberState, TransportState } from "./LocalMember";
|
||||
|
||||
/** The local member's state, collapsed to what a host shows. */
|
||||
@@ -14,7 +14,7 @@ export function sessionStatus(
|
||||
joinRequested: boolean,
|
||||
connected: boolean,
|
||||
reconnecting: boolean,
|
||||
): SessionConnectionStatus {
|
||||
): ConnectionStatus {
|
||||
if (fatalError(state) !== null) return "disconnected";
|
||||
if (state === TransportState.Waiting) return "waitingForTransport";
|
||||
if (connected) return "connected";
|
||||
@@ -23,7 +23,7 @@ export function sessionStatus(
|
||||
}
|
||||
|
||||
/** The error that stops the session, if the state holds one. */
|
||||
export function fatalError(state: LocalMemberState): RtcSessionError | null {
|
||||
export function fatalError(state: LocalMemberState): MatrixRTCError | null {
|
||||
if (state === TransportState.Waiting) return null;
|
||||
if (state instanceof Error) return state;
|
||||
if (state.matrix instanceof Error) return state.matrix;
|
||||
|
||||
@@ -7,29 +7,29 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { RtcSessionError } from "../api";
|
||||
import { toRtcSessionError } from "./errors";
|
||||
import { MatrixRTCError } from "../api";
|
||||
import { toMatrixRTCError } from "./errors";
|
||||
|
||||
describe("toRtcSessionError", () => {
|
||||
it("keeps an RtcSessionError as it is", () => {
|
||||
const error = new RtcSessionError("gone");
|
||||
expect(toRtcSessionError(error)).toBe(error);
|
||||
describe("toMatrixRTCError", () => {
|
||||
it("keeps an MatrixRTCError as it is", () => {
|
||||
const error = new MatrixRTCError("gone");
|
||||
expect(toMatrixRTCError(error)).toBe(error);
|
||||
});
|
||||
|
||||
it("wraps another error as the cause", () => {
|
||||
const cause = new Error("boom");
|
||||
const error = toRtcSessionError(cause);
|
||||
expect(error).toBeInstanceOf(RtcSessionError);
|
||||
const error = toMatrixRTCError(cause);
|
||||
expect(error).toBeInstanceOf(MatrixRTCError);
|
||||
expect(error.message).toBe("boom");
|
||||
expect(error.cause).toBe(cause);
|
||||
});
|
||||
|
||||
it("falls back to the code of an error without a message", () => {
|
||||
const cause = Object.assign(new Error(""), { code: "SFU_ERROR" });
|
||||
expect(toRtcSessionError(cause).message).toBe("SFU_ERROR");
|
||||
expect(toMatrixRTCError(cause).message).toBe("SFU_ERROR");
|
||||
});
|
||||
|
||||
it("stringifies anything else", () => {
|
||||
expect(toRtcSessionError("nope").message).toBe("nope");
|
||||
expect(toMatrixRTCError("nope").message).toBe("nope");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,14 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { RtcSessionError } from "../api";
|
||||
import { MatrixRTCError } from "../api";
|
||||
|
||||
/** Wraps whatever a backend threw into the SDK's error type, keeping it as the cause. */
|
||||
export function toRtcSessionError(error: unknown): RtcSessionError {
|
||||
if (error instanceof RtcSessionError) return error;
|
||||
export function toMatrixRTCError(error: unknown): MatrixRTCError {
|
||||
if (error instanceof MatrixRTCError) return error;
|
||||
if (error instanceof Error)
|
||||
return new RtcSessionError(messageOf(error), { cause: error });
|
||||
return new RtcSessionError(String(error));
|
||||
return new MatrixRTCError(messageOf(error), { cause: error });
|
||||
return new MatrixRTCError(String(error));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user