mirror of
https://github.com/vector-im/element-call.git
synced 2026-10-16 23:45:50 +00:00
renames + rewriting docs
This commit is contained in:
Timo K.
+10
-9
@@ -3,10 +3,11 @@
|
|||||||
`@element-hq/matrixrtc-sdk` is the call model under Element Call, on its own:
|
`@element-hq/matrixrtc-sdk` is the call model under Element Call, on its own:
|
||||||
MatrixRTC memberships and transports, LiveKit connections, publishing, E2EE keys and
|
MatrixRTC memberships and transports, LiveKit connections, publishing, E2EE keys and
|
||||||
the media of every member, as observables. It has no UI. Element Call's own
|
the media of every member, as observables. It has no UI. Element Call's own
|
||||||
`CallViewModel` is meant to become one consumer of it; the design is in
|
`CallViewModel` is meant to become one consumer of it. The design is in
|
||||||
[`sdk-plan.md`](../sdk-plan.md).
|
[`SdkArchitecture.md`](./SdkArchitecture.md), the migration from the view model
|
||||||
|
in [`SdkMigration.md`](./SdkMigration.md).
|
||||||
|
|
||||||
**Status:** first implementation. `createRtcSession` joins the session, connects to
|
**Status:** first implementation. `createMatrixRTCClient` joins the session, connects to
|
||||||
the transport, publishes the local media and exposes every member's media; the
|
the transport, publishes the local media and exposes every member's media; the
|
||||||
development harness and its e2e tests in `playwright/sdk` drive it. The
|
development harness and its e2e tests in `playwright/sdk` drive it. The
|
||||||
implementation still imports the building blocks it shares with Element Call from
|
implementation still imports the building blocks it shares with Element Call from
|
||||||
@@ -17,17 +18,17 @@ implementation still imports the building blocks it shares with Element Call fro
|
|||||||
```ts
|
```ts
|
||||||
import {
|
import {
|
||||||
constant,
|
constant,
|
||||||
createRtcSession,
|
createMatrixRTCClient,
|
||||||
E2eeType,
|
E2eeType,
|
||||||
MatrixRTCMode,
|
MatrixRTCMode,
|
||||||
ObservableScope,
|
ObservableScope,
|
||||||
} from "@element-hq/matrixrtc-sdk";
|
} from "@element-hq/matrixrtc-sdk";
|
||||||
|
|
||||||
const scope = new ObservableScope();
|
const scope = new ObservableScope();
|
||||||
const session = createRtcSession(
|
const rtcClient = createMatrixRTCClient(
|
||||||
scope,
|
scope,
|
||||||
client, // a matrix-js-sdk MatrixClient, logged in and syncing
|
client, // a matrix-js-sdk MatrixClient, logged in and syncing
|
||||||
room, // the matrix-js-sdk Room to hold the session in, from client.getRoom()
|
room, // the matrix-js-sdk Room, from client.getRoom() once the join has synced
|
||||||
{
|
{
|
||||||
microphoneEnabled$: constant(true),
|
microphoneEnabled$: constant(true),
|
||||||
cameraEnabled$: constant(true),
|
cameraEnabled$: constant(true),
|
||||||
@@ -40,15 +41,15 @@ const session = createRtcSession(
|
|||||||
matrixRTCMode: MatrixRTCMode.Compatibility,
|
matrixRTCMode: MatrixRTCMode.Compatibility,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
session.join();
|
rtcClient.join();
|
||||||
|
|
||||||
session.remoteMembers$.subscribe((members) => {
|
rtcClient.remoteMembers$.subscribe((members) => {
|
||||||
// each member has displayName$, media$ and more; a media track is rendered
|
// each member has displayName$, media$ and more; a media track is rendered
|
||||||
// by handing it a <video> or <audio> element: track.attach(element)
|
// by handing it a <video> or <audio> element: track.attach(element)
|
||||||
});
|
});
|
||||||
|
|
||||||
// later
|
// later
|
||||||
session.leave();
|
rtcClient.leave();
|
||||||
scope.end();
|
scope.end();
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,570 @@
|
|||||||
|
# MatrixRTC SDK architecture
|
||||||
|
|
||||||
|
`@element-hq/matrixrtc-sdk` is the model under Element Call, on its own: one
|
||||||
|
`MatrixRTCClient` per room that joins and leaves the MatrixRTC session, discovers
|
||||||
|
and connects to the media transports, publishes the local media, distributes the
|
||||||
|
encryption keys, and hands out every member with its media as observables. It has
|
||||||
|
no UI, reads no configuration and no settings, and never names LiveKit in its
|
||||||
|
interface.
|
||||||
|
|
||||||
|
Element Call becomes one consumer of it: `CallViewModel` keeps what makes a session
|
||||||
|
a _call_ (ringing, auto-leave, layout, sounds, reactions, settings, the host
|
||||||
|
bridge) and takes everything else from the client. The dependency direction is
|
||||||
|
therefore `Element Call → sdk`, never the reverse. The migration from today's
|
||||||
|
`CallViewModel` to that shape is in [`SdkMigration.md`](./SdkMigration.md).
|
||||||
|
|
||||||
|
## The architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
host application (Element Call, a whiteboard, watch-together, …)
|
||||||
|
│
|
||||||
|
│ createMatrixRTCClient(scope, matrixClient, room, localMedia, options)
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ MatrixRTCClient │
|
||||||
|
│ │
|
||||||
|
│ public API, all Behaviors: │
|
||||||
|
│ join() leave() status$ connected$ reconnecting$ fatalError$ │
|
||||||
|
│ localMember$ remoteMembers$ memberCount$ │
|
||||||
|
│ keyRotationSuppressed$ connectedTransports$ │
|
||||||
|
│ │
|
||||||
|
│ per member RTCMember { id userId deviceId membership$ displayName$ │
|
||||||
|
│ avatarUrl$ transport$ media$ } │
|
||||||
|
│ per media MemberMedia { speaking$ microphone$ camera$ screenShare$ … } │
|
||||||
|
│ per track MediaTrack { muted$ encrypted$ stats$ attach() detach() } │
|
||||||
|
│ │
|
||||||
|
│ ┌──────────────────────────┐ memberships ┌───────────────────────────┐ │
|
||||||
|
│ │ MatrixRTC core │ ───────────────▶ │ transports │ │
|
||||||
|
│ │ (js-sdk MatrixRTCSession,│ transports │ │ │
|
||||||
|
│ │ later the rust-rtc │ │ ConnectionManager │ │
|
||||||
|
│ │ crate behind a bridge) │ │ one Connection per │ │
|
||||||
|
│ │ │ │ transport url │ │
|
||||||
|
│ │ memberships from room │ │ Connection │ │
|
||||||
|
│ │ state and sticky events│ │ OpenID → JWT → LiveKit │ │
|
||||||
|
│ │ own membership, delayed │ │ room, participants, │ │
|
||||||
|
│ │ leave, retries │ media keys │ tracks │ │
|
||||||
|
│ │ media keys over │ ───────────────▶ │ Publisher │ │
|
||||||
|
│ │ to-device messages │ (key provider) │ local tracks, upstream │ │
|
||||||
|
│ └──────────────────────────┘ └───────────────────────────┘ │
|
||||||
|
│ │ │ │
|
||||||
|
│ └──── RTCMember = membership + participant ─┘ │
|
||||||
|
│ matched on rtcBackendIdentity = LiveKit identity │
|
||||||
|
└──────────────────────────────────────────────────────────────────────────────┘
|
||||||
|
│ │
|
||||||
|
▼ ▼
|
||||||
|
MatrixClient (sync, room state, to-device, OpenID) LiveKit SFU
|
||||||
|
```
|
||||||
|
|
||||||
|
The left box is MatrixRTC: who is in the session, with which transport, and the
|
||||||
|
keys. The right box is media: a connection to every transport any member
|
||||||
|
advertises, and our own publication on the one we advertise. The client is the
|
||||||
|
join of the two: a member exists once its membership does, and gets its media once
|
||||||
|
a participant with the matching identity shows up on the member's transport.
|
||||||
|
|
||||||
|
### How Element Call uses it
|
||||||
|
|
||||||
|
```
|
||||||
|
Element Call, as app, widget or component
|
||||||
|
┌────────────────────────────────────────────────────────────────┐
|
||||||
|
│ views: InCallView, tiles, header, footer, settings │
|
||||||
|
│ ▲ Behaviors │
|
||||||
|
│ CallViewModel │
|
||||||
|
│ layout, ringing and notifications, auto-leave, sounds, │
|
||||||
|
│ reactions, hand raise, settings, audio routing, host bridge │
|
||||||
|
│ ▲ status$, localMember$, remoteMembers$, MemberMedia │
|
||||||
|
│ MatrixRTCClient @element-hq/matrixrtc-sdk │
|
||||||
|
└────────────────────────────────────────────────────────────────┘
|
||||||
|
│ MatrixClient + Room │ livekit-client
|
||||||
|
▼ ▼
|
||||||
|
homeserver SFU
|
||||||
|
```
|
||||||
|
|
||||||
|
All three modes already hold a `MatrixClient` and a `Room` by the time a call
|
||||||
|
starts: standalone after `useLoadGroupCall`, the component from the host's `client`
|
||||||
|
prop, the widget from `createRoomWidgetClient`. The view model passes them on and
|
||||||
|
builds its media view models from `RTCMember` and `MemberMedia` instead of from
|
||||||
|
LiveKit participants.
|
||||||
|
|
||||||
|
### Joining, in order
|
||||||
|
|
||||||
|
```
|
||||||
|
host MatrixRTCClient homeserver / JWT service LiveKit SFU
|
||||||
|
│ create ───▶│ │ │
|
||||||
|
│ │ GET rtc/transports ──────────────▶│ │
|
||||||
|
│ │◀── preferred transport ───────────│ │
|
||||||
|
│ │ OpenID token, then /sfu/get ─────▶│ │
|
||||||
|
│ │◀── sfu url, jwt, alias, identity ─│ = resolved$ of the │
|
||||||
|
│ │ │ local transport │
|
||||||
|
│ │ connect(url, jwt) ───────────────────────────────────────▶│
|
||||||
|
│ join() ───▶│ │ │
|
||||||
|
│ │ membership state event + delayed leave ──▶│ │
|
||||||
|
│ │ create tracks, resume upstream ──────────────────────────▶│
|
||||||
|
│ │ media key to every other member (to-device) ──▶│ │
|
||||||
|
│ │◀── other memberships (sync) ──────│◀── participants ────────│
|
||||||
|
│◀─ status$ "connected", localMember$, remoteMembers$ with media$ ──────────│
|
||||||
|
```
|
||||||
|
|
||||||
|
The transport is resolved and connected before `join()`, so that the membership
|
||||||
|
can name it and the first frames go out as soon as the state event is sent. Until
|
||||||
|
`join()` the tracks exist but their upstream is paused: a host can show a preview
|
||||||
|
without anyone hearing it. `leave()` pauses the upstream again and sends the leave.
|
||||||
|
|
||||||
|
## Vocabulary
|
||||||
|
|
||||||
|
| Word | Meaning here | Elsewhere |
|
||||||
|
| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ |
|
||||||
|
| client | a `MatrixRTCClient`: one room, one slot, one local member | `MatrixClient` is always written out |
|
||||||
|
| session | the MatrixRTC session in the room: the set of memberships. Only used for the js-sdk object, which stays inside the SDK as `JsSdkRTCSession` | MSC4143 |
|
||||||
|
| membership | one device's MatrixRTC state event, the js-sdk `CallMembership`. The js-sdk name is kept until it renames it | |
|
||||||
|
| member | an `RTCMember`: a membership plus what the SDK derives from it (display name, transport, media) | Element Call's "matrixLivekitMember" |
|
||||||
|
| participant | a LiveKit participant, the media side of a member. Never leaves the SDK | |
|
||||||
|
| transport | where media is exchanged: a LiveKit service url today, described by `TransportMetadata` | MSC4143 "focus" |
|
||||||
|
| application, slot | what the session is for (`m.call`) and which one of them in the room (`ROOM`) | MSC4143 |
|
||||||
|
|
||||||
|
Names use `RTC` in capitals, as the js-sdk does: `MatrixRTCClient`, `RTCMember`,
|
||||||
|
`matrixRTCMode`.
|
||||||
|
|
||||||
|
## Reactive primitives
|
||||||
|
|
||||||
|
The whole public API is made of `Behavior`s: an rxjs observable that always holds a
|
||||||
|
current value (`.value`) and replays it on subscription. A consumer subscribes to
|
||||||
|
what it renders and reads `.value` where it needs the present state. Everything is
|
||||||
|
bound to an `ObservableScope` the host creates; ending the scope ends the client,
|
||||||
|
its connections and its tracks. Both come from the SDK (`Behavior`, `constant`,
|
||||||
|
`ObservableScope`); the host needs rxjs and nothing else.
|
||||||
|
|
||||||
|
## What a host must provide
|
||||||
|
|
||||||
|
Found the hard way while building the harness; each one fails in a way that looks
|
||||||
|
like an SDK bug.
|
||||||
|
|
||||||
|
- **The room the sync loop maintains.** `client.getRoom(roomId)` once the join has
|
||||||
|
synced. `joinRoom` returns a detached copy for a room joined just now, and a
|
||||||
|
client built on it never sees a member.
|
||||||
|
- **Power to send the membership event.** Every member needs to write
|
||||||
|
`org.matrix.msc3401.call.member` (and the sticky event in Matrix 2.0 mode). A
|
||||||
|
plain room only lets its creator send state; Element Call's rooms set the level
|
||||||
|
to 0.
|
||||||
|
- **An encrypted room for per-participant keys.** Media keys travel in encrypted
|
||||||
|
to-device messages, which the crypto only delivers to devices it tracks, which
|
||||||
|
are the members of encrypted rooms. In an unencrypted room the key never arrives
|
||||||
|
and every remote track decodes nothing.
|
||||||
|
- **A logged-in, syncing client with crypto set up**, and the transport advertised
|
||||||
|
by the homeserver (`rtc/transports`). The SDK has no fallback url of its own.
|
||||||
|
|
||||||
|
## The client: `MatrixRTCClient`
|
||||||
|
|
||||||
|
Rule of thumb for what belongs here: if it would be the same for a different
|
||||||
|
application on top of the same MatrixRTC session, it is in the client. If it is
|
||||||
|
specific to calls, or depends on the window, a setting, a platform, or a React
|
||||||
|
context, it is in the view model.
|
||||||
|
|
||||||
|
### Creating one
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { type MatrixClient, type Room } from "matrix-js-sdk";
|
||||||
|
import { type RTCNotificationType } from "matrix-js-sdk/lib/matrixrtc";
|
||||||
|
|
||||||
|
export interface MatrixRTCClientOptions {
|
||||||
|
encryptionSystem: EncryptionSystem;
|
||||||
|
/** Resolved by the host; the SDK reads neither config.json nor settings. */
|
||||||
|
matrixRTCMode: MatrixRTCMode;
|
||||||
|
/**
|
||||||
|
* MSC4075 notification sent with the join. A parameter of the MatrixRTC
|
||||||
|
* join itself, so it is here even though it is named after calls; reacting
|
||||||
|
* to a notification (ringing, timeouts, declines) is the application's job.
|
||||||
|
*/
|
||||||
|
sendNotificationType?: RTCNotificationType;
|
||||||
|
/** The application the session belongs to, as named in the membership. Default `m.call`. */
|
||||||
|
application?: string;
|
||||||
|
/** The application's slot in the room. Default `ROOM`. */
|
||||||
|
slot?: string;
|
||||||
|
/**
|
||||||
|
* Whatever the application wants to say about itself in the membership.
|
||||||
|
* Opaque to the SDK. Today the js-sdk carries one key, `m.call.intent`
|
||||||
|
* (`"audio"` or `"video"`); anything else is dropped until it can.
|
||||||
|
*/
|
||||||
|
applicationData?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* What the local member publishes. The SDK publishes what it is told; device
|
||||||
|
* enumeration, permission prompts and the lobby preview stay with the host.
|
||||||
|
*/
|
||||||
|
export interface LocalMediaInputs {
|
||||||
|
microphoneEnabled$: Behavior<boolean>;
|
||||||
|
cameraEnabled$: Behavior<boolean>;
|
||||||
|
audioInputDeviceId$: Behavior<string | undefined>;
|
||||||
|
videoInputDeviceId$: Behavior<string | undefined>;
|
||||||
|
/** Background blur and the like. */
|
||||||
|
videoProcessor$: Behavior<TrackProcessor<Track.Kind.Video> | undefined>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Takes the whole MatrixClient, not a narrow pick, and finds the MatrixRTC
|
||||||
|
* session itself. The js-sdk is the MatrixRTC implementation today; when the
|
||||||
|
* rust-rtc crate replaces it, the SDK has to bridge the crate to the js-sdk
|
||||||
|
* client (events, state, to-device, delayed events, OpenID), and only the SDK
|
||||||
|
* knows what that bridge needs. Holding the client keeps that change inside
|
||||||
|
* the SDK. The js-sdk session is not part of the public API.
|
||||||
|
*
|
||||||
|
* The room has to be the one the client's sync loop maintains; see "What a
|
||||||
|
* host must provide".
|
||||||
|
*/
|
||||||
|
export function createMatrixRTCClient(
|
||||||
|
scope: ObservableScope,
|
||||||
|
client: MatrixClient,
|
||||||
|
room: Room,
|
||||||
|
localMedia: LocalMediaInputs,
|
||||||
|
options: MatrixRTCClientOptions,
|
||||||
|
): MatrixRTCClient;
|
||||||
|
```
|
||||||
|
|
||||||
|
### The client
|
||||||
|
|
||||||
|
```ts
|
||||||
|
export type ConnectionStatus =
|
||||||
|
| "waitingForTransport"
|
||||||
|
| "connecting"
|
||||||
|
| "connected"
|
||||||
|
| "reconnecting"
|
||||||
|
| "disconnected";
|
||||||
|
|
||||||
|
/** An error raised by the client. `cause` holds the backend's error, for a host that knows the backend. */
|
||||||
|
export class MatrixRTCError extends Error {}
|
||||||
|
|
||||||
|
export interface MatrixRTCClient {
|
||||||
|
join(): void;
|
||||||
|
leave(): void;
|
||||||
|
/** The local member's state machine, collapsed to what a host shows. */
|
||||||
|
status$: Behavior<ConnectionStatus>;
|
||||||
|
/** Connected to the homeserver, the session and the local transport, all three. */
|
||||||
|
connected$: Behavior<boolean>;
|
||||||
|
/** Connected once, and currently not. */
|
||||||
|
reconnecting$: Behavior<boolean>;
|
||||||
|
/**
|
||||||
|
* A transport, Matrix or connection error that stops the session. Null
|
||||||
|
* while fine. A failed publication is not fatal: the member can still
|
||||||
|
* receive, so it stays on the local member's media.
|
||||||
|
*/
|
||||||
|
fatalError$: Behavior<MatrixRTCError | null>;
|
||||||
|
|
||||||
|
/** Null until our own membership has been seen in the room. */
|
||||||
|
localMember$: Behavior<LocalRTCMember | null>;
|
||||||
|
remoteMembers$: Behavior<RemoteRTCMember[]>;
|
||||||
|
/** `remoteMembers.length`, plus one for the local member once it exists. One per membership, so one user may count several times. */
|
||||||
|
memberCount$: Behavior<number>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether the session has grown large enough that MatrixRTC has stopped
|
||||||
|
* rotating the media encryption key.
|
||||||
|
*/
|
||||||
|
keyRotationSuppressed$: Behavior<boolean>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transports the client currently holds a live connection to. The
|
||||||
|
* connections themselves stay internal; this is what a developer panel
|
||||||
|
* needs.
|
||||||
|
*/
|
||||||
|
connectedTransports$: Behavior<TransportMetadata[]>;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Members
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
|
||||||
|
|
||||||
|
export interface RTCMember {
|
||||||
|
local: boolean;
|
||||||
|
/**
|
||||||
|
* The identity the media backend knows this member by: the membership's
|
||||||
|
* `rtcBackendIdentity`, which is `${userId}:${deviceId}` before sticky
|
||||||
|
* events and a hash of the member id in Matrix 2.0 mode.
|
||||||
|
*/
|
||||||
|
id: string;
|
||||||
|
userId: string;
|
||||||
|
deviceId: string;
|
||||||
|
membership$: Behavior<CallMembership>;
|
||||||
|
/**
|
||||||
|
* Matrix room state rather than MatrixRTC, but the SDK already holds the
|
||||||
|
* room and every consumer wants them next to the membership. Disambiguated
|
||||||
|
* among the members of the session.
|
||||||
|
*/
|
||||||
|
displayName$: Behavior<string>;
|
||||||
|
avatarUrl$: Behavior<string | undefined>;
|
||||||
|
/** Which transport this member is on; undefined when the membership names none. */
|
||||||
|
transport$: Behavior<TransportMetadata | undefined>;
|
||||||
|
/**
|
||||||
|
* Null while the member has a transport but no participant has shown up on
|
||||||
|
* it yet ("waiting for media"). Hand raise and reactions are room events
|
||||||
|
* keyed by member and stay with the host for now; they are the obvious next
|
||||||
|
* fields here.
|
||||||
|
*/
|
||||||
|
media$: Behavior<MemberMedia | null>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RemoteRTCMember extends RTCMember {
|
||||||
|
local: false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LocalRTCMember extends RTCMember {
|
||||||
|
local: true;
|
||||||
|
media$: Behavior<LocalMemberMedia | null>;
|
||||||
|
sharingScreen$: Behavior<boolean>;
|
||||||
|
/** Null when the platform cannot share a screen (no getDisplayMedia). Hiding the button is the host's policy. */
|
||||||
|
toggleScreenSharing: (() => void) | null;
|
||||||
|
/** The last error from toggling, until dismissed. The user closing the picker is not one. */
|
||||||
|
screenShareError$: Behavior<Error | null>;
|
||||||
|
dismissScreenShareError(): void;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Transports
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { type Transport } from "matrix-js-sdk/lib/matrixrtc";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One transport advertised in a membership. Transport independent: `type` and
|
||||||
|
* `id` are all the SDK needs; `raw` and `resolved$` are there for a
|
||||||
|
* backend-specific developer panel and for connection diagnostics. One object
|
||||||
|
* per transport, however many memberships name it.
|
||||||
|
*/
|
||||||
|
export interface TransportMetadata {
|
||||||
|
/** `"livekit"` today. */
|
||||||
|
type: string;
|
||||||
|
/** Stable key, unique per transport in the session. For LiveKit, the service url. */
|
||||||
|
id: string;
|
||||||
|
/** The transport object as it appears in the membership, e.g. `{ type, livekit_service_url }`. */
|
||||||
|
raw: Transport;
|
||||||
|
/**
|
||||||
|
* What the backend had to fetch before it could connect. Undefined until
|
||||||
|
* the connection has resolved it, and again after the connection stops or
|
||||||
|
* fails. The local transport resolves before the join, since the membership
|
||||||
|
* needs it; remote ones when their connection starts.
|
||||||
|
*/
|
||||||
|
resolved$: Behavior<ResolvedTransport | undefined>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ResolvedTransport =
|
||||||
|
| {
|
||||||
|
type: "livekit";
|
||||||
|
/** The SFU websocket url, as opposed to the JWT service url in `raw`. */
|
||||||
|
url: string;
|
||||||
|
/** A secret: fit for a developer panel, never for a log line. */
|
||||||
|
token: string;
|
||||||
|
roomAlias: string;
|
||||||
|
identity: string;
|
||||||
|
}
|
||||||
|
| { type: string; [key: string]: unknown };
|
||||||
|
```
|
||||||
|
|
||||||
|
### Call-specific things kept out of the client
|
||||||
|
|
||||||
|
Checked against `CallViewModel` and its options. These stay in Element Call because
|
||||||
|
they only make sense for a call:
|
||||||
|
|
||||||
|
| Today | Why it is call-specific | Where it lives instead |
|
||||||
|
| ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- |
|
||||||
|
| `autoLeave$` / `AutoLeaveReason` (`"allOthersLeft" \| "timeout" \| "decline"`) | hanging up because the other side declined or everyone left is call etiquette, not session semantics | `CallViewModel.leave$`, built from `remoteMembers$` and the room timeline |
|
||||||
|
| `ringAttempts$`, `createCallNotificationLifecycle$`, `createSentCallNotification$`, `createReceivedDecline$` | ringing, pickup timeouts and decline events are MSC4075 call-notify UX | `src/state/CallViewModel/CallNotificationLifecycle.ts`, fed by `remoteMembers$` and the Matrix room |
|
||||||
|
| `waitForCallPickup`, `autoLeaveWhenOthersLeft` options | inputs to the above | `CallViewModelOptions` |
|
||||||
|
| `hideScreensharing` option | a UI policy; the session can always share a screen if the platform can | `CallViewModel` nulls `toggleScreenSharing` |
|
||||||
|
| `ringingVm$`, `ringingStatusLocation` | view models | `CallViewModel` |
|
||||||
|
| `hostBridge.hangUp$`, `hangup()`, `userHangup$` | widget / host integration | `CallViewModel` |
|
||||||
|
| `ElementCallError` | carries Element Call translation keys | the view model wraps `MatrixRTCError`, reading `cause` |
|
||||||
|
| `participantCount$` sound limits, join/leave sounds | call UX | `CallViewModel` |
|
||||||
|
| `callIntent` | what a membership says about the call | `applicationData["m.call.intent"]`, set by the host |
|
||||||
|
|
||||||
|
Kept despite the name: `sendNotificationType`, because the js-sdk join sends the
|
||||||
|
notification as part of entering the session. Only _reacting_ to it is
|
||||||
|
call-specific.
|
||||||
|
|
||||||
|
## Media: `MemberMedia`
|
||||||
|
|
||||||
|
Every member has media. Today, in Element Call, that media _is_ a `livekit-client`
|
||||||
|
participant, handed straight through to the view models. In the SDK the participant
|
||||||
|
is wrapped in a `MemberMedia`, so that nothing outside the SDK imports
|
||||||
|
`Participant`, `TrackPublication`, `Track` or `TrackReference`. The adapter in
|
||||||
|
`sdk/src/media/` is the only place that reads a participant.
|
||||||
|
|
||||||
|
```ts
|
||||||
|
export type MediaSource =
|
||||||
|
"microphone" | "camera" | "screenShare" | "screenShareAudio";
|
||||||
|
|
||||||
|
export type MediaStreamStats =
|
||||||
|
RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined;
|
||||||
|
|
||||||
|
/** One published track of a member. */
|
||||||
|
export interface MediaTrack {
|
||||||
|
source: MediaSource;
|
||||||
|
kind: "audio" | "video";
|
||||||
|
/** Stable for the life of the track. */
|
||||||
|
id: string;
|
||||||
|
muted$: Behavior<boolean>;
|
||||||
|
/** False when the SFU reports the track as unencrypted. */
|
||||||
|
encrypted$: Behavior<boolean>;
|
||||||
|
/**
|
||||||
|
* Inbound statistics for a remote track, outbound for a local one. Polled
|
||||||
|
* once a second, but only while someone is subscribed, so a statistics
|
||||||
|
* panel costs nothing while closed.
|
||||||
|
*/
|
||||||
|
stats$: Behavior<MediaStreamStats>;
|
||||||
|
/**
|
||||||
|
* Rendering. The view hands its <video> or <audio> element over; the SDK
|
||||||
|
* sets its stream and, for video, registers the size and on-screen
|
||||||
|
* observers that pick a simulcast layer and pause the subscription while
|
||||||
|
* the element is hidden. attach is idempotent per element; detach has to be
|
||||||
|
* called before the element leaves the DOM so those observers are released.
|
||||||
|
*/
|
||||||
|
attach(element: HTMLMediaElement): void;
|
||||||
|
detach(element: HTMLMediaElement): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AudioMediaTrack extends MediaTrack {
|
||||||
|
kind: "audio";
|
||||||
|
/** Route playback through Web Audio, for earpiece pan and gain. Undefined resets. */
|
||||||
|
setAudioContext(ctx: AudioContext | undefined, plugins?: AudioNode[]): void;
|
||||||
|
/** No-op for our own audio, which is never played back. */
|
||||||
|
setVolume(volume: number): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VideoMediaTrack extends MediaTrack {
|
||||||
|
kind: "video";
|
||||||
|
/** For mirroring and switching cameras. Undefined for remote tracks. */
|
||||||
|
facingMode$?: Behavior<"user" | "environment" | undefined>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type EncryptionError = "MissingKey" | "InvalidKey";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The media of one member, backed by a LiveKit participant inside the SDK.
|
||||||
|
* There is no identity field: the LiveKit identity is the member's `id`, and
|
||||||
|
* the SDK does the matching before a MemberMedia exists.
|
||||||
|
*/
|
||||||
|
export interface MemberMedia {
|
||||||
|
local: boolean;
|
||||||
|
speaking$: Behavior<boolean>;
|
||||||
|
screenShareEnabled$: Behavior<boolean>;
|
||||||
|
|
||||||
|
microphone$: Behavior<AudioMediaTrack | undefined>;
|
||||||
|
camera$: Behavior<VideoMediaTrack | undefined>;
|
||||||
|
screenShare$: Behavior<VideoMediaTrack | undefined>;
|
||||||
|
screenShareAudio$: Behavior<AudioMediaTrack | undefined>;
|
||||||
|
|
||||||
|
/** Emits when the SFU reports a key problem for this member. */
|
||||||
|
encryptionError$: Observable<EncryptionError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LocalMemberMedia extends MemberMedia {
|
||||||
|
local: true;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Rendering: `attach` and `detach`
|
||||||
|
|
||||||
|
The view passes its DOM element into the SDK and the SDK does the rest; there is no
|
||||||
|
`MediaStreamTrack` on the interface. The reason is adaptive stream: Element Call
|
||||||
|
runs with `adaptiveStream: true`, which means LiveKit decides the simulcast layer
|
||||||
|
from the size of the attached video element and pauses the subscription at the
|
||||||
|
SFU while no attached element is on screen. Those size and visibility observers are
|
||||||
|
set up inside LiveKit's `attach`. Handing the element over is the only way to keep
|
||||||
|
that without reimplementing the observers outside the SDK. The e2e tests check both
|
||||||
|
effects: a larger tile receives a higher resolution, a hidden tile stops receiving
|
||||||
|
frames.
|
||||||
|
|
||||||
|
What `attach` covers for free, so the host does not have to: `srcObject` and
|
||||||
|
`autoplay` setup with the Safari and Firefox black-video workarounds; `play()` and
|
||||||
|
the autoplay-blocked signal; following the track through mute, reconnect and
|
||||||
|
re-subscription into every attached element; the Web Audio graph for earpiece pan
|
||||||
|
and gain; the adaptive stream observers.
|
||||||
|
|
||||||
|
The interface is web-only. A non-DOM host would need a different media surface; if
|
||||||
|
one shows up, a `MediaStreamTrack` accessor is added next to `attach`, not instead
|
||||||
|
of it.
|
||||||
|
|
||||||
|
## Packaging
|
||||||
|
|
||||||
|
`sdk/` is packaged exactly as `component/` is: a `package.json` with peer
|
||||||
|
dependencies (`matrix-js-sdk`, `livekit-client`, `rxjs`), `exports` with types,
|
||||||
|
type emission through `sdk/tsconfig.build.json`, an externals list in
|
||||||
|
`vite-sdk.config.ts` that `pnpm lint:externals` enforces, and installation from the
|
||||||
|
repository as a git dependency with a `path:` suffix. `pnpm build:sdk` produces
|
||||||
|
`sdk/dist`. It is not a pnpm workspace package yet; the root is not a workspace, and
|
||||||
|
moving to one later carries the package name, entry point and externals over
|
||||||
|
unchanged.
|
||||||
|
|
||||||
|
Element Call consumes the SDK by its package name from day one, so the swap to a
|
||||||
|
published dependency is one line: `tsconfig.json` `paths` and the `resolve.alias`
|
||||||
|
in `vitePluginsConfig` both map `@element-hq/matrixrtc-sdk` to `sdk/index.ts`. The
|
||||||
|
alias has to live in `vitePluginsConfig`, which every config is built on. Put only
|
||||||
|
into the app's own config, the harness resolves the name through `sdk/package.json`
|
||||||
|
to whatever stale `sdk/dist` is lying around.
|
||||||
|
|
||||||
|
Layout:
|
||||||
|
|
||||||
|
```
|
||||||
|
sdk/
|
||||||
|
index.ts the entry point: re-exports the API and the primitives
|
||||||
|
src/api.ts the public types above
|
||||||
|
src/session/ MatrixRTCClient, the local member, publisher, connections, transports, join
|
||||||
|
src/media/ the LiveKit adapter: MemberMedia and MediaTrack
|
||||||
|
src/utils/ LazyBehavior, mapScoped, error wrapping
|
||||||
|
dev/ the harness (below)
|
||||||
|
SdkArchitecture.md this document
|
||||||
|
SdkMigration.md how Element Call gets from CallViewModel to the client
|
||||||
|
```
|
||||||
|
|
||||||
|
An import boundary is still to be added to the local lint plugin: nothing under
|
||||||
|
`sdk/` may import `src/`, `component/`, `playwright/`, React, i18n, Compound or the
|
||||||
|
settings. Until the shared modules have moved (see the migration document) the
|
||||||
|
SDK imports them from `src/`, so the rule cannot be turned on yet.
|
||||||
|
|
||||||
|
## Dev harness: `sdk/dev/`
|
||||||
|
|
||||||
|
A web app that uses the SDK and nothing else: plain DOM, no React, no Compound, no
|
||||||
|
i18n, imports only `@element-hq/matrixrtc-sdk`, `matrix-js-sdk` and `rxjs`. It is
|
||||||
|
where the SDK is run during development and what the e2e tests in `playwright/sdk`
|
||||||
|
drive. If something cannot be done in the harness without reaching into `src/`, the
|
||||||
|
SDK API is missing something.
|
||||||
|
|
||||||
|
What it does: log in with a password, or with an access token the URL hands it (the
|
||||||
|
tests register users through the admin API and skip `/login`, which the dev
|
||||||
|
homeserver rate-limits); join the room and wait for the sync to hold it; create the
|
||||||
|
client with microphone and camera enabled; show one tile per member with the display
|
||||||
|
name, a `<video>` attached from `media$.camera$` and, for remote members, an
|
||||||
|
`<audio>` from `media$.microphone$`; toggle the microphone and camera; leave.
|
||||||
|
|
||||||
|
Every media element carries the track's state as `data-*` attributes (`muted`,
|
||||||
|
`encrypted`, `frameWidth`, `frames`), which is what the tests read to check mute
|
||||||
|
propagation, encryption, adaptive resolution and paused subscriptions. That is more
|
||||||
|
than the smallest consumer needs, and the price of testing the SDK through its
|
||||||
|
public API only.
|
||||||
|
|
||||||
|
## Open decisions
|
||||||
|
|
||||||
|
- **Timings.** Delayed-leave timings, the sync grace period, the key rotation
|
||||||
|
delay and participant limit come from `DEFAULT_CONFIG` today. The SDK must not
|
||||||
|
read `config.json`, so they become an option (`timings`, with these defaults).
|
||||||
|
Delegating the delayed leave to the SFU is not implemented, since it needs the
|
||||||
|
same timings; the js-sdk restarts the delayed event itself meanwhile.
|
||||||
|
- **`LocalMediaInputs` as behaviors or methods.** Behaviors match how the mute
|
||||||
|
states drive the publisher and keep a lobby's pre-join state in one place;
|
||||||
|
methods are the more conventional SDK shape. Two things are missing either way:
|
||||||
|
creating the tracks before `join()` for a preview (today they are created at the
|
||||||
|
join), and screen share capture options, which are fixed.
|
||||||
|
- **`applicationData` after the join.** The option is read once. The implementation
|
||||||
|
still updates `m.call.intent` from the camera input, as Element Call does; that
|
||||||
|
moves to the host once the data can be updated, as a behavior or a setter.
|
||||||
|
- **Backend overrides.** Tests of the client itself need to inject the transport,
|
||||||
|
the connection factory and the LiveKit room. They were planned as
|
||||||
|
`options.backend`, grouped so that no LiveKit type appears on the options; not
|
||||||
|
yet added, so `createMatrixRTCClient` is covered by the e2e tests only.
|
||||||
|
- **Raw local state.** The local member's state machine (`LocalMemberState`) stays
|
||||||
|
internal; `status$` and `fatalError$` are its public view. A developer panel may
|
||||||
|
want it as `LocalRTCMember.state$`.
|
||||||
|
- **Developer panel.** `resolved$.identity` covers the local identity and
|
||||||
|
`remoteMembers$` filtered by `transport$.id` the remote list. Anything beyond
|
||||||
|
that (LiveKit room state, connection quality) needs an opaque `debug$` per
|
||||||
|
transport.
|
||||||
@@ -0,0 +1,328 @@
|
|||||||
|
# From `CallViewModel` to `MatrixRTCClient`
|
||||||
|
|
||||||
|
How Element Call gets from today's `CallViewModel`, which does everything, to a
|
||||||
|
view model that consumes the SDK. The target shape is in
|
||||||
|
[`SdkArchitecture.md`](./SdkArchitecture.md); this document is the inventory of
|
||||||
|
what moves, what stays, and how far along it is. It refers to source files and
|
||||||
|
symbols, so it ages with the code; the architecture document should not.
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
| Slice | State |
|
||||||
|
| -------------------------------------------------------------------------------------------------------------- | -------------------- |
|
||||||
|
| 1. Scaffold: `sdk/` packaging, the interface, the harness, the smoke test | done, October 2026 |
|
||||||
|
| 2. First implementation of `createMatrixRTCClient` under `sdk/src`, the media adapter, the e2e suite | done, 1 October 2026 |
|
||||||
|
| 3. Move the shared modules out of `src/` into `sdk/src/`, leave re-exports behind, turn on the import boundary | next |
|
||||||
|
| 4. `createCallViewModel$` consumes the client; media view models take `media$` | after 3 |
|
||||||
|
| 5. Remove the re-exports, `sdk-target-based-on-call-view-model/` and the old `build:sdk` output | last |
|
||||||
|
|
||||||
|
Slice 2 was built without touching `src/`, so the SDK imports its building blocks
|
||||||
|
from there for now. Nothing in Element Call consumes the SDK yet.
|
||||||
|
|
||||||
|
## What the SDK still imports from `src/`
|
||||||
|
|
||||||
|
These move into `sdk/src/` in slice 3, each with its tests, leaving a one-line
|
||||||
|
re-export in `src` until Element Call's importers are updated.
|
||||||
|
|
||||||
|
| Module | Used for | Notes for the move |
|
||||||
|
| ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| `state/Behavior.ts`, `state/ObservableScope.ts`, `utils/observable.ts` | the reactive primitives | exported from `sdk/index.ts` already; 37 importers in `src` and `component` keep working through the re-export |
|
||||||
|
| `state/SessionBehaviors.ts` | `createMemberships$`, `membershipsAndTransports$`, `createKeyRotationSuppressed$` | as is |
|
||||||
|
| `state/CallViewModel/remoteMembers/Connection.ts` | `Connection`, `ConnectionState` | the SDK subclasses it as `ResolvedConnection` to keep the JWT answer; fold that in |
|
||||||
|
| `state/CallViewModel/remoteMembers/ConnectionFactory.ts` | the `ConnectionFactory` interface only | `ECConnectionFactory` stays behind: it reads settings and `MediaDevices`; the SDK has `LivekitConnectionFactory` |
|
||||||
|
| `state/CallViewModel/remoteMembers/ConnectionManager.ts` | `createConnectionManager$` | as is |
|
||||||
|
| `state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts` | `createRemoteMatrixLivekitMembers$`, the membership-to-participant match | the `TaggedParticipant` type goes; the SDK maps the result to `RTCMember` |
|
||||||
|
| `state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts` | `createRoomMembers$`, `createMatrixMemberMetadata$` | depends on `utils/displayname.ts`, which moves with it |
|
||||||
|
| `state/CallViewModel/localMember/HomeserverConnected.ts` | `createHomeserverConnected$` | reads `Config.get()` unless the grace period is passed; the SDK passes it |
|
||||||
|
| `state/CallViewModel/localMember/LocalTransport.ts` | the `LocalTransport` type only | `getLocalTransport` reads config and settings; the SDK has its own in `session/LocalTransport.ts` |
|
||||||
|
| `state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts` | discovery through `rtc/transports` | takes a resolved config for the fallback url; the SDK passes `DEFAULT_CONFIG`, which has none |
|
||||||
|
| `state/CallViewModel/localMember/LocalMember.ts` | `observeSharingScreen$` only | the SDK has its own local member; the rest stays until slice 4 |
|
||||||
|
| `livekit/openIDSFU.ts` | `getSFUConfigWithOpenID`, `SFUConfig` | reads `Config.get()` only on the delegated delayed leave path, which the SDK does not use |
|
||||||
|
| `livekit/options.ts` | `buildLiveKitOptions` | takes the media quality config as a parameter; `getLiveKitOptions` stays behind |
|
||||||
|
| `e2ee/matrixKeyProvider.ts`, `e2ee/e2eeType.ts`, `e2ee/sharedKeyManagement.ts` | the key provider, `E2eeType`, the `EncryptionSystem` type | `sharedKeyManagement.ts` also holds React hooks; split the type out |
|
||||||
|
| `config/ConfigOptions.ts` | `MatrixRTCMode`, `DEFAULT_CONFIG` for the timings | the enum moves; the timings become an option (see the architecture document's open decisions) |
|
||||||
|
| `utils/errors.ts` | the error classes the modules above throw | they carry translation keys, so they stay; the SDK wraps them as `MatrixRTCError` with `cause` |
|
||||||
|
|
||||||
|
Rewritten in the SDK rather than moved, because the originals read settings,
|
||||||
|
config, analytics or the host bridge: the local member (`session/LocalMember.ts`),
|
||||||
|
the publisher (`session/Publisher.ts`), the connection factory and LiveKit room
|
||||||
|
options (`session/ConnectionFactory.ts`), transport discovery
|
||||||
|
(`session/LocalTransport.ts`), the join (`session/joinJsSdkSession.ts`) and the
|
||||||
|
key provider selection (`session/KeyProvider.ts`).
|
||||||
|
|
||||||
|
## What becomes what in `createCallViewModel$`
|
||||||
|
|
||||||
|
Everything below `createCallViewModel$` that is not presentation and not in the
|
||||||
|
call-specific table of the architecture document. Grouped by what it is today in
|
||||||
|
`src/state/CallViewModel/CallViewModel.ts`.
|
||||||
|
|
||||||
|
| Binding today | In the SDK |
|
||||||
|
| ---------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- |
|
||||||
|
| `livekitKeyProvider` via `getE2eeKeyProvider` | internal, keyed off `options.encryptionSystem` |
|
||||||
|
| `matrixRTCMode` resolution from `Config.get()` | an input; resolution stays in the caller |
|
||||||
|
| `matrixRTCSession` parameter | internal: the session manager for the slot, `getRoomSession(room)` |
|
||||||
|
| `memberships$`, `membershipsAndTransports` | internal; surfaces as `RTCMember.transport$` |
|
||||||
|
| `ownMembershipIdentity` | internal; surfaces as `RTCMember.id` |
|
||||||
|
| `localTransport$` via `getLocalTransport` | internal |
|
||||||
|
| `connectionFactory`, `connectionManager` | internal; surfaces as `connectedTransports$` |
|
||||||
|
| `remoteMatrixLivekitMembers$` | `remoteMembers$` |
|
||||||
|
| `localMembership` via `createLocalMembership$`, `enterRTCSession`, `Publisher`, `delayId$` | internal; surfaces as `join`, `leave`, `status$`, `connected$`, `reconnecting$`, the screen share fields |
|
||||||
|
| `localRtcMembership$`, `localMatrixLivekitMember$` | `localMember$` |
|
||||||
|
| `matrixLivekitMembers$`, `participantCount$` | `memberCount$` |
|
||||||
|
| `matrixRoomMembers$`, `matrixMemberMetadataStore` | `RTCMember.displayName$` / `avatarUrl$` |
|
||||||
|
| `allConnections$` | `connectedTransports$` |
|
||||||
|
| `livekitRoomItems$` | dropped; audio renders from `remoteMembers$[].media$` |
|
||||||
|
| `keyRotationSuppressed$` | same |
|
||||||
|
| `sharingScreen$`, `toggleScreenSharing` | `localMember$.sharingScreen$`, `.toggleScreenSharing` |
|
||||||
|
| `errors$` and the `fatalError$` mapping | `fatalError$`, as `MatrixRTCError` |
|
||||||
|
| `join`, `leave`, `connected$`, `reconnecting$`, `screenShareError$`, `dismissScreenShareError` | same names on `MatrixRTCClient` / `LocalRTCMember` |
|
||||||
|
| `callIntent` option, `updateCallIntent` from the camera | `applicationData["m.call.intent"]`; the camera sync is still in the SDK, see the open decisions |
|
||||||
|
|
||||||
|
Changes to the call sites that the rename brings: `localMember$` and
|
||||||
|
`remoteMembers$` replace `localMatrixLivekitMember$` and
|
||||||
|
`remoteMatrixLivekitMembers$`; `connection$` on a member goes away, since the view
|
||||||
|
model only ever used `livekitRoom` (for `waitingForMedia$` and encryption errors,
|
||||||
|
both on `MemberMedia` now) and `transport.livekit_service_url` (now
|
||||||
|
`transport$.id`); `connectedTransports$` replaces `allConnections$`, since the
|
||||||
|
developer panel only needed which transports exist and whether they are up.
|
||||||
|
|
||||||
|
## What stays in `CallViewModel`
|
||||||
|
|
||||||
|
Everything in the call-specific table, plus everything that takes `userMedia$` or a
|
||||||
|
UI input and turns it into something to draw:
|
||||||
|
|
||||||
|
- Media view models (`createWrappedUserMedia` and below), built from
|
||||||
|
`remoteMembers$` / `localMember$` instead of participants.
|
||||||
|
- `CallNotificationLifecycle.ts` in full: `ringAttempts$`, `autoLeave$`,
|
||||||
|
`ringingMedia$`, `ringingStatusLocation`.
|
||||||
|
- `leave$` (= `autoLeave$` + `hangup()` + `hostBridge.hangUp$`), `hangup()`.
|
||||||
|
- `screenShares$`.
|
||||||
|
- Sounds: `joinSoundEffect$`, `leaveSoundEffect$`, `newHandRaised$`,
|
||||||
|
`newScreenShare$`, `audibleReactions$`, `visibleReactions$`.
|
||||||
|
- `handsRaised$`, `reactions$`, until they move onto `RTCMember`.
|
||||||
|
- All layout: `spotlightSpeaker$`, `grid$`, `spotlight$`, `pip*`, `windowMode$`,
|
||||||
|
`layoutSwitchVm`, every `*LayoutMedia$`, `layout$`, `visibleTiles$`,
|
||||||
|
`tileStoreGeneration$`, `overflowing$`, `spotlightExpanded$`.
|
||||||
|
- Chrome: `showHeader$`, `showFooter$`, `edgeToEdge$`, `showModals$`,
|
||||||
|
`settingsOpen$`, `showSpotlightIndicators$`, `showSpeakingIndicators$`,
|
||||||
|
`showNameTags$`, tap and hover subjects.
|
||||||
|
- Audio routing: `earpieceMode$`, `audioOutputSwitcher$`, which read
|
||||||
|
`MediaDevices`.
|
||||||
|
- `CallViewModelOptions` keeps `hostBridge`, `controlledAudioDevices`, `header`,
|
||||||
|
`showControls`, `hideScreensharing`, `waitForCallPickup`,
|
||||||
|
`autoLeaveWhenOthersLeft`, `windowSize$`, `toggleScreensharing` (test override),
|
||||||
|
and forwards `encryptionSystem`, `matrixRTCMode`, `sendNotificationType` and
|
||||||
|
`applicationData` to `MatrixRTCClientOptions`.
|
||||||
|
|
||||||
|
## The media view models and the participant
|
||||||
|
|
||||||
|
The goal of slice 4 on the media side: nothing in Element Call imports
|
||||||
|
`Participant`, `RemoteParticipant`, `LocalParticipant`, `TrackPublication`,
|
||||||
|
`Track` or `TrackReference`. This is where the participant flows today, and what
|
||||||
|
each use becomes.
|
||||||
|
|
||||||
|
### Where the participant flows today
|
||||||
|
|
||||||
|
Start: `remoteMatrixLivekitMembers$` in `CallViewModel.ts`, built by
|
||||||
|
`createRemoteMatrixLivekitMembers$` in
|
||||||
|
`src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts`, as
|
||||||
|
`participant: { type: "remote"; value$: Behavior<RemoteParticipant | null> }`.
|
||||||
|
Consumers, in order of depth:
|
||||||
|
|
||||||
|
1. `MatrixLivekitMembers.ts` matches `participant.identity` against
|
||||||
|
`membership.rtcBackendIdentity`; logs `participant.sid`.
|
||||||
|
2. `CallViewModel.ts` `livekitRoomItems$` reads `isLocal`, `identity`, hands
|
||||||
|
`connection.livekitRoom` plus identities to `LivekitRoomAudioRenderer`.
|
||||||
|
3. `CallViewModel.ts` `userMedia$` passes the whole `TaggedParticipant` into
|
||||||
|
`createWrappedUserMedia`.
|
||||||
|
4. `src/state/media/WrappedUserMediaViewModel.ts`: `observeParticipantEvents(p,
|
||||||
|
TrackPublished, TrackUnpublished, LocalTrackPublished, LocalTrackUnpublished)`
|
||||||
|
then `p.isScreenShareEnabled` to spawn screen share view models.
|
||||||
|
5. `src/state/media/UserMediaViewModel.ts`: `observeParticipantMedia(p)` for
|
||||||
|
`microphoneTrack?.isMuted` / `cameraTrack?.isMuted`; `IsSpeakingChanged` →
|
||||||
|
`p.isSpeaking`; `observeRtpStreamStats$(p, source, type)`.
|
||||||
|
6. `src/state/media/MemberMediaViewModel.ts`: `observeTrackReference$(p, source)`
|
||||||
|
for audio and video; `publication.isEncrypted`; `p.isLocal`; `p.identity`
|
||||||
|
matched inside `RoomEvent.EncryptionError` messages; inbound RTP frame counters.
|
||||||
|
7. `src/state/media/RemoteUserMediaViewModel.ts`, `RemoteScreenShareViewModel.ts`:
|
||||||
|
`p.setVolume(v)` / `p.setVolume(v, ScreenShareAudio)`; `participant === null`
|
||||||
|
means "waiting for media".
|
||||||
|
8. `src/state/observeTrackReference.ts`: `p.getTrackPublication(source)` wrapped as
|
||||||
|
`{ participant, publication, source }`.
|
||||||
|
9. `src/state/media/observeRtpStreamStats.ts`: `publication.track` must be
|
||||||
|
`RemoteTrack | LocalTrack`, then `track.getRTCStatsReport()`.
|
||||||
|
10. Views: `src/tile/MediaView.tsx` renders `<VideoTrack trackRef={video}>` from
|
||||||
|
`@livekit/components-react`; `src/livekit/MatrixAudioRenderer.tsx` uses
|
||||||
|
`useTracks` on the LiveKit room, filters on `ref.participant.isLocal` /
|
||||||
|
`.identity`, and calls `RemoteAudioTrack.setAudioContext` /
|
||||||
|
`setWebAudioPlugins`.
|
||||||
|
|
||||||
|
Local-only extras, same shape but on `LocalParticipant`
|
||||||
|
(`localMember/LocalMember.ts`, `Publisher.ts`,
|
||||||
|
`src/state/media/LocalUserMediaViewModel.ts`): `trackPublications`,
|
||||||
|
`audioTrackPublications`, `setScreenShareEnabled`, `setMicrophoneEnabled`,
|
||||||
|
`setCameraEnabled`, `enableCameraAndMicrophone`, `isMicrophoneEnabled`,
|
||||||
|
`isCameraEnabled`, `unpublishTrack`, and on the camera `LocalVideoTrack`:
|
||||||
|
`facingModeFromLocalTrack`, `restartTrack`, `mediaStreamTrack`,
|
||||||
|
`TrackEvent.Restarted` / `Muted`.
|
||||||
|
|
||||||
|
### Participant API actually used, and what covers it
|
||||||
|
|
||||||
|
| LiveKit surface | Used for | Used in | Now |
|
||||||
|
| --------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | ----------- | ------------------------------------------------ |
|
||||||
|
| `identity` | match to `rtcBackendIdentity`; audio renderer allow-list; E2EE error matching; logs | 1, 2, 6, 10 | matched inside the SDK; `RTCMember.id` |
|
||||||
|
| `sid` | logs only | 1 | SDK logs |
|
||||||
|
| `isLocal` | skip local audio; encryption status short-circuit | 2, 6, 10 | `MemberMedia.local` |
|
||||||
|
| `isSpeaking` + `IsSpeakingChanged` | `speaking$` | 5 | `MemberMedia.speaking$` |
|
||||||
|
| `isScreenShareEnabled` + publish events | spawn screen share VM | 4 | `MemberMedia.screenShareEnabled$` |
|
||||||
|
| `observeParticipantMedia(p)` → `microphoneTrack?.isMuted`, `cameraTrack?.isMuted` | `audioEnabled$`, `videoEnabled$` | 5 | `microphone$` / `camera$` → `muted$` |
|
||||||
|
| `getTrackPublication(source)` for the four sources | `TrackReference` for video, encryption, stats, screen share audio presence | 6, 7, 8 | the four track behaviors |
|
||||||
|
| `publication.isEncrypted` | `unencryptedWarning$` | 6 | `MediaTrack.encrypted$` |
|
||||||
|
| `publication.track.getRTCStatsReport()` | `audioStreamStats$`, `videoStreamStats$`, "receiving okay" heuristic | 5, 6, 9 | `MediaTrack.stats$` |
|
||||||
|
| `setVolume(v)`, `setVolume(v, ScreenShareAudio)` | `VolumeControls` sink | 7 | `AudioMediaTrack.setVolume` |
|
||||||
|
| `TrackReference` → `<VideoTrack>` | render video | 10 | `MediaTrack.attach` on a `<video>` |
|
||||||
|
| room `useTracks` + `RemoteAudioTrack.setAudioContext/setWebAudioPlugins` | render audio with earpiece pan/gain | 10 | one `<audio>` per audio track, `setAudioContext` |
|
||||||
|
| room `RoomEvent.EncryptionError` with `identity` + `"MissingKey"`/`"InvalidKey"` | `encryptionStatus$` | 6 | `MemberMedia.encryptionError$` |
|
||||||
|
|
||||||
|
Everything else in the media view models (`observeSpeaker$`, `VolumeControls`,
|
||||||
|
`SortingBin`, hand raise, reactions, display name, avatar, aspect ratio) already
|
||||||
|
works on plain booleans and observables and does not change.
|
||||||
|
|
||||||
|
### How the call sites map onto `MemberMedia`
|
||||||
|
|
||||||
|
- `audioEnabled$` = `microphone$ → track?.muted$ === false`; `videoEnabled$`
|
||||||
|
likewise from `camera$`.
|
||||||
|
- `unencryptedWarning$` = any of the four tracks with `encrypted$ === false`.
|
||||||
|
- `encryptionStatus$` = `encryptionError$` plus a "frames decoded" check on
|
||||||
|
`camera$.stats$` / `microphone$.stats$`; same logic, no participant.
|
||||||
|
- `screenShares$` in `WrappedUserMediaViewModel` keys off `screenShareEnabled$`.
|
||||||
|
- `VolumeControls` sink = `microphone$.value?.setVolume` /
|
||||||
|
`screenShareAudio$.value?.setVolume`.
|
||||||
|
- `waitingForMedia$` = transport present and `media === null`, unchanged
|
||||||
|
semantics; `TaggedParticipant` goes away.
|
||||||
|
- `video$: Behavior<TrackReference | undefined>` on `BaseMemberMediaViewModel`
|
||||||
|
becomes `Behavior<VideoMediaTrack | undefined>`; `MediaView` replaces
|
||||||
|
`<VideoTrack trackRef>` with a `<video>` plus `attach` / `detach` in an effect,
|
||||||
|
keeping the `onResize` callback for the aspect ratio. `VideoPreview.tsx` already
|
||||||
|
follows this pattern.
|
||||||
|
- `LivekitRoomAudioRenderer` stops using `useTracks` on the room and renders one
|
||||||
|
`<audio>` per `microphone$` / `screenShareAudio$` across `remoteMembers$`. That
|
||||||
|
removes the "no matching matrix call member" warning path, since an unmatched
|
||||||
|
track can no longer be seen.
|
||||||
|
- The `showConnectionStats` setting gates the subscription to `stats$` in the view
|
||||||
|
model; the SDK polls only while subscribed.
|
||||||
|
- Stories and tests mock `attach` by setting `srcObject` from
|
||||||
|
`canvas.captureStream()` on the element, or by doing nothing and asserting the
|
||||||
|
call.
|
||||||
|
|
||||||
|
## Packaging, as it was planned
|
||||||
|
|
||||||
|
`sdk/` mirrors `component/` file for file. The table that drove the scaffold, kept
|
||||||
|
for the record:
|
||||||
|
|
||||||
|
| `component/` | `sdk/` | Differences |
|
||||||
|
| ------------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| `package.json` (`@element-hq/element-call-component`) | `package.json` (`@element-hq/matrixrtc-sdk`) | peers: `matrix-js-sdk`, `livekit-client`, `rxjs`. No React, no Compound. `exports`: `"."` only. No `./style.css`, no `sideEffects`. Same `prepare`. |
|
||||||
|
| `pnpm-workspace.yaml` (marker, `autoInstallPeers: false`) | same | none |
|
||||||
|
| `tsconfig.build.json` | same | `include: ["./index.ts"]` |
|
||||||
|
| `index.tsx` + `api.ts` | `index.ts` + `src/api.ts` | one entry; no separate types entry until a host asks |
|
||||||
|
| `host.ts`, `localization.ts`, `ElementCall.module.css`, `build/scopeStylesToRoot.ts` | none | no host bridge, no i18n, no styles |
|
||||||
|
| `dev/` on port 3001 | `dev/` on port 3002 via `vite-sdk-dev.config.ts` | plain DOM |
|
||||||
|
| `vite-component.config.ts` | `vite-sdk.config.ts` | `vitePluginsConfig({ mode, html: false })`, `outDir: "sdk/dist"`, externals = js-sdk and LiveKit entries plus `rxjs` |
|
||||||
|
| root scripts `build:component*` | `build:sdk*` | `build:sdk:types` = `tsc -p sdk/tsconfig.build.json` |
|
||||||
|
| root `tsconfig.json` `include` | adds `./sdk/**/*.ts` | |
|
||||||
|
| `scripts/check-component-externals.mjs` | parameterised by config; `sdk` added to the sources, `sdk/dev` excluded | |
|
||||||
|
| `knip.ts` | `vite-sdk.config.ts`, `vite-sdk-dev.config.ts`, `sdk/index.ts` as entry | |
|
||||||
|
| `vitest.config.ts` unit project | adds `sdk/**/*.test.ts` | |
|
||||||
|
| `package.json` `lint:oxlint` paths | adds `sdk` | |
|
||||||
|
|
||||||
|
Why not a pnpm workspace package yet: the root has no `packages:` in
|
||||||
|
`pnpm-workspace.yaml`, so a workspace split means restructuring the app, the
|
||||||
|
component's `prepare` trick, CI and the lockfile at the same time as moving the
|
||||||
|
code, and dev-time HMR across workspace packages needs the same source alias
|
||||||
|
anyway. It is the right end state if the SDK gets a release cadence of its own.
|
||||||
|
|
||||||
|
## Reference: the `CallViewModel` public API before the migration
|
||||||
|
|
||||||
|
Source: `src/state/CallViewModel/CallViewModel.ts`. Only consumer outside tests:
|
||||||
|
`src/room/InCallView.tsx`. A snapshot, so that the split can be checked against
|
||||||
|
what was there.
|
||||||
|
|
||||||
|
### Factory
|
||||||
|
|
||||||
|
`createCallViewModel$(scope, matrixRTCSession, matrixRoom, mediaDevices, muteStates, options, handsRaisedSubject$, reactionsSubject$, trackProcessorState$): CallViewModel`
|
||||||
|
|
||||||
|
- `scope: ObservableScope`: ending the scope is one of the two ways to leave.
|
||||||
|
- `matrixRTCSession: MatrixRTCSession`, `matrixRoom: MatrixRoom`: one room per view model.
|
||||||
|
- `mediaDevices: MediaDevices`, `muteStates: MuteStates`
|
||||||
|
- `options: CallViewModelOptions` (below)
|
||||||
|
- `handsRaisedSubject$: Observable<Record<string, RaisedHandInfo>>`
|
||||||
|
- `reactionsSubject$: Observable<Record<string, ReactionInfo>>`
|
||||||
|
- `trackProcessorState$: Behavior<ProcessorState>`
|
||||||
|
|
||||||
|
Helper: `callViewModelOptionsFromParams(params: UrlParams)` picks
|
||||||
|
`controlledAudioDevices`, `header`, `showControls`, `hideScreensharing`,
|
||||||
|
`sendNotificationType`, `callIntent` out of the URL params. `autoLeaveWhenOthersLeft`
|
||||||
|
and `waitForCallPickup` are left to the caller.
|
||||||
|
|
||||||
|
### `CallViewModelOptions`
|
||||||
|
|
||||||
|
| Field | Type | Note |
|
||||||
|
| ----------------------------------------------------------------------------------------------------------- | --------------------------- | --------------------------------------------------- |
|
||||||
|
| `encryptionSystem` | `EncryptionSystem` | required |
|
||||||
|
| `hostBridge?` | `HostBridge` | host can hang up / observe join+leave; default none |
|
||||||
|
| `controlledAudioDevices?` | `boolean` | host controls audio output devices |
|
||||||
|
| `header?` | `HeaderStyle` | default `Standard` |
|
||||||
|
| `showControls?` | `boolean` | default `true` |
|
||||||
|
| `hideScreensharing?` | `boolean` | default `false` |
|
||||||
|
| `sendNotificationType?` | `RTCNotificationType` | notification sent on join |
|
||||||
|
| `callIntent?` | `RTCCallIntent` | |
|
||||||
|
| `autoLeaveWhenOthersLeft?` | `boolean` | |
|
||||||
|
| `waitForCallPickup?` | `boolean` | telephone-style ringing UI |
|
||||||
|
| `windowSize$` | `Behavior<{width; height}>` | required; drives `WindowMode` |
|
||||||
|
| `matrixRTCMode?` | `MatrixRTCMode` | MatrixRTC version / compat mode |
|
||||||
|
| `livekitRoomFactory?`, `connectionState$?`, `localTransport?`, `connectionFactory?`, `toggleScreensharing?` | | test overrides |
|
||||||
|
|
||||||
|
### The interface, grouped
|
||||||
|
|
||||||
|
**Lifecycle / connection** (to the client): `join()`, `leave()`, `hangup()`,
|
||||||
|
`leave$: Observable<"user" | AutoLeaveReason>`, `autoLeave$`, `connected$`,
|
||||||
|
`reconnecting$`, `allConnections$: Behavior<ConnectionManagerData>`,
|
||||||
|
`fatalError$: Behavior<ElementCallError | null>`, `keyRotationSuppressed$`.
|
||||||
|
|
||||||
|
**Members / participants** (to the client): `participantCount$`,
|
||||||
|
`localMatrixLivekitMember$`, `remoteMatrixLivekitMembers$`, `livekitRoomItems$`,
|
||||||
|
`handsRaised$`, `reactions$` (keyed by `${userId}:${deviceId}`).
|
||||||
|
|
||||||
|
**Screen sharing** (capability to the client, error UI stays): `toggleScreenSharing`,
|
||||||
|
`sharingScreen$`, `screenShareError$`, `dismissScreenShareError()`.
|
||||||
|
|
||||||
|
**Ringing** (stays): `ringingVm$`, `ringingStatusLocation`.
|
||||||
|
|
||||||
|
**Audio routing** (stays): `earpieceMode$`, `audioOutputSwitcher$`.
|
||||||
|
|
||||||
|
**Sounds and transient events** (stay): `joinSoundEffect$`, `leaveSoundEffect$`,
|
||||||
|
`newHandRaised$`, `newScreenShare$`, `audibleReactions$`, `visibleReactions$`,
|
||||||
|
`MAX_PARTICIPANT_COUNT_FOR_SOUND = 8`, `THROTTLE_SOUND_EFFECT_MS = 500`.
|
||||||
|
|
||||||
|
**Layout and tiles** (stay): `layout$`, `tileStoreGeneration$`,
|
||||||
|
`showSpotlightIndicators$`, `showSpeakingIndicators$`, `showNameTags$`,
|
||||||
|
`spotlightExpanded$`, `toggleSpotlightExpanded$`, `layoutSwitchVm$`.
|
||||||
|
|
||||||
|
**Chrome visibility and UI interaction** (stay): `showHeader$`, `showFooter$`,
|
||||||
|
`edgeToEdge$`, `overflowing$`, `showModals$`, `settingsOpen$`, `setSettingsOpen$`,
|
||||||
|
`tapScreen()`, `tapControls()`, `hoverScreen()`, `unhoverScreen()`.
|
||||||
|
|
||||||
|
### Pending rename noted in the source
|
||||||
|
|
||||||
|
A TODO above `CallViewModelOptions` proposes: member/membership → `rtcMember`,
|
||||||
|
participant → `livekitParticipant`, matrixLivekitItem → `callMember`; and in the
|
||||||
|
js-sdk `callMembership` → `rtcMembership`. The SDK's vocabulary (architecture
|
||||||
|
document, "Vocabulary") settles the first three; the js-sdk rename is still open.
|
||||||
|
|
||||||
|
### Build targets before the migration
|
||||||
|
|
||||||
|
`vite-sdk.config.ts` (`pnpm build:sdk`), `vite-component.config.ts`,
|
||||||
|
`vite-embedded.config.ts`, `vite.config.ts`. The old SDK target
|
||||||
|
(`sdk-target-based-on-call-view-model/`) is the one whose dependency direction the
|
||||||
|
migration flips; it goes in slice 5.
|
||||||
+12
-12
@@ -21,13 +21,13 @@ import {
|
|||||||
} from "rxjs";
|
} from "rxjs";
|
||||||
import {
|
import {
|
||||||
constant,
|
constant,
|
||||||
createRtcSession,
|
createMatrixRTCClient,
|
||||||
E2eeType,
|
E2eeType,
|
||||||
MatrixRTCMode,
|
MatrixRTCMode,
|
||||||
type MediaTrack,
|
type MediaTrack,
|
||||||
ObservableScope,
|
ObservableScope,
|
||||||
type RtcMember,
|
type RTCMember,
|
||||||
type RtcSession,
|
type MatrixRTCClient,
|
||||||
} from "@element-hq/matrixrtc-sdk";
|
} from "@element-hq/matrixrtc-sdk";
|
||||||
|
|
||||||
import { createSession, joinRoom, type Login } from "./session";
|
import { createSession, joinRoom, type Login } from "./session";
|
||||||
@@ -73,7 +73,7 @@ async function start(
|
|||||||
const scope = new ObservableScope();
|
const scope = new ObservableScope();
|
||||||
const microphoneEnabled$ = new BehaviorSubject(true);
|
const microphoneEnabled$ = new BehaviorSubject(true);
|
||||||
const cameraEnabled$ = new BehaviorSubject(true);
|
const cameraEnabled$ = new BehaviorSubject(true);
|
||||||
const session = createRtcSession(
|
const rtcClient = createMatrixRTCClient(
|
||||||
scope,
|
scope,
|
||||||
client,
|
client,
|
||||||
room,
|
room,
|
||||||
@@ -89,20 +89,20 @@ async function start(
|
|||||||
matrixRTCMode: MatrixRTCMode.Compatibility,
|
matrixRTCMode: MatrixRTCMode.Compatibility,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
session.status$.pipe(scope.bind()).subscribe((s) => {
|
rtcClient.status$.pipe(scope.bind()).subscribe((s) => {
|
||||||
status.textContent = s;
|
status.textContent = s;
|
||||||
});
|
});
|
||||||
session.fatalError$.pipe(scope.bind()).subscribe((error) => {
|
rtcClient.fatalError$.pipe(scope.bind()).subscribe((error) => {
|
||||||
if (error !== null) status.textContent = `Error: ${error.message}`;
|
if (error !== null) status.textContent = `Error: ${error.message}`;
|
||||||
});
|
});
|
||||||
showMembers(scope, session);
|
showMembers(scope, rtcClient);
|
||||||
session.join();
|
rtcClient.join();
|
||||||
|
|
||||||
toggle(buttons.microphone, microphoneEnabled$);
|
toggle(buttons.microphone, microphoneEnabled$);
|
||||||
toggle(buttons.camera, cameraEnabled$);
|
toggle(buttons.camera, cameraEnabled$);
|
||||||
buttons.leave.hidden = false;
|
buttons.leave.hidden = false;
|
||||||
buttons.leave.onclick = (): void => {
|
buttons.leave.onclick = (): void => {
|
||||||
session.leave();
|
rtcClient.leave();
|
||||||
scope.end();
|
scope.end();
|
||||||
members.replaceChildren();
|
members.replaceChildren();
|
||||||
for (const button of Object.values(buttons)) button.hidden = true;
|
for (const button of Object.values(buttons)) button.hidden = true;
|
||||||
@@ -125,9 +125,9 @@ function toggle(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function showMembers(scope: ObservableScope, session: RtcSession): void {
|
function showMembers(scope: ObservableScope, rtcClient: MatrixRTCClient): void {
|
||||||
const tiles = new Map<string, HTMLElement>();
|
const tiles = new Map<string, HTMLElement>();
|
||||||
combineLatest([session.localMember$, session.remoteMembers$])
|
combineLatest([rtcClient.localMember$, rtcClient.remoteMembers$])
|
||||||
.pipe(scope.bind())
|
.pipe(scope.bind())
|
||||||
.subscribe(([local, remote]) => {
|
.subscribe(([local, remote]) => {
|
||||||
const current = local === null ? remote : [local, ...remote];
|
const current = local === null ? remote : [local, ...remote];
|
||||||
@@ -145,7 +145,7 @@ function showMembers(scope: ObservableScope, session: RtcSession): void {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function memberTile(scope: ObservableScope, member: RtcMember): HTMLElement {
|
function memberTile(scope: ObservableScope, member: RTCMember): HTMLElement {
|
||||||
const tile = document.createElement("section");
|
const tile = document.createElement("section");
|
||||||
tile.dataset.testid = "member";
|
tile.dataset.testid = "member";
|
||||||
tile.dataset.userId = member.userId;
|
tile.dataset.userId = member.userId;
|
||||||
|
|||||||
+1
-1
@@ -24,4 +24,4 @@ export { type EncryptionSystem } from "../src/e2ee/sharedKeyManagement";
|
|||||||
export { MatrixRTCMode } from "../src/config/ConfigOptions";
|
export { MatrixRTCMode } from "../src/config/ConfigOptions";
|
||||||
|
|
||||||
export * from "./src/api";
|
export * from "./src/api";
|
||||||
export { createRtcSession } from "./src/session/RtcSession";
|
export { createMatrixRTCClient } from "./src/session/MatrixRTCClient";
|
||||||
|
|||||||
+25
-17
@@ -6,13 +6,12 @@ Please see LICENSE in the repository root for full details.
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The public types of the SDK. `createRtcSession` in `session/RtcSession.ts`
|
* The public types of the SDK. `createMatrixRTCClient` in `session/MatrixRTCClient.ts`
|
||||||
* is the only way to obtain an implementation of them.
|
* is the only way to obtain an implementation of them.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import {
|
import {
|
||||||
type CallMembership,
|
type CallMembership,
|
||||||
type RTCCallIntent,
|
|
||||||
type RTCNotificationType,
|
type RTCNotificationType,
|
||||||
type Transport,
|
type Transport,
|
||||||
} from "matrix-js-sdk/lib/matrixrtc";
|
} from "matrix-js-sdk/lib/matrixrtc";
|
||||||
@@ -26,17 +25,26 @@ import { type MatrixRTCMode } from "../../src/config/ConfigOptions";
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Session
|
// Session
|
||||||
|
|
||||||
export interface RtcSessionOptions {
|
export interface MatrixRTCClientOptions {
|
||||||
encryptionSystem: EncryptionSystem;
|
encryptionSystem: EncryptionSystem;
|
||||||
/** Resolved by the host; the SDK reads neither config.json nor settings. */
|
/** Resolved by the host; the SDK reads neither config.json nor settings. */
|
||||||
matrixRTCMode: MatrixRTCMode;
|
matrixRTCMode: MatrixRTCMode;
|
||||||
/**
|
/**
|
||||||
* MSC4075 notification sent with the join. Parameters of the MatrixRTC join
|
* MSC4075 notification sent with the join. A parameter of the MatrixRTC
|
||||||
* itself, so they are here even though they are named after calls; reacting
|
* join itself, so it is here even though it is named after calls; reacting
|
||||||
* to a notification (ringing, timeouts, declines) is the application's job.
|
* to a notification (ringing, timeouts, declines) is the application's job.
|
||||||
*/
|
*/
|
||||||
sendNotificationType?: RTCNotificationType;
|
sendNotificationType?: RTCNotificationType;
|
||||||
callIntent?: RTCCallIntent;
|
/** The application the session belongs to, as named in the membership. Default `m.call`. */
|
||||||
|
application?: string;
|
||||||
|
/** The application's slot in the room. Default `ROOM`. */
|
||||||
|
slot?: string;
|
||||||
|
/**
|
||||||
|
* Whatever the application wants to say about itself in the membership.
|
||||||
|
* Opaque to the SDK. Today the js-sdk carries one key, `m.call.intent`
|
||||||
|
* (`"audio"` or `"video"`); anything else is dropped until it can.
|
||||||
|
*/
|
||||||
|
applicationData?: Record<string, unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** What the local member publishes. */
|
/** What the local member publishes. */
|
||||||
@@ -49,7 +57,7 @@ export interface LocalMediaInputs {
|
|||||||
videoProcessor$: Behavior<TrackProcessor<Track.Kind.Video> | undefined>;
|
videoProcessor$: Behavior<TrackProcessor<Track.Kind.Video> | undefined>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type SessionConnectionStatus =
|
export type ConnectionStatus =
|
||||||
| "waitingForTransport"
|
| "waitingForTransport"
|
||||||
| "connecting"
|
| "connecting"
|
||||||
| "connected"
|
| "connected"
|
||||||
@@ -60,10 +68,10 @@ export type SessionConnectionStatus =
|
|||||||
* An error raised by the session. `cause` holds the underlying error, which
|
* An error raised by the session. `cause` holds the underlying error, which
|
||||||
* lets a host that knows the backend tell the failures apart.
|
* lets a host that knows the backend tell the failures apart.
|
||||||
*/
|
*/
|
||||||
export class RtcSessionError extends Error {
|
export class MatrixRTCError extends Error {
|
||||||
public constructor(message: string, options?: ErrorOptions) {
|
public constructor(message: string, options?: ErrorOptions) {
|
||||||
super(message, options);
|
super(message, options);
|
||||||
this.name = "RtcSessionError";
|
this.name = "MatrixRTCError";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,18 +81,18 @@ export class RtcSessionError extends Error {
|
|||||||
* detached copy `joinRoom` returns for a room joined just now never receives
|
* detached copy `joinRoom` returns for a room joined just now never receives
|
||||||
* the state the members are read from.
|
* the state the members are read from.
|
||||||
*/
|
*/
|
||||||
export interface RtcSession {
|
export interface MatrixRTCClient {
|
||||||
join(): void;
|
join(): void;
|
||||||
leave(): void;
|
leave(): void;
|
||||||
/** Collapsed view of the local member's state machine. */
|
/** Collapsed view of the local member's state machine. */
|
||||||
status$: Behavior<SessionConnectionStatus>;
|
status$: Behavior<ConnectionStatus>;
|
||||||
connected$: Behavior<boolean>;
|
connected$: Behavior<boolean>;
|
||||||
reconnecting$: Behavior<boolean>;
|
reconnecting$: Behavior<boolean>;
|
||||||
/** A transport, Matrix or connection error that stops the session. */
|
/** A transport, Matrix or connection error that stops the session. */
|
||||||
fatalError$: Behavior<RtcSessionError | null>;
|
fatalError$: Behavior<MatrixRTCError | null>;
|
||||||
|
|
||||||
localMember$: Behavior<LocalRtcMember | null>;
|
localMember$: Behavior<LocalRTCMember | null>;
|
||||||
remoteMembers$: Behavior<RemoteRtcMember[]>;
|
remoteMembers$: Behavior<RemoteRTCMember[]>;
|
||||||
/** `remoteMembers.length`, plus one for the local member once it exists. */
|
/** `remoteMembers.length`, plus one for the local member once it exists. */
|
||||||
memberCount$: Behavior<number>;
|
memberCount$: Behavior<number>;
|
||||||
|
|
||||||
@@ -135,7 +143,7 @@ export type ResolvedTransport =
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Members
|
// Members
|
||||||
|
|
||||||
export interface RtcMember {
|
export interface RTCMember {
|
||||||
local: boolean;
|
local: boolean;
|
||||||
/** The identity the media backend knows this member by. */
|
/** The identity the media backend knows this member by. */
|
||||||
id: string;
|
id: string;
|
||||||
@@ -153,11 +161,11 @@ export interface RtcMember {
|
|||||||
media$: Behavior<MemberMedia | null>;
|
media$: Behavior<MemberMedia | null>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RemoteRtcMember extends RtcMember {
|
export interface RemoteRTCMember extends RTCMember {
|
||||||
local: false;
|
local: false;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LocalRtcMember extends RtcMember {
|
export interface LocalRTCMember extends RTCMember {
|
||||||
local: true;
|
local: true;
|
||||||
media$: Behavior<LocalMemberMedia | null>;
|
media$: Behavior<LocalMemberMedia | null>;
|
||||||
sharingScreen$: Behavior<boolean>;
|
sharingScreen$: Behavior<boolean>;
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
|
|||||||
|
|
||||||
import { type BaseKeyProvider, ExternalE2EEKeyProvider } from "livekit-client";
|
import { type BaseKeyProvider, ExternalE2EEKeyProvider } from "livekit-client";
|
||||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||||
import { type MatrixRTCSession as JsSdkRtcSession } from "matrix-js-sdk/lib/matrixrtc";
|
import { type MatrixRTCSession as JsSdkRTCSession } from "matrix-js-sdk/lib/matrixrtc";
|
||||||
|
|
||||||
import { E2eeType } from "../../../src/e2ee/e2eeType";
|
import { E2eeType } from "../../../src/e2ee/e2eeType";
|
||||||
import { MatrixKeyProvider } from "../../../src/e2ee/matrixKeyProvider";
|
import { MatrixKeyProvider } from "../../../src/e2ee/matrixKeyProvider";
|
||||||
@@ -16,7 +16,7 @@ import { type EncryptionSystem } from "../../../src/e2ee/sharedKeyManagement";
|
|||||||
/** The LiveKit key provider for the encryption the host asked for, or none. */
|
/** The LiveKit key provider for the encryption the host asked for, or none. */
|
||||||
export function createKeyProvider(
|
export function createKeyProvider(
|
||||||
encryptionSystem: EncryptionSystem,
|
encryptionSystem: EncryptionSystem,
|
||||||
session: JsSdkRtcSession,
|
session: JsSdkRTCSession,
|
||||||
logger: Logger,
|
logger: Logger,
|
||||||
): BaseKeyProvider | undefined {
|
): BaseKeyProvider | undefined {
|
||||||
switch (encryptionSystem.kind) {
|
switch (encryptionSystem.kind) {
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import {
|
|||||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||||
import {
|
import {
|
||||||
type LivekitTransport,
|
type LivekitTransport,
|
||||||
type MatrixRTCSession as JsSdkRtcSession,
|
type MatrixRTCSession as JsSdkRTCSession,
|
||||||
type Status as RTCSessionStatus,
|
type Status as RTCSessionStatus,
|
||||||
} from "matrix-js-sdk/lib/matrixrtc";
|
} from "matrix-js-sdk/lib/matrixrtc";
|
||||||
import { deepCompare } from "matrix-js-sdk/lib/utils";
|
import { deepCompare } from "matrix-js-sdk/lib/utils";
|
||||||
@@ -41,8 +41,8 @@ import {
|
|||||||
ConnectionState,
|
ConnectionState,
|
||||||
} from "../../../src/state/CallViewModel/remoteMembers/Connection";
|
} from "../../../src/state/CallViewModel/remoteMembers/Connection";
|
||||||
import { type IConnectionManager } from "../../../src/state/CallViewModel/remoteMembers/ConnectionManager";
|
import { type IConnectionManager } from "../../../src/state/CallViewModel/remoteMembers/ConnectionManager";
|
||||||
import { type RtcSessionError } from "../api";
|
import { type MatrixRTCError } from "../api";
|
||||||
import { toRtcSessionError } from "../utils/errors";
|
import { toMatrixRTCError } from "../utils/errors";
|
||||||
import { type LocalTransport } from "./LocalTransport";
|
import { type LocalTransport } from "./LocalTransport";
|
||||||
import { type Publisher } from "./Publisher";
|
import { type Publisher } from "./Publisher";
|
||||||
|
|
||||||
@@ -56,16 +56,16 @@ export enum PublishState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export type LocalMemberMediaState =
|
export type LocalMemberMediaState =
|
||||||
| { connection: ConnectionState | RtcSessionError }
|
| { connection: ConnectionState | MatrixRTCError }
|
||||||
| PublishState
|
| PublishState
|
||||||
| RtcSessionError;
|
| MatrixRTCError;
|
||||||
|
|
||||||
export type LocalMemberState =
|
export type LocalMemberState =
|
||||||
| RtcSessionError
|
| MatrixRTCError
|
||||||
| TransportState.Waiting
|
| TransportState.Waiting
|
||||||
| {
|
| {
|
||||||
media: LocalMemberMediaState;
|
media: LocalMemberMediaState;
|
||||||
matrix: RtcSessionError | RTCSessionStatus;
|
matrix: MatrixRTCError | RTCSessionStatus;
|
||||||
};
|
};
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
@@ -78,7 +78,7 @@ interface Props {
|
|||||||
/** The membership manager giving up on keeping the membership alive. */
|
/** The membership manager giving up on keeping the membership alive. */
|
||||||
membershipManagerError$: Observable<unknown>;
|
membershipManagerError$: Observable<unknown>;
|
||||||
matrixRTCSession: Pick<
|
matrixRTCSession: Pick<
|
||||||
JsSdkRtcSession,
|
JsSdkRTCSession,
|
||||||
"updateCallIntent" | "leaveRoomSession"
|
"updateCallIntent" | "leaveRoomSession"
|
||||||
>;
|
>;
|
||||||
cameraEnabled$: Behavior<boolean>;
|
cameraEnabled$: Behavior<boolean>;
|
||||||
@@ -121,10 +121,10 @@ export function createLocalMembership$({
|
|||||||
}: Props): LocalMembership {
|
}: Props): LocalMembership {
|
||||||
const logger = parentLogger.getChild("[LocalMember]");
|
const logger = parentLogger.getChild("[LocalMember]");
|
||||||
|
|
||||||
const fatalTransportError$ = new Subject<RtcSessionError>();
|
const fatalTransportError$ = new Subject<MatrixRTCError>();
|
||||||
const localTransport$ = localTransportWithErrors$.pipe(
|
const localTransport$ = localTransportWithErrors$.pipe(
|
||||||
catchError((e: unknown) => {
|
catchError((e: unknown) => {
|
||||||
fatalTransportError$.next(toRtcSessionError(e));
|
fatalTransportError$.next(toMatrixRTCError(e));
|
||||||
return NEVER;
|
return NEVER;
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
@@ -147,8 +147,8 @@ export function createLocalMembership$({
|
|||||||
|
|
||||||
const joinRequested$ = new BehaviorSubject(false);
|
const joinRequested$ = new BehaviorSubject(false);
|
||||||
const publisher$ = new BehaviorSubject<Publisher | null>(null);
|
const publisher$ = new BehaviorSubject<Publisher | null>(null);
|
||||||
const publishError$ = new BehaviorSubject<RtcSessionError | null>(null);
|
const publishError$ = new BehaviorSubject<MatrixRTCError | null>(null);
|
||||||
const matrixError$ = new BehaviorSubject<RtcSessionError | null>(null);
|
const matrixError$ = new BehaviorSubject<MatrixRTCError | null>(null);
|
||||||
|
|
||||||
scope.reconcile(connection$, async (connection) => {
|
scope.reconcile(connection$, async (connection) => {
|
||||||
if (connection === null) return;
|
if (connection === null) return;
|
||||||
@@ -171,7 +171,7 @@ export function createLocalMembership$({
|
|||||||
} else if (publisher.shouldPublish) await publisher.stopPublishing();
|
} else if (publisher.shouldPublish) await publisher.stopPublishing();
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (publishError$.value === null)
|
if (publishError$.value === null)
|
||||||
publishError$.next(toRtcSessionError(e));
|
publishError$.next(toMatrixRTCError(e));
|
||||||
else logger.error("Another publish error", e);
|
else logger.error("Another publish error", e);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -185,8 +185,7 @@ export function createLocalMembership$({
|
|||||||
joinMatrixRTC(transport.transport);
|
joinMatrixRTC(transport.transport);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logger.error("Failed to enter the session", e);
|
logger.error("Failed to enter the session", e);
|
||||||
if (matrixError$.value === null)
|
if (matrixError$.value === null) matrixError$.next(toMatrixRTCError(e));
|
||||||
matrixError$.next(toRtcSessionError(e));
|
|
||||||
}
|
}
|
||||||
return Promise.resolve(async (): Promise<void> => {
|
return Promise.resolve(async (): Promise<void> => {
|
||||||
try {
|
try {
|
||||||
@@ -200,7 +199,7 @@ export function createLocalMembership$({
|
|||||||
|
|
||||||
membershipManagerError$.pipe(scope.bind()).subscribe((e) => {
|
membershipManagerError$.pipe(scope.bind()).subscribe((e) => {
|
||||||
logger.error("The membership manager stopped", e);
|
logger.error("The membership manager stopped", e);
|
||||||
if (matrixError$.value === null) matrixError$.next(toRtcSessionError(e));
|
if (matrixError$.value === null) matrixError$.next(toMatrixRTCError(e));
|
||||||
});
|
});
|
||||||
|
|
||||||
cameraEnabled$.pipe(scope.bind()).subscribe((videoEnabled) => {
|
cameraEnabled$.pipe(scope.bind()).subscribe((videoEnabled) => {
|
||||||
@@ -224,7 +223,7 @@ export function createLocalMembership$({
|
|||||||
return {
|
return {
|
||||||
connection:
|
connection:
|
||||||
connectionState instanceof Error
|
connectionState instanceof Error
|
||||||
? toRtcSessionError(connectionState)
|
? toMatrixRTCError(connectionState)
|
||||||
: (connectionState ?? ConnectionState.Initialized),
|
: (connectionState ?? ConnectionState.Initialized),
|
||||||
};
|
};
|
||||||
return shouldPublish
|
return shouldPublish
|
||||||
|
|||||||
@@ -6,8 +6,11 @@ Please see LICENSE in the repository root for full details.
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { type MatrixClient, type Room } from "matrix-js-sdk";
|
import { type MatrixClient, type Room } from "matrix-js-sdk";
|
||||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||||
import { MatrixRTCSessionEvent } from "matrix-js-sdk/lib/matrixrtc";
|
import {
|
||||||
|
MatrixRTCSessionEvent,
|
||||||
|
MatrixRTCSessionManager,
|
||||||
|
} from "matrix-js-sdk/lib/matrixrtc";
|
||||||
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
|
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
|
||||||
import { v4 as uuidv4 } from "uuid";
|
import { v4 as uuidv4 } from "uuid";
|
||||||
import { combineLatest, from, fromEvent, map } from "rxjs";
|
import { combineLatest, from, fromEvent, map } from "rxjs";
|
||||||
@@ -29,20 +32,20 @@ import {
|
|||||||
import { filterBehavior, generateItems } from "../../../src/utils/observable";
|
import { filterBehavior, generateItems } from "../../../src/utils/observable";
|
||||||
import {
|
import {
|
||||||
type LocalMediaInputs,
|
type LocalMediaInputs,
|
||||||
type LocalRtcMember,
|
type LocalRTCMember,
|
||||||
type RemoteRtcMember,
|
type RemoteRTCMember,
|
||||||
type RtcSession,
|
type MatrixRTCClient,
|
||||||
RtcSessionError,
|
MatrixRTCError,
|
||||||
type RtcSessionOptions,
|
type MatrixRTCClientOptions,
|
||||||
} from "../api";
|
} from "../api";
|
||||||
import { LivekitConnectionFactory } from "./ConnectionFactory";
|
import { LivekitConnectionFactory } from "./ConnectionFactory";
|
||||||
import { joinRtcSession, sessionTimings } from "./joinRtcSession";
|
import { joinJsSdkSession, sessionTimings } from "./joinJsSdkSession";
|
||||||
import { createKeyProvider } from "./KeyProvider";
|
import { createKeyProvider } from "./KeyProvider";
|
||||||
import { createLocalMembership$ } from "./LocalMember";
|
import { createLocalMembership$ } from "./LocalMember";
|
||||||
import { getLocalTransport } from "./LocalTransport";
|
import { getLocalTransport } from "./LocalTransport";
|
||||||
import {
|
import {
|
||||||
createLocalRtcMember,
|
createLocalRTCMember,
|
||||||
createRemoteRtcMember,
|
createRemoteRTCMember,
|
||||||
membershipKeys,
|
membershipKeys,
|
||||||
} from "./Members";
|
} from "./Members";
|
||||||
import { Publisher } from "./Publisher";
|
import { Publisher } from "./Publisher";
|
||||||
@@ -57,21 +60,26 @@ import { createTransportRegistry } from "./Transports";
|
|||||||
* js-sdk client, and only the SDK knows what that bridge needs. Holding the
|
* js-sdk client, and only the SDK knows what that bridge needs. Holding the
|
||||||
* client keeps that change inside the SDK.
|
* client keeps that change inside the SDK.
|
||||||
*/
|
*/
|
||||||
export function createRtcSession(
|
export function createMatrixRTCClient(
|
||||||
scope: ObservableScope,
|
scope: ObservableScope,
|
||||||
client: MatrixClient,
|
client: MatrixClient,
|
||||||
room: Room,
|
room: Room,
|
||||||
localMedia: LocalMediaInputs,
|
localMedia: LocalMediaInputs,
|
||||||
options: RtcSessionOptions,
|
options: MatrixRTCClientOptions,
|
||||||
): RtcSession {
|
): MatrixRTCClient {
|
||||||
const logger = rootLogger.getChild("[RtcSession]");
|
const logger = rootLogger.getChild("[MatrixRTCClient]");
|
||||||
const userId = client.getUserId();
|
const userId = client.getUserId();
|
||||||
const deviceId = client.getDeviceId();
|
const deviceId = client.getDeviceId();
|
||||||
if (!(userId && deviceId))
|
if (!(userId && deviceId))
|
||||||
throw new RtcSessionError("The client has to be logged in");
|
throw new MatrixRTCError("The client has to be logged in");
|
||||||
const { encryptionSystem, matrixRTCMode } = options;
|
const { encryptionSystem, matrixRTCMode } = options;
|
||||||
|
|
||||||
const jsSdkSession = client.matrixRTC.getRoomSession(room);
|
const jsSdkSession = sessionManager(
|
||||||
|
scope,
|
||||||
|
client,
|
||||||
|
options,
|
||||||
|
logger,
|
||||||
|
).getRoomSession(room);
|
||||||
const keyProvider = createKeyProvider(encryptionSystem, jsSdkSession, logger);
|
const keyProvider = createKeyProvider(encryptionSystem, jsSdkSession, logger);
|
||||||
|
|
||||||
const ownMembershipIdentity: CallMembershipIdentityParts = {
|
const ownMembershipIdentity: CallMembershipIdentityParts = {
|
||||||
@@ -136,11 +144,11 @@ export function createRtcSession(
|
|||||||
),
|
),
|
||||||
),
|
),
|
||||||
joinMatrixRTC: (transport) =>
|
joinMatrixRTC: (transport) =>
|
||||||
joinRtcSession(jsSdkSession, ownMembershipIdentity, transport, {
|
joinJsSdkSession(jsSdkSession, ownMembershipIdentity, transport, {
|
||||||
encryptMedia: keyProvider !== undefined,
|
encryptMedia: keyProvider !== undefined,
|
||||||
matrixRTCMode,
|
matrixRTCMode,
|
||||||
sendNotificationType: options.sendNotificationType,
|
sendNotificationType: options.sendNotificationType,
|
||||||
callIntent: options.callIntent,
|
applicationData: options.applicationData,
|
||||||
}),
|
}),
|
||||||
membershipManagerError$: fromEvent(
|
membershipManagerError$: fromEvent(
|
||||||
jsSdkSession,
|
jsSdkSession,
|
||||||
@@ -161,7 +169,7 @@ export function createRtcSession(
|
|||||||
encryptionSystem,
|
encryptionSystem,
|
||||||
};
|
};
|
||||||
|
|
||||||
const remoteMembers$ = scope.behavior<RemoteRtcMember[]>(
|
const remoteMembers$ = scope.behavior<RemoteRTCMember[]>(
|
||||||
createRemoteMatrixLivekitMembers$({
|
createRemoteMatrixLivekitMembers$({
|
||||||
scope,
|
scope,
|
||||||
membershipsWithTransport$,
|
membershipsWithTransport$,
|
||||||
@@ -170,7 +178,7 @@ export function createRtcSession(
|
|||||||
}).pipe(
|
}).pipe(
|
||||||
map(({ value }) => value),
|
map(({ value }) => value),
|
||||||
generateItems(
|
generateItems(
|
||||||
"RtcSession remoteMembers",
|
"MatrixRTCClient remoteMembers",
|
||||||
function* (members) {
|
function* (members) {
|
||||||
for (const member of members)
|
for (const member of members)
|
||||||
yield {
|
yield {
|
||||||
@@ -179,7 +187,7 @@ export function createRtcSession(
|
|||||||
};
|
};
|
||||||
},
|
},
|
||||||
(memberScope, member$) =>
|
(memberScope, member$) =>
|
||||||
createRemoteRtcMember(memberScope, member$.value, context),
|
createRemoteRTCMember(memberScope, member$.value, context),
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
@@ -196,9 +204,9 @@ export function createRtcSession(
|
|||||||
filterBehavior((membership) => membership !== null),
|
filterBehavior((membership) => membership !== null),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
const localMember$ = scope.behavior<LocalRtcMember | null>(
|
const localMember$ = scope.behavior<LocalRTCMember | null>(
|
||||||
mapScoped(scope, localMembership$, (memberScope, membership$) =>
|
mapScoped(scope, localMembership$, (memberScope, membership$) =>
|
||||||
createLocalRtcMember(memberScope, membership$, localMembership, context),
|
createLocalRTCMember(memberScope, membership$, localMembership, context),
|
||||||
).pipe(map((member) => member ?? null)),
|
).pipe(map((member) => member ?? null)),
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -233,3 +241,24 @@ export function createRtcSession(
|
|||||||
connectedTransports$: transports.connected$,
|
connectedTransports$: transports.connected$,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The js-sdk keeps one session manager per slot. The client comes with the
|
||||||
|
* manager for the default slot; any other slot gets a manager of its own,
|
||||||
|
* which lives as long as the scope.
|
||||||
|
*/
|
||||||
|
function sessionManager(
|
||||||
|
scope: ObservableScope,
|
||||||
|
client: MatrixClient,
|
||||||
|
{ application = "m.call", slot = "ROOM" }: MatrixRTCClientOptions,
|
||||||
|
logger: Logger,
|
||||||
|
): MatrixRTCSessionManager {
|
||||||
|
if (application === "m.call" && slot === "ROOM") return client.matrixRTC;
|
||||||
|
const manager = new MatrixRTCSessionManager(logger, client, {
|
||||||
|
application,
|
||||||
|
id: slot,
|
||||||
|
});
|
||||||
|
manager.start();
|
||||||
|
scope.onEnd(() => manager.stop());
|
||||||
|
return manager;
|
||||||
|
}
|
||||||
+11
-11
@@ -20,10 +20,10 @@ import { type createMatrixMemberMetadata$ } from "../../../src/state/CallViewMod
|
|||||||
import { type RemoteMatrixLivekitMember } from "../../../src/state/CallViewModel/remoteMembers/MatrixLivekitMembers";
|
import { type RemoteMatrixLivekitMember } from "../../../src/state/CallViewModel/remoteMembers/MatrixLivekitMembers";
|
||||||
import { type EncryptionSystem } from "../../../src/e2ee/sharedKeyManagement";
|
import { type EncryptionSystem } from "../../../src/e2ee/sharedKeyManagement";
|
||||||
import {
|
import {
|
||||||
type LocalRtcMember,
|
type LocalRTCMember,
|
||||||
type MemberMedia,
|
type MemberMedia,
|
||||||
type RemoteRtcMember,
|
type RemoteRTCMember,
|
||||||
type RtcMember,
|
type RTCMember,
|
||||||
} from "../api";
|
} from "../api";
|
||||||
import {
|
import {
|
||||||
createLivekitMemberMedia,
|
createLivekitMemberMedia,
|
||||||
@@ -51,13 +51,13 @@ export function membershipKeys(
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createRemoteRtcMember(
|
export function createRemoteRTCMember(
|
||||||
scope: ObservableScope,
|
scope: ObservableScope,
|
||||||
member: RemoteMatrixLivekitMember,
|
member: RemoteMatrixLivekitMember,
|
||||||
context: MemberContext,
|
context: MemberContext,
|
||||||
): RemoteRtcMember {
|
): RemoteRTCMember {
|
||||||
return {
|
return {
|
||||||
...createRtcMember(scope, member.membership$, context),
|
...createRTCMember(scope, member.membership$, context),
|
||||||
local: false,
|
local: false,
|
||||||
media$: mediaFor(
|
media$: mediaFor(
|
||||||
scope,
|
scope,
|
||||||
@@ -74,14 +74,14 @@ export function createRemoteRtcMember(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createLocalRtcMember(
|
export function createLocalRTCMember(
|
||||||
scope: ObservableScope,
|
scope: ObservableScope,
|
||||||
membership$: Behavior<CallMembership>,
|
membership$: Behavior<CallMembership>,
|
||||||
localMembership: LocalMembership,
|
localMembership: LocalMembership,
|
||||||
context: MemberContext,
|
context: MemberContext,
|
||||||
): LocalRtcMember {
|
): LocalRTCMember {
|
||||||
return {
|
return {
|
||||||
...createRtcMember(scope, membership$, context),
|
...createRTCMember(scope, membership$, context),
|
||||||
local: true,
|
local: true,
|
||||||
media$: mediaFor(
|
media$: mediaFor(
|
||||||
scope,
|
scope,
|
||||||
@@ -102,11 +102,11 @@ export function createLocalRtcMember(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function createRtcMember(
|
function createRTCMember(
|
||||||
scope: ObservableScope,
|
scope: ObservableScope,
|
||||||
membership$: Behavior<CallMembership>,
|
membership$: Behavior<CallMembership>,
|
||||||
{ metadata, transports }: MemberContext,
|
{ metadata, transports }: MemberContext,
|
||||||
): Omit<RtcMember, "local" | "media$"> {
|
): Omit<RTCMember, "local" | "media$"> {
|
||||||
const { userId, deviceId, rtcBackendIdentity } = membership$.value;
|
const { userId, deviceId, rtcBackendIdentity } = membership$.value;
|
||||||
return {
|
return {
|
||||||
id: rtcBackendIdentity,
|
id: rtcBackendIdentity,
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
type LivekitTransport,
|
type LivekitTransport,
|
||||||
type MatrixRTCSession as JsSdkRtcSession,
|
type MatrixRTCSession as JsSdkRTCSession,
|
||||||
type RTCCallIntent,
|
type RTCCallIntent,
|
||||||
type RTCNotificationType,
|
type RTCNotificationType,
|
||||||
} from "matrix-js-sdk/lib/matrixrtc";
|
} from "matrix-js-sdk/lib/matrixrtc";
|
||||||
@@ -22,7 +22,7 @@ interface Options {
|
|||||||
encryptMedia: boolean;
|
encryptMedia: boolean;
|
||||||
matrixRTCMode: MatrixRTCMode;
|
matrixRTCMode: MatrixRTCMode;
|
||||||
sendNotificationType?: RTCNotificationType;
|
sendNotificationType?: RTCNotificationType;
|
||||||
callIntent?: RTCCallIntent;
|
applicationData?: Record<string, unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The session's timing defaults; a later slice makes them an input. */
|
/** The session's timing defaults; a later slice makes them an input. */
|
||||||
@@ -36,12 +36,21 @@ export const sessionTimings = {
|
|||||||
* alive and retries on failure.
|
* alive and retries on failure.
|
||||||
* @throws If the join could not be started.
|
* @throws If the join could not be started.
|
||||||
*/
|
*/
|
||||||
export function joinRtcSession(
|
export function joinJsSdkSession(
|
||||||
session: JsSdkRtcSession,
|
session: JsSdkRTCSession,
|
||||||
ownMembershipIdentity: CallMembershipIdentityParts,
|
ownMembershipIdentity: CallMembershipIdentityParts,
|
||||||
transport: LivekitTransport,
|
transport: LivekitTransport,
|
||||||
{ encryptMedia, matrixRTCMode, sendNotificationType, callIntent }: Options,
|
{
|
||||||
|
encryptMedia,
|
||||||
|
matrixRTCMode,
|
||||||
|
sendNotificationType,
|
||||||
|
applicationData,
|
||||||
|
}: Options,
|
||||||
): void {
|
): void {
|
||||||
|
// The one piece of application data the js-sdk can put in a membership
|
||||||
|
const intent = applicationData?.["m.call.intent"];
|
||||||
|
const callIntent =
|
||||||
|
typeof intent === "string" ? (intent as RTCCallIntent) : undefined;
|
||||||
const timings = sessionTimings.delayed_leave;
|
const timings = sessionTimings.delayed_leave;
|
||||||
// Give up on the network as soon as either the sync has been down for the
|
// Give up on the network as soon as either the sync has been down for the
|
||||||
// grace period or the delayed leave has probably been sent, whichever is
|
// grace period or the delayed leave has probably been sent, whichever is
|
||||||
@@ -9,7 +9,7 @@ import { describe, expect, it } from "vitest";
|
|||||||
import { Status } from "matrix-js-sdk/lib/matrixrtc";
|
import { Status } from "matrix-js-sdk/lib/matrixrtc";
|
||||||
|
|
||||||
import { ConnectionState } from "../../../src/state/CallViewModel/remoteMembers/Connection";
|
import { ConnectionState } from "../../../src/state/CallViewModel/remoteMembers/Connection";
|
||||||
import { RtcSessionError } from "../api";
|
import { MatrixRTCError } from "../api";
|
||||||
import {
|
import {
|
||||||
type LocalMemberState,
|
type LocalMemberState,
|
||||||
PublishState,
|
PublishState,
|
||||||
@@ -25,7 +25,7 @@ const connecting: LocalMemberState = {
|
|||||||
media: { connection: ConnectionState.LivekitConnecting },
|
media: { connection: ConnectionState.LivekitConnecting },
|
||||||
matrix: Status.Connecting,
|
matrix: Status.Connecting,
|
||||||
};
|
};
|
||||||
const error = new RtcSessionError("gone");
|
const error = new MatrixRTCError("gone");
|
||||||
|
|
||||||
describe("sessionStatus", () => {
|
describe("sessionStatus", () => {
|
||||||
it.each<[string, LocalMemberState, boolean, boolean, boolean, string]>([
|
it.each<[string, LocalMemberState, boolean, boolean, boolean, string]>([
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
|||||||
Please see LICENSE in the repository root for full details.
|
Please see LICENSE in the repository root for full details.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { type RtcSessionError, type SessionConnectionStatus } from "../api";
|
import { type MatrixRTCError, type ConnectionStatus } from "../api";
|
||||||
import { type LocalMemberState, TransportState } from "./LocalMember";
|
import { type LocalMemberState, TransportState } from "./LocalMember";
|
||||||
|
|
||||||
/** The local member's state, collapsed to what a host shows. */
|
/** The local member's state, collapsed to what a host shows. */
|
||||||
@@ -14,7 +14,7 @@ export function sessionStatus(
|
|||||||
joinRequested: boolean,
|
joinRequested: boolean,
|
||||||
connected: boolean,
|
connected: boolean,
|
||||||
reconnecting: boolean,
|
reconnecting: boolean,
|
||||||
): SessionConnectionStatus {
|
): ConnectionStatus {
|
||||||
if (fatalError(state) !== null) return "disconnected";
|
if (fatalError(state) !== null) return "disconnected";
|
||||||
if (state === TransportState.Waiting) return "waitingForTransport";
|
if (state === TransportState.Waiting) return "waitingForTransport";
|
||||||
if (connected) return "connected";
|
if (connected) return "connected";
|
||||||
@@ -23,7 +23,7 @@ export function sessionStatus(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** The error that stops the session, if the state holds one. */
|
/** The error that stops the session, if the state holds one. */
|
||||||
export function fatalError(state: LocalMemberState): RtcSessionError | null {
|
export function fatalError(state: LocalMemberState): MatrixRTCError | null {
|
||||||
if (state === TransportState.Waiting) return null;
|
if (state === TransportState.Waiting) return null;
|
||||||
if (state instanceof Error) return state;
|
if (state instanceof Error) return state;
|
||||||
if (state.matrix instanceof Error) return state.matrix;
|
if (state.matrix instanceof Error) return state.matrix;
|
||||||
|
|||||||
@@ -7,29 +7,29 @@ Please see LICENSE in the repository root for full details.
|
|||||||
|
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
import { RtcSessionError } from "../api";
|
import { MatrixRTCError } from "../api";
|
||||||
import { toRtcSessionError } from "./errors";
|
import { toMatrixRTCError } from "./errors";
|
||||||
|
|
||||||
describe("toRtcSessionError", () => {
|
describe("toMatrixRTCError", () => {
|
||||||
it("keeps an RtcSessionError as it is", () => {
|
it("keeps an MatrixRTCError as it is", () => {
|
||||||
const error = new RtcSessionError("gone");
|
const error = new MatrixRTCError("gone");
|
||||||
expect(toRtcSessionError(error)).toBe(error);
|
expect(toMatrixRTCError(error)).toBe(error);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("wraps another error as the cause", () => {
|
it("wraps another error as the cause", () => {
|
||||||
const cause = new Error("boom");
|
const cause = new Error("boom");
|
||||||
const error = toRtcSessionError(cause);
|
const error = toMatrixRTCError(cause);
|
||||||
expect(error).toBeInstanceOf(RtcSessionError);
|
expect(error).toBeInstanceOf(MatrixRTCError);
|
||||||
expect(error.message).toBe("boom");
|
expect(error.message).toBe("boom");
|
||||||
expect(error.cause).toBe(cause);
|
expect(error.cause).toBe(cause);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("falls back to the code of an error without a message", () => {
|
it("falls back to the code of an error without a message", () => {
|
||||||
const cause = Object.assign(new Error(""), { code: "SFU_ERROR" });
|
const cause = Object.assign(new Error(""), { code: "SFU_ERROR" });
|
||||||
expect(toRtcSessionError(cause).message).toBe("SFU_ERROR");
|
expect(toMatrixRTCError(cause).message).toBe("SFU_ERROR");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("stringifies anything else", () => {
|
it("stringifies anything else", () => {
|
||||||
expect(toRtcSessionError("nope").message).toBe("nope");
|
expect(toMatrixRTCError("nope").message).toBe("nope");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -5,14 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
|||||||
Please see LICENSE in the repository root for full details.
|
Please see LICENSE in the repository root for full details.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { RtcSessionError } from "../api";
|
import { MatrixRTCError } from "../api";
|
||||||
|
|
||||||
/** Wraps whatever a backend threw into the SDK's error type, keeping it as the cause. */
|
/** Wraps whatever a backend threw into the SDK's error type, keeping it as the cause. */
|
||||||
export function toRtcSessionError(error: unknown): RtcSessionError {
|
export function toMatrixRTCError(error: unknown): MatrixRTCError {
|
||||||
if (error instanceof RtcSessionError) return error;
|
if (error instanceof MatrixRTCError) return error;
|
||||||
if (error instanceof Error)
|
if (error instanceof Error)
|
||||||
return new RtcSessionError(messageOf(error), { cause: error });
|
return new MatrixRTCError(messageOf(error), { cause: error });
|
||||||
return new RtcSessionError(String(error));
|
return new MatrixRTCError(String(error));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user