Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-11-10 01:02:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor media sdk api

Browse Source
This commit is contained in:
Timo K.
2026-11-05 16:57:39 +01:00
parent 06f07fb808
commit de16cb4e37
15 changed files with 390 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
playwright/sdk/data.spec.ts
+3 -2
View File
@@ -11,7 +11,8 @@ import { startPair, tileOf } from "./harness";
test.describe.configure({ timeout: 300_000 });
test("a message sent beside the media reaches the peer, and only the peer", async ({
// The LiveKit backend carries no data tracks yet, so publishing one rejects
test.fixme("a message sent on a data track reaches the peer, and only the peer", async ({
browser,
}) => {
const { pages, users } = await startPair(browser, "sdkdata");
@@ -34,7 +35,7 @@ test("a message sent beside the media reaches the peer, and only the peer", asyn
timeout: 30_000,
},
);
await expect(message).toHaveAttribute("data-topic", "chat");
await expect(message).toHaveAttribute("data-name", "chat");
await expect(message).toHaveAttribute("data-user-id", users[0].userId);
// Our own messages are not echoed back
await expect(sender.getByTestId("message")).toHaveCount(0);
sdk-target-based-on-call-view-model/main.ts
+54 -14
View File
@@ -16,11 +16,25 @@ Please see LICENSE in the repository root for full details.
*/
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { EMPTY, filter, map, type Observable, of, switchMap, take } from "rxjs";
import {
distinct,
distinctUntilChanged,
EMPTY,
filter,
map,
mergeMap,
type Observable,
of,
switchMap,
take,
takeUntil,
} from "rxjs";
import {
type Behavior,
createRTCSlot,
type DataTrack,
E2eeType,
type LocalDataTrack,
type LocalRTCMember,
ObservableScope,
type RemoteRTCMember,
@@ -45,7 +59,7 @@ import { getUrlParams } from "../src/UrlParams";
import { observeElementSize$ } from "../src/utils/elementSize";
import { initializeWidget } from "../src/widget";
/** The data channel topic this bundle has always used, so that old and new builds talk. */
/** The name of our data track: the topic this bundle has always used, so that old and new builds talk. */
const TOPIC = "matrixRTC";
export interface MatrixRTCSdk {
@@ -59,9 +73,9 @@ export interface MatrixRTCSdk {
localMember$: Behavior<LocalRTCMember | null>;
/** The other members of the slot, with the media the participation carries for them. */
remoteMembers$: Behavior<RemoteRTCMember[]>;
/** What other members sent with `sendData`. */
/** What other members sent with `sendData`, on their data track. */
data$: Observable<{ rtcBackendIdentity: string; data: string }>;
/** Sends to every member on our transport. Rejects while not joined. */
/** Sends on our data track, published with the first message. Rejects while not joined. */
sendData: (data: unknown) => Promise<void>;
sendRoomMessage: (message: string) => Promise<void>;
}
@@ -100,6 +114,9 @@ export async function createMatrixRTCSdk(
id,
});
let vm: CallViewModel | undefined;
let dataTrack: Promise<LocalDataTrack> | undefined;
const encoder = new TextEncoder();
const decoder = new TextDecoder();
const failed =
(what: string) =>
(e: unknown): void =>
@@ -115,6 +132,8 @@ export async function createMatrixRTCSdk(
),
);
const remoteMembers$ = over((p) => p.remoteMembers$, []);
return {
join: (): void => {
if (slot.participation$.value !== null) return;
@@ -122,6 +141,7 @@ export async function createMatrixRTCSdk(
widget.api
.setAlwaysOnScreen(true)
.catch(failed("Failed to make the widget sticky"));
dataTrack = undefined;
const participation = slot.join({
...rtcParticipationOptions(urlParams),
publish: initialPublishRequests(muteStates, mediaDevices, capture),
@@ -169,20 +189,40 @@ export async function createMatrixRTCSdk(
false,
),
localMember$: over((p) => p.localMember$, null),
remoteMembers$: over((p) => p.remoteMembers$, []),
// Attested members only: the SDK has matched the sender
data$: slot.participation$.pipe(
switchMap((p) => p?.data$ ?? EMPTY),
filter(({ topic }) => topic === TOPIC),
map(({ member, text }) => ({
rtcBackendIdentity: member.rtcBackendIdentity,
data: text,
})),
remoteMembers$,
data$: remoteMembers$.pipe(
// Each member once, for as long as it stays: a data track is only
// subscribed while observed, so it is not dropped on every list change
mergeMap((members) => members),
distinct(),
mergeMap((member) =>
member.tracks$.pipe(
map((tracks) =>
tracks?.find(
(t): t is DataTrack => t.kind === "data" && t.name === TOPIC,
),
),
distinctUntilChanged(),
switchMap((track) => track?.data$ ?? EMPTY),
map((payload) => ({
rtcBackendIdentity: member.rtcBackendIdentity,
data: decoder.decode(payload),
})),
takeUntil(
remoteMembers$.pipe(filter((members) => !members.includes(member))),
),
),
),
),
sendData: async (data): Promise<void> => {
const participation = slot.participation$.value;
if (participation === null) throw new Error("Not joined");
await participation.sendData(TOPIC, JSON.stringify(data));
dataTrack ??= participation.publish({ source: "data", name: TOPIC });
const track = await dataTrack.catch((e: unknown) => {
dataTrack = undefined;
throw e;
});
await track.send(encoder.encode(JSON.stringify(data)));
},
sendRoomMessage: async (message): Promise<void> => {
await client.sendTextMessage(room.roomId, message);
sdk/SdkArchitecture.md
+59 -36
View File
@@ -25,7 +25,7 @@ Call's `src/` to get there is listed in [`SdkMigration.md`](./SdkMigration.md).
│ │
│ public API, all Behaviors: │
│ join() leave() state$ │
│ localMember$ remoteMembers$ memberCount$ sendData() data$ │
│ localMember$ remoteMembers$ memberCount$ publish(data) │
│ keyRotationSuppressed$ connectedTransports$ │
│ │
│ per member RTCMember { id userId deviceId membership$ displayName$ │
@@ -146,12 +146,12 @@ the sections below still describe `MatrixRTCClient` until that lands.
| Word | Meaning here | Elsewhere |
| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- |
| slot | an `RTCSlot`: the namespace in a room that memberships join into, `application` plus `id`. It exists while the room does, with nobody in it too. Holds `members$`, its open or closed `status$`, and `join()` | the js-sdk `MatrixRTCSession` is one per slot; the MSC4143 slot event |
| participation | an `RTCParticipation`: the span between our `join()` of a slot and our `leave()`. Holds the connection and its `state$`, `localMember$`, `remoteMembers$`, the connected transports and the data channel. Made by `slot.join()`, never reused | "session" is not used here, because in the js-sdk it means the slot |
| participation | an `RTCParticipation`: the span between our `join()` of a slot and our `leave()`. Holds the connection and its `state$`, `localMember$`, `remoteMembers$`, and the connected transports. Made by `slot.join()`, never reused | "session" is not used here, because in the js-sdk it means the slot |
| member | an `RTCMember`: one membership in a slot with what the SDK derives from it, `rtcBackendIdentity`, `userId`, `deviceId`, `memberId`, `displayName$`, `avatarUrl$`, `transport$` and `applicationData$`. No media. What the slot hands out in `members$` | the js-sdk `CallMembership` behind it, which never leaves the SDK; Element Call's old "matrixLivekitMember" |
| local member, remote member | `LocalRTCMember` and `RemoteRTCMember`: an `RTCMember` plus its `MemberMedia`, which only a participation can carry. The local one's tracks carry the controls. What the participation hands out in `localMember$` and `remoteMembers$` | LiveKit `LocalParticipant` and `RemoteParticipant`, which never leave the SDK |
| media | a `MemberMedia`: what a member sends once it has arrived on its transport, `tracks$`, null while waiting, and `encryptionError$` beside it. The media backend supplies exactly these two fields per member | Element Call's `MediaViewModel` is a view of it |
| track | a `MediaTrack`: one publication of a member, audio or video, with `source`, `kind`, `id` and the behaviors on it. `LocalAudioMediaTrack` and `LocalVideoMediaTrack` add the controls over our own | LiveKit `TrackPublication` and `Track`, which never leave the SDK |
| publish request | a `PublishRequest`: a source, where to capture it from and how to encode it. Given at the join in `publish`, or later to `publish` on the participation | |
| track | a `Track`: one publication of a member. A `MediaTrack` is audio or video, with `source`, `kind`, `id` and the behaviors on it; a `DataTrack` is a named channel of bytes. The `Local…` variants add the controls over our own, and `send` for data | LiveKit `TrackPublication` and `Track`, which never leave the SDK |
| publish request | a `PublishRequest`: a source, where to capture it from and how to encode it. Given at the join in `publish`, or later to `publish` on the participation. A `DataPublishRequest` names a data track, and only goes to `publish` | |
| attach, detach | handing a `<video>` or `<audio>` element to a track and taking it back. The SDK owns the stream and the observers on the element | LiveKit `Track.attach` |
| transport | an `RTCTransport`: where media is exchanged, a LiveKit service url today. Carries the raw js-sdk `Transport` from the membership and, in `resolved$`, what the backend fetched to connect to it | MSC4143 "focus" |
| media backend | a `MediaBackend`: what carries the media, behind `sdk/src/media-backend/api.ts`. One per participation, chosen with `backend` in the join options; LiveKit under `sdk/src/media-backend/livekit/` is the one that exists | "SFU", a LiveKit room, a peer connection |
@@ -340,28 +340,6 @@ export interface MatrixRTCClient {
* the browser's choice, never calls it.
*/
setAudioOutputDeviceId(deviceId: string): Promise<void>;
/**
* Sends a short text to every member on the local transport, over a
* reliable data channel beside the media. A packet holds about 15 KiB;
* anything larger belongs in a room event. Rejects while the local
* transport is not connected. Encrypted on the wire like the media, but
* not end to end with the media key.
*/
sendData(topic: string, text: string): Promise<void>;
/**
* What remote members sent with `sendData`, on every transport the client
* is connected to. A message from an identity that is not a member is
* dropped, so a host only ever hears from attested members.
*/
data$: Observable<DataMessage>;
}
/** One `sendData` call as it arrives at a remote member. */
export interface DataMessage {
member: RemoteRTCMember;
topic: string;
text: string;
}
```
@@ -452,11 +430,15 @@ export interface LocalRTCMember extends RTCMember {
* user closing the picker. One publication per source: publishing a source
* again unmutes it.
*/
publish(request: DataPublishRequest): Promise<LocalDataTrack>;
publish(
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack>;
/** Removes one of our tracks; a screen share takes its audio with it. */
unpublish(id: string): Promise<void>;
/**
* Removes one of our tracks, as `publish` handed it out; a screen share
* takes its audio with it. The track's controls and `send` reject from then on.
*/
unpublish(track: LocalTrack): Promise<void>;
}
```
@@ -557,6 +539,7 @@ export interface MediaBackend {
membership$: Behavior<RTCMembership>,
): MemberMediaFields;
readonly connections$: Behavior<BackendConnection[]>; // transport, state, resolved; diagnostics
// Until LiveKit carries data tracks; then data travels as tracks in `tracks$`
sendData(topic: string, text: string): Promise<void>;
readonly data$: Observable<DataPacket>;
}
@@ -707,14 +690,52 @@ export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack {
setProcessor(processor: VideoProcessor | undefined): Promise<void>;
}
/** A named channel of bytes a member publishes beside its media. */
export interface DataPublishRequest {
source: "data";
/** Unique per member; what receivers find the track by. */
name: string;
/** Whether every message arrives, in order. Default true. */
reliable?: boolean;
}
/** One data track of a member; it exists only on a member, so its sender is attested. */
export interface DataTrack {
source: "data";
kind: "data";
id: string;
name: string;
reliable: boolean;
/** Whether payloads are end-to-end encrypted with the media key. */
encrypted: boolean;
/** What the member sends from now on, subscribed while observed. */
data$: Observable<Uint8Array>;
}
/** Our own data track; its `data$` replays what we sent. */
export interface LocalDataTrack extends DataTrack {
send(payload: Uint8Array): Promise<void>;
}
export type Track = AudioMediaTrack | VideoMediaTrack | DataTrack;
export type LocalTrack =
LocalAudioMediaTrack | LocalVideoMediaTrack | LocalDataTrack;
export type EncryptionError = "MissingKey" | "InvalidKey";
/** The member's first track of a source; undefined while there is none, or no media yet. */
export function trackBySource$<S extends MediaSource>(
scope: ObservableScope,
tracks$: Behavior<MediaTrack[] | null>,
tracks$: Behavior<readonly (MediaTrack | DataTrack)[] | null>,
source: S,
): Behavior<TrackOfSource<S> | undefined>;
/** The member's data track of this name; undefined while there is none. */
export function dataTrackByName$(
scope: ObservableScope,
tracks$: Behavior<readonly (MediaTrack | DataTrack)[] | null>,
name: string,
): Behavior<DataTrack | undefined>;
```
### Rendering: `attach` and `detach`
@@ -822,13 +843,15 @@ public API only.
- **Developer panel.** `resolved$` on each connected transport covers what the
panel shows today. Anything beyond that (LiveKit room state, connection
quality) needs an opaque `debug$` per transport.
- **Data beside the media.** `sendData`/`data$` ride on LiveKit's reliable data
packets: topic-agnostic, matched to members by identity, received on every
connection but sent only on the local transport, so a member on another
transport does not hear us. They are transport-encrypted, not end to end:
LiveKit can encrypt data packets with the media key, but that is not wired
through the key provider yet. A host that needs either reach or end-to-end
encryption uses room events.
- **Data beside the media.** Data travels as tracks: a member publishes a named
`DataTrack` with a `DataPublishRequest`, it shows up in the member's `tracks$`
next to the media, and receivers find it with `dataTrackByName$`. Reliability
and encryption are per track. The public API is in place, but the LiveKit
backend does not carry data tracks yet, so publishing one rejects. Its old
packet channel (`MediaBackend.sendData`/`data$`) is still behind the seam,
and the mapping onto LiveKit replaces it: reliable tracks on data packets,
lossy ones on LiveKit data tracks, announced so that a track exists before
its first message, encrypted through the key provider.
- **Unencrypted rooms with per-participant keys.** Keys only reach devices the
crypto tracks, which are the members of encrypted rooms. If the SDK is to work
in an unencrypted room, tracking the members' devices becomes its job.
sdk/dev/main.ts
+42 -21
View File
@@ -12,12 +12,16 @@ Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/lib/logger";
import { combineLatest, type Observable, of, switchMap } from "rxjs";
import { combineLatest, EMPTY, type Observable, of, switchMap } from "rxjs";
import {
type AudioMediaTrack,
createRTCSlot,
dataTrackByName$,
E2eeType,
type LocalAudioMediaTrack,
type LocalDataTrack,
type LocalRTCMember,
type LocalVideoMediaTrack,
MatrixRTCMode,
ObservableScope,
type RemoteRTCMember,
@@ -82,7 +86,7 @@ async function start(
state.kind === "failed" ? `Error: ${state.error.message}` : state.kind;
});
showMembers(scope, rtcParticipation);
showMessages(scope, rtcParticipation);
showMessages(rtcParticipation);
toggle(buttons.microphone, async (enabled) =>
setPublished(rtcParticipation, "microphone", enabled),
@@ -120,10 +124,11 @@ async function setPublished(
enabled: boolean,
): Promise<boolean> {
const track = rtcParticipation.localMember$.value?.tracks$.value?.find(
(t) => t.source === source,
(t): t is LocalAudioMediaTrack | LocalVideoMediaTrack =>
t.source === source,
);
if (track && source === "screenShare") {
if (!enabled) await rtcParticipation.unpublish(source);
if (!enabled) await rtcParticipation.unpublish(track);
return enabled;
}
if (track) return track.setEnabled(enabled);
@@ -145,30 +150,44 @@ function toggle(
};
}
/** Sends what the form holds on the "chat" topic, and lists what arrives. */
function showMessages(
scope: ObservableScope,
rtcParticipation: RTCParticipation,
): void {
/**
* Sends what the form holds on our "chat" data track, published with the
* first message. What arrives is listed per member, in `memberTile`.
*/
function showMessages(rtcParticipation: RTCParticipation): void {
const encoder = new TextEncoder();
let chat: Promise<LocalDataTrack> | undefined;
dataForm.hidden = false;
dataForm.onsubmit = (event): void => {
event.preventDefault();
const text = new FormData(dataForm).get("text") as string;
rtcParticipation.sendData("chat", text).then(
() => dataForm.reset(),
(e: unknown) => {
status.textContent = `Error: ${e}`;
},
);
chat ??= rtcParticipation.publish({ source: "data", name: "chat" });
chat
.then(async (track) => track.send(encoder.encode(text)))
.then(
() => dataForm.reset(),
(e: unknown) => {
chat = undefined;
status.textContent = `Error: ${e}`;
},
);
};
rtcParticipation.data$
.pipe(scope.bind())
.subscribe(({ member, topic, text }) => {
}
/** Lists what a remote member sends on its "chat" data track. */
function listMessages(scope: ObservableScope, member: RemoteRTCMember): void {
const decoder = new TextDecoder();
dataTrackByName$(scope, member.tracks$, "chat")
.pipe(
switchMap((track) => track?.data$ ?? EMPTY),
scope.bind(),
)
.subscribe((payload) => {
const line = messages.appendChild(document.createElement("li"));
line.dataset.testid = "message";
line.dataset.topic = topic;
line.dataset.name = "chat";
line.dataset.userId = member.userId;
line.textContent = `${member.displayName$.value}: ${text}`;
line.textContent = `${member.displayName$.value}: ${decoder.decode(payload)}`;
});
}
@@ -218,12 +237,14 @@ function memberTile(
tile.appendChild(document.createElement("video")),
);
// Our own microphone would only echo
if (!member.local)
if (!member.local) {
render(
scope,
trackBySource$(scope, member.tracks$, "microphone"),
tile.appendChild(document.createElement("audio")),
);
listMessages(scope, member);
}
return tile;
}
sdk/index.ts
+5 -1
View File
@@ -55,7 +55,11 @@ export {
export * from "./src/api";
export * from "./src/media-api";
export * from "./src/media-backend/api";
export { trackBySource$, type TrackOfSource } from "./src/utils/tracks";
export {
dataTrackByName$,
trackBySource$,
type TrackOfSource,
} from "./src/utils/tracks";
export { createRTCSlot } from "./src/RTCSlot";
export {
createLivekitBackend,
sdk/src/RTCParticipation.test.ts
+28 -20
View File
@@ -18,7 +18,6 @@ import { BehaviorSubject, EMPTY, Subject } from "rxjs";
import { describe, expect, it, type Mock, vi } from "vitest";
import {
type DataMessage,
type ParticipationState,
type RTCMember,
type RTCParticipationOptions,
@@ -35,7 +34,11 @@ import {
MediaConnectionState,
type MediaKey,
} from "./media-backend/api";
import { type MemberMedia } from "./media-api";
import {
type LocalDataTrack,
type LocalVideoMediaTrack,
type MemberMedia,
} from "./media-api";
import { constant } from "./reactive/Behavior";
import { ObservableScope } from "./reactive/ObservableScope";
import { createRTCParticipation } from "./RTCParticipation";
@@ -414,27 +417,32 @@ describe("createRTCParticipation", () => {
scope.end();
});
it("delivers data from a member with its member and drops the rest", () => {
const { participation, members$, fake, scope } = setup();
members$.next([
member({
userId: "@bob:example.org",
memberId: "bob-member",
backendIdentities: ["bob-hashed", "bob-member"],
}),
]);
const messages: DataMessage[] = [];
participation.data$.subscribe((message) => messages.push(message));
it("rejects a data track until the backend carries them", async () => {
const { participation, fake, scope } = setup();
fake.data$.next({ senderId: "bob-hashed", topic: "chat", text: "hi" });
fake.data$.next({ senderId: "stranger", topic: "chat", text: "boo" });
await expect(
participation.publish({ source: "data", name: "chat" }),
).rejects.toThrow("not supported");
expect(fake.backend.local.publish).not.toHaveBeenCalled();
await expect(
participation.unpublish({ kind: "data" } as LocalDataTrack),
).rejects.toThrow("not supported");
expect(fake.backend.local.unpublish).not.toHaveBeenCalled();
scope.end();
});
expect(messages).toHaveLength(1);
expect(messages[0]).toMatchObject({
topic: "chat",
text: "hi",
member: { memberId: "bob-member" },
it("hands media publications to the backend by their source", async () => {
const { participation, fake, scope } = setup();
await participation.publish({ source: "camera" });
expect(fake.backend.local.publish).toHaveBeenCalledWith({
source: "camera",
});
await participation.unpublish({
kind: "video",
source: "screenShare",
} as LocalVideoMediaTrack);
expect(fake.backend.local.unpublish).toHaveBeenCalledWith("screenShare");
scope.end();
});
sdk/src/RTCParticipation.ts
+32 -22
View File
@@ -20,7 +20,6 @@ import { v4 as uuidv4 } from "uuid";
import {
combineLatest,
distinctUntilChanged,
filter,
from,
fromEvent,
map,
@@ -35,7 +34,6 @@ import { type ObservableScope } from "./reactive/ObservableScope";
import { generateItems } from "./reactive/observable";
import { mapScoped } from "./utils/mapScoped";
import {
type DataMessage,
type LocalRTCMember,
type ParticipationState,
type RemoteRTCMember,
@@ -45,6 +43,14 @@ import {
type RTCSlot,
type RTCSlotOptions,
} from "./api";
import {
type DataPublishRequest,
type LocalAudioMediaTrack,
type LocalDataTrack,
type LocalTrack,
type LocalVideoMediaTrack,
type PublishRequest,
} from "./media-api";
import {
type MediaBackend,
type MediaBackendContext,
@@ -245,22 +251,6 @@ export function createRTCParticipation(
).pipe(map((member) => member ?? null)),
);
// A packet from an identity that is not a member is dropped, so a host only
// ever hears from attested members
const data$ = backend.data$.pipe(
map(({ senderId, topic, text }): DataMessage | null => {
const member = remoteMembers$.value.find((m) =>
m.backendIdentities.includes(senderId),
);
if (member === undefined) {
logger.warn(`Dropping data from ${senderId}: not a member`);
return null;
}
return { member, topic, text };
}),
filter((message) => message !== null),
);
// Nothing leaves this device while it may already have been dropped from
// the session: the member would show as away while still being heard
localMembership.state$
@@ -285,6 +275,22 @@ export function createRTCParticipation(
),
);
function publish(request: DataPublishRequest): Promise<LocalDataTrack>;
function publish(
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack>;
async function publish(
request: DataPublishRequest | PublishRequest,
): Promise<LocalTrack> {
if (request.source === "data") throw dataTracksUnsupported();
return backend.local.publish(request);
}
const unpublish = async (track: LocalTrack): Promise<void> => {
if (track.kind === "data") throw dataTracksUnsupported();
await backend.local.unpublish(track.source);
};
const leave = (): void => {
if (state$.value.kind === "left") return;
localMembership.leave();
@@ -299,14 +305,12 @@ export function createRTCParticipation(
state$,
localMember$,
remoteMembers$,
publish: async (request) => backend.local.publish(request),
unpublish: async (source) => backend.local.unpublish(source),
publish,
unpublish,
keyRotationSuppressed$: createKeyRotationSuppressed$(scope, jsSdkSession),
connectedTransports$: transports.connected$,
setAudioOutputDeviceId: async (deviceId) =>
backend.setAudioOutputDeviceId(deviceId),
sendData: async (topic, text) => backend.sendData(topic, text),
data$,
},
};
}
@@ -336,3 +340,9 @@ function mediaKeys$(session: JsSdkRTCSession): Observable<MediaKey> {
return () => session.off(MatrixRTCSessionEvent.EncryptionKeyChanged, onKey);
});
}
// Until the media backend carries data tracks; see "Data beside the media" in
// SdkArchitecture.md
function dataTracksUnsupported(): Error {
return new Error("Data tracks are not supported by this backend yet");
}
sdk/src/api.ts
+12 -30
View File
@@ -15,15 +15,15 @@ import {
type RTCNotificationType,
type Transport,
} from "matrix-js-sdk/lib/matrixrtc";
import { type Observable } from "rxjs";
import { type Behavior } from "./reactive/Behavior";
import {
type DataPublishRequest,
type LocalAudioMediaTrack,
type LocalDataTrack,
type LocalMedia,
type LocalMemberMedia,
type LocalTrack,
type LocalVideoMediaTrack,
type MediaSource,
type MemberMedia,
type PublishRequest,
} from "./media-api";
@@ -176,15 +176,19 @@ export interface RTCParticipation {
remoteMembers$: Behavior<RemoteRTCMember[]>;
/**
* Publishes a source and resolves with its track, which also shows up in
* `localMember$.tracks$`. Works from the moment `join()` returns: before the
* transport is connected.
* Publishes a source, or a data track, and resolves with its track, which
* also shows up in `localMember$.tracks$`. Works from the moment `join()`
* returns: before the transport is connected.
*/
publish(request: DataPublishRequest): Promise<LocalDataTrack>;
publish(
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack>;
/** Removes one of our publications; a screen share takes its audio with it. */
unpublish(source: MediaSource): Promise<void>;
/**
* Removes one of our publications; a screen share takes its audio with it.
* The track's object is finished: its controls and `send` reject from then on.
*/
unpublish(track: LocalTrack): Promise<void>;
/**
* Whether the slot has grown large enough that MatrixRTC has stopped
@@ -201,28 +205,6 @@ export interface RTCParticipation {
* the browser's choice, never calls it.
*/
setAudioOutputDeviceId(deviceId: string): Promise<void>;
/**
* Sends a short text to every member on the local transport, over a
* reliable data channel beside the media. A packet holds about 15 KiB;
* anything larger belongs in a room event. Rejects while the local
* transport is not connected. Encrypted on the wire like the media, but
* not end to end with the media key.
*/
sendData(topic: string, text: string): Promise<void>;
/**
* What remote members sent with `sendData`, on every transport the
* participation is connected to. A message from an identity that is not a
* member is dropped, so a host only ever hears from attested members.
*/
data$: Observable<DataMessage>;
}
/** One `sendData` call as it arrives at a remote member. */
export interface DataMessage {
member: RemoteRTCMember;
topic: string;
text: string;
}
// ---------------------------------------------------------------------------
sdk/src/media-api.ts
+65 -7
View File
@@ -69,6 +69,21 @@ export type PublishRequest =
capture?: VideoCaptureSettings;
};
/** A named channel of bytes a member publishes beside its media. */
export interface DataPublishRequest {
source: "data";
/**
* What receivers find the track by, with `dataTrackByName$`. Unique per
* member: publishing a name that is published already rejects.
*/
name: string;
/**
* Whether every message arrives, in order. Default true. Lossy delivery is
* for high-rate state where only the newest value matters, such as cursors.
*/
reliable?: boolean;
}
export type MediaSource =
| "microphone"
| "camera"
@@ -82,7 +97,7 @@ export type MediaStreamStats =
| RTCOutboundRtpStreamStats
| undefined;
/** One published track of a member. */
/** One published audio or video track of a member. */
export interface MediaTrack {
source: MediaSource;
kind: "audio" | "video";
@@ -163,6 +178,48 @@ export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack {
setProcessor(processor: VideoProcessor | undefined): Promise<void>;
}
/**
* One data track of a member. It exists only on a member of the slot, so
* whatever arrives on it comes from an attested member.
*/
export interface DataTrack {
source: "data";
kind: "data";
/** Stable for the life of the track. */
id: string;
/** As the publisher named it in its `DataPublishRequest`. */
name: string;
reliable: boolean;
/** Whether payloads are end-to-end encrypted with the media key. */
encrypted: boolean;
/**
* What the member sends from now on. The track is subscribed while this is
* observed and not at all otherwise; nothing sent earlier is replayed.
*/
data$: Observable<Uint8Array>;
}
/**
* Our own data track. Its `data$` replays what we sent: each payload once
* `send` has handed it to the transport, so a host renders its own messages
* from the same stream as a remote member's.
*/
export interface LocalDataTrack extends DataTrack {
/**
* Sends to every member subscribed to this track. Rejects while the local
* transport is not connected, and where the payload is larger than the
* backend carries.
*/
send(payload: Uint8Array): Promise<void>;
}
/** Any track of a member: its media and its data. */
export type Track = AudioMediaTrack | VideoMediaTrack | DataTrack;
export type LocalTrack =
| LocalAudioMediaTrack
| LocalVideoMediaTrack
| LocalDataTrack;
export type EncryptionError = "MissingKey" | "InvalidKey";
/**
@@ -172,18 +229,19 @@ export type EncryptionError = "MissingKey" | "InvalidKey";
*/
export interface MemberMedia {
/**
* The member's tracks, in publication order, once it has shown up on its
* transport; null until then ("waiting for media"). An entry stays the same
* object for as long as the same publication is behind it. Which track is
* which is in its `source`; `trackBySource$` picks one out.
* The member's tracks, media and data, in publication order, once it has
* shown up on its transport; null until then ("waiting for media"). An
* entry stays the same object for as long as the same publication is
* behind it. Which track is which is in its `source`: `trackBySource$`
* picks a media track out, `dataTrackByName$` a data track.
*/
tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[] | null>;
tracks$: Behavior<Track[] | null>;
/** Emits when the SFU reports a key problem for this member. */
encryptionError$: Observable<EncryptionError>;
}
export interface LocalMemberMedia extends MemberMedia {
tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>;
tracks$: Behavior<LocalTrack[] | null>;
}
/**
sdk/src/media-backend/livekit/LivekitMemberMedia.ts
+8 -1
View File
@@ -32,7 +32,9 @@ import { E2eeType, type EncryptionSystem } from "../../encryption";
import { type Behavior } from "../../reactive/Behavior";
import {
type EncryptionError,
type LocalAudioMediaTrack,
type LocalMemberMedia,
type LocalVideoMediaTrack,
type MediaSource,
type MemberMedia,
} from "../../media-api";
@@ -82,12 +84,17 @@ export function createLivekitMemberMedia(
};
}
/** Media only: LiveKit carries no data tracks yet. */
export interface LivekitLocalMemberMedia extends LocalMemberMedia {
tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>;
}
export function createLocalLivekitMemberMedia(
scope: ObservableScope,
source$: Behavior<ParticipantSource<LocalParticipant>>,
encryptionSystem: EncryptionSystem,
setEnabled: (source: MediaSource, enabled: boolean) => Promise<boolean>,
): LocalMemberMedia {
): LivekitLocalMemberMedia {
return {
tracks$: memberTracks$(
scope,
sdk/src/media-backend/livekit/LocalMedia.ts
+2
View File
@@ -67,6 +67,8 @@ interface Props {
}
export interface LivekitLocalMedia extends LocalMediaBackend {
/** Media only: LiveKit carries no data tracks yet. */
tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>;
/** The connection the local member publishes on; the data channel sends on it. */
connection$: Behavior<Connection | null>;
setAudioOutputDeviceId(deviceId: string): Promise<void>;
sdk/src/utils/tracks.test.ts
+39 -2
View File
@@ -8,9 +8,9 @@ Please see LICENSE in the repository root for full details.
import { BehaviorSubject } from "rxjs";
import { describe, expect, it } from "vitest";
import { type MediaTrack } from "../media-api";
import { type DataTrack, type MediaTrack } from "../media-api";
import { testScope } from "./test";
import { trackBySource$ } from "./tracks";
import { dataTrackByName$, trackBySource$ } from "./tracks";
const camera = { source: "camera", kind: "video", id: "1" } as MediaTrack;
const microphone = {
@@ -39,3 +39,40 @@ describe("trackBySource$", () => {
expect(emissions).toEqual([camera]);
});
});
const chat = {
source: "data",
kind: "data",
id: "3",
name: "chat",
} as DataTrack;
const cursors = {
source: "data",
kind: "data",
id: "4",
name: "cursors",
} as DataTrack;
describe("dataTrackByName$", () => {
it("picks the data track of a name and follows the list", () => {
const tracks$ = new BehaviorSubject<(MediaTrack | DataTrack)[] | null>(
null,
);
const chat$ = dataTrackByName$(testScope(), tracks$, "chat");
expect(chat$.value).toBeUndefined();
tracks$.next([camera, cursors, chat]);
expect(chat$.value).toBe(chat);
tracks$.next([camera, cursors]);
expect(chat$.value).toBeUndefined();
});
it("does not emit when the list changes elsewhere", () => {
const tracks$ = new BehaviorSubject<(MediaTrack | DataTrack)[]>([chat]);
const chat$ = dataTrackByName$(testScope(), tracks$, "chat");
const emissions: unknown[] = [];
chat$.subscribe((t) => emissions.push(t));
tracks$.next([chat, camera]);
tracks$.next([microphone, chat, cursors]);
expect(emissions).toEqual([chat]);
});
});
sdk/src/utils/tracks.ts
+20 -1
View File
@@ -11,6 +11,7 @@ import { type Behavior } from "../reactive/Behavior";
import { type ObservableScope } from "../reactive/ObservableScope";
import {
type AudioMediaTrack,
type DataTrack,
type MediaSource,
type MediaTrack,
type VideoMediaTrack,
@@ -27,7 +28,7 @@ export type TrackOfSource<S extends MediaSource> = S extends
/** The member's first track of a source; undefined while there is none, or no media yet. */
export function trackBySource$<S extends MediaSource>(
scope: ObservableScope,
tracks$: Behavior<MediaTrack[] | null>,
tracks$: Behavior<readonly (MediaTrack | DataTrack)[] | null>,
source: S,
): Behavior<TrackOfSource<S> | undefined> {
return scope.behavior(
@@ -42,3 +43,21 @@ export function trackBySource$<S extends MediaSource>(
),
);
}
/** The member's data track of this name; undefined while there is none, or no media yet. */
export function dataTrackByName$(
scope: ObservableScope,
tracks$: Behavior<readonly (MediaTrack | DataTrack)[] | null>,
name: string,
): Behavior<DataTrack | undefined> {
return scope.behavior(
tracks$.pipe(
map((tracks) =>
tracks?.find(
(t): t is DataTrack => t.kind === "data" && t.name === name,
),
),
distinctUntilChanged(),
),
);
}
src/state/CallViewModel/CallViewModel.ts
+7 -2
View File
@@ -494,7 +494,12 @@ export function createCallViewModel$(
>(
rtcParticipation.localMember$.pipe(
switchMap((member) => member?.tracks$ ?? of(null)),
map((tracks) => tracks ?? []),
map((tracks) =>
(tracks ?? []).filter(
(t): t is LocalAudioMediaTrack | LocalVideoMediaTrack =>
t.kind !== "data",
),
),
),
);
// Our own tracks, which carry the controls remote ones lack
@@ -1454,7 +1459,7 @@ export function createCallViewModel$(
source: "screenShare",
capture: capture.screenShare,
})
: rtcParticipation.unpublish("screenShare")
: rtcParticipation.unpublish(track)
).catch((e: unknown) => {
logger.error(
`Screen share ${track === undefined ? "start" : "stop"} failed`,
src/utils/test.ts
+14 -9
View File
@@ -71,6 +71,8 @@ import {
type RTCParticipation,
type RTCSlot,
type ParticipationState,
type DataPublishRequest,
type LocalTrack,
type PublishRequest,
type RemoteRTCMember,
type TransportMetadata,
@@ -535,19 +537,22 @@ export function mockRTCParticipation(
state$,
localMember$,
remoteMembers$,
publish: vi.fn(async (request: PublishRequest) =>
Promise.resolve(
request.source === "microphone"
? mockAudioTrack()
: mockVideoTrack({ source: request.source }),
),
),
publish: vi.fn(
async (
request: PublishRequest | DataPublishRequest,
): Promise<LocalTrack> => {
if (request.source === "data") throw new Error("Not mocked");
return Promise.resolve(
request.source === "microphone"
? mockAudioTrack()
: mockVideoTrack({ source: request.source }),
);
},
) as RTCParticipation["publish"],
unpublish: vi.fn(async () => Promise.resolve()),
keyRotationSuppressed$,
connectedTransports$,
setAudioOutputDeviceId: vi.fn(async () => Promise.resolve()),
sendData: vi.fn(async () => Promise.resolve()),
data$: NEVER,
};
participation$.next(participation);
return participation;