From e2e504d307f97708929261f0ebc6534194769b28 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Fri, 6 Nov 2026 10:07:00 +0100 Subject: [PATCH] Data channel lk support --- playwright/sdk/data.spec.ts | 16 +- sdk/SdkArchitecture.md | 59 ++++-- sdk/dev/main.ts | 51 +++-- sdk/src/RTCParticipation.test.ts | 51 ++--- sdk/src/RTCParticipation.ts | 19 +- sdk/src/media-backend/api.ts | 30 +-- .../media-backend/livekit/ByteStreams.test.ts | 85 ++++++++ sdk/src/media-backend/livekit/ByteStreams.ts | 82 ++++++++ .../livekit/ConnectionFactory.test.ts | 1 + .../livekit/ConnectionFactory.ts | 21 +- .../media-backend/livekit/DataChannel.test.ts | 143 ------------- sdk/src/media-backend/livekit/DataChannel.ts | 119 ----------- .../media-backend/livekit/LivekitBackend.ts | 16 +- .../livekit/LivekitDataTrack.test.ts | 124 +++++++++++ .../media-backend/livekit/LivekitDataTrack.ts | 74 +++++++ .../livekit/LivekitMemberMedia.test.ts | 43 ++++ .../livekit/LivekitMemberMedia.ts | 65 ++++-- .../media-backend/livekit/LocalDataTracks.ts | 195 ++++++++++++++++++ .../media-backend/livekit/LocalMedia.test.ts | 138 ++++++++++++- sdk/src/media-backend/livekit/LocalMedia.ts | 62 ++++-- sdk/src/media-backend/livekit/Publisher.ts | 8 +- .../livekit/RemoteDataTracks.test.ts | 39 ++++ .../media-backend/livekit/RemoteDataTracks.ts | 60 ++++++ .../livekit/dataTrackName.test.ts | 28 +++ .../media-backend/livekit/dataTrackName.ts | 36 ++++ 25 files changed, 1156 insertions(+), 409 deletions(-) create mode 100644 sdk/src/media-backend/livekit/ByteStreams.test.ts create mode 100644 sdk/src/media-backend/livekit/ByteStreams.ts delete mode 100644 sdk/src/media-backend/livekit/DataChannel.test.ts delete mode 100644 sdk/src/media-backend/livekit/DataChannel.ts create mode 100644 sdk/src/media-backend/livekit/LivekitDataTrack.test.ts create mode 100644 sdk/src/media-backend/livekit/LivekitDataTrack.ts create mode 100644 sdk/src/media-backend/livekit/LocalDataTracks.ts create mode 100644 sdk/src/media-backend/livekit/RemoteDataTracks.test.ts create mode 100644 sdk/src/media-backend/livekit/RemoteDataTracks.ts create mode 100644 sdk/src/media-backend/livekit/dataTrackName.test.ts create mode 100644 sdk/src/media-backend/livekit/dataTrackName.ts diff --git a/playwright/sdk/data.spec.ts b/playwright/sdk/data.spec.ts index b14b80c36..b1e25d44a 100644 --- a/playwright/sdk/data.spec.ts +++ b/playwright/sdk/data.spec.ts @@ -11,19 +11,25 @@ import { startPair, tileOf } from "./harness"; test.describe.configure({ timeout: 300_000 }); -// The LiveKit backend carries no data tracks yet, so publishing one rejects -test.fixme("a message sent on a data track reaches the peer, and only the peer", async ({ +test("a message sent on a data track reaches the peer, and only the peer", async ({ browser, }) => { const { pages, users } = await startPair(browser, "sdkdata"); const [sender, receiver] = pages; // Both are in the call before anything is sent, so the receiver knows the - // sender as a member and does not drop the message + // sender as a member for (const page of pages) await expect(page.getByTestId("member")).toHaveCount(2, { timeout: 60_000, }); - await expect(tileOf(receiver, users[0])).toBeVisible(); + // The receiver listens on the sender's track, published with the join + await expect(tileOf(receiver, users[0])).toHaveAttribute( + "data-chat", + "true", + { + timeout: 30_000, + }, + ); await sender.getByPlaceholder("message").fill("hello from the SDK"); await sender.getByRole("button", { name: "Send" }).click(); @@ -36,6 +42,8 @@ test.fixme("a message sent on a data track reaches the peer, and only the peer", }, ); await expect(message).toHaveAttribute("data-name", "chat"); + // Decrypted with the sender's key, at the key index the stream carried + await expect(message).toHaveAttribute("data-encrypted", "true"); await expect(message).toHaveAttribute("data-user-id", users[0].userId); // Our own messages are not echoed back await expect(sender.getByTestId("message")).toHaveCount(0); diff --git a/sdk/SdkArchitecture.md b/sdk/SdkArchitecture.md index 4cc85a76a..8847d7f77 100644 --- a/sdk/SdkArchitecture.md +++ b/sdk/SdkArchitecture.md @@ -48,7 +48,7 @@ Call's `src/` to get there is listed in [`SdkMigration.md`](./SdkMigration.md). │ │ (LocalMembership) │ local.publish, │ Publisher: local tracks, │ │ │ │ transport registry │ setPublishing▶│ upstream paused until │ │ │ │ (TransportMetadata) │ ◀─ connections$ │ joined │ │ -│ │ data packet → member │ ◀─ data$ │ LivekitMemberMedia: │ │ +│ │ │ │ LivekitMemberMedia: │ │ │ │ │ │ participant → tracks │ │ │ └───────────────────────────┘ └───────────────────────────┘ │ │ │ @@ -539,9 +539,6 @@ export interface MediaBackend { membership$: Behavior, ): MemberMediaFields; readonly connections$: Behavior; // transport, state, resolved; diagnostics - // Until LiveKit carries data tracks; then data travels as tracks in `tracks$` - sendData(topic: string, text: string): Promise; - readonly data$: Observable; } ``` @@ -551,8 +548,7 @@ LiveKit backend only, a last resort of another member's SFU that lets us publish (`findRemotePublishLivekitTransport`, wired in by `RTCParticipation` rather than by the backend interface, since it is an MSC4195 matter), the membership state machine in `LocalMembership.ts` and its combination with -`local.connectionState$` into `state$`, the choice of delayed leave timings, -mapping a data packet's sender to a member, and the transport registry, which keys on the raw +`local.connectionState$` into `state$`, the choice of delayed leave timings, and the transport registry, which keys on the raw transport serialised with sorted keys and derives `connectedTransports$` and each `TransportMetadata.resolved$` from `connections$`. @@ -596,6 +592,43 @@ the local transport, and keys its connections on the transport plus the server name where the transport has a `url`. A JWT service is reached directly, so members of several homeservers advertising the same one share a connection. +### Data tracks on LiveKit + +A data track is a track like the media, in `tracks$`, so the backend carries it +the way it carries the media: `local.publish` takes a `DataPublishRequest`, and +`mediaFor` lists a remote member's data tracks after its media. Its sender is +attested by construction: a track only exists on a member. The session never +sees a key, a frame or a stream. + +Every data track is one LiveKit data track, whose publication announces it: a +receiver sees the track before its first message, and sees it go. LiveKit's +data tracks are lossy only, so the LiveKit name says how the payloads travel +(`dataTrackName.ts`): + +| Request | LiveKit name | Payloads | +| ----------------- | ----------------- | --------------------------------------------------------------- | +| `reliable: false` | `lossy:` | frames on the data track: unordered, no retransmits | +| `reliable: true` | `reliable:` | one byte stream per `send` on topic `reliable:`, in order | + +A track without a prefix, from a plain LiveKit publisher, is read as lossy. A +room takes one byte stream handler per topic, so `ByteStreams.ts` shares it +between every reliable track on that topic and filters by sender identity. + +Encryption is LiveKit's. The rooms are built with `encryption` rather than +`e2ee`, which encrypts the data channel as well as the media with the same +E2EE manager and our key provider. The sender encrypts with its current key +and puts the key index and IV on the wire: in each frame's E2EE extension for a +lossy track, in each packet's `EncryptedPacket` for a stream. The receiver +decrypts with the sender's key at that index. A frame that does not decrypt is +dropped. A stream cannot be checked: LiveKit 2.23 sends an encrypted packet +without its encryption type, so every received stream reads as unencrypted, +and a cleartext stream injected on the way would be accepted as the sender's. + +A local track is published on LiveKit once publishing is allowed, like the +media, and again on every new connection to the local transport; `publish` +resolves with that first publication. The object a host holds outlives the +connections. + ## Media: `tracks$` and `MediaTrack` Every member has media. Today, in Element Call, that media _is_ a `livekit-client` @@ -843,15 +876,11 @@ public API only. - **Developer panel.** `resolved$` on each connected transport covers what the panel shows today. Anything beyond that (LiveKit room state, connection quality) needs an opaque `debug$` per transport. -- **Data beside the media.** Data travels as tracks: a member publishes a named - `DataTrack` with a `DataPublishRequest`, it shows up in the member's `tracks$` - next to the media, and receivers find it with `dataTrackByName$`. Reliability - and encryption are per track. The public API is in place, but the LiveKit - backend does not carry data tracks yet, so publishing one rejects. Its old - packet channel (`MediaBackend.sendData`/`data$`) is still behind the seam, - and the mapping onto LiveKit replaces it: reliable tracks on data packets, - lossy ones on LiveKit data tracks, announced so that a track exists before - its first message, encrypted through the key provider. +- **Data before a receiver listens.** Nothing is replayed: a message sent + before a receiver has subscribed to the track, which it does once it has + seen the track published, does not reach it. Old builds of the legacy + widget bundle, which sent bare packets, do not hear new builds or the other + way round. - **Unencrypted rooms with per-participant keys.** Keys only reach devices the crypto tracks, which are the members of encrypted rooms. If the SDK is to work in an unencrypted room, tracking the members' devices becomes its job. diff --git a/sdk/dev/main.ts b/sdk/dev/main.ts index b5a7c435e..04822c652 100644 --- a/sdk/dev/main.ts +++ b/sdk/dev/main.ts @@ -12,14 +12,20 @@ Please see LICENSE in the repository root for full details. */ import { logger } from "matrix-js-sdk/lib/logger"; -import { combineLatest, EMPTY, type Observable, of, switchMap } from "rxjs"; +import { + combineLatest, + EMPTY, + map, + type Observable, + of, + switchMap, +} from "rxjs"; import { type AudioMediaTrack, createRTCSlot, dataTrackByName$, E2eeType, type LocalAudioMediaTrack, - type LocalDataTrack, type LocalRTCMember, type LocalVideoMediaTrack, MatrixRTCMode, @@ -151,41 +157,60 @@ function toggle( } /** - * Sends what the form holds on our "chat" data track, published with the - * first message. What arrives is listed per member, in `memberTile`. + * Publishes our "chat" data track with the join, so that the others hear the + * first message, and sends what the form holds on it. What arrives is listed + * per member, in `memberTile`. */ function showMessages(rtcParticipation: RTCParticipation): void { const encoder = new TextEncoder(); - let chat: Promise | undefined; + const chat = rtcParticipation.publish({ source: "data", name: "chat" }); + chat.catch((e: unknown) => { + status.textContent = `Error: ${e}`; + }); dataForm.hidden = false; dataForm.onsubmit = (event): void => { event.preventDefault(); const text = new FormData(dataForm).get("text") as string; - chat ??= rtcParticipation.publish({ source: "data", name: "chat" }); chat .then(async (track) => track.send(encoder.encode(text))) .then( () => dataForm.reset(), (e: unknown) => { - chat = undefined; status.textContent = `Error: ${e}`; }, ); }; } -/** Lists what a remote member sends on its "chat" data track. */ -function listMessages(scope: ObservableScope, member: RemoteRTCMember): void { +/** + * Lists what a remote member sends on its "chat" data track, and marks its + * tile while it has one. + */ +function listMessages( + scope: ObservableScope, + member: RemoteRTCMember, + tile: HTMLElement, +): void { const decoder = new TextDecoder(); - dataTrackByName$(scope, member.tracks$, "chat") + const chat$ = dataTrackByName$(scope, member.tracks$, "chat"); + chat$.pipe(scope.bind()).subscribe((track) => { + tile.dataset.chat = String(track !== undefined); + }); + chat$ .pipe( - switchMap((track) => track?.data$ ?? EMPTY), + switchMap( + (track) => + track?.data$.pipe( + map((payload) => ({ payload, encrypted: track.encrypted })), + ) ?? EMPTY, + ), scope.bind(), ) - .subscribe((payload) => { + .subscribe(({ payload, encrypted }) => { const line = messages.appendChild(document.createElement("li")); line.dataset.testid = "message"; line.dataset.name = "chat"; + line.dataset.encrypted = String(encrypted); line.dataset.userId = member.userId; line.textContent = `${member.displayName$.value}: ${decoder.decode(payload)}`; }); @@ -243,7 +268,7 @@ function memberTile( trackBySource$(scope, member.tracks$, "microphone"), tile.appendChild(document.createElement("audio")), ); - listMessages(scope, member); + listMessages(scope, member, tile); } return tile; } diff --git a/sdk/src/RTCParticipation.test.ts b/sdk/src/RTCParticipation.test.ts index d07918dfb..f4499b53d 100644 --- a/sdk/src/RTCParticipation.test.ts +++ b/sdk/src/RTCParticipation.test.ts @@ -14,7 +14,7 @@ import { type Transport, type UnstableLivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, EMPTY, Subject } from "rxjs"; +import { BehaviorSubject, EMPTY } from "rxjs"; import { describe, expect, it, type Mock, vi } from "vitest"; import { @@ -28,7 +28,6 @@ import { defaultSessionTimings, MatrixRTCMode } from "./config"; import { E2eeType } from "./encryption"; import { MatrixRTCError } from "./errors"; import { - type DataPacket, type MediaBackend, type MediaBackendContext, MediaConnectionState, @@ -106,7 +105,6 @@ interface FakeBackend { context: MediaBackendContext; prepared: PromiseWithResolvers<{ canDelegateDelayedLeave: boolean }>; connectionState$: BehaviorSubject; - data$: Subject; setPublishing: Mock<(publish: boolean) => void>; delegateDelayedLeave: Mock<(delayId: string) => Promise>; mediaFor: Mock<(scope: ObservableScope, member: RTCMember) => MemberMedia>; @@ -154,7 +152,6 @@ function setup( connectionState$: new BehaviorSubject( MediaConnectionState.Initialized, ), - data$: new Subject(), setPublishing: vi.fn(), delegateDelayedLeave: vi.fn(async () => Promise.resolve()), mediaFor: vi.fn(() => ({ @@ -182,8 +179,6 @@ function setup( mediaFor: fake.mediaFor, connections$: constant([]), setAudioOutputDeviceId: vi.fn(), - sendData: vi.fn(), - data$: fake.data$, }; fake.context = context; fake.backend = backend; @@ -417,32 +412,30 @@ describe("createRTCParticipation", () => { scope.end(); }); - it("rejects a data track until the backend carries them", async () => { + it("hands publications to the backend, data tracks too", async () => { const { participation, fake, scope } = setup(); - - await expect( - participation.publish({ source: "data", name: "chat" }), - ).rejects.toThrow("not supported"); - expect(fake.backend.local.publish).not.toHaveBeenCalled(); - await expect( - participation.unpublish({ kind: "data" } as LocalDataTrack), - ).rejects.toThrow("not supported"); - expect(fake.backend.local.unpublish).not.toHaveBeenCalled(); - scope.end(); - }); - - it("hands media publications to the backend by their source", async () => { - const { participation, fake, scope } = setup(); - - await participation.publish({ source: "camera" }); - expect(fake.backend.local.publish).toHaveBeenCalledWith({ - source: "camera", - }); - await participation.unpublish({ + const chat = { kind: "data", name: "chat" } as LocalDataTrack; + const screenShare = { kind: "video", source: "screenShare", - } as LocalVideoMediaTrack); - expect(fake.backend.local.unpublish).toHaveBeenCalledWith("screenShare"); + } as LocalVideoMediaTrack; + + await participation.publish({ source: "data", name: "chat" }); + await participation.publish({ source: "camera" }); + expect(fake.backend.local.publish).toHaveBeenNthCalledWith(1, { + source: "data", + name: "chat", + }); + expect(fake.backend.local.publish).toHaveBeenNthCalledWith(2, { + source: "camera", + }); + await participation.unpublish(chat); + await participation.unpublish(screenShare); + expect(fake.backend.local.unpublish).toHaveBeenNthCalledWith(1, chat); + expect(fake.backend.local.unpublish).toHaveBeenNthCalledWith( + 2, + screenShare, + ); scope.end(); }); diff --git a/sdk/src/RTCParticipation.ts b/sdk/src/RTCParticipation.ts index 4c17aeec9..92a06ce57 100644 --- a/sdk/src/RTCParticipation.ts +++ b/sdk/src/RTCParticipation.ts @@ -282,15 +282,12 @@ export function createRTCParticipation( async function publish( request: DataPublishRequest | PublishRequest, ): Promise { - if (request.source === "data") throw dataTracksUnsupported(); - return backend.local.publish(request); + // Narrowed for the overloads; both go to the backend alike + return request.source === "data" + ? backend.local.publish(request) + : backend.local.publish(request); } - const unpublish = async (track: LocalTrack): Promise => { - if (track.kind === "data") throw dataTracksUnsupported(); - await backend.local.unpublish(track.source); - }; - const leave = (): void => { if (state$.value.kind === "left") return; localMembership.leave(); @@ -306,7 +303,7 @@ export function createRTCParticipation( localMember$, remoteMembers$, publish, - unpublish, + unpublish: async (track) => backend.local.unpublish(track), keyRotationSuppressed$: createKeyRotationSuppressed$(scope, jsSdkSession), connectedTransports$: transports.connected$, setAudioOutputDeviceId: async (deviceId) => @@ -340,9 +337,3 @@ function mediaKeys$(session: JsSdkRTCSession): Observable { return () => session.off(MatrixRTCSessionEvent.EncryptionKeyChanged, onKey); }); } - -// Until the media backend carries data tracks; see "Data beside the media" in -// SdkArchitecture.md -function dataTracksUnsupported(): Error { - return new Error("Data tracks are not supported by this backend yet"); -} diff --git a/sdk/src/media-backend/api.ts b/sdk/src/media-backend/api.ts index 427548632..e6e4c303c 100644 --- a/sdk/src/media-backend/api.ts +++ b/sdk/src/media-backend/api.ts @@ -19,11 +19,13 @@ import { type Observable } from "rxjs"; import { type ResolvedTransport, type RTCMember } from "../api"; import { + type DataPublishRequest, type LocalAudioMediaTrack, + type LocalDataTrack, type LocalMedia, type LocalMemberMedia, + type LocalTrack, type LocalVideoMediaTrack, - type MediaSource, type MemberMedia, type PublishRequest, } from "../media-api"; @@ -106,13 +108,6 @@ export interface BackendConnection { resolved: ResolvedTransport | undefined; } -export interface DataPacket { - /** The backend identity of the sender. */ - senderId: string; - topic: string; - text: string; -} - export interface MediaBackend { /** * The membership transport types this backend serves, ordered decreasingly @@ -158,14 +153,6 @@ export interface MediaBackend { * exists keeps the device it was created with. */ setAudioOutputDeviceId(deviceId: string): Promise; - - /** - * A text channel beside the media: sent on the local transport, received on every one. - * This could well be just another track kind. Not needed for element call. just for the sdk target. - * Keeping it on the top level for visibility. Should be tackled in a future refactor. - */ - sendData(topic: string, text: string): Promise; - readonly data$: Observable; } /** `tracks$` is null until the connection carries a local participant. */ @@ -187,14 +174,17 @@ export interface LocalMediaBackend extends LocalMemberMedia { readonly publishError$: Behavior; /** - * Publishes a source and resolves with its track once it is in `tracks$`. - * Remembered across connections, so a request before the + * Publishes a source, or a data track, and resolves with its track once it + * is in `tracks$`. Remembered across connections, so a request before the * transport is up is applied once it is, and republished after a - * reconnection. Rejects where the device could not be used. + * reconnection. Rejects where the device could not be used. A data track + * is end-to-end encrypted by the backend wherever the media is; the session + * never sees how. */ + publish(request: DataPublishRequest): Promise; publish( request: PublishRequest, ): Promise; /** Removes one of our publications; a screen share takes its audio with it. */ - unpublish(source: MediaSource): Promise; + unpublish(track: LocalTrack): Promise; } diff --git a/sdk/src/media-backend/livekit/ByteStreams.test.ts b/sdk/src/media-backend/livekit/ByteStreams.test.ts new file mode 100644 index 000000000..cb0de106d --- /dev/null +++ b/sdk/src/media-backend/livekit/ByteStreams.test.ts @@ -0,0 +1,85 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type ByteStreamHandler, + type ByteStreamReader, + type Room as LivekitRoom, +} from "livekit-client"; +import { describe, expect, it, vi } from "vitest"; + +import { byteStreams$, type ReceivedBytes } from "./ByteStreams"; +import { flushPromises } from "../../utils/test"; + +function fakeRoom(): { + room: LivekitRoom; + handlers: Map; +} { + const handlers = new Map(); + const room = { + registerByteStreamHandler: vi.fn( + (topic: string, handler: ByteStreamHandler) => { + if (handlers.has(topic)) throw new Error("already registered"); + handlers.set(topic, handler); + }, + ), + unregisterByteStreamHandler: vi.fn((topic: string) => + handlers.delete(topic), + ), + } as unknown as LivekitRoom; + return { room, handlers }; +} + +function reader(chunks: number[][]): { + reader: ByteStreamReader; + finish: () => void; +} { + const done = Promise.withResolvers(); + return { + reader: { + readAll: async () => done.promise, + } as unknown as ByteStreamReader, + finish: (): void => done.resolve(chunks.map((c) => new Uint8Array(c))), + }; +} + +describe("byteStreams$", () => { + it("shares one handler per topic, released with the last observer", () => { + const { room, handlers } = fakeRoom(); + const first = byteStreams$(room, "reliable:chat").subscribe(); + const second = byteStreams$(room, "reliable:chat").subscribe(); + expect(room.registerByteStreamHandler).toHaveBeenCalledTimes(1); + first.unsubscribe(); + expect(handlers.has("reliable:chat")).toBe(true); + second.unsubscribe(); + expect(handlers.has("reliable:chat")).toBe(false); + }); + + it("joins the chunks and keeps the order the streams started in", async () => { + const { room, handlers } = fakeRoom(); + const received: ReceivedBytes[] = []; + const subscription = byteStreams$(room, "reliable:chat").subscribe((r) => + received.push(r), + ); + const handler = handlers.get("reliable:chat")!; + const one = reader([[1, 2], [3]]); + const two = reader([[4]]); + handler(one.reader, { identity: "alice" }); + handler(two.reader, { identity: "bob" }); + two.finish(); + await flushPromises(); + expect(received).toEqual([]); + one.finish(); + await flushPromises(); + + expect(received).toEqual([ + { identity: "alice", payload: new Uint8Array([1, 2, 3]) }, + { identity: "bob", payload: new Uint8Array([4]) }, + ]); + subscription.unsubscribe(); + }); +}); diff --git a/sdk/src/media-backend/livekit/ByteStreams.ts b/sdk/src/media-backend/livekit/ByteStreams.ts new file mode 100644 index 000000000..09b075a9a --- /dev/null +++ b/sdk/src/media-backend/livekit/ByteStreams.ts @@ -0,0 +1,82 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { + type ByteStreamReader, + type Room as LivekitRoom, +} from "livekit-client"; +import { catchError, concatMap, EMPTY, from, Observable, share } from "rxjs"; + +/** One byte stream, read to its end. */ +export interface ReceivedBytes { + /** The participant identity of the sender. */ + identity: string; + payload: Uint8Array; +} + +const byTopicByRoom = new WeakMap< + LivekitRoom, + Map> +>(); + +/** + * The byte streams a room receives on a topic, in the order they started. A + * room takes one handler per topic, so every observer of a topic shares it: + * it is registered with the first subscription and released with the last. + */ +export function byteStreams$( + room: LivekitRoom, + topic: string, +): Observable { + let byTopic = byTopicByRoom.get(room); + if (byTopic === undefined) { + byTopic = new Map(); + byTopicByRoom.set(room, byTopic); + } + let streams$ = byTopic.get(topic); + if (streams$ === undefined) { + streams$ = new Observable<{ reader: ByteStreamReader; identity: string }>( + (subscriber) => { + room.registerByteStreamHandler(topic, (reader, { identity }) => + subscriber.next({ reader, identity }), + ); + return (): void => room.unregisterByteStreamHandler(topic); + }, + ).pipe( + concatMap(({ reader, identity }) => + from(readStream(reader, identity)).pipe( + catchError((e: unknown) => { + rootLogger + .getChild("[ByteStreams]") + .warn(`Dropping a stream on ${topic} from ${identity}`, e); + return EMPTY; + }), + ), + ), + share(), + ); + byTopic.set(topic, streams$); + } + return streams$; +} + +async function readStream( + reader: ByteStreamReader, + identity: string, +): Promise { + const chunks = await reader.readAll(); + const payload = new Uint8Array( + chunks.reduce((length, chunk) => length + chunk.length, 0), + ); + let offset = 0; + for (const chunk of chunks) { + payload.set(chunk, offset); + offset += chunk.length; + } + return { identity, payload }; +} diff --git a/sdk/src/media-backend/livekit/ConnectionFactory.test.ts b/sdk/src/media-backend/livekit/ConnectionFactory.test.ts index 182248a51..28c0339f5 100644 --- a/sdk/src/media-backend/livekit/ConnectionFactory.test.ts +++ b/sdk/src/media-backend/livekit/ConnectionFactory.test.ts @@ -77,6 +77,7 @@ describe("LivekitConnectionFactory", () => { resolution: expect.objectContaining({ height: 720 }), }); expect(options.audioOutput).toEqual({ deviceId: undefined }); + expect(options.encryption).toBeUndefined(); expect(options.e2ee).toBeUndefined(); }); diff --git a/sdk/src/media-backend/livekit/ConnectionFactory.ts b/sdk/src/media-backend/livekit/ConnectionFactory.ts index eb8230060..a53835121 100644 --- a/sdk/src/media-backend/livekit/ConnectionFactory.ts +++ b/sdk/src/media-backend/livekit/ConnectionFactory.ts @@ -19,6 +19,7 @@ import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/En import { BehaviorSubject, combineLatest, map } from "rxjs"; import { type ResolvedTransport } from "../../api"; +import { watchRemoteDataTracks } from "./RemoteDataTracks"; import { type MediaQuality } from "../../config"; import { type PublishRequest } from "../../media-api"; import { type Behavior } from "../../reactive/Behavior"; @@ -128,12 +129,14 @@ export class LivekitConnectionFactory implements ConnectionFactory { client: this.client, scope, livekitRoomFactory: () => - new LivekitRoom( - roomOptions( - this.publish, - this.audioOutputDeviceId$.value, - this.keyProvider, - this.mediaQuality, + watchRemoteDataTracks( + new LivekitRoom( + roomOptions( + this.publish, + this.audioOutputDeviceId$.value, + this.keyProvider, + this.mediaQuality, + ), ), ), ownMembershipIdentity, @@ -169,8 +172,10 @@ function roomOptions( }, audioOutput: { deviceId: audioOutputDeviceId }, // Every room needs a worker of its own: one gets confused by streams from - // several rooms - e2ee: keyProvider && { keyProvider, worker: new E2EEWorker() }, + // several rooms. `encryption` rather than `e2ee`, so that LiveKit encrypts + // the data channel too, which carries the reliable data tracks; media is + // encrypted the same either way + encryption: keyProvider && { keyProvider, worker: new E2EEWorker() }, }; } diff --git a/sdk/src/media-backend/livekit/DataChannel.test.ts b/sdk/src/media-backend/livekit/DataChannel.test.ts deleted file mode 100644 index 9dff925b2..000000000 --- a/sdk/src/media-backend/livekit/DataChannel.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { EventEmitter } from "events"; -import { - ConnectionState as LivekitConnectionState, - type RemoteParticipant, - RoomEvent, -} from "livekit-client"; -import { BehaviorSubject } from "rxjs"; -import { describe, expect, it, vi } from "vitest"; - -import { Epoch } from "../../reactive/ObservableScope"; -import { constant } from "../../reactive/Behavior"; -import { type DataPacket } from "../api"; -import { type Connection, type ConnectionState } from "./Connection"; -import { ConnectionMap } from "./ConnectionMap"; -import { createDataChannel$ } from "./DataChannel"; -import { testScope } from "../../utils/test"; -import { exampleTransport } from "./test"; - -const encode = (text: string): Uint8Array => new TextEncoder().encode(text); - -const connected: ConnectionState = { - state: "authenticated", - livekitState: LivekitConnectionState.Connected, -}; - -function fakeConnection( - url: string, - state: ConnectionState = connected, -): { - connection: Connection; - room: EventEmitter; - publishData: ReturnType; -} { - const room = new EventEmitter(); - const publishData = vi.fn(async () => Promise.resolve()); - const connection = { - transport: { ...exampleTransport, livekit_service_url: url }, - serverName: "example.org", - state$: constant(state), - livekitRoom: Object.assign(room, { localParticipant: { publishData } }), - } as unknown as Connection; - return { connection, room, publishData }; -} - -function setup(connections: Connection[]): { - received: DataPacket[]; - connections$: BehaviorSubject>; - sendData: (topic: string, text: string) => Promise; -} { - const data = new ConnectionMap(); - for (const connection of connections) data.add(connection, []); - const connections$ = new BehaviorSubject(new Epoch(data)); - const { data$, sendData } = createDataChannel$({ - scope: testScope(), - connectionMap$: connections$, - connection$: constant(connections[0] ?? null), - }); - const received: DataPacket[] = []; - data$.subscribe((message) => received.push(message)); - return { received, connections$, sendData }; -} - -describe("data$", () => { - it("delivers a packet with its sender, topic and text", () => { - const { connection, room } = fakeConnection("https://sfu"); - const { received } = setup([connection]); - - room.emit( - RoomEvent.DataReceived, - encode("hello"), - { identity: "alice" } as RemoteParticipant, - undefined, - "chat", - ); - - expect(received).toEqual([ - { senderId: "alice", topic: "chat", text: "hello" }, - ]); - }); - - it("drops packets from the server", () => { - const { connection, room } = fakeConnection("https://sfu"); - const { received } = setup([connection]); - - room.emit(RoomEvent.DataReceived, encode("server"), undefined); - - expect(received).toEqual([]); - }); - - it("listens on every connection, including ones added later", () => { - const first = fakeConnection("https://sfu-1"); - const second = fakeConnection("https://sfu-2"); - const { received, connections$ } = setup([first.connection]); - - const data = new ConnectionMap(); - data.add(first.connection, []); - data.add(second.connection, []); - connections$.next(new Epoch(data, 1)); - - first.room.emit(RoomEvent.DataReceived, encode("one"), { - identity: "alice", - } as RemoteParticipant); - second.room.emit(RoomEvent.DataReceived, encode("two"), { - identity: "bob", - } as RemoteParticipant); - - expect(received.map((m) => [m.senderId, m.text])).toEqual([ - ["alice", "one"], - ["bob", "two"], - ]); - }); -}); - -describe("sendData", () => { - it("publishes the text as a reliable packet on the local connection", async () => { - const { connection, publishData } = fakeConnection("https://sfu"); - const { sendData } = setup([connection]); - - await sendData("chat", "hello"); - - expect(publishData).toHaveBeenCalledWith(encode("hello"), { - reliable: true, - topic: "chat", - }); - }); - - it("rejects while the local transport is not connected", async () => { - const { connection, publishData } = fakeConnection("https://sfu", { - state: "initialized", - }); - const { sendData } = setup([connection]); - - await expect(sendData("chat", "hello")).rejects.toThrow("not connected"); - expect(publishData).not.toHaveBeenCalled(); - }); -}); diff --git a/sdk/src/media-backend/livekit/DataChannel.ts b/sdk/src/media-backend/livekit/DataChannel.ts deleted file mode 100644 index 4849ab96b..000000000 --- a/sdk/src/media-backend/livekit/DataChannel.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* -Copyright 2026 Element Creations Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - ConnectionState as LivekitConnectionState, - type RemoteParticipant, - type Room as LivekitRoom, - RoomEvent, -} from "livekit-client"; -import { - distinctUntilChanged, - filter, - fromEvent, - map, - merge, - type Observable, - share, - switchMap, -} from "rxjs"; - -import { type Behavior } from "../../reactive/Behavior"; -import { - type Epoch, - type ObservableScope, -} from "../../reactive/ObservableScope"; -import { type DataPacket } from "../api"; -import { type Connection } from "./Connection"; -import { type ConnectionMap } from "./ConnectionMap"; - -interface Props { - scope: ObservableScope; - connectionMap$: Behavior>; - /** The local member's connection, which carries what we send. */ - connection$: Behavior; -} - -export interface DataChannel { - data$: Observable; - sendData: (topic: string, text: string) => Promise; -} - -/** - * A text channel beside the media: LiveKit's reliable data packets, received - * on every connection the client holds and sent on the local member's. The - * sender is reported by its identity; the packet's own claim of who sent it - * never reaches the host. - */ -export function createDataChannel$({ - scope, - connectionMap$, - connection$, -}: Props): DataChannel { - const decoder = new TextDecoder(); - const encoder = new TextEncoder(); - - const rooms$ = connectionMap$.pipe( - map(({ value }) => value.getConnections().map((c) => c.livekitRoom)), - distinctUntilChanged( - (a, b) => a.length === b.length && a.every((room, i) => room === b[i]), - ), - ); - - const data$ = rooms$.pipe( - switchMap((rooms) => merge(...rooms.map(receivedOn))), - map(({ payload, participant, topic }): DataPacket | null => - // Only the server sends without a participant; a host has no use for it - participant === undefined - ? null - : { - senderId: participant.identity, - topic: topic ?? "", - text: decoder.decode(payload), - }, - ), - filter((packet) => packet !== null), - scope.bind(), - share(), - ); - - const sendData = async (topic: string, text: string): Promise => { - const connection = connection$.value; - const state = connection?.state$.value; - if ( - connection === null || - state?.state !== "authenticated" || - state.livekitState !== LivekitConnectionState.Connected - ) - throw new Error("Cannot send data: the local transport is not connected"); - await connection.livekitRoom.localParticipant.publishData( - encoder.encode(text), - { reliable: true, topic }, - ); - }; - - return { data$, sendData }; -} - -interface Received { - payload: Uint8Array; - participant: RemoteParticipant | undefined; - topic: string | undefined; -} - -function receivedOn(room: LivekitRoom): Observable { - return fromEvent( - room, - RoomEvent.DataReceived, - ( - payload: Uint8Array, - participant?: RemoteParticipant, - _kind?: unknown, - topic?: string, - ): Received => ({ payload, participant, topic }), - ); -} diff --git a/sdk/src/media-backend/livekit/LivekitBackend.ts b/sdk/src/media-backend/livekit/LivekitBackend.ts index 075859374..ab7cb90e0 100644 --- a/sdk/src/media-backend/livekit/LivekitBackend.ts +++ b/sdk/src/media-backend/livekit/LivekitBackend.ts @@ -58,7 +58,6 @@ import { ResolvedConnection, } from "./ConnectionFactory"; import { toMediaConnectionState } from "./connectionState"; -import { createDataChannel$ } from "./DataChannel"; import { createKeyProvider } from "./KeyProvider"; import { createLivekitMemberMedia, @@ -263,7 +262,12 @@ export function createLivekitBackend( `${member.rtcBackendIdentity}: LiveKit participant missing on ${transportString}`, ); }); - return createLivekitMemberMedia(memberScope, source$, encryptionSystem); + return createLivekitMemberMedia( + memberScope, + source$, + encryptionSystem, + logger, + ); }; const connections$ = scope.behavior( @@ -276,12 +280,6 @@ export function createLivekitBackend( ), ); - const { data$, sendData } = createDataChannel$({ - scope, - connectionMap$, - connection$: localMedia_.connection$, - }); - return { transportTypes: ["m.livekit", "livekit"], prepareLocalTransport, @@ -290,8 +288,6 @@ export function createLivekitBackend( mediaFor, connections$, setAudioOutputDeviceId: localMedia_.setAudioOutputDeviceId, - sendData, - data$, }; } diff --git a/sdk/src/media-backend/livekit/LivekitDataTrack.test.ts b/sdk/src/media-backend/livekit/LivekitDataTrack.test.ts new file mode 100644 index 000000000..04011fc64 --- /dev/null +++ b/sdk/src/media-backend/livekit/LivekitDataTrack.test.ts @@ -0,0 +1,124 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { logger } from "matrix-js-sdk/lib/logger"; +import { + type ByteStreamHandler, + type ByteStreamReader, + type DataTrackFrame, + type RemoteDataTrack, + type Room as LivekitRoom, +} from "livekit-client"; +import { describe, expect, it, vi } from "vitest"; + +import { createLivekitDataTrack } from "./LivekitDataTrack"; +import { flushPromises } from "../../utils/test"; + +function remoteTrack( + name: string, + usesE2ee = true, +): { + track: RemoteDataTrack; + frames: ReadableStreamDefaultController[]; + cancel: ReturnType; +} { + const frames: ReadableStreamDefaultController[] = []; + const cancel = vi.fn(); + const track = { + info: { sid: "DT_1", name, usesE2ee }, + publisherIdentity: "alice", + subscribe: vi.fn( + () => + new ReadableStream({ + start: (controller): void => { + frames.push(controller); + }, + cancel, + }), + ), + } as unknown as RemoteDataTrack; + return { track, frames, cancel }; +} + +function roomWithStreams(): { + room: LivekitRoom; + send: (identity: string, bytes: number[]) => void; +} { + const handlers = new Map(); + return { + room: { + registerByteStreamHandler: (topic: string, h: ByteStreamHandler) => + handlers.set(topic, h), + unregisterByteStreamHandler: (topic: string) => handlers.delete(topic), + } as unknown as LivekitRoom, + send: (identity, bytes): void => + handlers.get("reliable:chat")!( + { + readAll: async () => Promise.resolve([new Uint8Array(bytes)]), + } as unknown as ByteStreamReader, + { identity }, + ), + }; +} + +describe("a remote data track", () => { + it("names itself without the LiveKit prefix", () => { + const { track } = remoteTrack("reliable:chat", false); + expect( + createLivekitDataTrack(roomWithStreams().room, track, logger), + ).toMatchObject({ + source: "data", + kind: "data", + id: "DT_1", + name: "chat", + reliable: true, + encrypted: false, + }); + }); + + it("subscribes a lossy track while observed, once for all observers", async () => { + const { track, frames, cancel } = remoteTrack("lossy:cursors"); + const dataTrack = createLivekitDataTrack( + roomWithStreams().room, + track, + logger, + ); + expect(track.subscribe).not.toHaveBeenCalled(); + + const received: Uint8Array[] = []; + const first = dataTrack.data$.subscribe((p) => received.push(p)); + const second = dataTrack.data$.subscribe(); + expect(track.subscribe).toHaveBeenCalledTimes(1); + frames[0].enqueue({ payload: new Uint8Array([7]) }); + await flushPromises(); + expect(received).toEqual([new Uint8Array([7])]); + + first.unsubscribe(); + expect(cancel).not.toHaveBeenCalled(); + second.unsubscribe(); + expect(cancel).toHaveBeenCalled(); + }); + + it("takes a reliable track's streams from its publisher only", async () => { + const { room, send } = roomWithStreams(); + const dataTrack = createLivekitDataTrack( + room, + remoteTrack("reliable:chat").track, + logger, + ); + const received: Uint8Array[] = []; + const subscription = dataTrack.data$.subscribe((p) => received.push(p)); + + send("bob", [1]); + send("alice", [2]); + await flushPromises(); + + // Bob's is not this track's + expect(received).toEqual([new Uint8Array([2])]); + subscription.unsubscribe(); + }); +}); diff --git a/sdk/src/media-backend/livekit/LivekitDataTrack.ts b/sdk/src/media-backend/livekit/LivekitDataTrack.ts new file mode 100644 index 000000000..2b072bd6d --- /dev/null +++ b/sdk/src/media-backend/livekit/LivekitDataTrack.ts @@ -0,0 +1,74 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type RemoteDataTrack, type Room as LivekitRoom } from "livekit-client"; +import { filter, map, Observable, share } from "rxjs"; + +import { type DataTrack } from "../../media-api"; +import { byteStreams$ } from "./ByteStreams"; +import { fromLivekitDataTrackName } from "./dataTrackName"; + +/** + * A remote member's data track. Decryption is LiveKit's: its E2EE manager + * reads the key index and IV off each frame or stream and decrypts with the + * sender's key, so nothing here sees a key. A frame of an encrypted track that + * does not decrypt is dropped by LiveKit. A stream is not checked: LiveKit + * 2.23 sends an encrypted packet without its encryption type, so a received + * stream reads as unencrypted either way, and a cleartext one would pass. + */ +export function createLivekitDataTrack( + room: LivekitRoom, + track: RemoteDataTrack, + logger: Logger, +): DataTrack { + const { name, reliable } = fromLivekitDataTrackName(track.info.name); + const encrypted = track.info.usesE2ee; + const data$ = reliable + ? byteStreams$(room, track.info.name).pipe( + filter(({ identity }) => identity === track.publisherIdentity), + map(({ payload }) => payload), + ) + : framesOf(track, logger); + return { + source: "data", + kind: "data", + id: track.info.sid, + name, + reliable, + encrypted, + data$, + }; +} + +/** The track's frames, subscribed at the SFU while observed. */ +function framesOf( + track: RemoteDataTrack, + logger: Logger, +): Observable { + return new Observable((subscriber) => { + const reader = track.subscribe().getReader(); + let cancelled = false; + void (async (): Promise => { + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + subscriber.next(value.payload); + } + } catch (e) { + // A failed subscription ends the stream; a host's pipeline survives it + if (!cancelled) logger.warn(`Data track ${track.info.name} ended`, e); + } + subscriber.complete(); + })(); + return (): void => { + cancelled = true; + reader.cancel().catch(() => {}); + }; + }).pipe(share()); +} diff --git a/sdk/src/media-backend/livekit/LivekitMemberMedia.test.ts b/sdk/src/media-backend/livekit/LivekitMemberMedia.test.ts index 04798d0c6..bef7ddc29 100644 --- a/sdk/src/media-backend/livekit/LivekitMemberMedia.test.ts +++ b/sdk/src/media-backend/livekit/LivekitMemberMedia.test.ts @@ -6,9 +6,12 @@ Please see LICENSE in the repository root for full details. */ import { EventEmitter } from "events"; +import { logger } from "matrix-js-sdk/lib/logger"; import { ParticipantEvent, + type RemoteDataTrack, type RemoteParticipant, + RoomEvent, type Room as LivekitRoom, Track, type TrackPublication, @@ -27,6 +30,7 @@ describe("createLivekitMemberMedia", () => { testScope(), constant({ participants: [participant], room }), { kind: E2eeType.NONE }, + logger, ); expect(media.tracks$.value).toEqual([]); @@ -54,6 +58,7 @@ describe("createLivekitMemberMedia", () => { testScope(), constant({ participants: [participant], room }), { kind: E2eeType.NONE }, + logger, ); emitter.emit( ParticipantEvent.TrackPublished, @@ -85,6 +90,44 @@ function publish( return publication; } +describe("a member's data tracks", () => { + it("follow the room's data track events, after the media", () => { + const { participant, publications, room, emitter } = fakes(); + const media = createLivekitMemberMedia( + testScope(), + constant({ participants: [participant], room }), + { kind: E2eeType.NONE }, + logger, + ); + const camera = publish(publications, "TR_1", Track.Source.Camera); + emitter.emit(ParticipantEvent.TrackPublished, camera); + + const chat = { + info: { sid: "DT_1", name: "reliable:chat", usesE2ee: true }, + publisherIdentity: participant.identity, + } as unknown as RemoteDataTrack; + (room as unknown as EventEmitter).emit(RoomEvent.DataTrackPublished, chat); + const [video, data] = media.tracks$.value!; + expect(video.source).toBe("camera"); + expect(data).toMatchObject({ + kind: "data", + name: "chat", + reliable: true, + encrypted: true, + }); + + // An unrelated media event keeps the same object + emitter.emit(ParticipantEvent.TrackMuted, camera); + expect(media.tracks$.value![1]).toBe(data); + + (room as unknown as EventEmitter).emit( + RoomEvent.DataTrackUnpublished, + "DT_1", + ); + expect(media.tracks$.value).toEqual([video]); + }); +}); + function fakes(): { participant: RemoteParticipant; publications: Map; diff --git a/sdk/src/media-backend/livekit/LivekitMemberMedia.ts b/sdk/src/media-backend/livekit/LivekitMemberMedia.ts index de0908a86..f075f84d4 100644 --- a/sdk/src/media-backend/livekit/LivekitMemberMedia.ts +++ b/sdk/src/media-backend/livekit/LivekitMemberMedia.ts @@ -9,6 +9,7 @@ import { observeParticipantMedia, roomEventSelector, } from "@livekit/components-core"; +import { type Logger } from "matrix-js-sdk/lib/logger"; import { type LocalParticipant, type Participant, @@ -31,13 +32,14 @@ import { generateItems } from "../../reactive/observable"; import { E2eeType, type EncryptionSystem } from "../../encryption"; import { type Behavior } from "../../reactive/Behavior"; import { + type DataTrack, type EncryptionError, - type LocalAudioMediaTrack, type LocalMemberMedia, - type LocalVideoMediaTrack, type MediaSource, type MemberMedia, } from "../../media-api"; +import { createLivekitDataTrack } from "./LivekitDataTrack"; +import { remoteDataTracks$ } from "./RemoteDataTracks"; import { createLivekitMediaTrack, createLocalLivekitMediaTrack, @@ -72,29 +74,32 @@ export function createLivekitMemberMedia( scope: ObservableScope, source$: Behavior, encryptionSystem: EncryptionSystem, + logger: Logger, ): MemberMedia { + const media$ = memberTracks$( + scope, + source$, + (trackScope, participant, publication, room) => + createLivekitMediaTrack(trackScope, participant, publication, room), + ); + const data$ = memberDataTracks$(scope, source$, logger); return { - tracks$: memberTracks$( - scope, - source$, - (trackScope, participant, publication, room) => - createLivekitMediaTrack(trackScope, participant, publication, room), + // The data tracks after the media, both null while there is no participant + tracks$: scope.behavior( + combineLatest([media$, data$], (media, data) => + media === null ? null : [...media, ...data], + ), ), encryptionError$: encryptionErrors$(source$, encryptionSystem), }; } -/** Media only: LiveKit carries no data tracks yet. */ -export interface LivekitLocalMemberMedia extends LocalMemberMedia { - tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>; -} - export function createLocalLivekitMemberMedia( scope: ObservableScope, source$: Behavior>, encryptionSystem: EncryptionSystem, setEnabled: (source: MediaSource, enabled: boolean) => Promise, -): LivekitLocalMemberMedia { +): LocalMemberMedia { return { tracks$: memberTracks$( scope, @@ -152,6 +157,40 @@ function memberTracks$

( ); } +/** + * One data track per LiveKit data track of the member's participants, each + * living as long as it is published. + */ +function memberDataTracks$( + scope: ObservableScope, + source$: Behavior, + logger: Logger, +): Behavior { + return scope.behavior( + source$.pipe( + switchMap((source) => { + if (source === null) return of([]); + const { participants, room } = source; + const identities = new Set(participants.map((p) => p.identity)); + return remoteDataTracks$(room).pipe( + map((tracks) => + tracks.filter((t) => identities.has(t.publisherIdentity)), + ), + generateItems( + `${[...identities].join(",")} dataTracks$`, + function* (tracks) { + for (const track of tracks) + yield { keys: [track.info.sid], data: track }; + }, + (_trackScope, track$) => + createLivekitDataTrack(room, track$.value, logger), + ), + ); + }), + ), + ); +} + function encryptionErrors$( source$: Behavior, encryptionSystem: EncryptionSystem, diff --git a/sdk/src/media-backend/livekit/LocalDataTracks.ts b/sdk/src/media-backend/livekit/LocalDataTracks.ts new file mode 100644 index 000000000..d19b6468e --- /dev/null +++ b/sdk/src/media-backend/livekit/LocalDataTracks.ts @@ -0,0 +1,195 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + ConnectionState as LivekitConnectionState, + type LocalDataTrack as LivekitLocalDataTrack, + type LocalParticipant, +} from "livekit-client"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { + BehaviorSubject, + combineLatest, + distinctUntilChanged, + first, + map, + type Observable, + of, + startWith, + Subject, + switchMap, +} from "rxjs"; + +import { type DataPublishRequest, type LocalDataTrack } from "../../media-api"; +import { type Behavior } from "../../reactive/Behavior"; +import { ObservableScope } from "../../reactive/ObservableScope"; +import { type Connection } from "./Connection"; +import { toLivekitDataTrackName } from "./dataTrackName"; + +interface Props { + scope: ObservableScope; + /** The connection the local member publishes on. */ + connection$: Behavior; + /** Whether anything may leave this device: joined and the homeserver reachable. */ + publishing$: Observable; + /** Whether the rooms carry a key provider, so LiveKit encrypts what we send. */ + encrypted: boolean; + logger: Logger; +} + +export interface LocalDataTracks { + tracks$: Behavior; + /** + * Resolves once the track is published on LiveKit, which waits for the + * join. Rejects where a track of that name is published already, or where + * LiveKit refuses it. + */ + publish(request: DataPublishRequest): Promise; + unpublish(track: LocalDataTrack): void; +} + +interface Published { + track: LivekitLocalDataTrack; + participant: LocalParticipant; +} + +interface Entry { + track: LocalDataTrack; + finish: () => void; +} + +/** + * The local member's data tracks. Each is one LiveKit data track, published + * on every connection to the local transport while publishing is allowed, so + * a track requested before the join goes out once we are joined. The object a + * host holds outlives the connections. Encryption is LiveKit's: its E2EE + * manager encrypts with our current key and writes the key index and IV into + * each frame, or each stream for a reliable track. + */ +export function createLocalDataTracks({ + scope, + connection$, + publishing$, + encrypted, + logger, +}: Props): LocalDataTracks { + const entries = new Map(); + const tracks$ = new BehaviorSubject([]); + const update = (): void => + tracks$.next([...entries.values()].map(({ track }) => track)); + scope.onEnd(() => { + for (const entry of entries.values()) entry.finish(); + }); + + // Where to publish: the local connection once it has connected, while + // publishing is allowed + const target$ = scope.behavior( + combineLatest([connection$, publishing$]).pipe( + switchMap(([connection, publishing]) => + connection === null || !publishing + ? of(null) + : connection.state$.pipe( + first( + (s) => + s.state === "authenticated" && + s.livekitState === LivekitConnectionState.Connected, + ), + map(() => connection), + startWith(null), + ), + ), + distinctUntilChanged(), + ), + ); + + const publish = async ({ + name, + reliable = true, + }: DataPublishRequest): Promise => { + if (entries.has(name)) + throw new Error(`A data track named ${name} is published already`); + const firstPublished = Promise.withResolvers(); + const livekitName = toLivekitDataTrackName(name, reliable); + const trackScope = new ObservableScope(); + const published$ = new BehaviorSubject(null); + const sent$ = new Subject(); + let finished = false; + + trackScope.reconcile(target$, async (connection) => { + if (connection === null) return; + const participant = connection.livekitRoom.localParticipant; + let track: LivekitLocalDataTrack; + try { + track = await participant.publishDataTrack({ name: livekitName }); + } catch (e) { + logger.error(`Failed to publish the data track ${name}`, e); + firstPublished.reject(e); + return; + } + published$.next({ track, participant }); + firstPublished.resolve(); + return async (): Promise => { + published$.next(null); + await track.unpublish().catch((e: unknown) => { + logger.warn(`Failed to unpublish the data track ${name}`, e); + }); + }; + }); + + const track: LocalDataTrack = { + source: "data", + kind: "data", + id: `data:${name}`, + name, + reliable, + encrypted, + data$: sent$.asObservable(), + send: async (payload): Promise => { + if (finished) throw new Error(`The data track ${name} is unpublished`); + const published = published$.value; + if (published === null) + throw new Error( + `Cannot send on ${name}: not published on the transport yet`, + ); + if (reliable) + await published.participant.sendBytes(payload, { + topic: livekitName, + }); + else + await published.track.tryPush({ payload: new Uint8Array(payload) }); + sent$.next(payload); + }, + }; + + entries.set(name, { + track, + finish: (): void => { + finished = true; + trackScope.end(); + sent$.complete(); + }, + }); + update(); + try { + await firstPublished.promise; + } catch (e) { + unpublish(track); + throw e; + } + return track; + }; + + const unpublish = (track: LocalDataTrack): void => { + const entry = entries.get(track.name); + if (entry?.track !== track) return; + entries.delete(track.name); + entry.finish(); + update(); + }; + + return { tracks$: scope.behavior(tracks$), publish, unpublish }; +} diff --git a/sdk/src/media-backend/livekit/LocalMedia.test.ts b/sdk/src/media-backend/livekit/LocalMedia.test.ts index 26ab05bd2..9ddc91d99 100644 --- a/sdk/src/media-backend/livekit/LocalMedia.test.ts +++ b/sdk/src/media-backend/livekit/LocalMedia.test.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { ConnectionState as LivekitConnectionState, + type LocalDataTrack as LivekitLocalDataTrack, type LocalParticipant, type LocalTrack, type LocalTrackPublication, @@ -28,6 +29,10 @@ import { type Connection, type ConnectionState } from "./Connection"; import { ConnectionMap } from "./ConnectionMap"; import { createLivekitLocalMedia } from "./LocalMedia"; import { exampleTransport } from "./test"; +import { + type LocalAudioMediaTrack, + type LocalDataTrack, +} from "../../media-api"; const localTransport: TransportLocator = { transport: exampleTransport, @@ -270,9 +275,8 @@ describe("createLivekitLocalMedia", () => { media.setPublishing(true); await flushPromises(); - await expect(media.tracks$.value![0].setEnabled(false)).resolves.toBe( - false, - ); + const microphone = media.tracks$.value![0] as LocalAudioMediaTrack; + await expect(microphone.setEnabled(false)).resolves.toBe(false); expect(first.participant.setMicrophoneEnabled).toHaveBeenLastCalledWith( false, ); @@ -293,10 +297,16 @@ describe("createLivekitLocalMedia", () => { await flushPromises(); await media.publish({ source: "screenShare" }); - await media.unpublish("screenShareAudio"); + await media.unpublish({ + kind: "audio", + source: "screenShareAudio", + } as LocalAudioMediaTrack); expect(participant.setScreenShareEnabled).toHaveBeenLastCalledWith(false); - await media.unpublish("microphone"); + await media.unpublish({ + kind: "audio", + source: "microphone", + } as LocalAudioMediaTrack); expect(participant.unpublishTrack).toHaveBeenCalledOnce(); expect(media.tracks$.value).toEqual([]); }); @@ -385,6 +395,116 @@ function fakeLocalTrack(source: Track.Source): LocalTrack { return t as unknown as LocalTrack; } +describe("local data tracks", () => { + function joined(): ReturnType & FakeConnection { + const fixture = setup(); + const fake = fakeConnection(); + fixture.connectionMap$.next(mapOf(fake.connection)); + fixture.localTransport$.next(localTransport); + return { ...fixture, ...fake }; + } + + it("publishes on LiveKit only once publishing is allowed, and again on a new connection", async () => { + const { media, participant, connectionMap$ } = joined(); + let published = false; + const publishing = media + .publish({ source: "data", name: "chat" }) + .then((track) => { + published = true; + return track; + }); + await flushPromises(); + expect(participant.publishDataTrack).not.toHaveBeenCalled(); + expect(published).toBe(false); + // Listed from the request on, like a media publication + const listed = media.tracks$.value!.find((t) => t.kind === "data"); + expect(listed).toMatchObject({ name: "chat", reliable: true }); + + media.setPublishing(true); + const chat = await publishing; + expect(chat).toBe(listed); + expect(participant.publishDataTrack).toHaveBeenCalledWith({ + name: "reliable:chat", + }); + + const next = fakeConnection(); + connectionMap$.next(mapOf(next.connection)); + await flushPromises(); + expect(next.participant.publishDataTrack).toHaveBeenCalledWith({ + name: "reliable:chat", + }); + // The host's object outlives the connection + expect(media.tracks$.value).toContain(chat); + }); + + it("sends reliable data as a byte stream and lossy data as frames, replaying what was sent", async () => { + const { media, participant } = joined(); + media.setPublishing(true); + const chat = await media.publish({ source: "data", name: "chat" }); + const cursors = await media.publish({ + source: "data", + name: "cursors", + reliable: false, + }); + await flushPromises(); + const sent: Uint8Array[] = []; + chat.data$.subscribe((p) => sent.push(p)); + + await chat.send(new Uint8Array([1])); + expect(participant.sendBytes).toHaveBeenCalledWith(new Uint8Array([1]), { + topic: "reliable:chat", + }); + expect(sent).toEqual([new Uint8Array([1])]); + + await cursors.send(new Uint8Array([2])); + const livekitCursors = await vi.mocked(participant.publishDataTrack).mock + .results[1].value; + expect(livekitCursors.tryPush).toHaveBeenCalledWith({ + payload: new Uint8Array([2]), + }); + }); + + it("rejects a second track of the same name, or one LiveKit refuses", async () => { + const { media, participant } = joined(); + media.setPublishing(true); + await media.publish({ source: "data", name: "chat" }); + await expect( + media.publish({ source: "data", name: "chat", reliable: false }), + ).rejects.toThrow("published already"); + + vi.mocked(participant.publishDataTrack).mockRejectedValueOnce( + new Error("DuplicateName"), + ); + await expect( + media.publish({ source: "data", name: "cursors" }), + ).rejects.toThrow("DuplicateName"); + expect( + media.tracks$.value!.filter((t) => t.kind === "data").map((t) => t.name), + ).toEqual(["chat"]); + }); + + it("finishes a track on unpublish", async () => { + const { media, participant } = joined(); + media.setPublishing(true); + const chat: LocalDataTrack = await media.publish({ + source: "data", + name: "chat", + }); + await flushPromises(); + let completed = false; + chat.data$.subscribe({ complete: () => (completed = true) }); + + await media.unpublish(chat); + await flushPromises(); + const livekitChat = await vi.mocked(participant.publishDataTrack).mock + .results[0].value; + expect(livekitChat.unpublish).toHaveBeenCalled(); + expect(completed).toBe(true); + expect(media.tracks$.value).not.toContain(chat); + await expect(chat.send(new Uint8Array([1]))).rejects.toThrow("unpublished"); + }); +}); + /** * The slice of a local participant the publisher and the local tracks use: * enabling a source publishes a fake track at once, muting an existing one @@ -442,6 +562,14 @@ function fakeLocalParticipant(): LocalParticipant { const participant = { ...emitter, + publishDataTrack: vi.fn(async ({ name }: { name: string }) => + Promise.resolve({ + name, + tryPush: vi.fn(async () => Promise.resolve()), + unpublish: vi.fn(async () => Promise.resolve()), + } as unknown as LivekitLocalDataTrack), + ), + sendBytes: vi.fn(async () => Promise.resolve({})), identity: "@alice:example.org:DEVICE", isLocal: true, isSpeaking: false, diff --git a/sdk/src/media-backend/livekit/LocalMedia.ts b/sdk/src/media-backend/livekit/LocalMedia.ts index 85fdc40f4..9437670e8 100644 --- a/sdk/src/media-backend/livekit/LocalMedia.ts +++ b/sdk/src/media-backend/livekit/LocalMedia.ts @@ -26,13 +26,16 @@ import { } from "rxjs"; import { + type DataPublishRequest, type LocalAudioMediaTrack, + type LocalDataTrack, type LocalMedia, + type LocalTrack, type LocalVideoMediaTrack, type MediaSource, type PublishRequest, } from "../../media-api"; -import { type EncryptionSystem } from "../../encryption"; +import { E2eeType, type EncryptionSystem } from "../../encryption"; import { FailToStartLivekitConnection, toMatrixRTCError } from "../../errors"; import { type Behavior } from "../../reactive/Behavior"; import { @@ -50,6 +53,7 @@ import { } from "./LivekitMemberMedia"; import { livekitSources } from "./LivekitMediaTrack"; import { type DesiredMedia, Publisher } from "./Publisher"; +import { createLocalDataTracks } from "./LocalDataTracks"; import { toMediaConnectionState } from "./connectionState"; interface Props { @@ -67,9 +71,7 @@ interface Props { } export interface LivekitLocalMedia extends LocalMediaBackend { - /** Media only: LiveKit carries no data tracks yet. */ - tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>; - /** The connection the local member publishes on; the data channel sends on it. */ + /** The connection the local member publishes on. */ connection$: Behavior; setAudioOutputDeviceId(deviceId: string): Promise; } @@ -173,14 +175,42 @@ export function createLivekitLocalMedia({ return result; }; - const { tracks$, encryptionError$ } = createLocalLivekitMemberMedia( + const { tracks$: mediaTracks$, encryptionError$ } = + createLocalLivekitMemberMedia( + scope, + participantAndRoom$(scope, participant$, connection$), + encryptionSystem, + setEnabled, + ); + const dataTracks = createLocalDataTracks({ scope, - participantAndRoom$(scope, participant$, connection$), - encryptionSystem, - setEnabled, + connection$, + publishing$, + encrypted: encryptionSystem.kind !== E2eeType.NONE, + logger, + }); + // The data tracks after the media, both null while there is no participant + const tracks$ = scope.behavior( + combineLatest([mediaTracks$, dataTracks.tracks$], (media, data) => + media === null ? null : [...media, ...data], + ), ); - const publish = async ( + function publish(request: DataPublishRequest): Promise; + function publish( + request: PublishRequest, + ): Promise; + async function publish( + request: DataPublishRequest | PublishRequest, + ): Promise { + if (request.source === "data") + return dataTracks.publish(request).catch((e: unknown) => { + throw toMatrixRTCError(e); + }); + return publishMedia(request); + } + + const publishMedia = async ( request: PublishRequest, ): Promise => { desired.set(request.source, { request, enabled: true }); @@ -197,8 +227,9 @@ export function createLivekitLocalMedia({ } }; - const unpublish = async (source: MediaSource): Promise => { - const published = publishedSource(source); + const unpublish = async (track: LocalTrack): Promise => { + if (track.kind === "data") return dataTracks.unpublish(track); + const published = publishedSource(track.source); if (published === undefined) return; desired.delete(published); await publisher$.value?.unpublish(published); @@ -261,11 +292,16 @@ function publication$( } function trackWithId$( - tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>, + tracks$: Behavior, id: string, ): Observable { return tracks$.pipe( - map((tracks) => tracks?.find((track) => track.id === id)), + map((tracks) => + tracks?.find( + (track): track is LocalAudioMediaTrack | LocalVideoMediaTrack => + track.kind !== "data" && track.id === id, + ), + ), filter((track) => track !== undefined), ); } diff --git a/sdk/src/media-backend/livekit/Publisher.ts b/sdk/src/media-backend/livekit/Publisher.ts index 247b1256b..99dcd7281 100644 --- a/sdk/src/media-backend/livekit/Publisher.ts +++ b/sdk/src/media-backend/livekit/Publisher.ts @@ -72,9 +72,11 @@ export class Publisher { private readonly media?: LocalMedia, ) { this.room = room; - room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => { - this.logger.error("Failed to enable E2EE on the room", e); - }); + room + .setE2EEEnabled(room.options.encryption !== undefined) + ?.catch((e: Error) => { + this.logger.error("Failed to enable E2EE on the room", e); + }); this.onLocalTrackPublished = this.onLocalTrackPublished.bind(this); room.localParticipant.on( ParticipantEvent.LocalTrackPublished, diff --git a/sdk/src/media-backend/livekit/RemoteDataTracks.test.ts b/sdk/src/media-backend/livekit/RemoteDataTracks.test.ts new file mode 100644 index 000000000..74b0b6cf2 --- /dev/null +++ b/sdk/src/media-backend/livekit/RemoteDataTracks.test.ts @@ -0,0 +1,39 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { EventEmitter } from "events"; +import { + type RemoteDataTrack, + type Room as LivekitRoom, + RoomEvent, +} from "livekit-client"; +import { describe, expect, it } from "vitest"; + +import { remoteDataTracks$, watchRemoteDataTracks } from "./RemoteDataTracks"; + +const track = (sid: string): RemoteDataTrack => + ({ info: { sid }, publisherIdentity: "alice" }) as RemoteDataTrack; + +describe("remoteDataTracks$", () => { + it("keeps what the room announced before anyone observed it", () => { + const emitter = new EventEmitter(); + const room = watchRemoteDataTracks(emitter as unknown as LivekitRoom); + // As in a join response: announced before any member asks + emitter.emit(RoomEvent.DataTrackPublished, track("DT_1")); + emitter.emit(RoomEvent.DataTrackPublished, track("DT_2")); + emitter.emit(RoomEvent.DataTrackUnpublished, "DT_1"); + + const seen: string[][] = []; + remoteDataTracks$(room).subscribe((tracks) => + seen.push(tracks.map((t) => t.info.sid)), + ); + expect(seen).toEqual([["DT_2"]]); + + emitter.emit(RoomEvent.Disconnected); + expect(seen.at(-1)).toEqual([]); + }); +}); diff --git a/sdk/src/media-backend/livekit/RemoteDataTracks.ts b/sdk/src/media-backend/livekit/RemoteDataTracks.ts new file mode 100644 index 000000000..90d4a4b98 --- /dev/null +++ b/sdk/src/media-backend/livekit/RemoteDataTracks.ts @@ -0,0 +1,60 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type RemoteDataTrack, + type Room as LivekitRoom, + RoomEvent, +} from "livekit-client"; +import { BehaviorSubject, type Observable } from "rxjs"; + +const tracksByRoom = new WeakMap< + LivekitRoom, + BehaviorSubject +>(); + +/** + * Starts keeping the room's remote data tracks, from its own events. Called + * as the room is created: LiveKit announces the tracks in the join response + * before the participants that publish them exist, so a participant's + * `dataTracks` misses them, and it adds a track to that map only after + * announcing it. + */ +export function watchRemoteDataTracks(room: LivekitRoom): LivekitRoom { + remoteDataTracksOf(room); + return room; +} + +/** Every remote data track the room knows of, as LiveKit announced them. */ +export function remoteDataTracks$( + room: LivekitRoom, +): Observable { + return remoteDataTracksOf(room); +} + +function remoteDataTracksOf( + room: LivekitRoom, +): BehaviorSubject { + let tracks$ = tracksByRoom.get(room); + if (tracks$ === undefined) { + const subject$ = new BehaviorSubject([]); + room.on(RoomEvent.DataTrackPublished, (track: RemoteDataTrack) => + subject$.next([ + ...subject$.value.filter((t) => t.info.sid !== track.info.sid), + track, + ]), + ); + room.on(RoomEvent.DataTrackUnpublished, (sid: string) => + subject$.next(subject$.value.filter((t) => t.info.sid !== sid)), + ); + // A new session announces its tracks again + room.on(RoomEvent.Disconnected, () => subject$.next([])); + tracksByRoom.set(room, subject$); + tracks$ = subject$; + } + return tracks$; +} diff --git a/sdk/src/media-backend/livekit/dataTrackName.test.ts b/sdk/src/media-backend/livekit/dataTrackName.test.ts new file mode 100644 index 000000000..cd2942323 --- /dev/null +++ b/sdk/src/media-backend/livekit/dataTrackName.test.ts @@ -0,0 +1,28 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, it } from "vitest"; + +import { + fromLivekitDataTrackName, + toLivekitDataTrackName, +} from "./dataTrackName"; + +describe("data track names on LiveKit", () => { + it.each([true, false])("round-trips reliable=%s", (reliable) => { + expect( + fromLivekitDataTrackName(toLivekitDataTrackName("chat:room", reliable)), + ).toEqual({ name: "chat:room", reliable }); + }); + + it("reads a name without a prefix as lossy", () => { + expect(fromLivekitDataTrackName("telemetry")).toEqual({ + name: "telemetry", + reliable: false, + }); + }); +}); diff --git a/sdk/src/media-backend/livekit/dataTrackName.ts b/sdk/src/media-backend/livekit/dataTrackName.ts new file mode 100644 index 000000000..12217d8fc --- /dev/null +++ b/sdk/src/media-backend/livekit/dataTrackName.ts @@ -0,0 +1,36 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +/** + * How a data track is named on LiveKit. Every data track of a member is one + * LiveKit data track, which announces it; the prefix says how its payloads + * travel. A lossy track pushes them as frames on the data track itself. A + * reliable one sends each as a byte stream on a topic of the same name, + * because LiveKit's data tracks are lossy only. + */ + +const lossyPrefix = "lossy:"; +const reliablePrefix = "reliable:"; + +export function toLivekitDataTrackName( + name: string, + reliable: boolean, +): string { + return (reliable ? reliablePrefix : lossyPrefix) + name; +} + +/** A track without a prefix, from a plain LiveKit publisher, is lossy under its own name. */ +export function fromLivekitDataTrackName(livekitName: string): { + name: string; + reliable: boolean; +} { + if (livekitName.startsWith(reliablePrefix)) + return { name: livekitName.slice(reliablePrefix.length), reliable: true }; + if (livekitName.startsWith(lossyPrefix)) + return { name: livekitName.slice(lossyPrefix.length), reliable: false }; + return { name: livekitName, reliable: false }; +}