From 18e0065086c03031f079a23724e6dcff5e587656 Mon Sep 17 00:00:00 2001 From: "Timo K." Date: Sun, 4 Oct 2026 12:20:33 +0200 Subject: [PATCH] specific call tracks -> generic tracks --- sdk/MemberMediaPlan.md | 66 ++++++++++++++++++++ sdk/dev/main.ts | 32 ++++++---- sdk/index.ts | 1 + sdk/src/api.ts | 24 +++++--- sdk/src/media/LivekitMediaTrack.test.ts | 47 ++++++++++---- sdk/src/media/LivekitMediaTrack.ts | 60 +++++++++++------- sdk/src/media/LivekitMemberMedia.ts | 81 +++++++------------------ sdk/src/utils/tracks.ts | 44 ++++++++++++++ 8 files changed, 240 insertions(+), 115 deletions(-) create mode 100644 sdk/src/utils/tracks.ts diff --git a/sdk/MemberMediaPlan.md b/sdk/MemberMediaPlan.md index 67b9e92bc..87c35ed56 100644 --- a/sdk/MemberMediaPlan.md +++ b/sdk/MemberMediaPlan.md @@ -314,3 +314,69 @@ three are the ones this plan makes visible. | `UnknownCallError` in `errors.ts` | Name only. | Rename with the two `Livekit*` errors in slice 11 of `SdkMigration.md`. | | `observeSharingScreen$` in `LocalMember.ts` | Reads `participant.isScreenShareEnabled`. | Goes with `sharingScreen$`, or reads `tracks$` from the local media in the meantime. | | `sendNotificationType` | Kept on purpose; see "Call-specific things kept out of the client" in `SdkArchitecture.md`. | Nothing. | + +## Plan for the refactors above + +Four slices, in dependency order. Each is one PR, green on its own, with the +`playwright/sdk` specs and the dev harness as the proof that behaviour is kept. +Element Call follows on its own branch after each. + +### 1. Local track controls + +The local member's tracks become objects a host can act on, instead of methods +named after a call's devices. + +- **Add** `LocalAudioMediaTrack` and `LocalVideoMediaTrack` (`local: true`) in + `api.ts`, with `setEnabled(enabled)`, `setDevice(deviceId)` and, on video, + `switchFacingMode()` and `setProcessor(processor)`. `tracks$` on + `LocalMemberMedia` is typed to them. +- **Remove** `LocalMemberMedia.switchCamera`, and `audioInputDeviceId$`, + `videoInputDeviceId$`, `videoProcessor$` from `LocalMediaInputs`. The host + subscribes to its settings and calls the track; the SDK stops watching + behaviors it does not own. `audioOutputDeviceId$` stays: playback is not a + track. +- **Keep** `setMicrophoneEnabled` / `setCameraEnabled` for now: they also cover + "no track yet", which slice 2 solves. +- **Touches** `LivekitMediaTrack.ts`, `Publisher.ts` (the processor and device + code moves onto the track), `MatrixRTCClient.ts`, the harness buttons. + +### 2. `publish` and `unpublish` + +One way to put media on the session, whatever it is. + +- **Add** to `LocalRTCMember`: `publish(request): Promise` and + `unpublish(id): Promise`. A request is `{ source, deviceId?, capture? }` + for `"microphone"` and `"camera"`, `{ source: "screenShare", audio?, capture? +}` for a screen. Failures reject the promise with a `MatrixRTCError`. +- **Remove** `sharingScreen$`, `toggleScreenSharing`, `screenShareError$`, + `dismissScreenShareError`, `observeSharingScreen$`, and the two + `set*Enabled` methods on the client. "Sharing" is `tracks$.some(...)` on the + local media; "enable" is `publish` or `track.setEnabled`. +- **Replace** `LocalMediaInputs.microphoneEnabled` / `cameraEnabled` with + `publish: PublishRequest[]`, applied once the transport is up, as today. + `CaptureSettings` becomes the `capture` of a request; `MediaQuality` keeps its + `config.json` shape and is mapped to per-request capture at the edge. +- **Touches** `LocalMember.ts` (the screen share block and `setEnabled`), + `Publisher.ts` (iterates requests instead of enumerating three sources), + `livekitOptions.ts`, `lifecycle.spec.ts` and `media.spec.ts` for the new + buttons. + +### 3. Names that say "call" + +No behaviour change; one PR of renames and type narrowing. + +- `RTCMember.membership$` is typed as an SDK `RTCMembership`: the subset of + `CallMembership` hosts read (`userId`, `deviceId`, `memberId`, + `rtcBackendIdentity`, `getTransport()`, `applicationData`). The js-sdk object + is still what is behind it. +- `UnknownCallError` is renamed with the two `Livekit*` errors, as slice 11 of + `SdkMigration.md` already plans; `index.ts` drops its alias. +- `applicationData` is passed through whole once the js-sdk accepts more than + `m.call.intent`; until then the README says which key survives. + +### Not planned + +- `MediaSource` keeps its five values. A `name` on `MediaTrack` (LiveKit's + `trackName`) is a one-line addition when the first non-call application asks + for it; adding it now would be a guess at that application's needs. +- `sendNotificationType` stays, as `SdkArchitecture.md` already decided. diff --git a/sdk/dev/main.ts b/sdk/dev/main.ts index d28cc468b..75445c2db 100644 --- a/sdk/dev/main.ts +++ b/sdk/dev/main.ts @@ -14,14 +14,16 @@ Please see LICENSE in the repository root for full details. import { logger } from "matrix-js-sdk/lib/logger"; import { combineLatest, type Observable, of, switchMap } from "rxjs"; import { + type AudioMediaTrack, constant, createMatrixRTCClient, E2eeType, MatrixRTCMode, - type MediaTrack, ObservableScope, type RTCMember, type MatrixRTCClient, + trackBySource$, + type VideoMediaTrack, } from "@element-hq/matrixrtc-sdk"; import { createSession, joinRoom, type Login } from "./session"; @@ -185,21 +187,21 @@ function memberTile(scope: ObservableScope, member: RTCMember): HTMLElement { name.textContent = n; }); - const camera$ = member.media$.pipe( - switchMap((media) => media?.camera$ ?? of(undefined)), + const tracks$ = scope.behavior( + member.media$.pipe(switchMap((media) => media?.tracks$ ?? of([]))), + ); + render( + scope, + trackBySource$(scope, tracks$, "camera"), + tile.appendChild(document.createElement("video")), ); - render(scope, camera$, tile.appendChild(document.createElement("video"))); // Our own microphone would only echo - if (!member.local) { - const microphone$ = member.media$.pipe( - switchMap((media) => media?.microphone$ ?? of(undefined)), - ); + if (!member.local) render( scope, - microphone$, + trackBySource$(scope, tracks$, "microphone"), tile.appendChild(document.createElement("audio")), ); - } return tile; } @@ -209,10 +211,10 @@ function memberTile(scope: ObservableScope, member: RTCMember): HTMLElement { */ function render( scope: ObservableScope, - track$: Observable, + track$: Observable, element: HTMLMediaElement, ): void { - let attached: MediaTrack | undefined; + let attached: AudioMediaTrack | VideoMediaTrack | undefined; track$.pipe(scope.bind()).subscribe((track) => { attached?.detach(element); attached = track; @@ -230,7 +232,7 @@ function render( }); }; const of$ = ( - pick: (track: MediaTrack) => Observable, + pick: (track: AudioMediaTrack | VideoMediaTrack) => Observable, ): Observable => track$.pipe(switchMap((track) => (track ? pick(track) : of(undefined)))); label( @@ -241,6 +243,10 @@ function render( "encrypted", of$((t) => t.encrypted$), ); + label( + "active", + of$((t) => (t.kind === "audio" ? t.isActive$ : of(undefined))), + ); const stats$ = of$((t) => t.stats$); label("frameWidth", stats$.pipe(switchMap((s) => of(frames(s)?.frameWidth)))); label("frames", stats$.pipe(switchMap((s) => of(frames(s)?.count)))); diff --git a/sdk/index.ts b/sdk/index.ts index 4e9b29ec9..d3eab3b0d 100644 --- a/sdk/index.ts +++ b/sdk/index.ts @@ -53,6 +53,7 @@ export { } from "./src/errors"; export * from "./src/api"; +export { trackBySource$, type TrackOfSource } from "./src/utils/tracks"; export { createMatrixRTCClient } from "./src/session/MatrixRTCClient"; // For a developer panel to check a transport url before using it export { getSFUConfigWithOpenID as authenticateWithTransport } from "./src/session/openIDSFU"; diff --git a/sdk/src/api.ts b/sdk/src/api.ts index ba571c928..4b9f0a1d3 100644 --- a/sdk/src/api.ts +++ b/sdk/src/api.ts @@ -253,7 +253,9 @@ export type MediaSource = | "microphone" | "camera" | "screenShare" - | "screenShareAudio"; + | "screenShareAudio" + /** Published without a source; the application knows what it is. */ + | "unknown"; export type MediaStreamStats = | RTCInboundRtpStreamStats @@ -284,6 +286,13 @@ export interface MediaTrack { export interface AudioMediaTrack extends MediaTrack { kind: "audio"; + /** + * Whether the track carries sound right now, as the backend measures it. + * False while muted. A call reads this on the microphone track and calls it + * "speaking". LiveKit measures it per member, so every audio track of a + * member reports the same value. + */ + isActive$: Behavior; /** Route playback through Web Audio, for earpiece pan and gain. Undefined resets. */ setAudioContext(ctx: AudioContext | undefined, plugins?: AudioNode[]): void; setVolume(volume: number): void; @@ -303,13 +312,12 @@ export type EncryptionError = "MissingKey" | "InvalidKey"; */ export interface MemberMedia { local: boolean; - speaking$: Behavior; - screenShareEnabled$: Behavior; - - microphone$: Behavior; - camera$: Behavior; - screenShare$: Behavior; - screenShareAudio$: Behavior; + /** + * One entry per published track, in publication order. An entry stays the + * same object for as long as the same publication is behind it. Which + * track is which is in its `source`; `trackBySource$` picks one out. + */ + tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[]>; /** Emits when the SFU reports a key problem for this member. */ encryptionError$: Observable; diff --git a/sdk/src/media/LivekitMediaTrack.test.ts b/sdk/src/media/LivekitMediaTrack.test.ts index f6adda3ed..87f97e6e2 100644 --- a/sdk/src/media/LivekitMediaTrack.test.ts +++ b/sdk/src/media/LivekitMediaTrack.test.ts @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { EventEmitter } from "events"; import { ParticipantEvent, + RemoteAudioTrack, type RemoteParticipant, type Room as LivekitRoom, Track, @@ -34,7 +35,6 @@ describe("createLivekitMediaTrack", () => { participant, publication as unknown as TrackPublication, room, - "camera", ); expect(track.kind).toBe("video"); expect(track.source).toBe("camera"); @@ -52,7 +52,6 @@ describe("createLivekitMediaTrack", () => { participant, publication as unknown as TrackPublication, room, - "camera", ); track.attach(element); track.attach(element); @@ -76,7 +75,6 @@ describe("createLivekitMediaTrack", () => { participant, publication as unknown as TrackPublication, room, - "camera", ); track.attach(element); scope.end(); @@ -90,7 +88,6 @@ describe("createLivekitMediaTrack", () => { participant, publication as unknown as TrackPublication, room, - "camera", ); expect(track.muted$.value).toBe(false); publication.isMuted = true; @@ -107,19 +104,35 @@ describe("createLivekitMediaTrack", () => { participant, publication as unknown as TrackPublication, room, - "microphone", ) as AudioMediaTrack; track.setVolume(0.5); - expect(participant.setVolume).toHaveBeenCalledWith( - 0.5, - Track.Source.Microphone, - ); + expect(publication.track!.setVolume).toHaveBeenCalledWith(0.5); + }); + + it("is active while the member speaks and the track is not muted", () => { + const { participant, publication, room, emitter } = fakes({ + kind: Track.Kind.Audio, + }); + const track = createLivekitMediaTrack( + scope, + participant, + publication as unknown as TrackPublication, + room, + ) as AudioMediaTrack; + expect(track.isActive$.value).toBe(false); + participant.isSpeaking = true; + emitter.emit(ParticipantEvent.IsSpeakingChanged, true); + expect(track.isActive$.value).toBe(true); + publication.isMuted = true; + emitter.emit(ParticipantEvent.TrackMuted, publication); + expect(track.isActive$.value).toBe(false); }); }); type FakeTrack = Track & { attach: ReturnType; detach: ReturnType; + setVolume: ReturnType; }; type FakePublication = Omit & { @@ -128,14 +141,20 @@ type FakePublication = Omit & { }; function fakeTrack(): FakeTrack { - return { attach: vi.fn(), detach: vi.fn() } as unknown as FakeTrack; + return Object.assign(Object.create(RemoteAudioTrack.prototype), { + attach: vi.fn(), + detach: vi.fn(), + setVolume: vi.fn(), + setAudioContext: vi.fn(), + setWebAudioPlugins: vi.fn(), + }) as FakeTrack; } function fakes({ track = fakeTrack(), kind = Track.Kind.Video, }: { track?: FakeTrack | undefined; kind?: Track.Kind } = {}): { - participant: RemoteParticipant & { setVolume: ReturnType }; + participant: RemoteParticipant & { isSpeaking: boolean }; publication: FakePublication; room: LivekitRoom; emitter: EventEmitter; @@ -143,13 +162,15 @@ function fakes({ const emitter = new EventEmitter(); const participant = Object.assign(emitter, { isLocal: false, + isSpeaking: false, identity: "@alice:example.org:DEVICE", - setVolume: vi.fn(), getTrackPublication: (): TrackPublication => publication as unknown as TrackPublication, - }) as unknown as RemoteParticipant & { setVolume: ReturnType }; + }) as unknown as RemoteParticipant & { isSpeaking: boolean }; const publication = Object.assign(emitter, { kind, + source: + kind === Track.Kind.Audio ? Track.Source.Microphone : Track.Source.Camera, trackSid: "TR_1", isMuted: false, isEncrypted: true, diff --git a/sdk/src/media/LivekitMediaTrack.ts b/sdk/src/media/LivekitMediaTrack.ts index b4b065d74..bd60504f7 100644 --- a/sdk/src/media/LivekitMediaTrack.ts +++ b/sdk/src/media/LivekitMediaTrack.ts @@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { + observeParticipantEvents, observeParticipantMedia, roomEventSelector, } from "@livekit/components-core"; @@ -14,8 +15,8 @@ import { LocalTrack, LocalVideoTrack, type Participant, + ParticipantEvent, RemoteAudioTrack, - type RemoteParticipant, RemoteTrack, type Room as LivekitRoom, RoomEvent, @@ -24,6 +25,7 @@ import { type TrackPublication, } from "livekit-client"; import { + combineLatest, distinctUntilChanged, fromEvent, interval, @@ -46,11 +48,12 @@ import { } from "../api"; import { LazyBehavior } from "../utils/LazyBehavior"; -export const livekitSources: Record = { - microphone: Track.Source.Microphone, - camera: Track.Source.Camera, - screenShare: Track.Source.ScreenShare, - screenShareAudio: Track.Source.ScreenShareAudio, +const sources: Record = { + [Track.Source.Microphone]: "microphone", + [Track.Source.Camera]: "camera", + [Track.Source.ScreenShare]: "screenShare", + [Track.Source.ScreenShareAudio]: "screenShareAudio", + [Track.Source.Unknown]: "unknown", }; // One timer for every track so that a large session does not keep hundreds of @@ -67,7 +70,6 @@ export function createLivekitMediaTrack( participant: Participant, publication: TrackPublication, room: LivekitRoom, - source: MediaSource, ): AudioMediaTrack | VideoMediaTrack { const mediaChanged$ = observeParticipantMedia(participant); const track$: Behavior = scope.behavior( @@ -85,17 +87,19 @@ export function createLivekitMediaTrack( const attached = new Set(); let audioContext: AudioContext | undefined; let audioPlugins: AudioNode[] = []; - const applyAudioContext = (track: Track | undefined): void => { + let volume: number | undefined; + const applyAudioSettings = (track: Track | undefined): void => { if (!(track instanceof RemoteAudioTrack)) return; track.setAudioContext(audioContext); track.setWebAudioPlugins(audioPlugins); + if (volume !== undefined) track.setVolume(volume); }; let current = track$.value; track$.pipe(scope.bind()).subscribe((track) => { for (const element of attached) current?.detach(element); current = track; - applyAudioContext(track); + applyAudioSettings(track); for (const element of attached) track?.attach(element); }); scope.onEnd(() => { @@ -103,14 +107,15 @@ export function createLivekitMediaTrack( attached.clear(); }); + const muted$ = scope.behavior( + mediaChanged$.pipe(map(() => publication.isMuted)), + publication.isMuted, + ); const base: MediaTrack = { - source, + source: sources[publication.source], kind: publication.kind === Track.Kind.Audio ? "audio" : "video", id: publication.trackSid, - muted$: scope.behavior( - mediaChanged$.pipe(map(() => publication.isMuted)), - publication.isMuted, - ), + muted$, encrypted$: scope.behavior( merge( mediaChanged$, @@ -140,19 +145,30 @@ export function createLivekitMediaTrack( return { ...base, kind: "audio", + // LiveKit detects speakers per participant, so this is the member's + // activity, narrowed to the tracks that can be carrying it + isActive$: scope.behavior( + combineLatest( + [ + observeParticipantEvents( + participant, + ParticipantEvent.IsSpeakingChanged, + ).pipe(map((p) => p.isSpeaking)), + muted$, + ], + (speaking, muted) => speaking && !muted, + ), + participant.isSpeaking && !publication.isMuted, + ), setAudioContext: (ctx, plugins = []) => { audioContext = ctx; audioPlugins = plugins; - applyAudioContext(current); + applyAudioSettings(current); }, - setVolume: (volume) => { + setVolume: (v) => { // Our own audio is never played back, so there is nothing to scale - if (participant.isLocal) return; - const remote = participant as RemoteParticipant; - if (source === "microphone") - remote.setVolume(volume, Track.Source.Microphone); - else if (source === "screenShareAudio") - remote.setVolume(volume, Track.Source.ScreenShareAudio); + volume = v; + applyAudioSettings(current); }, }; diff --git a/sdk/src/media/LivekitMemberMedia.ts b/sdk/src/media/LivekitMemberMedia.ts index 06c8bcd6c..b6a058a32 100644 --- a/sdk/src/media/LivekitMemberMedia.ts +++ b/sdk/src/media/LivekitMemberMedia.ts @@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details. */ import { - observeParticipantEvents, observeParticipantMedia, roomEventSelector, } from "@livekit/components-core"; @@ -15,28 +14,22 @@ import { type LocalParticipant, LocalVideoTrack, type Participant, - ParticipantEvent, type Room as LivekitRoom, RoomEvent, Track, - type TrackPublication, } from "livekit-client"; -import { distinctUntilChanged, filter, map, type Observable } from "rxjs"; +import { filter, map, type Observable } from "rxjs"; -import { type Behavior } from "../reactive/Behavior"; import { type ObservableScope } from "../reactive/ObservableScope"; +import { generateItems } from "../reactive/observable"; import { E2eeType } from "../encryption"; import { type EncryptionSystem } from "../encryption"; import { - type AudioMediaTrack, type EncryptionError, type LocalMemberMedia, - type MediaSource, type MemberMedia, - type VideoMediaTrack, } from "../api"; -import { mapScoped } from "../utils/mapScoped"; -import { createLivekitMediaTrack, livekitSources } from "./LivekitMediaTrack"; +import { createLivekitMediaTrack } from "./LivekitMediaTrack"; /** * A participant as `MemberMedia`. The only place, with `LivekitMediaTrack`, @@ -49,31 +42,28 @@ export function createLivekitMemberMedia( room: LivekitRoom, encryptionSystem: EncryptionSystem, ): MemberMedia { - const mediaChanged$ = observeParticipantMedia(participant); - const track$ = ( - source: MediaSource, - ): Behavior => - memberTrack$(scope, participant, room, source, mediaChanged$) as Behavior< - T | undefined - >; - return { local: participant.isLocal, - speaking$: scope.behavior( - observeParticipantEvents( - participant, - ParticipantEvent.IsSpeakingChanged, - ).pipe(map((p) => p.isSpeaking)), - participant.isSpeaking, + // One track per publication, each living as long as its publication is + // in the participant's map + tracks$: scope.behavior( + observeParticipantMedia(participant).pipe( + generateItems( + `${participant.identity} tracks$`, + function* () { + for (const publication of participant.trackPublications.values()) + yield { keys: [publication.trackSid], data: publication }; + }, + (trackScope, publication$) => + createLivekitMediaTrack( + trackScope, + participant, + publication$.value, + room, + ), + ), + ), ), - screenShareEnabled$: scope.behavior( - mediaChanged$.pipe(map((media) => media.isScreenShareEnabled)), - participant.isScreenShareEnabled, - ), - microphone$: track$("microphone"), - camera$: track$("camera"), - screenShare$: track$("screenShare"), - screenShareAudio$: track$("screenShareAudio"), encryptionError$: encryptionErrors$( scope, participant, @@ -105,33 +95,6 @@ export function createLocalLivekitMemberMedia( }; } -/** The member's track for a source, as long as the same publication is behind it. */ -function memberTrack$( - scope: ObservableScope, - participant: Participant, - room: LivekitRoom, - source: MediaSource, - mediaChanged$: Observable, -): Behavior { - const publication = (): TrackPublication | undefined => - participant.getTrackPublication(livekitSources[source]); - return mapScoped( - scope, - scope.behavior( - mediaChanged$.pipe(map(publication), distinctUntilChanged()), - publication(), - ), - (trackScope, publication) => - createLivekitMediaTrack( - trackScope, - participant, - publication, - room, - source, - ), - ); -} - function encryptionErrors$( scope: ObservableScope, participant: Participant, diff --git a/sdk/src/utils/tracks.ts b/sdk/src/utils/tracks.ts new file mode 100644 index 000000000..f2ce65418 --- /dev/null +++ b/sdk/src/utils/tracks.ts @@ -0,0 +1,44 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { distinctUntilChanged, map } from "rxjs"; + +import { type Behavior } from "../reactive/Behavior"; +import { type ObservableScope } from "../reactive/ObservableScope"; +import { + type AudioMediaTrack, + type MediaSource, + type MediaTrack, + type VideoMediaTrack, +} from "../api"; + +export type TrackOfSource = S extends + | "microphone" + | "screenShareAudio" + ? AudioMediaTrack + : S extends "camera" | "screenShare" + ? VideoMediaTrack + : AudioMediaTrack | VideoMediaTrack; + +/** The member's first track of a source; undefined while there is none. */ +export function trackBySource$( + scope: ObservableScope, + tracks$: Behavior, + source: S, +): Behavior | undefined> { + return scope.behavior( + tracks$.pipe( + map( + (tracks) => + tracks.find((t) => t.source === source) as + | TrackOfSource + | undefined, + ), + distinctUntilChanged(), + ), + ); +}