specific call tracks -> generic tracks

2026-10-16 23:45:50 +00:00 · 2026-10-08 13:53:57 +02:00
parent d7ab2b27b6
commit 18e0065086
8 changed files with 240 additions and 115 deletions
@@ -314,3 +314,69 @@ three are the ones this plan makes visible.
 | `UnknownCallError` in `errors.ts`                                                                                                 | Name only.                                                                                                                                                                    | Rename with the two `Livekit*` errors in slice 11 of `SdkMigration.md`.                                                                                                          |
 | `observeSharingScreen$` in `LocalMember.ts`                                                                                       | Reads `participant.isScreenShareEnabled`.                                                                                                                                     | Goes with `sharingScreen$`, or reads `tracks$` from the local media in the meantime.                                                                                             |
 | `sendNotificationType`                                                                                                            | Kept on purpose; see "Call-specific things kept out of the client" in `SdkArchitecture.md`.                                                                                   | Nothing.                                                                                                                                                                         |
+
+## Plan for the refactors above
+
+Four slices, in dependency order. Each is one PR, green on its own, with the
+`playwright/sdk` specs and the dev harness as the proof that behaviour is kept.
+Element Call follows on its own branch after each.
+
+### 1. Local track controls
+
+The local member's tracks become objects a host can act on, instead of methods
+named after a call's devices.
+
+- **Add** `LocalAudioMediaTrack` and `LocalVideoMediaTrack` (`local: true`) in
+  `api.ts`, with `setEnabled(enabled)`, `setDevice(deviceId)` and, on video,
+  `switchFacingMode()` and `setProcessor(processor)`. `tracks$` on
+  `LocalMemberMedia` is typed to them.
+- **Remove** `LocalMemberMedia.switchCamera`, and `audioInputDeviceId$`,
+  `videoInputDeviceId$`, `videoProcessor$` from `LocalMediaInputs`. The host
+  subscribes to its settings and calls the track; the SDK stops watching
+  behaviors it does not own. `audioOutputDeviceId$` stays: playback is not a
+  track.
+- **Keep** `setMicrophoneEnabled` / `setCameraEnabled` for now: they also cover
+  "no track yet", which slice 2 solves.
+- **Touches** `LivekitMediaTrack.ts`, `Publisher.ts` (the processor and device
+  code moves onto the track), `MatrixRTCClient.ts`, the harness buttons.
+
+### 2. `publish` and `unpublish`
+
+One way to put media on the session, whatever it is.
+
+- **Add** to `LocalRTCMember`: `publish(request): Promise<LocalMediaTrack>` and
+  `unpublish(id): Promise<void>`. A request is `{ source, deviceId?, capture? }`
+  for `"microphone"` and `"camera"`, `{ source: "screenShare", audio?, capture?
+}` for a screen. Failures reject the promise with a `MatrixRTCError`.
+- **Remove** `sharingScreen$`, `toggleScreenSharing`, `screenShareError$`,
+  `dismissScreenShareError`, `observeSharingScreen$`, and the two
+  `set*Enabled` methods on the client. "Sharing" is `tracks$.some(...)` on the
+  local media; "enable" is `publish` or `track.setEnabled`.
+- **Replace** `LocalMediaInputs.microphoneEnabled` / `cameraEnabled` with
+  `publish: PublishRequest[]`, applied once the transport is up, as today.
+  `CaptureSettings` becomes the `capture` of a request; `MediaQuality` keeps its
+  `config.json` shape and is mapped to per-request capture at the edge.
+- **Touches** `LocalMember.ts` (the screen share block and `setEnabled`),
+  `Publisher.ts` (iterates requests instead of enumerating three sources),
+  `livekitOptions.ts`, `lifecycle.spec.ts` and `media.spec.ts` for the new
+  buttons.
+
+### 3. Names that say "call"
+
+No behaviour change; one PR of renames and type narrowing.
+
+- `RTCMember.membership$` is typed as an SDK `RTCMembership`: the subset of
+  `CallMembership` hosts read (`userId`, `deviceId`, `memberId`,
+  `rtcBackendIdentity`, `getTransport()`, `applicationData`). The js-sdk object
+  is still what is behind it.
+- `UnknownCallError` is renamed with the two `Livekit*` errors, as slice 11 of
+  `SdkMigration.md` already plans; `index.ts` drops its alias.
+- `applicationData` is passed through whole once the js-sdk accepts more than
+  `m.call.intent`; until then the README says which key survives.
+
+### Not planned
+
+- `MediaSource` keeps its five values. A `name` on `MediaTrack` (LiveKit's
+  `trackName`) is a one-line addition when the first non-call application asks
+  for it; adding it now would be a guess at that application's needs.
+- `sendNotificationType` stays, as `SdkArchitecture.md` already decided.
@@ -14,14 +14,16 @@ Please see LICENSE in the repository root for full details.
 import { logger } from "matrix-js-sdk/lib/logger";
 import { combineLatest, type Observable, of, switchMap } from "rxjs";
 import {
+  type AudioMediaTrack,
  constant,
  createMatrixRTCClient,
  E2eeType,
  MatrixRTCMode,
-  type MediaTrack,
  ObservableScope,
  type RTCMember,
  type MatrixRTCClient,
+  trackBySource$,
+  type VideoMediaTrack,
 } from "@element-hq/matrixrtc-sdk";

 import { createSession, joinRoom, type Login } from "./session";
@@ -185,21 +187,21 @@ function memberTile(scope: ObservableScope, member: RTCMember): HTMLElement {
    name.textContent = n;
  });

-  const camera$ = member.media$.pipe(
-    switchMap((media) => media?.camera$ ?? of(undefined)),
-  );
-  render(scope, camera$, tile.appendChild(document.createElement("video")));
-  // Our own microphone would only echo
-  if (!member.local) {
-    const microphone$ = member.media$.pipe(
-      switchMap((media) => media?.microphone$ ?? of(undefined)),
+  const tracks$ = scope.behavior(
+    member.media$.pipe(switchMap((media) => media?.tracks$ ?? of([]))),
  );
  render(
    scope,
-      microphone$,
+    trackBySource$(scope, tracks$, "camera"),
+    tile.appendChild(document.createElement("video")),
+  );
+  // Our own microphone would only echo
+  if (!member.local)
+    render(
+      scope,
+      trackBySource$(scope, tracks$, "microphone"),
      tile.appendChild(document.createElement("audio")),
    );
-  }
  return tile;
 }

@@ -209,10 +211,10 @@ function memberTile(scope: ObservableScope, member: RTCMember): HTMLElement {
 */
 function render(
  scope: ObservableScope,
-  track$: Observable<MediaTrack | undefined>,
+  track$: Observable<AudioMediaTrack | VideoMediaTrack | undefined>,
  element: HTMLMediaElement,
 ): void {
-  let attached: MediaTrack | undefined;
+  let attached: AudioMediaTrack | VideoMediaTrack | undefined;
  track$.pipe(scope.bind()).subscribe((track) => {
    attached?.detach(element);
    attached = track;
@@ -230,7 +232,7 @@ function render(
    });
  };
  const of$ = <T>(
-    pick: (track: MediaTrack) => Observable<T>,
+    pick: (track: AudioMediaTrack | VideoMediaTrack) => Observable<T>,
  ): Observable<T | undefined> =>
    track$.pipe(switchMap((track) => (track ? pick(track) : of(undefined))));
  label(
@@ -241,6 +243,10 @@ function render(
    "encrypted",
    of$((t) => t.encrypted$),
  );
+  label(
+    "active",
+    of$((t) => (t.kind === "audio" ? t.isActive$ : of(undefined))),
+  );
  const stats$ = of$((t) => t.stats$);
  label("frameWidth", stats$.pipe(switchMap((s) => of(frames(s)?.frameWidth))));
  label("frames", stats$.pipe(switchMap((s) => of(frames(s)?.count))));
@@ -53,6 +53,7 @@ export {
 } from "./src/errors";

 export * from "./src/api";
+export { trackBySource$, type TrackOfSource } from "./src/utils/tracks";
 export { createMatrixRTCClient } from "./src/session/MatrixRTCClient";
 // For a developer panel to check a transport url before using it
 export { getSFUConfigWithOpenID as authenticateWithTransport } from "./src/session/openIDSFU";
@@ -253,7 +253,9 @@ export type MediaSource =
  | "microphone"
  | "camera"
  | "screenShare"
-  | "screenShareAudio";
+  | "screenShareAudio"
+  /** Published without a source; the application knows what it is. */
+  | "unknown";

 export type MediaStreamStats =
  | RTCInboundRtpStreamStats
@@ -284,6 +286,13 @@ export interface MediaTrack {

 export interface AudioMediaTrack extends MediaTrack {
  kind: "audio";
+  /**
+   * Whether the track carries sound right now, as the backend measures it.
+   * False while muted. A call reads this on the microphone track and calls it
+   * "speaking". LiveKit measures it per member, so every audio track of a
+   * member reports the same value.
+   */
+  isActive$: Behavior<boolean>;
  /** Route playback through Web Audio, for earpiece pan and gain. Undefined resets. */
  setAudioContext(ctx: AudioContext | undefined, plugins?: AudioNode[]): void;
  setVolume(volume: number): void;
@@ -303,13 +312,12 @@ export type EncryptionError = "MissingKey" | "InvalidKey";
 */
 export interface MemberMedia {
  local: boolean;
-  speaking$: Behavior<boolean>;
-  screenShareEnabled$: Behavior<boolean>;
-
-  microphone$: Behavior<AudioMediaTrack | undefined>;
-  camera$: Behavior<VideoMediaTrack | undefined>;
-  screenShare$: Behavior<VideoMediaTrack | undefined>;
-  screenShareAudio$: Behavior<AudioMediaTrack | undefined>;
+  /**
+   * One entry per published track, in publication order. An entry stays the
+   * same object for as long as the same publication is behind it. Which
+   * track is which is in its `source`; `trackBySource$` picks one out.
+   */
+  tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[]>;

  /** Emits when the SFU reports a key problem for this member. */
  encryptionError$: Observable<EncryptionError>;
@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
 import { EventEmitter } from "events";
 import {
  ParticipantEvent,
+  RemoteAudioTrack,
  type RemoteParticipant,
  type Room as LivekitRoom,
  Track,
@@ -34,7 +35,6 @@ describe("createLivekitMediaTrack", () => {
      participant,
      publication as unknown as TrackPublication,
      room,
-      "camera",
    );
    expect(track.kind).toBe("video");
    expect(track.source).toBe("camera");
@@ -52,7 +52,6 @@ describe("createLivekitMediaTrack", () => {
      participant,
      publication as unknown as TrackPublication,
      room,
-      "camera",
    );
    track.attach(element);
    track.attach(element);
@@ -76,7 +75,6 @@ describe("createLivekitMediaTrack", () => {
      participant,
      publication as unknown as TrackPublication,
      room,
-      "camera",
    );
    track.attach(element);
    scope.end();
@@ -90,7 +88,6 @@ describe("createLivekitMediaTrack", () => {
      participant,
      publication as unknown as TrackPublication,
      room,
-      "camera",
    );
    expect(track.muted$.value).toBe(false);
    publication.isMuted = true;
@@ -107,19 +104,35 @@ describe("createLivekitMediaTrack", () => {
      participant,
      publication as unknown as TrackPublication,
      room,
-      "microphone",
    ) as AudioMediaTrack;
    track.setVolume(0.5);
-    expect(participant.setVolume).toHaveBeenCalledWith(
-      0.5,
-      Track.Source.Microphone,
-    );
+    expect(publication.track!.setVolume).toHaveBeenCalledWith(0.5);
+  });
+
+  it("is active while the member speaks and the track is not muted", () => {
+    const { participant, publication, room, emitter } = fakes({
+      kind: Track.Kind.Audio,
+    });
+    const track = createLivekitMediaTrack(
+      scope,
+      participant,
+      publication as unknown as TrackPublication,
+      room,
+    ) as AudioMediaTrack;
+    expect(track.isActive$.value).toBe(false);
+    participant.isSpeaking = true;
+    emitter.emit(ParticipantEvent.IsSpeakingChanged, true);
+    expect(track.isActive$.value).toBe(true);
+    publication.isMuted = true;
+    emitter.emit(ParticipantEvent.TrackMuted, publication);
+    expect(track.isActive$.value).toBe(false);
  });
 });

 type FakeTrack = Track & {
  attach: ReturnType<typeof vi.fn>;
  detach: ReturnType<typeof vi.fn>;
+  setVolume: ReturnType<typeof vi.fn>;
 };

 type FakePublication = Omit<TrackPublication, "track" | "isMuted"> & {
@@ -128,14 +141,20 @@ type FakePublication = Omit<TrackPublication, "track" | "isMuted"> & {
 };

 function fakeTrack(): FakeTrack {
-  return { attach: vi.fn(), detach: vi.fn() } as unknown as FakeTrack;
+  return Object.assign(Object.create(RemoteAudioTrack.prototype), {
+    attach: vi.fn(),
+    detach: vi.fn(),
+    setVolume: vi.fn(),
+    setAudioContext: vi.fn(),
+    setWebAudioPlugins: vi.fn(),
+  }) as FakeTrack;
 }

 function fakes({
  track = fakeTrack(),
  kind = Track.Kind.Video,
 }: { track?: FakeTrack | undefined; kind?: Track.Kind } = {}): {
-  participant: RemoteParticipant & { setVolume: ReturnType<typeof vi.fn> };
+  participant: RemoteParticipant & { isSpeaking: boolean };
  publication: FakePublication;
  room: LivekitRoom;
  emitter: EventEmitter;
@@ -143,13 +162,15 @@ function fakes({
  const emitter = new EventEmitter();
  const participant = Object.assign(emitter, {
    isLocal: false,
+    isSpeaking: false,
    identity: "@alice:example.org:DEVICE",
-    setVolume: vi.fn(),
    getTrackPublication: (): TrackPublication =>
      publication as unknown as TrackPublication,
-  }) as unknown as RemoteParticipant & { setVolume: ReturnType<typeof vi.fn> };
+  }) as unknown as RemoteParticipant & { isSpeaking: boolean };
  const publication = Object.assign(emitter, {
    kind,
+    source:
+      kind === Track.Kind.Audio ? Track.Source.Microphone : Track.Source.Camera,
    trackSid: "TR_1",
    isMuted: false,
    isEncrypted: true,
@@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details.
 */

 import {
+  observeParticipantEvents,
  observeParticipantMedia,
  roomEventSelector,
 } from "@livekit/components-core";
@@ -14,8 +15,8 @@ import {
  LocalTrack,
  LocalVideoTrack,
  type Participant,
+  ParticipantEvent,
  RemoteAudioTrack,
-  type RemoteParticipant,
  RemoteTrack,
  type Room as LivekitRoom,
  RoomEvent,
@@ -24,6 +25,7 @@ import {
  type TrackPublication,
 } from "livekit-client";
 import {
+  combineLatest,
  distinctUntilChanged,
  fromEvent,
  interval,
@@ -46,11 +48,12 @@ import {
 } from "../api";
 import { LazyBehavior } from "../utils/LazyBehavior";

-export const livekitSources: Record<MediaSource, Track.Source> = {
-  microphone: Track.Source.Microphone,
-  camera: Track.Source.Camera,
-  screenShare: Track.Source.ScreenShare,
-  screenShareAudio: Track.Source.ScreenShareAudio,
+const sources: Record<Track.Source, MediaSource> = {
+  [Track.Source.Microphone]: "microphone",
+  [Track.Source.Camera]: "camera",
+  [Track.Source.ScreenShare]: "screenShare",
+  [Track.Source.ScreenShareAudio]: "screenShareAudio",
+  [Track.Source.Unknown]: "unknown",
 };

 // One timer for every track so that a large session does not keep hundreds of
@@ -67,7 +70,6 @@ export function createLivekitMediaTrack(
  participant: Participant,
  publication: TrackPublication,
  room: LivekitRoom,
-  source: MediaSource,
 ): AudioMediaTrack | VideoMediaTrack {
  const mediaChanged$ = observeParticipantMedia(participant);
  const track$: Behavior<Track | undefined> = scope.behavior(
@@ -85,17 +87,19 @@ export function createLivekitMediaTrack(
  const attached = new Set<HTMLMediaElement>();
  let audioContext: AudioContext | undefined;
  let audioPlugins: AudioNode[] = [];
-  const applyAudioContext = (track: Track | undefined): void => {
+  let volume: number | undefined;
+  const applyAudioSettings = (track: Track | undefined): void => {
    if (!(track instanceof RemoteAudioTrack)) return;
    track.setAudioContext(audioContext);
    track.setWebAudioPlugins(audioPlugins);
+    if (volume !== undefined) track.setVolume(volume);
  };

  let current = track$.value;
  track$.pipe(scope.bind()).subscribe((track) => {
    for (const element of attached) current?.detach(element);
    current = track;
-    applyAudioContext(track);
+    applyAudioSettings(track);
    for (const element of attached) track?.attach(element);
  });
  scope.onEnd(() => {
@@ -103,14 +107,15 @@ export function createLivekitMediaTrack(
    attached.clear();
  });

-  const base: MediaTrack = {
-    source,
-    kind: publication.kind === Track.Kind.Audio ? "audio" : "video",
-    id: publication.trackSid,
-    muted$: scope.behavior(
+  const muted$ = scope.behavior(
    mediaChanged$.pipe(map(() => publication.isMuted)),
    publication.isMuted,
-    ),
+  );
+  const base: MediaTrack = {
+    source: sources[publication.source],
+    kind: publication.kind === Track.Kind.Audio ? "audio" : "video",
+    id: publication.trackSid,
+    muted$,
    encrypted$: scope.behavior(
      merge(
        mediaChanged$,
@@ -140,19 +145,30 @@ export function createLivekitMediaTrack(
    return {
      ...base,
      kind: "audio",
+      // LiveKit detects speakers per participant, so this is the member's
+      // activity, narrowed to the tracks that can be carrying it
+      isActive$: scope.behavior(
+        combineLatest(
+          [
+            observeParticipantEvents(
+              participant,
+              ParticipantEvent.IsSpeakingChanged,
+            ).pipe(map((p) => p.isSpeaking)),
+            muted$,
+          ],
+          (speaking, muted) => speaking && !muted,
+        ),
+        participant.isSpeaking && !publication.isMuted,
+      ),
      setAudioContext: (ctx, plugins = []) => {
        audioContext = ctx;
        audioPlugins = plugins;
-        applyAudioContext(current);
+        applyAudioSettings(current);
      },
-      setVolume: (volume) => {
+      setVolume: (v) => {
        // Our own audio is never played back, so there is nothing to scale
-        if (participant.isLocal) return;
-        const remote = participant as RemoteParticipant;
-        if (source === "microphone")
-          remote.setVolume(volume, Track.Source.Microphone);
-        else if (source === "screenShareAudio")
-          remote.setVolume(volume, Track.Source.ScreenShareAudio);
+        volume = v;
+        applyAudioSettings(current);
      },
    };

@@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details.
 */

 import {
-  observeParticipantEvents,
  observeParticipantMedia,
  roomEventSelector,
 } from "@livekit/components-core";
@@ -15,28 +14,22 @@ import {
  type LocalParticipant,
  LocalVideoTrack,
  type Participant,
-  ParticipantEvent,
  type Room as LivekitRoom,
  RoomEvent,
  Track,
-  type TrackPublication,
 } from "livekit-client";
-import { distinctUntilChanged, filter, map, type Observable } from "rxjs";
+import { filter, map, type Observable } from "rxjs";

-import { type Behavior } from "../reactive/Behavior";
 import { type ObservableScope } from "../reactive/ObservableScope";
+import { generateItems } from "../reactive/observable";
 import { E2eeType } from "../encryption";
 import { type EncryptionSystem } from "../encryption";
 import {
-  type AudioMediaTrack,
  type EncryptionError,
  type LocalMemberMedia,
-  type MediaSource,
  type MemberMedia,
-  type VideoMediaTrack,
 } from "../api";
-import { mapScoped } from "../utils/mapScoped";
-import { createLivekitMediaTrack, livekitSources } from "./LivekitMediaTrack";
+import { createLivekitMediaTrack } from "./LivekitMediaTrack";

 /**
 * A participant as `MemberMedia`. The only place, with `LivekitMediaTrack`,
@@ -49,31 +42,28 @@ export function createLivekitMemberMedia(
  room: LivekitRoom,
  encryptionSystem: EncryptionSystem,
 ): MemberMedia {
-  const mediaChanged$ = observeParticipantMedia(participant);
-  const track$ = <T extends AudioMediaTrack | VideoMediaTrack>(
-    source: MediaSource,
-  ): Behavior<T | undefined> =>
-    memberTrack$(scope, participant, room, source, mediaChanged$) as Behavior<
-      T | undefined
-    >;
-
  return {
    local: participant.isLocal,
-    speaking$: scope.behavior(
-      observeParticipantEvents(
+    // One track per publication, each living as long as its publication is
+    // in the participant's map
+    tracks$: scope.behavior(
+      observeParticipantMedia(participant).pipe(
+        generateItems(
+          `${participant.identity} tracks$`,
+          function* () {
+            for (const publication of participant.trackPublications.values())
+              yield { keys: [publication.trackSid], data: publication };
+          },
+          (trackScope, publication$) =>
+            createLivekitMediaTrack(
+              trackScope,
              participant,
-        ParticipantEvent.IsSpeakingChanged,
-      ).pipe(map((p) => p.isSpeaking)),
-      participant.isSpeaking,
+              publication$.value,
+              room,
+            ),
+        ),
      ),
-    screenShareEnabled$: scope.behavior(
-      mediaChanged$.pipe(map((media) => media.isScreenShareEnabled)),
-      participant.isScreenShareEnabled,
    ),
-    microphone$: track$<AudioMediaTrack>("microphone"),
-    camera$: track$<VideoMediaTrack>("camera"),
-    screenShare$: track$<VideoMediaTrack>("screenShare"),
-    screenShareAudio$: track$<AudioMediaTrack>("screenShareAudio"),
    encryptionError$: encryptionErrors$(
      scope,
      participant,
@@ -105,33 +95,6 @@ export function createLocalLivekitMemberMedia(
  };
 }

-/** The member's track for a source, as long as the same publication is behind it. */
-function memberTrack$(
-  scope: ObservableScope,
-  participant: Participant,
-  room: LivekitRoom,
-  source: MediaSource,
-  mediaChanged$: Observable<unknown>,
-): Behavior<AudioMediaTrack | VideoMediaTrack | undefined> {
-  const publication = (): TrackPublication | undefined =>
-    participant.getTrackPublication(livekitSources[source]);
-  return mapScoped(
-    scope,
-    scope.behavior(
-      mediaChanged$.pipe(map(publication), distinctUntilChanged()),
-      publication(),
-    ),
-    (trackScope, publication) =>
-      createLivekitMediaTrack(
-        trackScope,
-        participant,
-        publication,
-        room,
-        source,
-      ),
-  );
-}
-
 function encryptionErrors$(
  scope: ObservableScope,
  participant: Participant,
@@ -0,0 +1,44 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { distinctUntilChanged, map } from "rxjs";
+
+import { type Behavior } from "../reactive/Behavior";
+import { type ObservableScope } from "../reactive/ObservableScope";
+import {
+  type AudioMediaTrack,
+  type MediaSource,
+  type MediaTrack,
+  type VideoMediaTrack,
+} from "../api";
+
+export type TrackOfSource<S extends MediaSource> = S extends
+  | "microphone"
+  | "screenShareAudio"
+  ? AudioMediaTrack
+  : S extends "camera" | "screenShare"
+    ? VideoMediaTrack
+    : AudioMediaTrack | VideoMediaTrack;
+
+/** The member's first track of a source; undefined while there is none. */
+export function trackBySource$<S extends MediaSource>(
+  scope: ObservableScope,
+  tracks$: Behavior<MediaTrack[]>,
+  source: S,
+): Behavior<TrackOfSource<S> | undefined> {
+  return scope.behavior(
+    tracks$.pipe(
+      map(
+        (tracks) =>
+          tracks.find((t) => t.source === source) as
+            | TrackOfSource<S>
+            | undefined,
+      ),
+      distinctUntilChanged(),
+    ),
+  );
+}