specific call tracks -> generic tracks

sdk/MemberMediaPlan.md

@@ -314,3 +314,69 @@ three are the ones this plan makes visible.
 | `UnknownCallError` in `errors.ts`                                                                                                 | Name only.                                                                                                                                                                    | Rename with the two `Livekit*` errors in slice 11 of `SdkMigration.md`.                                                                                                          |
 | `observeSharingScreen$` in `LocalMember.ts`                                                                                       | Reads `participant.isScreenShareEnabled`.                                                                                                                                     | Goes with `sharingScreen$`, or reads `tracks$` from the local media in the meantime.                                                                                             |
 | `sendNotificationType`                                                                                                            | Kept on purpose; see "Call-specific things kept out of the client" in `SdkArchitecture.md`.                                                                                   | Nothing.                                                                                                                                                                         |
+
+## Plan for the refactors above
+
+Four slices, in dependency order. Each is one PR, green on its own, with the
+`playwright/sdk` specs and the dev harness as the proof that behaviour is kept.
+Element Call follows on its own branch after each.
+
+### 1. Local track controls
+
+The local member's tracks become objects a host can act on, instead of methods
+named after a call's devices.
+
+- **Add** `LocalAudioMediaTrack` and `LocalVideoMediaTrack` (`local: true`) in
+  `api.ts`, with `setEnabled(enabled)`, `setDevice(deviceId)` and, on video,
+  `switchFacingMode()` and `setProcessor(processor)`. `tracks$` on
+  `LocalMemberMedia` is typed to them.
+- **Remove** `LocalMemberMedia.switchCamera`, and `audioInputDeviceId$`,
+  `videoInputDeviceId$`, `videoProcessor$` from `LocalMediaInputs`. The host
+  subscribes to its settings and calls the track; the SDK stops watching
+  behaviors it does not own. `audioOutputDeviceId$` stays: playback is not a
+  track.
+- **Keep** `setMicrophoneEnabled` / `setCameraEnabled` for now: they also cover
+  "no track yet", which slice 2 solves.
+- **Touches** `LivekitMediaTrack.ts`, `Publisher.ts` (the processor and device
+  code moves onto the track), `MatrixRTCClient.ts`, the harness buttons.
+
+### 2. `publish` and `unpublish`
+
+One way to put media on the session, whatever it is.
+
+- **Add** to `LocalRTCMember`: `publish(request): Promise<LocalMediaTrack>` and
+  `unpublish(id): Promise<void>`. A request is `{ source, deviceId?, capture? }`
+  for `"microphone"` and `"camera"`, `{ source: "screenShare", audio?, capture?
+}` for a screen. Failures reject the promise with a `MatrixRTCError`.
+- **Remove** `sharingScreen$`, `toggleScreenSharing`, `screenShareError$`,
+  `dismissScreenShareError`, `observeSharingScreen$`, and the two
+  `set*Enabled` methods on the client. "Sharing" is `tracks$.some(...)` on the
+  local media; "enable" is `publish` or `track.setEnabled`.
+- **Replace** `LocalMediaInputs.microphoneEnabled` / `cameraEnabled` with
+  `publish: PublishRequest[]`, applied once the transport is up, as today.
+  `CaptureSettings` becomes the `capture` of a request; `MediaQuality` keeps its
+  `config.json` shape and is mapped to per-request capture at the edge.
+- **Touches** `LocalMember.ts` (the screen share block and `setEnabled`),
+  `Publisher.ts` (iterates requests instead of enumerating three sources),
+  `livekitOptions.ts`, `lifecycle.spec.ts` and `media.spec.ts` for the new
+  buttons.
+
+### 3. Names that say "call"
+
+No behaviour change; one PR of renames and type narrowing.
+
+- `RTCMember.membership$` is typed as an SDK `RTCMembership`: the subset of
+  `CallMembership` hosts read (`userId`, `deviceId`, `memberId`,
+  `rtcBackendIdentity`, `getTransport()`, `applicationData`). The js-sdk object
+  is still what is behind it.
+- `UnknownCallError` is renamed with the two `Livekit*` errors, as slice 11 of
+  `SdkMigration.md` already plans; `index.ts` drops its alias.
+- `applicationData` is passed through whole once the js-sdk accepts more than
+  `m.call.intent`; until then the README says which key survives.
+
+### Not planned
+
+- `MediaSource` keeps its five values. A `name` on `MediaTrack` (LiveKit's
+  `trackName`) is a one-line addition when the first non-call application asks
+  for it; adding it now would be a guess at that application's needs.
+- `sendNotificationType` stays, as `SdkArchitecture.md` already decided.

sdk/dev/main.ts

@@ -14,14 +14,16 @@ Please see LICENSE in the repository root for full details.
 import { logger } from "matrix-js-sdk/lib/logger";
 import { combineLatest, type Observable, of, switchMap } from "rxjs";
 import {
+  type AudioMediaTrack,
   constant,
   createMatrixRTCClient,
   E2eeType,
   MatrixRTCMode,
-  type MediaTrack,
   ObservableScope,
   type RTCMember,
   type MatrixRTCClient,
+  trackBySource$,
+  type VideoMediaTrack,
 } from "@element-hq/matrixrtc-sdk";
 
 import { createSession, joinRoom, type Login } from "./session";
@@ -185,21 +187,21 @@ function memberTile(scope: ObservableScope, member: RTCMember): HTMLElement {
     name.textContent = n;
   });
 
-  const camera$ = member.media$.pipe(
+  const tracks$ = scope.behavior(
-    switchMap((media) => media?.camera$ ?? of(undefined)),
+    member.media$.pipe(switchMap((media) => media?.tracks$ ?? of([]))),
+  );
+  render(
+    scope,
+    trackBySource$(scope, tracks$, "camera"),
+    tile.appendChild(document.createElement("video")),
   );
-  render(scope, camera$, tile.appendChild(document.createElement("video")));
   // Our own microphone would only echo
-  if (!member.local) {
+  if (!member.local)
-    const microphone$ = member.media$.pipe(
-      switchMap((media) => media?.microphone$ ?? of(undefined)),
-    );
     render(
       scope,
-      microphone$,
+      trackBySource$(scope, tracks$, "microphone"),
       tile.appendChild(document.createElement("audio")),
     );
-  }
   return tile;
 }
 
@@ -209,10 +211,10 @@ function memberTile(scope: ObservableScope, member: RTCMember): HTMLElement {
  */
 function render(
   scope: ObservableScope,
-  track$: Observable<MediaTrack | undefined>,
+  track$: Observable<AudioMediaTrack | VideoMediaTrack | undefined>,
   element: HTMLMediaElement,
 ): void {
-  let attached: MediaTrack | undefined;
+  let attached: AudioMediaTrack | VideoMediaTrack | undefined;
   track$.pipe(scope.bind()).subscribe((track) => {
     attached?.detach(element);
     attached = track;
@@ -230,7 +232,7 @@ function render(
     });
   };
   const of$ = <T>(
-    pick: (track: MediaTrack) => Observable<T>,
+    pick: (track: AudioMediaTrack | VideoMediaTrack) => Observable<T>,
   ): Observable<T | undefined> =>
     track$.pipe(switchMap((track) => (track ? pick(track) : of(undefined))));
   label(
@@ -241,6 +243,10 @@ function render(
     "encrypted",
     of$((t) => t.encrypted$),
   );
+  label(
+    "active",
+    of$((t) => (t.kind === "audio" ? t.isActive$ : of(undefined))),
+  );
   const stats$ = of$((t) => t.stats$);
   label("frameWidth", stats$.pipe(switchMap((s) => of(frames(s)?.frameWidth))));
   label("frames", stats$.pipe(switchMap((s) => of(frames(s)?.count))));

sdk/index.ts

@@ -53,6 +53,7 @@ export {
 } from "./src/errors";
 
 export * from "./src/api";
+export { trackBySource$, type TrackOfSource } from "./src/utils/tracks";
 export { createMatrixRTCClient } from "./src/session/MatrixRTCClient";
 // For a developer panel to check a transport url before using it
 export { getSFUConfigWithOpenID as authenticateWithTransport } from "./src/session/openIDSFU";

sdk/src/api.ts

@@ -253,7 +253,9 @@ export type MediaSource =
   | "microphone"
   | "camera"
   | "screenShare"
-  | "screenShareAudio";
+  | "screenShareAudio"
+  /** Published without a source; the application knows what it is. */
+  | "unknown";
 
 export type MediaStreamStats =
   | RTCInboundRtpStreamStats
@@ -284,6 +286,13 @@ export interface MediaTrack {
 
 export interface AudioMediaTrack extends MediaTrack {
   kind: "audio";
+  /**
+   * Whether the track carries sound right now, as the backend measures it.
+   * False while muted. A call reads this on the microphone track and calls it
+   * "speaking". LiveKit measures it per member, so every audio track of a
+   * member reports the same value.
+   */
+  isActive$: Behavior<boolean>;
   /** Route playback through Web Audio, for earpiece pan and gain. Undefined resets. */
   setAudioContext(ctx: AudioContext | undefined, plugins?: AudioNode[]): void;
   setVolume(volume: number): void;
@@ -303,13 +312,12 @@ export type EncryptionError = "MissingKey" | "InvalidKey";
  */
 export interface MemberMedia {
   local: boolean;
-  speaking$: Behavior<boolean>;
+  /**
-  screenShareEnabled$: Behavior<boolean>;
+   * One entry per published track, in publication order. An entry stays the
-
+   * same object for as long as the same publication is behind it. Which
-  microphone$: Behavior<AudioMediaTrack | undefined>;
+   * track is which is in its `source`; `trackBySource$` picks one out.
-  camera$: Behavior<VideoMediaTrack | undefined>;
+   */
-  screenShare$: Behavior<VideoMediaTrack | undefined>;
+  tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[]>;
-  screenShareAudio$: Behavior<AudioMediaTrack | undefined>;
 
   /** Emits when the SFU reports a key problem for this member. */
   encryptionError$: Observable<EncryptionError>;

sdk/src/media/LivekitMediaTrack.test.ts

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
 import { EventEmitter } from "events";
 import {
   ParticipantEvent,
+  RemoteAudioTrack,
   type RemoteParticipant,
   type Room as LivekitRoom,
   Track,
@@ -34,7 +35,6 @@ describe("createLivekitMediaTrack", () => {
       participant,
       publication as unknown as TrackPublication,
       room,
-      "camera",
     );
     expect(track.kind).toBe("video");
     expect(track.source).toBe("camera");
@@ -52,7 +52,6 @@ describe("createLivekitMediaTrack", () => {
       participant,
       publication as unknown as TrackPublication,
       room,
-      "camera",
     );
     track.attach(element);
     track.attach(element);
@@ -76,7 +75,6 @@ describe("createLivekitMediaTrack", () => {
       participant,
       publication as unknown as TrackPublication,
       room,
-      "camera",
     );
     track.attach(element);
     scope.end();
@@ -90,7 +88,6 @@ describe("createLivekitMediaTrack", () => {
       participant,
       publication as unknown as TrackPublication,
       room,
-      "camera",
     );
     expect(track.muted$.value).toBe(false);
     publication.isMuted = true;
@@ -107,19 +104,35 @@ describe("createLivekitMediaTrack", () => {
       participant,
       publication as unknown as TrackPublication,
       room,
-      "microphone",
     ) as AudioMediaTrack;
     track.setVolume(0.5);
-    expect(participant.setVolume).toHaveBeenCalledWith(
+    expect(publication.track!.setVolume).toHaveBeenCalledWith(0.5);
-      0.5,
+  });
-      Track.Source.Microphone,
+
-    );
+  it("is active while the member speaks and the track is not muted", () => {
+    const { participant, publication, room, emitter } = fakes({
+      kind: Track.Kind.Audio,
+    });
+    const track = createLivekitMediaTrack(
+      scope,
+      participant,
+      publication as unknown as TrackPublication,
+      room,
+    ) as AudioMediaTrack;
+    expect(track.isActive$.value).toBe(false);
+    participant.isSpeaking = true;
+    emitter.emit(ParticipantEvent.IsSpeakingChanged, true);
+    expect(track.isActive$.value).toBe(true);
+    publication.isMuted = true;
+    emitter.emit(ParticipantEvent.TrackMuted, publication);
+    expect(track.isActive$.value).toBe(false);
   });
 });
 
 type FakeTrack = Track & {
   attach: ReturnType<typeof vi.fn>;
   detach: ReturnType<typeof vi.fn>;
+  setVolume: ReturnType<typeof vi.fn>;
 };
 
 type FakePublication = Omit<TrackPublication, "track" | "isMuted"> & {
@@ -128,14 +141,20 @@ type FakePublication = Omit<TrackPublication, "track" | "isMuted"> & {
 };
 
 function fakeTrack(): FakeTrack {
-  return { attach: vi.fn(), detach: vi.fn() } as unknown as FakeTrack;
+  return Object.assign(Object.create(RemoteAudioTrack.prototype), {
+    attach: vi.fn(),
+    detach: vi.fn(),
+    setVolume: vi.fn(),
+    setAudioContext: vi.fn(),
+    setWebAudioPlugins: vi.fn(),
+  }) as FakeTrack;
 }
 
 function fakes({
   track = fakeTrack(),
   kind = Track.Kind.Video,
 }: { track?: FakeTrack | undefined; kind?: Track.Kind } = {}): {
-  participant: RemoteParticipant & { setVolume: ReturnType<typeof vi.fn> };
+  participant: RemoteParticipant & { isSpeaking: boolean };
   publication: FakePublication;
   room: LivekitRoom;
   emitter: EventEmitter;
@@ -143,13 +162,15 @@ function fakes({
   const emitter = new EventEmitter();
   const participant = Object.assign(emitter, {
     isLocal: false,
+    isSpeaking: false,
     identity: "@alice:example.org:DEVICE",
-    setVolume: vi.fn(),
     getTrackPublication: (): TrackPublication =>
       publication as unknown as TrackPublication,
-  }) as unknown as RemoteParticipant & { setVolume: ReturnType<typeof vi.fn> };
+  }) as unknown as RemoteParticipant & { isSpeaking: boolean };
   const publication = Object.assign(emitter, {
     kind,
+    source:
+      kind === Track.Kind.Audio ? Track.Source.Microphone : Track.Source.Camera,
     trackSid: "TR_1",
     isMuted: false,
     isEncrypted: true,

sdk/src/media/LivekitMediaTrack.ts

@@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
+  observeParticipantEvents,
   observeParticipantMedia,
   roomEventSelector,
 } from "@livekit/components-core";
@@ -14,8 +15,8 @@ import {
   LocalTrack,
   LocalVideoTrack,
   type Participant,
+  ParticipantEvent,
   RemoteAudioTrack,
-  type RemoteParticipant,
   RemoteTrack,
   type Room as LivekitRoom,
   RoomEvent,
@@ -24,6 +25,7 @@ import {
   type TrackPublication,
 } from "livekit-client";
 import {
+  combineLatest,
   distinctUntilChanged,
   fromEvent,
   interval,
@@ -46,11 +48,12 @@ import {
 } from "../api";
 import { LazyBehavior } from "../utils/LazyBehavior";
 
-export const livekitSources: Record<MediaSource, Track.Source> = {
+const sources: Record<Track.Source, MediaSource> = {
-  microphone: Track.Source.Microphone,
+  [Track.Source.Microphone]: "microphone",
-  camera: Track.Source.Camera,
+  [Track.Source.Camera]: "camera",
-  screenShare: Track.Source.ScreenShare,
+  [Track.Source.ScreenShare]: "screenShare",
-  screenShareAudio: Track.Source.ScreenShareAudio,
+  [Track.Source.ScreenShareAudio]: "screenShareAudio",
+  [Track.Source.Unknown]: "unknown",
 };
 
 // One timer for every track so that a large session does not keep hundreds of
@@ -67,7 +70,6 @@ export function createLivekitMediaTrack(
   participant: Participant,
   publication: TrackPublication,
   room: LivekitRoom,
-  source: MediaSource,
 ): AudioMediaTrack | VideoMediaTrack {
   const mediaChanged$ = observeParticipantMedia(participant);
   const track$: Behavior<Track | undefined> = scope.behavior(
@@ -85,17 +87,19 @@ export function createLivekitMediaTrack(
   const attached = new Set<HTMLMediaElement>();
   let audioContext: AudioContext | undefined;
   let audioPlugins: AudioNode[] = [];
-  const applyAudioContext = (track: Track | undefined): void => {
+  let volume: number | undefined;
+  const applyAudioSettings = (track: Track | undefined): void => {
     if (!(track instanceof RemoteAudioTrack)) return;
     track.setAudioContext(audioContext);
     track.setWebAudioPlugins(audioPlugins);
+    if (volume !== undefined) track.setVolume(volume);
   };
 
   let current = track$.value;
   track$.pipe(scope.bind()).subscribe((track) => {
     for (const element of attached) current?.detach(element);
     current = track;
-    applyAudioContext(track);
+    applyAudioSettings(track);
     for (const element of attached) track?.attach(element);
   });
   scope.onEnd(() => {
@@ -103,14 +107,15 @@ export function createLivekitMediaTrack(
     attached.clear();
   });
 
+  const muted$ = scope.behavior(
+    mediaChanged$.pipe(map(() => publication.isMuted)),
+    publication.isMuted,
+  );
   const base: MediaTrack = {
-    source,
+    source: sources[publication.source],
     kind: publication.kind === Track.Kind.Audio ? "audio" : "video",
     id: publication.trackSid,
-    muted$: scope.behavior(
+    muted$,
-      mediaChanged$.pipe(map(() => publication.isMuted)),
-      publication.isMuted,
-    ),
     encrypted$: scope.behavior(
       merge(
         mediaChanged$,
@@ -140,19 +145,30 @@ export function createLivekitMediaTrack(
     return {
       ...base,
       kind: "audio",
+      // LiveKit detects speakers per participant, so this is the member's
+      // activity, narrowed to the tracks that can be carrying it
+      isActive$: scope.behavior(
+        combineLatest(
+          [
+            observeParticipantEvents(
+              participant,
+              ParticipantEvent.IsSpeakingChanged,
+            ).pipe(map((p) => p.isSpeaking)),
+            muted$,
+          ],
+          (speaking, muted) => speaking && !muted,
+        ),
+        participant.isSpeaking && !publication.isMuted,
+      ),
       setAudioContext: (ctx, plugins = []) => {
         audioContext = ctx;
         audioPlugins = plugins;
-        applyAudioContext(current);
+        applyAudioSettings(current);
       },
-      setVolume: (volume) => {
+      setVolume: (v) => {
         // Our own audio is never played back, so there is nothing to scale
-        if (participant.isLocal) return;
+        volume = v;
-        const remote = participant as RemoteParticipant;
+        applyAudioSettings(current);
-        if (source === "microphone")
-          remote.setVolume(volume, Track.Source.Microphone);
-        else if (source === "screenShareAudio")
-          remote.setVolume(volume, Track.Source.ScreenShareAudio);
       },
     };
 

sdk/src/media/LivekitMemberMedia.ts

@@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
-  observeParticipantEvents,
   observeParticipantMedia,
   roomEventSelector,
 } from "@livekit/components-core";
@@ -15,28 +14,22 @@ import {
   type LocalParticipant,
   LocalVideoTrack,
   type Participant,
-  ParticipantEvent,
   type Room as LivekitRoom,
   RoomEvent,
   Track,
-  type TrackPublication,
 } from "livekit-client";
-import { distinctUntilChanged, filter, map, type Observable } from "rxjs";
+import { filter, map, type Observable } from "rxjs";
 
-import { type Behavior } from "../reactive/Behavior";
 import { type ObservableScope } from "../reactive/ObservableScope";
+import { generateItems } from "../reactive/observable";
 import { E2eeType } from "../encryption";
 import { type EncryptionSystem } from "../encryption";
 import {
-  type AudioMediaTrack,
   type EncryptionError,
   type LocalMemberMedia,
-  type MediaSource,
   type MemberMedia,
-  type VideoMediaTrack,
 } from "../api";
-import { mapScoped } from "../utils/mapScoped";
+import { createLivekitMediaTrack } from "./LivekitMediaTrack";
-import { createLivekitMediaTrack, livekitSources } from "./LivekitMediaTrack";
 
 /**
  * A participant as `MemberMedia`. The only place, with `LivekitMediaTrack`,
@@ -49,31 +42,28 @@ export function createLivekitMemberMedia(
   room: LivekitRoom,
   encryptionSystem: EncryptionSystem,
 ): MemberMedia {
-  const mediaChanged$ = observeParticipantMedia(participant);
-  const track$ = <T extends AudioMediaTrack | VideoMediaTrack>(
-    source: MediaSource,
-  ): Behavior<T | undefined> =>
-    memberTrack$(scope, participant, room, source, mediaChanged$) as Behavior<
-      T | undefined
-    >;
-
   return {
     local: participant.isLocal,
-    speaking$: scope.behavior(
+    // One track per publication, each living as long as its publication is
-      observeParticipantEvents(
+    // in the participant's map
-        participant,
+    tracks$: scope.behavior(
-        ParticipantEvent.IsSpeakingChanged,
+      observeParticipantMedia(participant).pipe(
-      ).pipe(map((p) => p.isSpeaking)),
+        generateItems(
-      participant.isSpeaking,
+          `${participant.identity} tracks$`,
+          function* () {
+            for (const publication of participant.trackPublications.values())
+              yield { keys: [publication.trackSid], data: publication };
+          },
+          (trackScope, publication$) =>
+            createLivekitMediaTrack(
+              trackScope,
+              participant,
+              publication$.value,
+              room,
+            ),
+        ),
+      ),
     ),
-    screenShareEnabled$: scope.behavior(
-      mediaChanged$.pipe(map((media) => media.isScreenShareEnabled)),
-      participant.isScreenShareEnabled,
-    ),
-    microphone$: track$<AudioMediaTrack>("microphone"),
-    camera$: track$<VideoMediaTrack>("camera"),
-    screenShare$: track$<VideoMediaTrack>("screenShare"),
-    screenShareAudio$: track$<AudioMediaTrack>("screenShareAudio"),
     encryptionError$: encryptionErrors$(
       scope,
       participant,
@@ -105,33 +95,6 @@ export function createLocalLivekitMemberMedia(
   };
 }
 
-/** The member's track for a source, as long as the same publication is behind it. */
-function memberTrack$(
-  scope: ObservableScope,
-  participant: Participant,
-  room: LivekitRoom,
-  source: MediaSource,
-  mediaChanged$: Observable<unknown>,
-): Behavior<AudioMediaTrack | VideoMediaTrack | undefined> {
-  const publication = (): TrackPublication | undefined =>
-    participant.getTrackPublication(livekitSources[source]);
-  return mapScoped(
-    scope,
-    scope.behavior(
-      mediaChanged$.pipe(map(publication), distinctUntilChanged()),
-      publication(),
-    ),
-    (trackScope, publication) =>
-      createLivekitMediaTrack(
-        trackScope,
-        participant,
-        publication,
-        room,
-        source,
-      ),
-  );
-}
-
 function encryptionErrors$(
   scope: ObservableScope,
   participant: Participant,

sdk/src/utils/tracks.ts

@@ -0,0 +1,44 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { distinctUntilChanged, map } from "rxjs";
+
+import { type Behavior } from "../reactive/Behavior";
+import { type ObservableScope } from "../reactive/ObservableScope";
+import {
+  type AudioMediaTrack,
+  type MediaSource,
+  type MediaTrack,
+  type VideoMediaTrack,
+} from "../api";
+
+export type TrackOfSource<S extends MediaSource> = S extends
+  | "microphone"
+  | "screenShareAudio"
+  ? AudioMediaTrack
+  : S extends "camera" | "screenShare"
+    ? VideoMediaTrack
+    : AudioMediaTrack | VideoMediaTrack;
+
+/** The member's first track of a source; undefined while there is none. */
+export function trackBySource$<S extends MediaSource>(
+  scope: ObservableScope,
+  tracks$: Behavior<MediaTrack[]>,
+  source: S,
+): Behavior<TrackOfSource<S> | undefined> {
+  return scope.behavior(
+    tracks$.pipe(
+      map(
+        (tracks) =>
+          tracks.find((t) => t.source === source) as
+            | TrackOfSource<S>
+            | undefined,
+      ),
+      distinctUntilChanged(),
+    ),
+  );
+}