specific call tracks -> generic tracks

sdk/src/api.ts

@@ -253,7 +253,9 @@ export type MediaSource =
   | "microphone"
   | "camera"
   | "screenShare"
-  | "screenShareAudio";
+  | "screenShareAudio"
+  /** Published without a source; the application knows what it is. */
+  | "unknown";
 
 export type MediaStreamStats =
   | RTCInboundRtpStreamStats
@@ -284,6 +286,13 @@ export interface MediaTrack {
 
 export interface AudioMediaTrack extends MediaTrack {
   kind: "audio";
+  /**
+   * Whether the track carries sound right now, as the backend measures it.
+   * False while muted. A call reads this on the microphone track and calls it
+   * "speaking". LiveKit measures it per member, so every audio track of a
+   * member reports the same value.
+   */
+  isActive$: Behavior<boolean>;
   /** Route playback through Web Audio, for earpiece pan and gain. Undefined resets. */
   setAudioContext(ctx: AudioContext | undefined, plugins?: AudioNode[]): void;
   setVolume(volume: number): void;
@@ -303,13 +312,12 @@ export type EncryptionError = "MissingKey" | "InvalidKey";
  */
 export interface MemberMedia {
   local: boolean;
-  speaking$: Behavior<boolean>;
-  screenShareEnabled$: Behavior<boolean>;
-
-  microphone$: Behavior<AudioMediaTrack | undefined>;
-  camera$: Behavior<VideoMediaTrack | undefined>;
-  screenShare$: Behavior<VideoMediaTrack | undefined>;
-  screenShareAudio$: Behavior<AudioMediaTrack | undefined>;
+  /**
+   * One entry per published track, in publication order. An entry stays the
+   * same object for as long as the same publication is behind it. Which
+   * track is which is in its `source`; `trackBySource$` picks one out.
+   */
+  tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[]>;
 
   /** Emits when the SFU reports a key problem for this member. */
   encryptionError$: Observable<EncryptionError>;

sdk/src/media/LivekitMediaTrack.test.ts

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
 import { EventEmitter } from "events";
 import {
   ParticipantEvent,
+  RemoteAudioTrack,
   type RemoteParticipant,
   type Room as LivekitRoom,
   Track,
@@ -34,7 +35,6 @@ describe("createLivekitMediaTrack", () => {
       participant,
       publication as unknown as TrackPublication,
       room,
-      "camera",
     );
     expect(track.kind).toBe("video");
     expect(track.source).toBe("camera");
@@ -52,7 +52,6 @@ describe("createLivekitMediaTrack", () => {
       participant,
       publication as unknown as TrackPublication,
       room,
-      "camera",
     );
     track.attach(element);
     track.attach(element);
@@ -76,7 +75,6 @@ describe("createLivekitMediaTrack", () => {
       participant,
       publication as unknown as TrackPublication,
       room,
-      "camera",
     );
     track.attach(element);
     scope.end();
@@ -90,7 +88,6 @@ describe("createLivekitMediaTrack", () => {
       participant,
       publication as unknown as TrackPublication,
       room,
-      "camera",
     );
     expect(track.muted$.value).toBe(false);
     publication.isMuted = true;
@@ -107,19 +104,35 @@ describe("createLivekitMediaTrack", () => {
       participant,
       publication as unknown as TrackPublication,
       room,
-      "microphone",
     ) as AudioMediaTrack;
     track.setVolume(0.5);
-    expect(participant.setVolume).toHaveBeenCalledWith(
-      0.5,
-      Track.Source.Microphone,
-    );
+    expect(publication.track!.setVolume).toHaveBeenCalledWith(0.5);
+  });
+
+  it("is active while the member speaks and the track is not muted", () => {
+    const { participant, publication, room, emitter } = fakes({
+      kind: Track.Kind.Audio,
+    });
+    const track = createLivekitMediaTrack(
+      scope,
+      participant,
+      publication as unknown as TrackPublication,
+      room,
+    ) as AudioMediaTrack;
+    expect(track.isActive$.value).toBe(false);
+    participant.isSpeaking = true;
+    emitter.emit(ParticipantEvent.IsSpeakingChanged, true);
+    expect(track.isActive$.value).toBe(true);
+    publication.isMuted = true;
+    emitter.emit(ParticipantEvent.TrackMuted, publication);
+    expect(track.isActive$.value).toBe(false);
   });
 });
 
 type FakeTrack = Track & {
   attach: ReturnType<typeof vi.fn>;
   detach: ReturnType<typeof vi.fn>;
+  setVolume: ReturnType<typeof vi.fn>;
 };
 
 type FakePublication = Omit<TrackPublication, "track" | "isMuted"> & {
@@ -128,14 +141,20 @@ type FakePublication = Omit<TrackPublication, "track" | "isMuted"> & {
 };
 
 function fakeTrack(): FakeTrack {
-  return { attach: vi.fn(), detach: vi.fn() } as unknown as FakeTrack;
+  return Object.assign(Object.create(RemoteAudioTrack.prototype), {
+    attach: vi.fn(),
+    detach: vi.fn(),
+    setVolume: vi.fn(),
+    setAudioContext: vi.fn(),
+    setWebAudioPlugins: vi.fn(),
+  }) as FakeTrack;
 }
 
 function fakes({
   track = fakeTrack(),
   kind = Track.Kind.Video,
 }: { track?: FakeTrack | undefined; kind?: Track.Kind } = {}): {
-  participant: RemoteParticipant & { setVolume: ReturnType<typeof vi.fn> };
+  participant: RemoteParticipant & { isSpeaking: boolean };
   publication: FakePublication;
   room: LivekitRoom;
   emitter: EventEmitter;
@@ -143,13 +162,15 @@ function fakes({
   const emitter = new EventEmitter();
   const participant = Object.assign(emitter, {
     isLocal: false,
+    isSpeaking: false,
     identity: "@alice:example.org:DEVICE",
-    setVolume: vi.fn(),
     getTrackPublication: (): TrackPublication =>
       publication as unknown as TrackPublication,
-  }) as unknown as RemoteParticipant & { setVolume: ReturnType<typeof vi.fn> };
+  }) as unknown as RemoteParticipant & { isSpeaking: boolean };
   const publication = Object.assign(emitter, {
     kind,
+    source:
+      kind === Track.Kind.Audio ? Track.Source.Microphone : Track.Source.Camera,
     trackSid: "TR_1",
     isMuted: false,
     isEncrypted: true,

sdk/src/media/LivekitMediaTrack.ts

@@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
+  observeParticipantEvents,
   observeParticipantMedia,
   roomEventSelector,
 } from "@livekit/components-core";
@@ -14,8 +15,8 @@ import {
   LocalTrack,
   LocalVideoTrack,
   type Participant,
+  ParticipantEvent,
   RemoteAudioTrack,
-  type RemoteParticipant,
   RemoteTrack,
   type Room as LivekitRoom,
   RoomEvent,
@@ -24,6 +25,7 @@ import {
   type TrackPublication,
 } from "livekit-client";
 import {
+  combineLatest,
   distinctUntilChanged,
   fromEvent,
   interval,
@@ -46,11 +48,12 @@ import {
 } from "../api";
 import { LazyBehavior } from "../utils/LazyBehavior";
 
-export const livekitSources: Record<MediaSource, Track.Source> = {
-  microphone: Track.Source.Microphone,
-  camera: Track.Source.Camera,
-  screenShare: Track.Source.ScreenShare,
-  screenShareAudio: Track.Source.ScreenShareAudio,
+const sources: Record<Track.Source, MediaSource> = {
+  [Track.Source.Microphone]: "microphone",
+  [Track.Source.Camera]: "camera",
+  [Track.Source.ScreenShare]: "screenShare",
+  [Track.Source.ScreenShareAudio]: "screenShareAudio",
+  [Track.Source.Unknown]: "unknown",
 };
 
 // One timer for every track so that a large session does not keep hundreds of
@@ -67,7 +70,6 @@ export function createLivekitMediaTrack(
   participant: Participant,
   publication: TrackPublication,
   room: LivekitRoom,
-  source: MediaSource,
 ): AudioMediaTrack | VideoMediaTrack {
   const mediaChanged$ = observeParticipantMedia(participant);
   const track$: Behavior<Track | undefined> = scope.behavior(
@@ -85,17 +87,19 @@ export function createLivekitMediaTrack(
   const attached = new Set<HTMLMediaElement>();
   let audioContext: AudioContext | undefined;
   let audioPlugins: AudioNode[] = [];
-  const applyAudioContext = (track: Track | undefined): void => {
+  let volume: number | undefined;
+  const applyAudioSettings = (track: Track | undefined): void => {
     if (!(track instanceof RemoteAudioTrack)) return;
     track.setAudioContext(audioContext);
     track.setWebAudioPlugins(audioPlugins);
+    if (volume !== undefined) track.setVolume(volume);
   };
 
   let current = track$.value;
   track$.pipe(scope.bind()).subscribe((track) => {
     for (const element of attached) current?.detach(element);
     current = track;
-    applyAudioContext(track);
+    applyAudioSettings(track);
     for (const element of attached) track?.attach(element);
   });
   scope.onEnd(() => {
@@ -103,14 +107,15 @@ export function createLivekitMediaTrack(
     attached.clear();
   });
 
+  const muted$ = scope.behavior(
+    mediaChanged$.pipe(map(() => publication.isMuted)),
+    publication.isMuted,
+  );
   const base: MediaTrack = {
-    source,
+    source: sources[publication.source],
     kind: publication.kind === Track.Kind.Audio ? "audio" : "video",
     id: publication.trackSid,
-    muted$: scope.behavior(
-      mediaChanged$.pipe(map(() => publication.isMuted)),
-      publication.isMuted,
-    ),
+    muted$,
     encrypted$: scope.behavior(
       merge(
         mediaChanged$,
@@ -140,19 +145,30 @@ export function createLivekitMediaTrack(
     return {
       ...base,
       kind: "audio",
+      // LiveKit detects speakers per participant, so this is the member's
+      // activity, narrowed to the tracks that can be carrying it
+      isActive$: scope.behavior(
+        combineLatest(
+          [
+            observeParticipantEvents(
+              participant,
+              ParticipantEvent.IsSpeakingChanged,
+            ).pipe(map((p) => p.isSpeaking)),
+            muted$,
+          ],
+          (speaking, muted) => speaking && !muted,
+        ),
+        participant.isSpeaking && !publication.isMuted,
+      ),
       setAudioContext: (ctx, plugins = []) => {
         audioContext = ctx;
         audioPlugins = plugins;
-        applyAudioContext(current);
+        applyAudioSettings(current);
       },
-      setVolume: (volume) => {
+      setVolume: (v) => {
         // Our own audio is never played back, so there is nothing to scale
-        if (participant.isLocal) return;
-        const remote = participant as RemoteParticipant;
-        if (source === "microphone")
-          remote.setVolume(volume, Track.Source.Microphone);
-        else if (source === "screenShareAudio")
-          remote.setVolume(volume, Track.Source.ScreenShareAudio);
+        volume = v;
+        applyAudioSettings(current);
       },
     };
 

sdk/src/media/LivekitMemberMedia.ts

@@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
-  observeParticipantEvents,
   observeParticipantMedia,
   roomEventSelector,
 } from "@livekit/components-core";
@@ -15,28 +14,22 @@ import {
   type LocalParticipant,
   LocalVideoTrack,
   type Participant,
-  ParticipantEvent,
   type Room as LivekitRoom,
   RoomEvent,
   Track,
-  type TrackPublication,
 } from "livekit-client";
-import { distinctUntilChanged, filter, map, type Observable } from "rxjs";
+import { filter, map, type Observable } from "rxjs";
 
-import { type Behavior } from "../reactive/Behavior";
 import { type ObservableScope } from "../reactive/ObservableScope";
+import { generateItems } from "../reactive/observable";
 import { E2eeType } from "../encryption";
 import { type EncryptionSystem } from "../encryption";
 import {
-  type AudioMediaTrack,
   type EncryptionError,
   type LocalMemberMedia,
-  type MediaSource,
   type MemberMedia,
-  type VideoMediaTrack,
 } from "../api";
-import { mapScoped } from "../utils/mapScoped";
-import { createLivekitMediaTrack, livekitSources } from "./LivekitMediaTrack";
+import { createLivekitMediaTrack } from "./LivekitMediaTrack";
 
 /**
  * A participant as `MemberMedia`. The only place, with `LivekitMediaTrack`,
@@ -49,31 +42,28 @@ export function createLivekitMemberMedia(
   room: LivekitRoom,
   encryptionSystem: EncryptionSystem,
 ): MemberMedia {
-  const mediaChanged$ = observeParticipantMedia(participant);
-  const track$ = <T extends AudioMediaTrack | VideoMediaTrack>(
-    source: MediaSource,
-  ): Behavior<T | undefined> =>
-    memberTrack$(scope, participant, room, source, mediaChanged$) as Behavior<
-      T | undefined
-    >;
-
   return {
     local: participant.isLocal,
-    speaking$: scope.behavior(
-      observeParticipantEvents(
-        participant,
-        ParticipantEvent.IsSpeakingChanged,
-      ).pipe(map((p) => p.isSpeaking)),
-      participant.isSpeaking,
+    // One track per publication, each living as long as its publication is
+    // in the participant's map
+    tracks$: scope.behavior(
+      observeParticipantMedia(participant).pipe(
+        generateItems(
+          `${participant.identity} tracks$`,
+          function* () {
+            for (const publication of participant.trackPublications.values())
+              yield { keys: [publication.trackSid], data: publication };
+          },
+          (trackScope, publication$) =>
+            createLivekitMediaTrack(
+              trackScope,
+              participant,
+              publication$.value,
+              room,
+            ),
+        ),
+      ),
     ),
-    screenShareEnabled$: scope.behavior(
-      mediaChanged$.pipe(map((media) => media.isScreenShareEnabled)),
-      participant.isScreenShareEnabled,
-    ),
-    microphone$: track$<AudioMediaTrack>("microphone"),
-    camera$: track$<VideoMediaTrack>("camera"),
-    screenShare$: track$<VideoMediaTrack>("screenShare"),
-    screenShareAudio$: track$<AudioMediaTrack>("screenShareAudio"),
     encryptionError$: encryptionErrors$(
       scope,
       participant,
@@ -105,33 +95,6 @@ export function createLocalLivekitMemberMedia(
   };
 }
 
-/** The member's track for a source, as long as the same publication is behind it. */
-function memberTrack$(
-  scope: ObservableScope,
-  participant: Participant,
-  room: LivekitRoom,
-  source: MediaSource,
-  mediaChanged$: Observable<unknown>,
-): Behavior<AudioMediaTrack | VideoMediaTrack | undefined> {
-  const publication = (): TrackPublication | undefined =>
-    participant.getTrackPublication(livekitSources[source]);
-  return mapScoped(
-    scope,
-    scope.behavior(
-      mediaChanged$.pipe(map(publication), distinctUntilChanged()),
-      publication(),
-    ),
-    (trackScope, publication) =>
-      createLivekitMediaTrack(
-        trackScope,
-        participant,
-        publication,
-        room,
-        source,
-      ),
-  );
-}
-
 function encryptionErrors$(
   scope: ObservableScope,
   participant: Participant,

sdk/src/utils/tracks.ts

@@ -0,0 +1,44 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { distinctUntilChanged, map } from "rxjs";
+
+import { type Behavior } from "../reactive/Behavior";
+import { type ObservableScope } from "../reactive/ObservableScope";
+import {
+  type AudioMediaTrack,
+  type MediaSource,
+  type MediaTrack,
+  type VideoMediaTrack,
+} from "../api";
+
+export type TrackOfSource<S extends MediaSource> = S extends
+  | "microphone"
+  | "screenShareAudio"
+  ? AudioMediaTrack
+  : S extends "camera" | "screenShare"
+    ? VideoMediaTrack
+    : AudioMediaTrack | VideoMediaTrack;
+
+/** The member's first track of a source; undefined while there is none. */
+export function trackBySource$<S extends MediaSource>(
+  scope: ObservableScope,
+  tracks$: Behavior<MediaTrack[]>,
+  source: S,
+): Behavior<TrackOfSource<S> | undefined> {
+  return scope.behavior(
+    tracks$.pipe(
+      map(
+        (tracks) =>
+          tracks.find((t) => t.source === source) as
+            | TrackOfSource<S>
+            | undefined,
+      ),
+      distinctUntilChanged(),
+    ),
+  );
+}