[SDK] make stats$ a polled method and add audioLevel$ to audio tracks

sdk/src/media-backend/livekit/LivekitMediaTrack.test.ts

@@ -12,6 +12,7 @@ import {
   RemoteAudioTrack,
   type RemoteParticipant,
   type Room as LivekitRoom,
+  RoomEvent,
   Track,
   TrackEvent,
   type TrackPublication,
@@ -30,14 +31,18 @@ describe("createLivekitMediaTrack", () => {
   beforeEach(() => {
     scope = new ObservableScope();
   });
-  afterEach(() => scope.end());
+  afterEach(() => {
+    scope.end();
+    vi.useRealTimers();
+  });
 
   it("describes the publication", () => {
-    const { participant, publication } = fakes();
+    const { participant, publication, room } = fakes();
     const track = createLivekitMediaTrack(
       scope,
       participant,
       publication as unknown as TrackPublication,
+      room,
     );
     expect(track.kind).toBe("video");
     expect(track.source).toBe("camera");
@@ -46,7 +51,7 @@ describe("createLivekitMediaTrack", () => {
   });
 
   it("attaches elements to the track, even one that arrives later", () => {
-    const { participant, publication, emitter } = fakes({
+    const { participant, publication, room, emitter } = fakes({
       track: undefined,
     });
     const element = document.createElement("video");
@@ -54,6 +59,7 @@ describe("createLivekitMediaTrack", () => {
       scope,
       participant,
       publication as unknown as TrackPublication,
+      room,
     );
     track.attach(element);
     track.attach(element);
@@ -69,13 +75,14 @@ describe("createLivekitMediaTrack", () => {
   });
 
   it("detaches everything when its scope ends", () => {
-    const { participant, publication } = fakes();
+    const { participant, publication, room } = fakes();
     const element = document.createElement("video");
     const scope = new ObservableScope();
     const track = createLivekitMediaTrack(
       scope,
       participant,
       publication as unknown as TrackPublication,
+      room,
     );
     track.attach(element);
     scope.end();
@@ -83,11 +90,12 @@ describe("createLivekitMediaTrack", () => {
   });
 
   it("follows the mute state", () => {
-    const { participant, publication, emitter } = fakes();
+    const { participant, publication, room, emitter } = fakes();
     const track = createLivekitMediaTrack(
       scope,
       participant,
       publication as unknown as TrackPublication,
+      room,
     );
     expect(track.muted$.value).toBe(false);
     publication.isMuted = true;
@@ -96,26 +104,28 @@ describe("createLivekitMediaTrack", () => {
   });
 
   it("scales a remote member's volume on the participant", () => {
-    const { participant, publication } = fakes({
+    const { participant, publication, room } = fakes({
       kind: Track.Kind.Audio,
     });
     const track = createLivekitMediaTrack(
       scope,
       participant,
       publication as unknown as TrackPublication,
+      room,
     ) as AudioMediaTrack;
     track.setVolume(0.5);
     expect(publication.track!.setVolume).toHaveBeenCalledWith(0.5);
   });
 
   it("is active while the member speaks and the track is not muted", () => {
-    const { participant, publication, emitter } = fakes({
+    const { participant, publication, room, emitter } = fakes({
       kind: Track.Kind.Audio,
     });
     const track = createLivekitMediaTrack(
       scope,
       participant,
       publication as unknown as TrackPublication,
+      room,
     ) as AudioMediaTrack;
     expect(track.isActive$.value).toBe(false);
     participant.isSpeaking = true;
@@ -126,6 +136,57 @@ describe("createLivekitMediaTrack", () => {
     expect(track.isActive$.value).toBe(false);
   });
 
+  it("reports the audio level the room announces, 0 while muted", () => {
+    const { participant, publication, room, emitter } = fakes({
+      kind: Track.Kind.Audio,
+    });
+    const track = createLivekitMediaTrack(
+      scope,
+      participant,
+      publication as unknown as TrackPublication,
+      room,
+    ) as AudioMediaTrack;
+    expect(track.audioLevel$.value).toBe(0);
+    participant.audioLevel = 0.4;
+    (room as unknown as EventEmitter).emit(RoomEvent.ActiveSpeakersChanged, [
+      participant,
+    ]);
+    expect(track.audioLevel$.value).toBe(0.4);
+    publication.isMuted = true;
+    emitter.emit(ParticipantEvent.TrackMuted, publication);
+    expect(track.audioLevel$.value).toBe(0);
+  });
+
+  it("polls statistics at the given interval, only while subscribed", async () => {
+    vi.useFakeTimers();
+    const { participant, publication, room } = fakes();
+    const stats = { type: "inbound-rtp" } as RTCInboundRtpStreamStats;
+    publication.track!.getRTCStatsReport.mockResolvedValue(
+      new Map([["s", stats]]),
+    );
+    const track = createLivekitMediaTrack(
+      scope,
+      participant,
+      publication as unknown as TrackPublication,
+      room,
+    );
+    const stats$ = track.stats$(500);
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(publication.track!.getRTCStatsReport).not.toHaveBeenCalled();
+
+    const seen = vi.fn();
+    const subscription = stats$.subscribe(seen);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(publication.track!.getRTCStatsReport).toHaveBeenCalledTimes(1);
+    expect(seen).toHaveBeenLastCalledWith(stats);
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(publication.track!.getRTCStatsReport).toHaveBeenCalledTimes(3);
+
+    subscription.unsubscribe();
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(publication.track!.getRTCStatsReport).toHaveBeenCalledTimes(3);
+  });
+
   it("gives our own track its controls", async () => {
     const { participant, publication, room } = fakes({
       kind: Track.Kind.Audio,
@@ -152,6 +213,7 @@ type FakeTrack = Track & {
   attach: ReturnType<typeof vi.fn>;
   detach: ReturnType<typeof vi.fn>;
   setVolume: ReturnType<typeof vi.fn>;
+  getRTCStatsReport: ReturnType<typeof vi.fn>;
 };
 
 type FakePublication = Omit<TrackPublication, "track" | "isMuted"> & {
@@ -166,6 +228,7 @@ function fakeTrack(): FakeTrack {
     setVolume: vi.fn(),
     setAudioContext: vi.fn(),
     setWebAudioPlugins: vi.fn(),
+    getRTCStatsReport: vi.fn().mockResolvedValue(undefined),
   }) as FakeTrack;
 }
 
@@ -173,7 +236,7 @@ function fakes({
   track = fakeTrack(),
   kind = Track.Kind.Video,
 }: { track?: FakeTrack | undefined; kind?: Track.Kind } = {}): {
-  participant: RemoteParticipant & { isSpeaking: boolean };
+  participant: RemoteParticipant & { isSpeaking: boolean; audioLevel: number };
   publication: FakePublication;
   room: LivekitRoom;
   emitter: EventEmitter;
@@ -182,10 +245,14 @@ function fakes({
   const participant = Object.assign(emitter, {
     isLocal: false,
     isSpeaking: false,
+    audioLevel: 0,
     identity: "@alice:example.org:DEVICE",
     getTrackPublication: (): TrackPublication =>
       publication as unknown as TrackPublication,
-  }) as unknown as RemoteParticipant & { isSpeaking: boolean };
+  }) as unknown as RemoteParticipant & {
+    isSpeaking: boolean;
+    audioLevel: number;
+  };
   const publication = Object.assign(emitter, {
     kind,
     source:

sdk/src/media-backend/livekit/LivekitMediaTrack.ts

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
 import {
   observeParticipantEvents,
   observeParticipantMedia,
+  roomEventSelector,
 } from "@livekit/components-core";
 import {
   facingModeFromLocalTrack,
@@ -19,6 +20,7 @@ import {
   RemoteAudioTrack,
   RemoteTrack,
   type Room as LivekitRoom,
+  RoomEvent,
   Track,
   TrackEvent,
   type TrackPublication,
@@ -27,13 +29,13 @@ import {
   combineLatest,
   distinctUntilChanged,
   fromEvent,
-  interval,
   map,
   merge,
+  type Observable,
   of,
-  share,
   startWith,
   switchMap,
+  timer,
 } from "rxjs";
 
 import { type Behavior } from "../../reactive/Behavior";
@@ -48,7 +50,6 @@ import {
   type VideoMediaTrack,
 } from "../../media-api";
 import { MatrixRTCError } from "../../errors";
-import { LazyBehavior } from "../../utils/LazyBehavior";
 import { convertToLivekitProcessor } from "./videoProcessor";
 
 export const mediaSources: Record<Track.Source, MediaSource> = {
@@ -67,10 +68,6 @@ export const livekitSources: Record<MediaSource, Track.Source> = {
   unknown: Track.Source.Unknown,
 };
 
-// One timer for every track so that a large session does not keep hundreds of
-// them, each firing a statistics request, in the event loop.
-const refreshStats$ = interval(1000).pipe(startWith(0), share());
-
 /**
  * One publication of a participant as a `MediaTrack`. The publication is
  * fixed; the track behind it may come and go (a remote track arrives on
@@ -80,6 +77,7 @@ export function createLivekitMediaTrack(
   scope: ObservableScope,
   participant: Participant,
   publication: TrackPublication,
+  room: LivekitRoom,
 ): AudioMediaTrack | VideoMediaTrack {
   const mediaChanged$ = observeParticipantMedia(participant);
   const track$ = publicationTrack$(scope, participant, publication);
@@ -117,13 +115,11 @@ export function createLivekitMediaTrack(
     id: publication.trackSid,
     muted$,
     encrypted: publication.isEncrypted,
-    stats$: new LazyBehavior<MediaStreamStats>(
-      refreshStats$.pipe(
+    stats$: (intervalMs): Observable<MediaStreamStats> =>
+      timer(0, intervalMs).pipe(
         switchMap(async () => rtpStreamStats(publication, participant.isLocal)),
         scope.bind(),
       ),
-      undefined,
-    ),
     attach: (element) => {
       if (attached.has(element)) return;
       attached.add(element);
@@ -154,6 +150,20 @@ export function createLivekitMediaTrack(
         ),
         participant.isSpeaking && !publication.isMuted,
       ),
+      // The level has no event of its own: it is written to the participant
+      // with each speaker update, which the room announces
+      audioLevel$: scope.behavior(
+        combineLatest(
+          [
+            roomEventSelector(room, RoomEvent.ActiveSpeakersChanged).pipe(
+              map(() => participant.audioLevel),
+              startWith(participant.audioLevel),
+            ),
+            muted$,
+          ],
+          (level, muted) => (muted ? 0 : level),
+        ),
+      ),
       setAudioContext: (ctx, plugins = []) => {
         audioContext = ctx;
         audioPlugins = plugins;
@@ -181,7 +191,7 @@ export function createLocalLivekitMediaTrack(
   room: LivekitRoom,
   setEnabled: (source: MediaSource, enabled: boolean) => Promise<boolean>,
 ): LocalAudioMediaTrack | LocalVideoMediaTrack {
-  const base = createLivekitMediaTrack(scope, participant, publication);
+  const base = createLivekitMediaTrack(scope, participant, publication, room);
   const local = {
     setEnabled: async (enabled: boolean) => setEnabled(base.source, enabled),
     setDevice: async (deviceId: string): Promise<void> => {

sdk/src/media-backend/livekit/LivekitMemberMedia.ts

@@ -54,8 +54,8 @@ export function createLivekitMemberMedia(
     tracks$: memberTracks$(
       scope,
       source$,
-      (trackScope, participant, publication) =>
-        createLivekitMediaTrack(trackScope, participant, publication),
+      (trackScope, participant, publication, room) =>
+        createLivekitMediaTrack(trackScope, participant, publication, room),
     ),
     encryptionError$: encryptionErrors$(source$, encryptionSystem),
   };