Performance: Fit video to frame without polling RTP stats

It turns out that the timers which repeatedly poll the RTP stats of each video track all run independently of each other and can add up to a small but constant sink of CPU. Meanwhile we can replace these timers with HTMLVideoElement 'resize' event listeners, which is way more efficient and reacts instantly to orientation changes.

src/utils/videoFit.test.ts

@@ -5,259 +5,106 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { describe, expect, test, vi } from "vitest";
-import {
-  LocalTrack,
-  type LocalTrackPublication,
-  type RemoteTrackPublication,
-  Track,
-} from "livekit-client";
+import { describe, expect, test } from "vitest";
 
-import { ObservableScope } from "../state/ObservableScope";
-import { videoFit$, videoSizeFromParticipant$ } from "./videoFit";
-import { constant } from "../state/Behavior";
-import {
-  flushPromises,
-  mockLocalParticipant,
-  mockRemoteParticipant,
-} from "./test";
+import { autoVideoFit } from "./videoFit";
 
 describe("videoFit$ defaults", () => {
   test.each([
     {
-      videoSize: { width: 1920, height: 1080 },
-      tileSize: undefined,
+      videoAspectRatio: 1920 / 1080,
+      tileAspectRatio: NaN,
     },
     {
-      videoSize: { width: 1080, height: 1920 },
-      tileSize: undefined,
+      videoAspectRatio: 1080 / 1920,
+      tileAspectRatio: NaN,
     },
     {
-      videoSize: undefined,
-      tileSize: { width: 1920, height: 1080 },
+      videoAspectRatio: NaN,
+      tileAspectRatio: 1920 / 1080,
     },
     {
-      videoSize: undefined,
-      tileSize: { width: 1080, height: 1920 },
+      videoAspectRatio: NaN,
+      tileAspectRatio: 1080 / 1920,
     },
   ])(
-    "videoFit$ returns `cover` when videoSize is $videoSize and tileSize is $tileSize",
-    ({ videoSize, tileSize }) => {
-      const scope = new ObservableScope();
-      const videoSize$ = constant(videoSize);
-      const tileSize$ = constant(tileSize);
-
-      const fit = videoFit$(scope, videoSize$, tileSize$);
-      expect(fit.value).toBe("cover");
-    },
+    "videoFit$ returns `cover` when videoAspectRatio is $videoAspectRatio and tileAspectRatio is $tileAspectRatio",
+    ({ videoAspectRatio, tileAspectRatio }) =>
+      expect(autoVideoFit(videoAspectRatio, tileAspectRatio)).toBe("cover"),
   );
 });
 
-const VIDEO_480_L = { width: 640, height: 480 };
-const VIDEO_720_L = { width: 1280, height: 720 };
-const VIDEO_1080_L = { width: 1920, height: 1080 };
+const VIDEO_480_L = 640 / 480;
+const VIDEO_720_L = 1280 / 720;
+const VIDEO_1080_L = 1920 / 1080;
 
 // Some sizes from real world testing, which don't match the standard video sizes exactly
-const TILE_SIZE_1_L = { width: 180, height: 135 };
-const TILE_SIZE_3_P = { width: 379, height: 542 };
-const TILE_SIZE_4_L = { width: 957, height: 542 };
+const TILE_SIZE_1_L = 180 / 135;
+const TILE_SIZE_3_P = 379 / 542;
+const TILE_SIZE_4_L = 957 / 542;
 // This is the size of an iPhone Xr in portrait mode
-const TILE_SIZE_5_P = { width: 414, height: 896 };
+const TILE_SIZE_5_P = 414 / 896;
 
-export function invertSize(size: { width: number; height: number }): {
-  width: number;
-  height: number;
-} {
-  return {
-    width: size.height,
-    height: size.width,
-  };
+function inverse(ratio: number): number {
+  return 1 / ratio;
 }
 
 test.each([
   {
-    videoSize: VIDEO_480_L,
-    tileSize: TILE_SIZE_1_L,
+    videoAspectRatio: VIDEO_480_L,
+    tileAspectRatio: TILE_SIZE_1_L,
     expected: "cover",
   },
   {
-    videoSize: invertSize(VIDEO_480_L),
-    tileSize: TILE_SIZE_1_L,
+    videoAspectRatio: inverse(VIDEO_480_L),
+    tileAspectRatio: TILE_SIZE_1_L,
     expected: "contain",
   },
   {
-    videoSize: VIDEO_720_L,
-    tileSize: TILE_SIZE_4_L,
+    videoAspectRatio: VIDEO_720_L,
+    tileAspectRatio: TILE_SIZE_4_L,
     expected: "cover",
   },
   {
-    videoSize: invertSize(VIDEO_720_L),
-    tileSize: TILE_SIZE_4_L,
+    videoAspectRatio: inverse(VIDEO_720_L),
+    tileAspectRatio: TILE_SIZE_4_L,
     expected: "contain",
   },
   {
-    videoSize: invertSize(VIDEO_1080_L),
-    tileSize: TILE_SIZE_3_P,
+    videoAspectRatio: inverse(VIDEO_1080_L),
+    tileAspectRatio: TILE_SIZE_3_P,
     expected: "cover",
   },
   {
-    videoSize: VIDEO_1080_L,
-    tileSize: TILE_SIZE_5_P,
+    videoAspectRatio: VIDEO_1080_L,
+    tileAspectRatio: TILE_SIZE_5_P,
     expected: "contain",
   },
   {
-    videoSize: invertSize(VIDEO_1080_L),
-    tileSize: TILE_SIZE_5_P,
+    videoAspectRatio: inverse(VIDEO_1080_L),
+    tileAspectRatio: TILE_SIZE_5_P,
     expected: "cover",
   },
   {
     // square video
-    videoSize: { width: 400, height: 400 },
-    tileSize: VIDEO_480_L,
+    videoAspectRatio: 400 / 400,
+    tileAspectRatio: VIDEO_480_L,
     expected: "contain",
   },
   {
     // Should default to cover if the initial size is 0:0.
     // Or else it will cause a flash of "contain" mode until the real size is loaded, which can be jarring.
-    videoSize: VIDEO_480_L,
-    tileSize: { width: 0, height: 0 },
+    videoAspectRatio: VIDEO_480_L,
+    tileAspectRatio: 0 / 0,
     expected: "cover",
   },
   {
-    videoSize: { width: 0, height: 0 },
-    tileSize: VIDEO_480_L,
+    videoAspectRatio: 0 / 0,
+    tileAspectRatio: VIDEO_480_L,
     expected: "cover",
   },
 ])(
-  "videoFit$ returns $expected when videoSize is $videoSize and tileSize is $tileSize",
-  ({ videoSize, tileSize, expected }) => {
-    const scope = new ObservableScope();
-    const videoSize$ = constant(videoSize);
-    const tileSize$ = constant(tileSize);
-
-    const fit = videoFit$(scope, videoSize$, tileSize$);
-    expect(fit.value).toBe(expected);
-  },
+  "videoFit$ returns $expected when videoAspectRatio is $videoAspectRatio and tileAspectRatio is $tileAspectRatio",
+  ({ videoAspectRatio, tileAspectRatio, expected }) =>
+    expect(autoVideoFit(videoAspectRatio, tileAspectRatio)).toBe(expected),
 );
-
-describe("extracting video size from participant stats", () => {
-  function createMockRtpStats(
-    isInbound: boolean,
-    props: Partial<RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats> = {},
-  ): RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats {
-    const baseStats = {
-      id: "mock-stats-id",
-      timestamp: Date.now(),
-      type: isInbound ? "inbound-rtp" : "outbound-rtp",
-      kind: "video",
-      ...props,
-    };
-
-    return baseStats as RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
-  }
-
-  test("get stats for local user", async () => {
-    const localParticipant = mockLocalParticipant({
-      identity: "@local:example.org:AAAAAA",
-    });
-
-    const mockReport: RTCStatsReport = new Map([
-      [
-        "OT01V639885149",
-        createMockRtpStats(false, {
-          frameWidth: 1280,
-          frameHeight: 720,
-        }),
-      ],
-    ]);
-
-    const track = {
-      source: Track.Source.Camera,
-      getRTCStatsReport: vi
-        .fn()
-        .mockImplementation(async () => Promise.resolve(mockReport)),
-    } as Partial<LocalTrack> as LocalTrack;
-
-    // Set up the prototype chain (there is an instanceof check in getRTCStatsReport)
-    Object.setPrototypeOf(track, LocalTrack.prototype);
-
-    localParticipant.getTrackPublication = vi
-      .fn()
-      .mockImplementation((source: Track.Source) => {
-        if (source === Track.Source.Camera) {
-          return {
-            track,
-          } as unknown as LocalTrackPublication;
-        } else {
-          return undefined;
-        }
-      });
-
-    const videoDimensions$ = videoSizeFromParticipant$(
-      constant(localParticipant),
-    );
-
-    const publishedDimensions: { width: number; height: number }[] = [];
-    videoDimensions$.subscribe((dimensions) => {
-      if (dimensions) publishedDimensions.push(dimensions);
-    });
-
-    await flushPromises();
-
-    const dimension = publishedDimensions.pop();
-    expect(dimension).toEqual({ width: 1280, height: 720 });
-  });
-
-  test("get stats for remote user", async () => {
-    // vi.useFakeTimers()
-    const remoteParticipant = mockRemoteParticipant({
-      identity: "@bob:example.org:AAAAAA",
-    });
-
-    const mockReport: RTCStatsReport = new Map([
-      [
-        "OT01V639885149",
-        createMockRtpStats(true, {
-          frameWidth: 480,
-          frameHeight: 640,
-        }),
-      ],
-    ]);
-
-    const track = {
-      source: Track.Source.Camera,
-      getRTCStatsReport: vi
-        .fn()
-        .mockImplementation(async () => Promise.resolve(mockReport)),
-    } as Partial<LocalTrack> as LocalTrack;
-
-    // Set up the prototype chain (there is an instanceof check in getRTCStatsReport)
-    Object.setPrototypeOf(track, LocalTrack.prototype);
-
-    remoteParticipant.getTrackPublication = vi
-      .fn()
-      .mockImplementation((source: Track.Source) => {
-        if (source === Track.Source.Camera) {
-          return {
-            track,
-          } as unknown as RemoteTrackPublication;
-        } else {
-          return undefined;
-        }
-      });
-
-    const videoDimensions$ = videoSizeFromParticipant$(
-      constant(remoteParticipant),
-    );
-
-    const publishedDimensions: { width: number; height: number }[] = [];
-    videoDimensions$.subscribe((dimensions) => {
-      if (dimensions) publishedDimensions.push(dimensions);
-    });
-
-    await flushPromises();
-
-    const dimension = publishedDimensions.pop();
-    expect(dimension).toEqual({ width: 480, height: 640 });
-  });
-});