Performance: Fit video to frame without polling RTP stats

It turns out that the timers which repeatedly poll the RTP stats of each video track all run independently of each other and can add up to a small but constant sink of CPU. Meanwhile we can replace these timers with HTMLVideoElement 'resize' event listeners, which is way more efficient and reacts instantly to orientation changes.

src/tile/GridTile.tsx

@@ -11,7 +11,6 @@ import {
   type ReactNode,
   type Ref,
   useCallback,
-  useEffect,
   useRef,
   useState,
 } from "react";
@@ -91,7 +90,6 @@ const RingingMediaTile: FC<RingingMediaTileProps> = ({
       }
       avatarStyle="translucent"
       videoEnabled={false}
-      videoFit="cover"
       mirror={false}
       {...props}
     />
@@ -141,19 +139,11 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
   const audioEnabled = useBehavior(vm.audioEnabled$);
   const videoEnabled = useBehavior(vm.videoEnabled$);
   const speaking = useBehavior(vm.speaking$);
-  const videoFit = useBehavior(vm.videoFit$);
 
   const rtcBackendIdentity = vm.rtcBackendIdentity;
   const handRaised = useBehavior(vm.handRaised$);
   const reaction = useBehavior(vm.reaction$);
 
-  // Whenever bounds change, inform the viewModel
-  useEffect(() => {
-    if (targetWidth > 0 && targetHeight > 0) {
-      vm.setTargetDimensions(targetWidth, targetHeight);
-    }
-  }, [targetWidth, targetHeight, vm]);
-
   const AudioIcon = playbackMuted
     ? VolumeOffSolidIcon
     : audioEnabled
@@ -190,7 +180,6 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
       userId={vm.userId}
       unencryptedWarning={unencryptedWarning}
       videoEnabled={videoEnabled}
-      videoFit={videoFit}
       className={classNames(className, styles.tile, {
         [styles.speaking]: showSpeaking,
         [styles.handRaised]: !showSpeaking && handRaised,
@@ -233,6 +222,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
       raisedHandOnClick={raisedHandOnClick}
       waitingForMedia={waitingForMedia}
       focusUrl={focusUrl}
+      setVideoAspectRatio={vm.setVideoAspectRatio}
       audioStreamStats={audioStreamStats}
       videoStreamStats={videoStreamStats}
       rtcBackendIdentity={rtcBackendIdentity}

src/tile/MediaView.test.tsx

@@ -33,7 +33,6 @@ describe("MediaView", () => {
   const baseProps: ComponentProps<typeof MediaView> = {
     displayName: "some name",
     videoEnabled: true,
-    videoFit: "contain",
     targetWidth: 300,
     targetHeight: 200,
     mirror: false,

src/tile/MediaView.tsx

@@ -7,7 +7,13 @@ Please see LICENSE in the repository root for full details.
 
 import { type TrackReferenceOrPlaceholder } from "@livekit/components-core";
 import { animated } from "@react-spring/web";
-import { type FC, type ComponentProps, type ReactNode } from "react";
+import {
+  type FC,
+  type ComponentProps,
+  type ReactNode,
+  type SyntheticEvent,
+  useState,
+} from "react";
 import { useTranslation } from "react-i18next";
 import classNames from "classnames";
 import { VideoTrack } from "@livekit/components-react";
@@ -26,6 +32,7 @@ import { type ReactionOption } from "../reactions";
 import { ReactionIndicator } from "../reactions/ReactionIndicator";
 import { RTCConnectionStats } from "../RTCConnectionStats";
 import videoPlaceholder from "../graphics/video-placeholder.gif";
+import { autoVideoFit } from "../utils/videoFit";
 
 interface Props extends ComponentProps<typeof animated.div> {
   className?: string;
@@ -33,7 +40,11 @@ interface Props extends ComponentProps<typeof animated.div> {
   targetWidth: number;
   targetHeight: number;
   video: TrackReferenceOrPlaceholder | undefined;
-  videoFit: "cover" | "contain";
+  /**
+   * How to fit the video content inside the tile. When undefined, MediaView
+   * chooses a smart default based on the aspect ratios of the tile and video.
+   */
+  videoFit?: "cover" | "contain";
   mirror: boolean;
   soundWaves?: boolean;
   userId: string;
@@ -55,8 +66,15 @@ interface Props extends ComponentProps<typeof animated.div> {
   audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
   videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
   rtcBackendIdentity?: string;
-  // The focus url, mainly for debugging purposes
+  /**
+   * The focus url, mainly for debugging purposes.
+   */
   focusUrl?: string;
+  /**
+   * Called whenever the aspect ratio of the video content becomes known or
+   * otherwise changes.
+   */
+  setVideoAspectRatio?: (ratio: number) => void;
 }
 
 export const MediaView: FC<Props> = ({
@@ -89,6 +107,7 @@ export const MediaView: FC<Props> = ({
   videoStreamStats,
   rtcBackendIdentity,
   focusUrl,
+  setVideoAspectRatio: setTheirVideoAspectRatio,
   ...props
 }) => {
   const { t } = useTranslation();
@@ -100,6 +119,22 @@ export const MediaView: FC<Props> = ({
       (soundWaves === undefined ? 0.5 : 0.38),
   );
 
+  const [videoAspectRatio, setOurVideoAspectRatio] = useState<number>(NaN);
+  const tileAspectRatio = targetWidth / targetHeight;
+
+  // Propagate video dimensions
+  const setVideoAspectRatio = (ratio: number) => {
+    setOurVideoAspectRatio(ratio);
+    setTheirVideoAspectRatio?.(ratio);
+  };
+  const videoRef = (el: HTMLVideoElement | null) => {
+    if (el !== null) setVideoAspectRatio(el.videoWidth / el.videoHeight);
+  };
+  const onResize = (ev: SyntheticEvent<HTMLVideoElement>) =>
+    setVideoAspectRatio(
+      ev.currentTarget.videoWidth / ev.currentTarget.videoHeight,
+    );
+
   const warnings = unencryptedWarning && (
     <Tooltip
       label={t("common.unencrypted")}
@@ -126,7 +161,9 @@ export const MediaView: FC<Props> = ({
       ref={ref}
       data-testid="videoTile"
       data-video-enabled={video && videoEnabled}
-      data-video-fit={videoFit}
+      data-video-fit={
+        videoFit ?? autoVideoFit(videoAspectRatio, tileAspectRatio)
+      }
       data-background={background}
       {...props}
     >
@@ -158,6 +195,8 @@ export const MediaView: FC<Props> = ({
             // Set the placeholder to a small transparent image. (On Android web
             // views the default poster image is particularly ugly.)
             poster={videoPlaceholder}
+            ref={videoRef}
+            onResize={onResize}
           />
         )}
       </div>

src/tile/SpotlightTile.tsx

@@ -68,6 +68,7 @@ interface SpotlightItemBaseProps {
   background: "solid" | "transparent";
   focusable: boolean;
   "aria-hidden"?: boolean;
+  setVideoAspectRatio?: (ratio: number) => void;
 }
 
 interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps {
@@ -77,7 +78,6 @@ interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps {
 }
 
 interface SpotlightUserMediaItemBaseProps extends SpotlightMemberMediaItemBaseProps {
-  videoFit: "contain" | "cover";
   videoEnabled: boolean;
   soundWaves: boolean | undefined;
 }
@@ -120,20 +120,12 @@ const SpotlightUserMediaItem: FC<SpotlightUserMediaItemProps> = ({
   targetHeight,
   ...props
 }) => {
-  const videoFit = useBehavior(vm.videoFit$);
   const videoEnabled = useBehavior(vm.videoEnabled$);
   const speaking = useBehavior(vm.speaking$);
 
-  // Whenever target bounds change, inform the viewModel
-  useEffect(() => {
-    if (targetWidth > 0 && targetHeight > 0) {
-      vm.setTargetDimensions(targetWidth, targetHeight);
-    }
-  }, [targetWidth, targetHeight, vm]);
-
   const baseProps: SpotlightUserMediaItemBaseProps &
     RefAttributes<HTMLDivElement> = {
-    videoFit,
+    setVideoAspectRatio: vm.setVideoAspectRatio,
     videoEnabled,
     soundWaves: props.background === "transparent" ? speaking : undefined,
     targetWidth,
@@ -227,7 +219,6 @@ const SpotlightRingingMediaItem: FC<SpotlightRingingMediaItemProps> = ({
       }
       avatarStyle="translucent"
       videoEnabled={false}
-      videoFit="cover"
       mirror={false}
       {...props}
     />