From 1e128c4da0fbb6d3f166fa70d6f158fbe8b87ff9 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 4 Aug 2026 19:01:20 +0200 Subject: [PATCH 1/4] Performance: Fit video to frame without polling RTP stats It turns out that the timers which repeatedly poll the RTP stats of each video track all run independently of each other and can add up to a small but constant sink of CPU. Meanwhile we can replace these timers with HTMLVideoElement 'resize' event listeners, which is way more efficient and reacts instantly to orientation changes. --- src/state/media/UserMediaViewModel.ts | 25 +-- src/state/media/observeRtpStreamStats.ts | 9 - src/tile/GridTile.tsx | 12 +- src/tile/MediaView.test.tsx | 1 - src/tile/MediaView.tsx | 47 ++++- src/tile/SpotlightTile.tsx | 13 +- src/utils/videoFit.test.ts | 243 +++++------------------ src/utils/videoFit.ts | 111 ++--------- 8 files changed, 111 insertions(+), 350 deletions(-) diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts index ea0331030..61d534656 100644 --- a/src/state/media/UserMediaViewModel.ts +++ b/src/state/media/UserMediaViewModel.ts @@ -33,7 +33,6 @@ import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel"; import { type ObservableScope } from "../ObservableScope"; import { showConnectionStats } from "../../settings/settings"; import { observeRtpStreamStats$ } from "./observeRtpStreamStats"; -import { videoFit$, videoSizeFromParticipant$ } from "../../utils/videoFit.ts"; /** * A participant's user media (i.e. their microphone and camera feed). @@ -47,7 +46,6 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel { speaking$: Behavior; audioEnabled$: Behavior; videoEnabled$: Behavior; - videoFit$: Behavior<"cover" | "contain">; videoOrientation$: Behavior<"landscape" | "portrait">; toggleCropVideo: () => void; /** @@ -63,12 +61,9 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel { RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined >; /** - * Set the target dimensions of the HTML element (final dimension after anim). - * This can be used to determine the best video fit (fit to frame / keep ratio). - * @param targetWidth - The target width of the HTML element displaying the video. - * @param targetHeight - The target height of the HTML element displaying the video. + * Set the aspect ratio of the video track to determine the orientation. */ - setTargetDimensions: (targetWidth: number, targetHeight: number) => void; + setVideoAspectRatio: (ratio: number) => void; } export interface BaseUserMediaInputs extends Omit< @@ -98,14 +93,8 @@ export function createBaseUserMedia( ), ); const toggleCropVideo$ = new Subject(); + const videoAspectRatio$ = new BehaviorSubject(NaN); - // The target size of the video element, used to determine the best video fit. - // The target size is the final size of the HTML element after any animations have completed. - const targetSize$ = new BehaviorSubject< - { width: number; height: number } | undefined - >(undefined); - - const videoSize$ = videoSizeFromParticipant$(participant$); return { ...createMemberMedia(scope, { ...inputs, @@ -132,13 +121,11 @@ export function createBaseUserMedia( media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), ), videoOrientation$: scope.behavior( - videoSize$.pipe( - map((s) => (s ? s.width / s.height : 1)), + videoAspectRatio$.pipe( map((aspect) => (aspect > 1 ? "landscape" : "portrait")), ), "portrait", ), - videoFit$: videoFit$(scope, videoSize$, targetSize$), toggleCropVideo: () => toggleCropVideo$.next(), rtcBackendIdentity, handRaised$, @@ -162,8 +149,6 @@ export function createBaseUserMedia( return observeRtpStreamStats$(p, Track.Source.Camera, statsType); }), ), - setTargetDimensions: (targetWidth: number, targetHeight: number): void => { - targetSize$.next({ width: targetWidth, height: targetHeight }); - }, + setVideoAspectRatio: (ratio) => videoAspectRatio$.next(ratio), }; } diff --git a/src/state/media/observeRtpStreamStats.ts b/src/state/media/observeRtpStreamStats.ts index 44181f857..5edc28ef7 100644 --- a/src/state/media/observeRtpStreamStats.ts +++ b/src/state/media/observeRtpStreamStats.ts @@ -69,12 +69,3 @@ export function observeInboundRtpStreamStats$( map((x) => x as RTCInboundRtpStreamStats | undefined), ); } - -export function observeOutboundRtpStreamStats$( - participant: Participant, - source: Track.Source, -): Observable { - return observeRtpStreamStats$(participant, source, "outbound-rtp").pipe( - map((x) => x as RTCOutboundRtpStreamStats | undefined), - ); -} diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 657bf0bc8..83d071eff 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -11,7 +11,6 @@ import { type ReactNode, type Ref, useCallback, - useEffect, useRef, useState, } from "react"; @@ -91,7 +90,6 @@ const RingingMediaTile: FC = ({ } avatarStyle="translucent" videoEnabled={false} - videoFit="cover" mirror={false} {...props} /> @@ -141,19 +139,11 @@ const UserMediaTile: FC = ({ const audioEnabled = useBehavior(vm.audioEnabled$); const videoEnabled = useBehavior(vm.videoEnabled$); const speaking = useBehavior(vm.speaking$); - const videoFit = useBehavior(vm.videoFit$); const rtcBackendIdentity = vm.rtcBackendIdentity; const handRaised = useBehavior(vm.handRaised$); const reaction = useBehavior(vm.reaction$); - // Whenever bounds change, inform the viewModel - useEffect(() => { - if (targetWidth > 0 && targetHeight > 0) { - vm.setTargetDimensions(targetWidth, targetHeight); - } - }, [targetWidth, targetHeight, vm]); - const AudioIcon = playbackMuted ? VolumeOffSolidIcon : audioEnabled @@ -190,7 +180,6 @@ const UserMediaTile: FC = ({ userId={vm.userId} unencryptedWarning={unencryptedWarning} videoEnabled={videoEnabled} - videoFit={videoFit} className={classNames(className, styles.tile, { [styles.speaking]: showSpeaking, [styles.handRaised]: !showSpeaking && handRaised, @@ -233,6 +222,7 @@ const UserMediaTile: FC = ({ raisedHandOnClick={raisedHandOnClick} waitingForMedia={waitingForMedia} focusUrl={focusUrl} + setVideoAspectRatio={vm.setVideoAspectRatio} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} rtcBackendIdentity={rtcBackendIdentity} diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 099cbaa22..660d37a27 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -33,7 +33,6 @@ describe("MediaView", () => { const baseProps: ComponentProps = { displayName: "some name", videoEnabled: true, - videoFit: "contain", targetWidth: 300, targetHeight: 200, mirror: false, diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index a00fb6cba..d57441687 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -7,7 +7,13 @@ Please see LICENSE in the repository root for full details. import { type TrackReferenceOrPlaceholder } from "@livekit/components-core"; import { animated } from "@react-spring/web"; -import { type FC, type ComponentProps, type ReactNode } from "react"; +import { + type FC, + type ComponentProps, + type ReactNode, + type SyntheticEvent, + useState, +} from "react"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; import { VideoTrack } from "@livekit/components-react"; @@ -26,6 +32,7 @@ import { type ReactionOption } from "../reactions"; import { ReactionIndicator } from "../reactions/ReactionIndicator"; import { RTCConnectionStats } from "../RTCConnectionStats"; import videoPlaceholder from "../graphics/video-placeholder.gif"; +import { autoVideoFit } from "../utils/videoFit"; interface Props extends ComponentProps { className?: string; @@ -33,7 +40,11 @@ interface Props extends ComponentProps { targetWidth: number; targetHeight: number; video: TrackReferenceOrPlaceholder | undefined; - videoFit: "cover" | "contain"; + /** + * How to fit the video content inside the tile. When undefined, MediaView + * chooses a smart default based on the aspect ratios of the tile and video. + */ + videoFit?: "cover" | "contain"; mirror: boolean; soundWaves?: boolean; userId: string; @@ -55,8 +66,15 @@ interface Props extends ComponentProps { audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; rtcBackendIdentity?: string; - // The focus url, mainly for debugging purposes + /** + * The focus url, mainly for debugging purposes. + */ focusUrl?: string; + /** + * Called whenever the aspect ratio of the video content becomes known or + * otherwise changes. + */ + setVideoAspectRatio?: (ratio: number) => void; } export const MediaView: FC = ({ @@ -89,6 +107,7 @@ export const MediaView: FC = ({ videoStreamStats, rtcBackendIdentity, focusUrl, + setVideoAspectRatio: setTheirVideoAspectRatio, ...props }) => { const { t } = useTranslation(); @@ -100,6 +119,22 @@ export const MediaView: FC = ({ (soundWaves === undefined ? 0.5 : 0.38), ); + const [videoAspectRatio, setOurVideoAspectRatio] = useState(NaN); + const tileAspectRatio = targetWidth / targetHeight; + + // Propagate video dimensions + const setVideoAspectRatio = (ratio: number) => { + setOurVideoAspectRatio(ratio); + setTheirVideoAspectRatio?.(ratio); + }; + const videoRef = (el: HTMLVideoElement | null) => { + if (el !== null) setVideoAspectRatio(el.videoWidth / el.videoHeight); + }; + const onResize = (ev: SyntheticEvent) => + setVideoAspectRatio( + ev.currentTarget.videoWidth / ev.currentTarget.videoHeight, + ); + const warnings = unencryptedWarning && ( = ({ ref={ref} data-testid="videoTile" data-video-enabled={video && videoEnabled} - data-video-fit={videoFit} + data-video-fit={ + videoFit ?? autoVideoFit(videoAspectRatio, tileAspectRatio) + } data-background={background} {...props} > @@ -158,6 +195,8 @@ export const MediaView: FC = ({ // Set the placeholder to a small transparent image. (On Android web // views the default poster image is particularly ugly.) poster={videoPlaceholder} + ref={videoRef} + onResize={onResize} /> )} diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index d21a7f5f5..036e044fe 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -68,6 +68,7 @@ interface SpotlightItemBaseProps { background: "solid" | "transparent"; focusable: boolean; "aria-hidden"?: boolean; + setVideoAspectRatio?: (ratio: number) => void; } interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps { @@ -77,7 +78,6 @@ interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps { } interface SpotlightUserMediaItemBaseProps extends SpotlightMemberMediaItemBaseProps { - videoFit: "contain" | "cover"; videoEnabled: boolean; soundWaves: boolean | undefined; } @@ -120,20 +120,12 @@ const SpotlightUserMediaItem: FC = ({ targetHeight, ...props }) => { - const videoFit = useBehavior(vm.videoFit$); const videoEnabled = useBehavior(vm.videoEnabled$); const speaking = useBehavior(vm.speaking$); - // Whenever target bounds change, inform the viewModel - useEffect(() => { - if (targetWidth > 0 && targetHeight > 0) { - vm.setTargetDimensions(targetWidth, targetHeight); - } - }, [targetWidth, targetHeight, vm]); - const baseProps: SpotlightUserMediaItemBaseProps & RefAttributes = { - videoFit, + setVideoAspectRatio: vm.setVideoAspectRatio, videoEnabled, soundWaves: props.background === "transparent" ? speaking : undefined, targetWidth, @@ -227,7 +219,6 @@ const SpotlightRingingMediaItem: FC = ({ } avatarStyle="translucent" videoEnabled={false} - videoFit="cover" mirror={false} {...props} /> diff --git a/src/utils/videoFit.test.ts b/src/utils/videoFit.test.ts index 5068526ba..ba46b6d84 100644 --- a/src/utils/videoFit.test.ts +++ b/src/utils/videoFit.test.ts @@ -5,259 +5,106 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, test, vi } from "vitest"; -import { - LocalTrack, - type LocalTrackPublication, - type RemoteTrackPublication, - Track, -} from "livekit-client"; +import { describe, expect, test } from "vitest"; -import { ObservableScope } from "../state/ObservableScope"; -import { videoFit$, videoSizeFromParticipant$ } from "./videoFit"; -import { constant } from "../state/Behavior"; -import { - flushPromises, - mockLocalParticipant, - mockRemoteParticipant, -} from "./test"; +import { autoVideoFit } from "./videoFit"; describe("videoFit$ defaults", () => { test.each([ { - videoSize: { width: 1920, height: 1080 }, - tileSize: undefined, + videoAspectRatio: 1920 / 1080, + tileAspectRatio: NaN, }, { - videoSize: { width: 1080, height: 1920 }, - tileSize: undefined, + videoAspectRatio: 1080 / 1920, + tileAspectRatio: NaN, }, { - videoSize: undefined, - tileSize: { width: 1920, height: 1080 }, + videoAspectRatio: NaN, + tileAspectRatio: 1920 / 1080, }, { - videoSize: undefined, - tileSize: { width: 1080, height: 1920 }, + videoAspectRatio: NaN, + tileAspectRatio: 1080 / 1920, }, ])( - "videoFit$ returns `cover` when videoSize is $videoSize and tileSize is $tileSize", - ({ videoSize, tileSize }) => { - const scope = new ObservableScope(); - const videoSize$ = constant(videoSize); - const tileSize$ = constant(tileSize); - - const fit = videoFit$(scope, videoSize$, tileSize$); - expect(fit.value).toBe("cover"); - }, + "videoFit$ returns `cover` when videoAspectRatio is $videoAspectRatio and tileAspectRatio is $tileAspectRatio", + ({ videoAspectRatio, tileAspectRatio }) => + expect(autoVideoFit(videoAspectRatio, tileAspectRatio)).toBe("cover"), ); }); -const VIDEO_480_L = { width: 640, height: 480 }; -const VIDEO_720_L = { width: 1280, height: 720 }; -const VIDEO_1080_L = { width: 1920, height: 1080 }; +const VIDEO_480_L = 640 / 480; +const VIDEO_720_L = 1280 / 720; +const VIDEO_1080_L = 1920 / 1080; // Some sizes from real world testing, which don't match the standard video sizes exactly -const TILE_SIZE_1_L = { width: 180, height: 135 }; -const TILE_SIZE_3_P = { width: 379, height: 542 }; -const TILE_SIZE_4_L = { width: 957, height: 542 }; +const TILE_SIZE_1_L = 180 / 135; +const TILE_SIZE_3_P = 379 / 542; +const TILE_SIZE_4_L = 957 / 542; // This is the size of an iPhone Xr in portrait mode -const TILE_SIZE_5_P = { width: 414, height: 896 }; +const TILE_SIZE_5_P = 414 / 896; -export function invertSize(size: { width: number; height: number }): { - width: number; - height: number; -} { - return { - width: size.height, - height: size.width, - }; +function inverse(ratio: number): number { + return 1 / ratio; } test.each([ { - videoSize: VIDEO_480_L, - tileSize: TILE_SIZE_1_L, + videoAspectRatio: VIDEO_480_L, + tileAspectRatio: TILE_SIZE_1_L, expected: "cover", }, { - videoSize: invertSize(VIDEO_480_L), - tileSize: TILE_SIZE_1_L, + videoAspectRatio: inverse(VIDEO_480_L), + tileAspectRatio: TILE_SIZE_1_L, expected: "contain", }, { - videoSize: VIDEO_720_L, - tileSize: TILE_SIZE_4_L, + videoAspectRatio: VIDEO_720_L, + tileAspectRatio: TILE_SIZE_4_L, expected: "cover", }, { - videoSize: invertSize(VIDEO_720_L), - tileSize: TILE_SIZE_4_L, + videoAspectRatio: inverse(VIDEO_720_L), + tileAspectRatio: TILE_SIZE_4_L, expected: "contain", }, { - videoSize: invertSize(VIDEO_1080_L), - tileSize: TILE_SIZE_3_P, + videoAspectRatio: inverse(VIDEO_1080_L), + tileAspectRatio: TILE_SIZE_3_P, expected: "cover", }, { - videoSize: VIDEO_1080_L, - tileSize: TILE_SIZE_5_P, + videoAspectRatio: VIDEO_1080_L, + tileAspectRatio: TILE_SIZE_5_P, expected: "contain", }, { - videoSize: invertSize(VIDEO_1080_L), - tileSize: TILE_SIZE_5_P, + videoAspectRatio: inverse(VIDEO_1080_L), + tileAspectRatio: TILE_SIZE_5_P, expected: "cover", }, { // square video - videoSize: { width: 400, height: 400 }, - tileSize: VIDEO_480_L, + videoAspectRatio: 400 / 400, + tileAspectRatio: VIDEO_480_L, expected: "contain", }, { // Should default to cover if the initial size is 0:0. // Or else it will cause a flash of "contain" mode until the real size is loaded, which can be jarring. - videoSize: VIDEO_480_L, - tileSize: { width: 0, height: 0 }, + videoAspectRatio: VIDEO_480_L, + tileAspectRatio: 0 / 0, expected: "cover", }, { - videoSize: { width: 0, height: 0 }, - tileSize: VIDEO_480_L, + videoAspectRatio: 0 / 0, + tileAspectRatio: VIDEO_480_L, expected: "cover", }, ])( - "videoFit$ returns $expected when videoSize is $videoSize and tileSize is $tileSize", - ({ videoSize, tileSize, expected }) => { - const scope = new ObservableScope(); - const videoSize$ = constant(videoSize); - const tileSize$ = constant(tileSize); - - const fit = videoFit$(scope, videoSize$, tileSize$); - expect(fit.value).toBe(expected); - }, + "videoFit$ returns $expected when videoAspectRatio is $videoAspectRatio and tileAspectRatio is $tileAspectRatio", + ({ videoAspectRatio, tileAspectRatio, expected }) => + expect(autoVideoFit(videoAspectRatio, tileAspectRatio)).toBe(expected), ); - -describe("extracting video size from participant stats", () => { - function createMockRtpStats( - isInbound: boolean, - props: Partial = {}, - ): RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats { - const baseStats = { - id: "mock-stats-id", - timestamp: Date.now(), - type: isInbound ? "inbound-rtp" : "outbound-rtp", - kind: "video", - ...props, - }; - - return baseStats as RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; - } - - test("get stats for local user", async () => { - const localParticipant = mockLocalParticipant({ - identity: "@local:example.org:AAAAAA", - }); - - const mockReport: RTCStatsReport = new Map([ - [ - "OT01V639885149", - createMockRtpStats(false, { - frameWidth: 1280, - frameHeight: 720, - }), - ], - ]); - - const track = { - source: Track.Source.Camera, - getRTCStatsReport: vi - .fn() - .mockImplementation(async () => Promise.resolve(mockReport)), - } as Partial as LocalTrack; - - // Set up the prototype chain (there is an instanceof check in getRTCStatsReport) - Object.setPrototypeOf(track, LocalTrack.prototype); - - localParticipant.getTrackPublication = vi - .fn() - .mockImplementation((source: Track.Source) => { - if (source === Track.Source.Camera) { - return { - track, - } as unknown as LocalTrackPublication; - } else { - return undefined; - } - }); - - const videoDimensions$ = videoSizeFromParticipant$( - constant(localParticipant), - ); - - const publishedDimensions: { width: number; height: number }[] = []; - videoDimensions$.subscribe((dimensions) => { - if (dimensions) publishedDimensions.push(dimensions); - }); - - await flushPromises(); - - const dimension = publishedDimensions.pop(); - expect(dimension).toEqual({ width: 1280, height: 720 }); - }); - - test("get stats for remote user", async () => { - // vi.useFakeTimers() - const remoteParticipant = mockRemoteParticipant({ - identity: "@bob:example.org:AAAAAA", - }); - - const mockReport: RTCStatsReport = new Map([ - [ - "OT01V639885149", - createMockRtpStats(true, { - frameWidth: 480, - frameHeight: 640, - }), - ], - ]); - - const track = { - source: Track.Source.Camera, - getRTCStatsReport: vi - .fn() - .mockImplementation(async () => Promise.resolve(mockReport)), - } as Partial as LocalTrack; - - // Set up the prototype chain (there is an instanceof check in getRTCStatsReport) - Object.setPrototypeOf(track, LocalTrack.prototype); - - remoteParticipant.getTrackPublication = vi - .fn() - .mockImplementation((source: Track.Source) => { - if (source === Track.Source.Camera) { - return { - track, - } as unknown as RemoteTrackPublication; - } else { - return undefined; - } - }); - - const videoDimensions$ = videoSizeFromParticipant$( - constant(remoteParticipant), - ); - - const publishedDimensions: { width: number; height: number }[] = []; - videoDimensions$.subscribe((dimensions) => { - if (dimensions) publishedDimensions.push(dimensions); - }); - - await flushPromises(); - - const dimension = publishedDimensions.pop(); - expect(dimension).toEqual({ width: 480, height: 640 }); - }); -}); diff --git a/src/utils/videoFit.ts b/src/utils/videoFit.ts index 39dc28c9f..2e9935c6a 100644 --- a/src/utils/videoFit.ts +++ b/src/utils/videoFit.ts @@ -5,107 +5,26 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { combineLatest, map, type Observable, of, switchMap } from "rxjs"; -import { - type LocalParticipant, - type RemoteParticipant, - Track, -} from "livekit-client"; - -import { type ObservableScope } from "../state/ObservableScope.ts"; -import { type Behavior } from "../state/Behavior.ts"; -import { - observeInboundRtpStreamStats$, - observeOutboundRtpStreamStats$, -} from "../state/media/observeRtpStreamStats"; - -type Size = { - width: number; - height: number; -}; - /** * Computes the appropriate video fit mode ("cover" or "contain") based on the aspect ratios of the video and the tile. * - If the video and tile have the same orientation (both landscape or both portrait), we use "cover" to fill the tile, even if it means cropping. * - If the video and tile have different orientations, we use "contain" to ensure the entire video is visible, even if it means letterboxing (black bars). - * @param scope - the ObservableScope to create the Behavior in - * @param videoSize$ - an Observable of the video size (width and height) or undefined if the size is not yet known (no data yet received). - * @param tileSize$ - an Observable of the tile size (width and height) or undefined if the size is not yet known (not yet rendered). */ -export function videoFit$( - scope: ObservableScope, - videoSize$: Observable, - tileSize$: Observable, -): Behavior<"cover" | "contain"> { - const fit$ = combineLatest([videoSize$, tileSize$]).pipe( - map(([videoSize, tileSize]) => { - if (!videoSize || !tileSize) { - // If we don't have the sizes, default to cover to avoid black bars. - // This is a reasonable default as it will ensure the video fills the tile, even if it means cropping. - return "cover"; - } - if ( - videoSize.width === 0 || - videoSize.height === 0 || - tileSize.width === 0 || - tileSize.height === 0 - ) { - // If we have invalid sizes (e.g. width or height is 0), default to cover to avoid black bars. - return "cover"; - } - const videoAspectRatio = videoSize.width / videoSize.height; - const tileAspectRatio = tileSize.width / tileSize.height; +export function autoVideoFit( + videoAspectRatio: number, + tileAspectRatio: number, +): "cover" | "contain" { + if (Number.isNaN(videoAspectRatio) || Number.isNaN(tileAspectRatio)) { + // If we have invalid sizes (e.g. width or height is 0), default to cover to avoid black bars. + return "cover"; + } - // If video is landscape (ratio > 1) and tile is portrait (ratio < 1) or vice versa, - // we want to use "contain" (fit) mode to avoid excessive cropping - const videoIsLandscape = videoAspectRatio > 1; - const tileIsLandscape = tileAspectRatio > 1; + // If video is landscape (ratio > 1) and tile is portrait (ratio < 1) or vice versa, + // we want to use "contain" (fit) mode to avoid excessive cropping + const videoIsLandscape = videoAspectRatio > 1; + const tileIsLandscape = tileAspectRatio > 1; - // If the orientations are the same, use the cover mode (Preserves the aspect ratio, and the image fills the container.) - // If they're not the same orientation, use the contain mode (Preserves the aspect ratio, but the image is letterboxed - black bars- to fit within the container.) - return videoIsLandscape === tileIsLandscape ? "cover" : "contain"; - }), - ); - - return scope.behavior(fit$, "cover"); -} - -/** - * Helper function to get the video size from a participant. - * It observes the participant's video track stats and extracts the frame width and height. - * @param participant$ - an Observable of a LocalParticipant or RemoteParticipant, or null if no participant is selected. - * @returns an Observable of the video size (width and height) or undefined if the size cannot be determined. - */ -export function videoSizeFromParticipant$( - participant$: Observable, -): Observable<{ width: number; height: number } | undefined> { - return participant$ - .pipe( - // If we have a participant, observe their video track stats. If not, return undefined. - switchMap((p) => { - if (!p) return of(undefined); - if (p.isLocal) { - return observeOutboundRtpStreamStats$(p, Track.Source.Camera); - } else { - return observeInboundRtpStreamStats$(p, Track.Source.Camera); - } - }), - ) - .pipe( - // Extract the frame width and height from the stats. If we don't have valid stats, return undefined. - map((stats) => { - if (!stats) return undefined; - if ( - // For video tracks, frameWidth and frameHeight should be numbers. If they're not, we can't determine the size. - typeof stats.frameWidth !== "number" || - typeof stats.frameHeight !== "number" - ) { - return undefined; - } - return { - width: stats.frameWidth, - height: stats.frameHeight, - }; - }), - ); + // If the orientations are the same, use the cover mode (Preserves the aspect ratio, and the image fills the container.) + // If they're not the same orientation, use the contain mode (Preserves the aspect ratio, but the image is letterboxed - black bars- to fit within the container.) + return videoIsLandscape === tileIsLandscape ? "cover" : "contain"; } From 8aead74ab94031ddde3f203721a0c1b4a3fb3f86 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 4 Aug 2026 19:15:48 +0200 Subject: [PATCH 2/4] Performance: Lower the RTP stats refresh interval Because we no longer use it for computing the fit to frame setting; it's only for developer tools. --- src/state/media/observeRtpStreamStats.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/state/media/observeRtpStreamStats.ts b/src/state/media/observeRtpStreamStats.ts index 5edc28ef7..d1640382a 100644 --- a/src/state/media/observeRtpStreamStats.ts +++ b/src/state/media/observeRtpStreamStats.ts @@ -32,9 +32,7 @@ export function observeRtpStreamStats$( > { return combineLatest([ observeTrackReference$(participant, source), - // The update frequency is high because we use this value to update the PiP orientation and the fit/fill video tile props based on that - // We want it to be responsive. For just the debug tools 1s would be sufficient. - interval(350).pipe(startWith(0)), + interval(1000).pipe(startWith(0)), ]).pipe( switchMap(async ([trackReference]) => { const track = trackReference?.publication?.track; From 33fc8997f8e5c85be340659064c078a0b589ee67 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 4 Aug 2026 19:21:33 +0200 Subject: [PATCH 3/4] Performance: Use a shared timer for polling RTP stats This is now only relevant in case the user has enabled the developer option to show advanced media statistics, but still an easy performance fix. --- src/state/media/observeRtpStreamStats.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/state/media/observeRtpStreamStats.ts b/src/state/media/observeRtpStreamStats.ts index d1640382a..afb0da62d 100644 --- a/src/state/media/observeRtpStreamStats.ts +++ b/src/state/media/observeRtpStreamStats.ts @@ -19,10 +19,15 @@ import { startWith, switchMap, map, + share, } from "rxjs"; import { observeTrackReference$ } from "../observeTrackReference"; +// Use a shared timer for all the stats observers so that we don't clog up the +// event loop with hundreds of timers in large calls +const refreshStats$ = interval(1000).pipe(share()); + export function observeRtpStreamStats$( participant: Participant, source: Track.Source, @@ -32,7 +37,7 @@ export function observeRtpStreamStats$( > { return combineLatest([ observeTrackReference$(participant, source), - interval(1000).pipe(startWith(0)), + refreshStats$.pipe(startWith(0)), ]).pipe( switchMap(async ([trackReference]) => { const track = trackReference?.publication?.track; From 64a5b2ee5c791974461a40472f6b25f55fd1accf Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 17 Aug 2026 11:19:43 +0200 Subject: [PATCH 4/4] Clarify how an invalid width and height manifest --- src/utils/videoFit.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/videoFit.ts b/src/utils/videoFit.ts index 2e9935c6a..16d7cb843 100644 --- a/src/utils/videoFit.ts +++ b/src/utils/videoFit.ts @@ -15,7 +15,8 @@ export function autoVideoFit( tileAspectRatio: number, ): "cover" | "contain" { if (Number.isNaN(videoAspectRatio) || Number.isNaN(tileAspectRatio)) { - // If we have invalid sizes (e.g. width or height is 0), default to cover to avoid black bars. + // If we have invalid sizes (e.g. useMeasure returns 0×0 on an initial render), + // default to cover to avoid black bars. return "cover"; }