Compare commits

...

22 Commits

Author SHA1 Message Date
Robin
b554fa63d1 Performance: Isolate tile store debug info in its own component
It's very uncommon to have this debug option enabled, and yet it currently causes the footer to re-render on every layout update, which is a small but avoidable cost.
2026-08-17 16:45:26 +02:00
Robin
b5bcb9aeed Merge pull request #4160 from alexmaras/room-joining/assume-public
Room joining: if join_rule is absent, assume public
2026-08-17 14:40:07 +02:00
Timo
3d3e6dcdcc Merge pull request #4171 from element-hq/toger5/fix-unclose-able-error-screen
Fix: error screen cannot be closed in embedded mode.
2026-08-17 14:16:37 +02:00
Timo K.
43ce68e80a Fix: error screen cannot be closed in embedded mode. 2026-08-17 12:08:30 +02:00
Robin
6d5f698357 Merge pull request #4151 from element-hq/more-more-performance
Performance: Fit video to frame without polling RTP stats
2026-08-17 11:36:37 +02:00
Robin
64a5b2ee5c Clarify how an invalid width and height manifest 2026-08-17 11:19:43 +02:00
Robin
c0d5f2d470 Merge branch 'livekit' into more-more-performance 2026-08-17 11:12:17 +02:00
Robin
4a5141406e Merge pull request #4152 from element-hq/more-more-more-performance
Performance: Disable animations when there are a large number of tiles
2026-08-17 11:11:39 +02:00
Robin
b036d1c0c0 Merge pull request #4169 from element-hq/renovate/major-compound
Update dependency @vector-im/compound-web to v10
2026-08-17 09:25:18 +02:00
Robin
0cfe6cb03d Merge pull request #4168 from element-hq/renovate/livekit-components
Update LiveKit components
2026-08-17 09:21:33 +02:00
renovate[bot]
8be8cf08bf Update ghcr.io/element-hq/element-web:develop Docker digest to abd6473 (#4165)
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
2026-08-17 09:19:40 +02:00
renovate[bot]
9a772131b6 Update dependency @vector-im/compound-web to v10 2026-08-17 00:38:58 +00:00
renovate[bot]
1cdd5d8db0 Update LiveKit components 2026-08-17 00:38:39 +00:00
renovate[bot]
fc9e13c922 Update ghcr.io/element-hq/element-web:develop Docker digest to 6176839 (#4163)
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
2026-08-14 12:49:32 +02:00
Robin
580897730a Merge pull request #4150 from element-hq/more-performance
Performance: Avoid re-rendering context menus so often
2026-08-12 17:30:13 +02:00
Alex Maras
1bb342e9e1 Room joining: if join_rule is absent, assume public 2026-08-11 21:24:38 +08:00
Robin
c90d1d8dda Expand GridTile test coverage 2026-08-05 12:08:18 +02:00
Robin
dd59519a85 Performance: Disable animations when there are a large number of tiles
I don't believe this affects the cost of running the useSprings hook, but it at least skips the animation frames that would otherwise follow, so it's worth a try.
2026-08-05 11:49:55 +02:00
Robin
33fc8997f8 Performance: Use a shared timer for polling RTP stats
This is now only relevant in case the user has enabled the developer option to show advanced media statistics, but still an easy performance fix.
2026-08-05 11:18:25 +02:00
Robin
8aead74ab9 Performance: Lower the RTP stats refresh interval
Because we no longer use it for computing the fit to frame setting; it's only for developer tools.
2026-08-05 11:18:22 +02:00
Robin
1e128c4da0 Performance: Fit video to frame without polling RTP stats
It turns out that the timers which repeatedly poll the RTP stats of each video track all run independently of each other and can add up to a small but constant sink of CPU. Meanwhile we can replace these timers with HTMLVideoElement 'resize' event listeners, which is way more efficient and reacts instantly to orientation changes.
2026-08-05 11:18:22 +02:00
Robin
e1df71817a Performance: Avoid re-rendering context menus so often
A change in a tile's speaking indicator could cause its entire tree of context menu components to re-render, which is expensive. Isolating the behavior subscriptions in their own component avoids this.
2026-08-04 17:57:25 +02:00
16 changed files with 292 additions and 428 deletions

View File

@@ -13,7 +13,7 @@ services:
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
element-web:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:f25f10f06646858a60be461ff0141992083786fcb817e3cb03521481d4d43033
image: ghcr.io/element-hq/element-web:develop@sha256:af44e1e18bdd363a3332a577c3af66a9dde4ad352508a42ee5012c4a9528b30d
element-web-1:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:f25f10f06646858a60be461ff0141992083786fcb817e3cb03521481d4d43033
image: ghcr.io/element-hq/element-web:develop@sha256:af44e1e18bdd363a3332a577c3af66a9dde4ad352508a42ee5012c4a9528b30d

View File

@@ -77,7 +77,7 @@
"@typescript-eslint/utils": "^8.61.0",
"@use-gesture/react": "^10.2.11",
"@vector-im/compound-design-tokens": "^10.0.0",
"@vector-im/compound-web": "^9.3.0",
"@vector-im/compound-web": "^10.0.0",
"@vitejs/plugin-react": "^6.0.2",
"@vitest/browser-playwright": "^4.1.5",
"@vitest/coverage-v8": "^4.0.18",

35
pnpm-lock.yaml generated
View File

@@ -39,10 +39,10 @@ importers:
version: 11.7.12
'@livekit/components-core':
specifier: ^0.12.0
version: 0.12.14(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)
version: 0.12.15(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)
'@livekit/components-react':
specifier: ^2.0.0
version: 2.9.23(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(tslib@2.8.1)
version: 2.9.24(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(tslib@2.8.1)
'@livekit/track-processors':
specifier: ^0.7.1
version: 0.7.2(@types/dom-mediacapture-transform@0.1.11)(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))
@@ -134,8 +134,8 @@ importers:
specifier: ^10.0.0
version: 10.2.3(@types/react@19.2.17)(react@19.2.8)
'@vector-im/compound-web':
specifier: ^9.3.0
version: 9.9.0(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(@vector-im/compound-design-tokens@10.2.3(@types/react@19.2.17)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
specifier: ^10.0.0
version: 10.0.1(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(@vector-im/compound-design-tokens@10.2.3(@types/react@19.2.17)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
'@vitejs/plugin-react':
specifier: ^6.0.2
version: 6.0.5(@rolldown/plugin-babel@0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@7.29.7)(rolldown@1.1.5)(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(sass@1.102.0)(terser@5.46.1)(yaml@2.9.0)))(babel-plugin-react-compiler@1.0.0)(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(sass@1.102.0)(terser@5.46.1)(yaml@2.9.0))
@@ -1159,15 +1159,15 @@ packages:
'@jridgewell/trace-mapping@0.3.31':
resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==}
'@livekit/components-core@0.12.14':
resolution: {integrity: sha512-6OKP/1Ok2fCZewDLKd3SzaTb7KvfZl/6hjggI+TgUdhp0Y7HBg3+tHA7YmhZRc0BO8wIyVy4VgTPn7qkLHt2nQ==}
'@livekit/components-core@0.12.15':
resolution: {integrity: sha512-bDkK+jkPMgyT4a5lZmdcuc/hZ8fZUrXW+qQPNHVeSpXCBMzk5q9QWzx+HPtTsPJ8hnncWGfoT81RnXHBsy+Z8g==}
engines: {node: '>=18'}
peerDependencies:
livekit-client: ^2.20.1
tslib: ^2.6.2
'@livekit/components-react@2.9.23':
resolution: {integrity: sha512-clO+0g/u3YBpuOvnAjUSAJBH/o7w+RpUAseswjiSML9rHrXlTUhwBNm8LPk+qN5GOU3A6lzsNnFYVpUHUBVOkg==}
'@livekit/components-react@2.9.24':
resolution: {integrity: sha512-qw5Oy1EfPg1f/xrKYrFielSrmtxY8BZ9mDt6299G5aikotQIztmldBfTNosrVS7VSNit7qeYfOFSyKaML20W6Q==}
engines: {node: '>=18'}
peerDependencies:
'@livekit/krisp-noise-filter': ^0.2.12 || ^0.3.0 || ^0.4.0
@@ -3251,8 +3251,8 @@ packages:
react:
optional: true
'@vector-im/compound-web@9.9.0':
resolution: {integrity: sha512-GjAEhzB/mt9MA7Y1ySe6/AlW26vodDu4hvTi1ThB5rTrt4KAiHMpvk+T56KV5Lwo0VvBxAyHMtm/IY5tkyH12A==}
'@vector-im/compound-web@10.0.1':
resolution: {integrity: sha512-CT3aGMalPSf53o3JUbBF7EA10Rfp+auftIuBwOUSna+hfZ/zPxGLTFcuCuOJ7L8hrzijM+V4I8L0PUkfyOczpg==}
peerDependencies:
'@fontsource/inconsolata': ^5
'@fontsource/inter': ^5
@@ -4462,6 +4462,9 @@ packages:
jose@6.2.4:
resolution: {integrity: sha512-N8acGzVsQy6M/fjFcxtysNc4Q379TcM5dM/qKkNtsHFji88yANnXTr7BLeP75iPnFwBfQzM/jg2BZ9+HZrHCZA==}
jose@6.2.9:
resolution: {integrity: sha512-XrchZOFZUl/T3vTwRe8XK+cJrGtMF4th1ARnDfwbBXFKThGhlsxEE4Zu03AD/bjJSt/9jT/mxrOCkJWOg77aPA==}
js-tokens@10.0.0:
resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==}
@@ -7061,7 +7064,7 @@ snapshots:
'@jridgewell/resolve-uri': 3.1.2
'@jridgewell/sourcemap-codec': 1.5.5
'@livekit/components-core@0.12.14(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)':
'@livekit/components-core@0.12.15(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)':
dependencies:
'@floating-ui/dom': 1.7.6
livekit-client: 2.21.0(@types/dom-mediacapture-record@1.0.22)
@@ -7069,12 +7072,12 @@ snapshots:
rxjs: 7.8.2
tslib: 2.8.1
'@livekit/components-react@2.9.23(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(tslib@2.8.1)':
'@livekit/components-react@2.9.24(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(tslib@2.8.1)':
dependencies:
'@livekit/components-core': 0.12.14(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)
'@livekit/components-core': 0.12.15(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)
clsx: 2.1.1
events: 3.3.0
jose: 6.2.4
jose: 6.2.9
livekit-client: 2.21.0(@types/dom-mediacapture-record@1.0.22)
react: 19.2.8
react-dom: 19.2.8(react@19.2.8)
@@ -8713,7 +8716,7 @@ snapshots:
'@types/react': 19.2.17
react: 19.2.8
'@vector-im/compound-web@9.9.0(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(@vector-im/compound-design-tokens@10.2.3(@types/react@19.2.17)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
'@vector-im/compound-web@10.0.1(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(@vector-im/compound-design-tokens@10.2.3(@types/react@19.2.17)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
dependencies:
'@floating-ui/react': 0.27.20(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
'@floating-ui/react-dom': 2.1.9(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
@@ -10052,6 +10055,8 @@ snapshots:
jose@6.2.4: {}
jose@6.2.9: {}
js-tokens@10.0.0: {}
js-tokens@4.0.0: {}

View File

@@ -26,6 +26,7 @@ import {
MediaMuteAndSwitchButton,
type MenuOptions,
} from "./MediaMuteAndSwitchButton";
import { type Behavior } from "../state/Behavior";
import { type ViewModel } from "../state/ViewModel";
import { useBehavior } from "../useBehavior";
import { type LayoutSwitchViewModel } from "../state/LayoutSwitchViewModel";
@@ -135,7 +136,6 @@ export const CallFooter: FC<FooterProps> = ({
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
const hangup = useBehavior(vm.hangup$);
const debugTileLayout = useBehavior(vm.debugTileLayout$);
const tileStoreGeneration = useBehavior(vm.tileStoreGeneration$);
const videoOptions = useBehavior(vm.videoOptions$);
const selectedVideo = useBehavior(vm.selectedVideo$);
const audioOptions = useBehavior(vm.audioOptions$);
@@ -283,7 +283,9 @@ export const CallFooter: FC<FooterProps> = ({
/>
</>
)}
{debugTileLayout ? `Tiles generation: ${tileStoreGeneration}` : undefined}
{debugTileLayout ? (
<TilesDebugInfo generation$={vm.tileStoreGeneration$} />
) : undefined}
</div>
);
@@ -316,3 +318,14 @@ export const CallFooter: FC<FooterProps> = ({
</div>
);
};
interface TilesDebugInfoProps {
generation$: Behavior<number | undefined>;
}
// Isolated in its own component since the layout generation updates frequently
// and we can avoid re-rendering the footer this way
const TilesDebugInfo: FC<TilesDebugInfoProps> = ({ generation$ }) => {
const generation = useBehavior(generation$);
return `Tiles generation: ${generation}`;
};

View File

@@ -41,6 +41,8 @@ import { TileWrapper } from "./TileWrapper";
import { usePrefersReducedMotion } from "../usePrefersReducedMotion";
import { useInitial } from "../useInitial";
const MAX_ANIMATED_TILES = 50; // Capped for performance reasons
interface Rect {
x: number;
y: number;
@@ -285,7 +287,6 @@ export function Grid<
const [visibleTilesCallback, setVisibleTilesCallback] =
useState<VisibleTilesCallback | null>(null);
const tiles = useInitial(() => new Map<string, Tile<TileModel>>());
const prefersReducedMotion = usePrefersReducedMotion();
const Slot: FC<SlotProps<TileModel>> = useMemo(
() =>
@@ -372,6 +373,10 @@ export function Grid<
// react-spring's imperative API during gestures to improve responsiveness
const dragState = useRef<DragState | null>(null);
// If true, disables animations
const immediate =
usePrefersReducedMotion() || placedTiles.length > MAX_ANIMATED_TILES;
const [tileTransitions, springRef] = useTransition(
placedTiles,
() => ({
@@ -389,9 +394,9 @@ export function Grid<
y,
width,
height,
immediate: prefersReducedMotion,
immediate,
}),
enter: { opacity: 1, scale: 1, immediate: prefersReducedMotion },
enter: { opacity: 1, scale: 1, immediate },
update: ({
id,
x,
@@ -406,9 +411,9 @@ export function Grid<
y,
width,
height,
immediate: prefersReducedMotion,
immediate,
},
leave: { opacity: 0, scale: 0, immediate: prefersReducedMotion },
leave: { opacity: 0, scale: 0, immediate },
config: { mass: 0.7, tension: 252, friction: 25 },
}),
// react-spring's types are bugged and can't infer the spring type
@@ -441,8 +446,7 @@ export function Grid<
y: tile.y,
width: tile.width,
height: tile.height,
immediate:
prefersReducedMotion || ((key): boolean => key === "zIndex"),
immediate: immediate || ((key): boolean => key === "zIndex"),
// Allow the tile's position to settle before pushing its
// z-index back down
delay: (key): number => (key === "zIndex" ? 500 : 0),
@@ -453,7 +457,7 @@ export function Grid<
x: tileX,
y: tileY,
immediate:
prefersReducedMotion ||
immediate ||
((key): boolean =>
key === "zIndex" || key === "x" || key === "y"),
},

View File

@@ -555,6 +555,9 @@ export const GroupCallView: FC<Props> = ({
}}
onError={(_error) => {
if (rtcSession.isJoined()) onLeft("error");
// If there is an error we need to be able to close the widget. This is done in `onLeft` as well
// We need it here explicitly in case rtcSession.isJoined is false.
void widget?.api.setAlwaysOnScreen(false);
}}
>
{body}

View File

@@ -280,7 +280,7 @@ export const useLoadGroupCall = (
);
}
if (
roomSummary === undefined ||
roomSummary?.join_rule === undefined ||
roomSummary.join_rule === JoinRule.Public
) {
room = await client.joinRoom(roomId, {

View File

@@ -33,7 +33,6 @@ import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel";
import { type ObservableScope } from "../ObservableScope";
import { showConnectionStats } from "../../settings/settings";
import { observeRtpStreamStats$ } from "./observeRtpStreamStats";
import { videoFit$, videoSizeFromParticipant$ } from "../../utils/videoFit.ts";
/**
* A participant's user media (i.e. their microphone and camera feed).
@@ -47,7 +46,6 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel {
speaking$: Behavior<boolean>;
audioEnabled$: Behavior<boolean>;
videoEnabled$: Behavior<boolean>;
videoFit$: Behavior<"cover" | "contain">;
videoOrientation$: Behavior<"landscape" | "portrait">;
toggleCropVideo: () => void;
/**
@@ -63,12 +61,9 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel {
RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
>;
/**
* Set the target dimensions of the HTML element (final dimension after anim).
* This can be used to determine the best video fit (fit to frame / keep ratio).
* @param targetWidth - The target width of the HTML element displaying the video.
* @param targetHeight - The target height of the HTML element displaying the video.
* Set the aspect ratio of the video track to determine the orientation.
*/
setTargetDimensions: (targetWidth: number, targetHeight: number) => void;
setVideoAspectRatio: (ratio: number) => void;
}
export interface BaseUserMediaInputs extends Omit<
@@ -98,14 +93,8 @@ export function createBaseUserMedia(
),
);
const toggleCropVideo$ = new Subject<void>();
const videoAspectRatio$ = new BehaviorSubject(NaN);
// The target size of the video element, used to determine the best video fit.
// The target size is the final size of the HTML element after any animations have completed.
const targetSize$ = new BehaviorSubject<
{ width: number; height: number } | undefined
>(undefined);
const videoSize$ = videoSizeFromParticipant$(participant$);
return {
...createMemberMedia(scope, {
...inputs,
@@ -132,13 +121,11 @@ export function createBaseUserMedia(
media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)),
),
videoOrientation$: scope.behavior(
videoSize$.pipe(
map((s) => (s ? s.width / s.height : 1)),
videoAspectRatio$.pipe(
map((aspect) => (aspect > 1 ? "landscape" : "portrait")),
),
"portrait",
),
videoFit$: videoFit$(scope, videoSize$, targetSize$),
toggleCropVideo: () => toggleCropVideo$.next(),
rtcBackendIdentity,
handRaised$,
@@ -162,8 +149,6 @@ export function createBaseUserMedia(
return observeRtpStreamStats$(p, Track.Source.Camera, statsType);
}),
),
setTargetDimensions: (targetWidth: number, targetHeight: number): void => {
targetSize$.next({ width: targetWidth, height: targetHeight });
},
setVideoAspectRatio: (ratio) => videoAspectRatio$.next(ratio),
};
}

View File

@@ -19,10 +19,15 @@ import {
startWith,
switchMap,
map,
share,
} from "rxjs";
import { observeTrackReference$ } from "../observeTrackReference";
// Use a shared timer for all the stats observers so that we don't clog up the
// event loop with hundreds of timers in large calls
const refreshStats$ = interval(1000).pipe(share());
export function observeRtpStreamStats$(
participant: Participant,
source: Track.Source,
@@ -32,9 +37,7 @@ export function observeRtpStreamStats$(
> {
return combineLatest([
observeTrackReference$(participant, source),
// The update frequency is high because we use this value to update the PiP orientation and the fit/fill video tile props based on that
// We want it to be responsive. For just the debug tools 1s would be sufficient.
interval(350).pipe(startWith(0)),
refreshStats$.pipe(startWith(0)),
]).pipe(
switchMap(async ([trackReference]) => {
const track = trackReference?.publication?.track;
@@ -69,12 +72,3 @@ export function observeInboundRtpStreamStats$(
map((x) => x as RTCInboundRtpStreamStats | undefined),
);
}
export function observeOutboundRtpStreamStats$(
participant: Participant,
source: Track.Source,
): Observable<RTCOutboundRtpStreamStats | undefined> {
return observeRtpStreamStats$(participant, source, "outbound-rtp").pipe(
map((x) => x as RTCOutboundRtpStreamStats | undefined),
);
}

View File

@@ -5,7 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RemoteTrackPublication } from "livekit-client";
import {
type LocalTrackPublication,
type RemoteTrackPublication,
} from "livekit-client";
import { test, expect } from "vitest";
import { act, render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
@@ -17,6 +20,9 @@ import {
mockRtcMembership,
mockRemoteMedia,
mockRemoteParticipant,
mockLocalMedia,
mockLocalParticipant,
mockMediaDevices,
} from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
@@ -54,7 +60,7 @@ const callVm = {
handsRaised$: constant({}),
} as Partial<CallViewModel> as CallViewModel;
test("GridTile is accessible", async () => {
test("GridTile displays remote media", async () => {
const vm = mockRemoteMedia(
mockRtcMembership("@alice:example.org", "AAAA"),
{
@@ -88,6 +94,40 @@ test("GridTile is accessible", async () => {
screen.getByText("Alice");
});
test("GridTile displays local media", async () => {
const vm = mockLocalMedia(
mockRtcMembership("@alice:example.org", "AAAA"),
{
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
mockLocalParticipant({
getTrackPublication: () =>
({}) as Partial<LocalTrackPublication> as LocalTrackPublication,
}),
mockMediaDevices({}),
);
const { container } = render(
<ReactionsSenderProvider vm={callVm} rtcSession={fakeRtcSession}>
<GridTile
vm={new GridTileViewModel(constant(vm))}
onOpenProfile={() => {}}
targetWidth={300}
targetHeight={200}
showSpeakingIndicators
showNameTags
showRingingStatus
showOutline
focusable
/>
</ReactionsSenderProvider>,
);
expect(await axe(container)).toHaveNoViolations();
// Name should be visible
screen.getByText("Alice");
});
test("GridTile displays ringing media", async () => {
const pickupState$ = new BehaviorSubject<
RingingMediaViewModel["pickupState$"]["value"]

View File

@@ -11,9 +11,9 @@ import {
type ReactNode,
type Ref,
useCallback,
useEffect,
useRef,
useState,
useMemo,
} from "react";
import { type animated } from "@react-spring/web";
import classNames from "classnames";
@@ -91,7 +91,6 @@ const RingingMediaTile: FC<RingingMediaTileProps> = ({
}
avatarStyle="translucent"
videoEnabled={false}
videoFit="cover"
mirror={false}
{...props}
/>
@@ -105,20 +104,22 @@ interface UserMediaTileProps extends TileProps {
playbackMuted: boolean;
waitingForMedia?: boolean;
primaryButton?: ReactNode;
menuStart?: ReactNode;
menuEnd?: ReactNode;
focusUrl: string | undefined;
}
const UserMediaTile: FC<UserMediaTileProps> = ({
/**
* A user media tile without a context menu.
*/
// The context menu is kept separate from this component for performance
// reasons (c.f. UserMediaTile)
const UserMediaTileInner: FC<UserMediaTileProps & { menu: ReactNode }> = ({
ref,
vm,
showSpeakingIndicators,
playbackMuted,
waitingForMedia,
primaryButton,
menuStart,
menuEnd,
menu,
className,
focusUrl,
displayName,
@@ -141,19 +142,11 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
const audioEnabled = useBehavior(vm.audioEnabled$);
const videoEnabled = useBehavior(vm.videoEnabled$);
const speaking = useBehavior(vm.speaking$);
const videoFit = useBehavior(vm.videoFit$);
const rtcBackendIdentity = vm.rtcBackendIdentity;
const handRaised = useBehavior(vm.handRaised$);
const reaction = useBehavior(vm.reaction$);
// Whenever bounds change, inform the viewModel
useEffect(() => {
if (targetWidth > 0 && targetHeight > 0) {
vm.setTargetDimensions(targetWidth, targetHeight);
}
}, [targetWidth, targetHeight, vm]);
const AudioIcon = playbackMuted
? VolumeOffSolidIcon
: audioEnabled
@@ -166,31 +159,32 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
: t("microphone_off");
const [menuOpen, setMenuOpen] = useState(false);
const menu = (
<>
{menuStart}
{/*
No additional menu item (used to be the manual fit to frame.
Placeholder for future menu items that should be placed here.
*/}
{menuEnd}
</>
const menuTrigger = useMemo(
() => (
<button
aria-label={t("common.options")}
tabIndex={focusable ? undefined : -1}
>
<OverflowHorizontalIcon aria-hidden width={20} height={20} />
</button>
),
[t, focusable],
);
const raisedHandOnClick = vm.local
? (): void => void toggleRaisedHand()
: undefined;
const raisedHandOnClick = useMemo(
() => (vm.local ? (): void => void toggleRaisedHand() : undefined),
[vm.local, toggleRaisedHand],
);
const showSpeaking = showSpeakingIndicators && speaking;
const tile = (
return (
<MediaView
ref={ref}
video={video}
userId={vm.userId}
unencryptedWarning={unencryptedWarning}
videoEnabled={videoEnabled}
videoFit={videoFit}
className={classNames(className, styles.tile, {
[styles.speaking]: showSpeaking,
[styles.handRaised]: !showSpeaking && handRaised,
@@ -213,14 +207,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
open={menuOpen}
onOpenChange={setMenuOpen}
title={displayName}
trigger={
<button
aria-label={t("common.options")}
tabIndex={focusable ? undefined : -1}
>
<OverflowHorizontalIcon aria-hidden width={20} height={20} />
</button>
}
trigger={menuTrigger}
side="left"
align="start"
>
@@ -233,6 +220,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
raisedHandOnClick={raisedHandOnClick}
waitingForMedia={waitingForMedia}
focusUrl={focusUrl}
setVideoAspectRatio={vm.setVideoAspectRatio}
audioStreamStats={audioStreamStats}
videoStreamStats={videoStreamStats}
rtcBackendIdentity={rtcBackendIdentity}
@@ -241,9 +229,37 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
{...props}
/>
);
};
/**
* A user media tile enhanced with a context menu.
*/
const UserMediaTile: FC<
UserMediaTileProps & { menuStart?: ReactNode; menuEnd?: ReactNode }
> = ({ menuStart, menuEnd, ...props }) => {
const menu = useMemo(
() => (
<>
{menuStart}
{/*
No additional menu item (used to be the manual fit to frame.
Placeholder for future menu items that should be placed here.
*/}
{menuEnd}
</>
),
[menuStart, menuEnd],
);
// ContextMenu is expensive to render, so we avoid subscribing to any
// frequently-changing behaviors here and instead keep them isolated in the
// UserMediaTileInner component
return (
<ContextMenu title={displayName} trigger={tile} hasAccessibleAlternative>
<ContextMenu
title={props.displayName}
trigger={<UserMediaTileInner {...props} menu={menu} />}
hasAccessibleAlternative
>
{menu}
</ContextMenu>
);
@@ -279,6 +295,29 @@ const LocalUserMediaTile: FC<LocalUserMediaTileProps> = ({
[vm, latestAlwaysShow],
);
const menuStart = useMemo(
() => (
<ToggleMenuItem
Icon={VisibilityOnIcon}
label={t("video_tile.always_show")}
checked={alwaysShow}
onSelect={onSelectAlwaysShow}
/>
),
[t, alwaysShow, onSelectAlwaysShow],
);
const menuEnd = useMemo(
() =>
onOpenProfile && (
<MenuItem
Icon={UserProfileIcon}
label={t("common.profile")}
onSelect={onOpenProfile}
/>
),
[t, onOpenProfile],
);
return (
<UserMediaTile
ref={ref}
@@ -297,23 +336,8 @@ const LocalUserMediaTile: FC<LocalUserMediaTileProps> = ({
</button>
)
}
menuStart={
<ToggleMenuItem
Icon={VisibilityOnIcon}
label={t("video_tile.always_show")}
checked={alwaysShow}
onSelect={onSelectAlwaysShow}
/>
}
menuEnd={
onOpenProfile && (
<MenuItem
Icon={UserProfileIcon}
label={t("common.profile")}
onSelect={onOpenProfile}
/>
)
}
menuStart={menuStart}
menuEnd={menuEnd}
focusable={focusable}
focusUrl={focusUrl}
{...props}

View File

@@ -33,7 +33,6 @@ describe("MediaView", () => {
const baseProps: ComponentProps<typeof MediaView> = {
displayName: "some name",
videoEnabled: true,
videoFit: "contain",
targetWidth: 300,
targetHeight: 200,
mirror: false,

View File

@@ -7,7 +7,13 @@ Please see LICENSE in the repository root for full details.
import { type TrackReferenceOrPlaceholder } from "@livekit/components-core";
import { animated } from "@react-spring/web";
import { type FC, type ComponentProps, type ReactNode } from "react";
import {
type FC,
type ComponentProps,
type ReactNode,
type SyntheticEvent,
useState,
} from "react";
import { useTranslation } from "react-i18next";
import classNames from "classnames";
import { VideoTrack } from "@livekit/components-react";
@@ -26,6 +32,7 @@ import { type ReactionOption } from "../reactions";
import { ReactionIndicator } from "../reactions/ReactionIndicator";
import { RTCConnectionStats } from "../RTCConnectionStats";
import videoPlaceholder from "../graphics/video-placeholder.gif";
import { autoVideoFit } from "../utils/videoFit";
interface Props extends ComponentProps<typeof animated.div> {
className?: string;
@@ -33,7 +40,11 @@ interface Props extends ComponentProps<typeof animated.div> {
targetWidth: number;
targetHeight: number;
video: TrackReferenceOrPlaceholder | undefined;
videoFit: "cover" | "contain";
/**
* How to fit the video content inside the tile. When undefined, MediaView
* chooses a smart default based on the aspect ratios of the tile and video.
*/
videoFit?: "cover" | "contain";
mirror: boolean;
soundWaves?: boolean;
userId: string;
@@ -55,8 +66,15 @@ interface Props extends ComponentProps<typeof animated.div> {
audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
rtcBackendIdentity?: string;
// The focus url, mainly for debugging purposes
/**
* The focus url, mainly for debugging purposes.
*/
focusUrl?: string;
/**
* Called whenever the aspect ratio of the video content becomes known or
* otherwise changes.
*/
setVideoAspectRatio?: (ratio: number) => void;
}
export const MediaView: FC<Props> = ({
@@ -89,6 +107,7 @@ export const MediaView: FC<Props> = ({
videoStreamStats,
rtcBackendIdentity,
focusUrl,
setVideoAspectRatio: setTheirVideoAspectRatio,
...props
}) => {
const { t } = useTranslation();
@@ -100,6 +119,22 @@ export const MediaView: FC<Props> = ({
(soundWaves === undefined ? 0.5 : 0.38),
);
const [videoAspectRatio, setOurVideoAspectRatio] = useState<number>(NaN);
const tileAspectRatio = targetWidth / targetHeight;
// Propagate video dimensions
const setVideoAspectRatio = (ratio: number) => {
setOurVideoAspectRatio(ratio);
setTheirVideoAspectRatio?.(ratio);
};
const videoRef = (el: HTMLVideoElement | null) => {
if (el !== null) setVideoAspectRatio(el.videoWidth / el.videoHeight);
};
const onResize = (ev: SyntheticEvent<HTMLVideoElement>) =>
setVideoAspectRatio(
ev.currentTarget.videoWidth / ev.currentTarget.videoHeight,
);
const warnings = unencryptedWarning && (
<Tooltip
label={t("common.unencrypted")}
@@ -126,7 +161,9 @@ export const MediaView: FC<Props> = ({
ref={ref}
data-testid="videoTile"
data-video-enabled={video && videoEnabled}
data-video-fit={videoFit}
data-video-fit={
videoFit ?? autoVideoFit(videoAspectRatio, tileAspectRatio)
}
data-background={background}
{...props}
>
@@ -158,6 +195,8 @@ export const MediaView: FC<Props> = ({
// Set the placeholder to a small transparent image. (On Android web
// views the default poster image is particularly ugly.)
poster={videoPlaceholder}
ref={videoRef}
onResize={onResize}
/>
)}
</div>

View File

@@ -68,6 +68,7 @@ interface SpotlightItemBaseProps {
background: "solid" | "transparent";
focusable: boolean;
"aria-hidden"?: boolean;
setVideoAspectRatio?: (ratio: number) => void;
}
interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps {
@@ -77,7 +78,6 @@ interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps {
}
interface SpotlightUserMediaItemBaseProps extends SpotlightMemberMediaItemBaseProps {
videoFit: "contain" | "cover";
videoEnabled: boolean;
soundWaves: boolean | undefined;
}
@@ -120,20 +120,12 @@ const SpotlightUserMediaItem: FC<SpotlightUserMediaItemProps> = ({
targetHeight,
...props
}) => {
const videoFit = useBehavior(vm.videoFit$);
const videoEnabled = useBehavior(vm.videoEnabled$);
const speaking = useBehavior(vm.speaking$);
// Whenever target bounds change, inform the viewModel
useEffect(() => {
if (targetWidth > 0 && targetHeight > 0) {
vm.setTargetDimensions(targetWidth, targetHeight);
}
}, [targetWidth, targetHeight, vm]);
const baseProps: SpotlightUserMediaItemBaseProps &
RefAttributes<HTMLDivElement> = {
videoFit,
setVideoAspectRatio: vm.setVideoAspectRatio,
videoEnabled,
soundWaves: props.background === "transparent" ? speaking : undefined,
targetWidth,
@@ -227,7 +219,6 @@ const SpotlightRingingMediaItem: FC<SpotlightRingingMediaItemProps> = ({
}
avatarStyle="translucent"
videoEnabled={false}
videoFit="cover"
mirror={false}
{...props}
/>

View File

@@ -5,259 +5,106 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, test, vi } from "vitest";
import {
LocalTrack,
type LocalTrackPublication,
type RemoteTrackPublication,
Track,
} from "livekit-client";
import { describe, expect, test } from "vitest";
import { ObservableScope } from "../state/ObservableScope";
import { videoFit$, videoSizeFromParticipant$ } from "./videoFit";
import { constant } from "../state/Behavior";
import {
flushPromises,
mockLocalParticipant,
mockRemoteParticipant,
} from "./test";
import { autoVideoFit } from "./videoFit";
describe("videoFit$ defaults", () => {
test.each([
{
videoSize: { width: 1920, height: 1080 },
tileSize: undefined,
videoAspectRatio: 1920 / 1080,
tileAspectRatio: NaN,
},
{
videoSize: { width: 1080, height: 1920 },
tileSize: undefined,
videoAspectRatio: 1080 / 1920,
tileAspectRatio: NaN,
},
{
videoSize: undefined,
tileSize: { width: 1920, height: 1080 },
videoAspectRatio: NaN,
tileAspectRatio: 1920 / 1080,
},
{
videoSize: undefined,
tileSize: { width: 1080, height: 1920 },
videoAspectRatio: NaN,
tileAspectRatio: 1080 / 1920,
},
])(
"videoFit$ returns `cover` when videoSize is $videoSize and tileSize is $tileSize",
({ videoSize, tileSize }) => {
const scope = new ObservableScope();
const videoSize$ = constant(videoSize);
const tileSize$ = constant(tileSize);
const fit = videoFit$(scope, videoSize$, tileSize$);
expect(fit.value).toBe("cover");
},
"videoFit$ returns `cover` when videoAspectRatio is $videoAspectRatio and tileAspectRatio is $tileAspectRatio",
({ videoAspectRatio, tileAspectRatio }) =>
expect(autoVideoFit(videoAspectRatio, tileAspectRatio)).toBe("cover"),
);
});
const VIDEO_480_L = { width: 640, height: 480 };
const VIDEO_720_L = { width: 1280, height: 720 };
const VIDEO_1080_L = { width: 1920, height: 1080 };
const VIDEO_480_L = 640 / 480;
const VIDEO_720_L = 1280 / 720;
const VIDEO_1080_L = 1920 / 1080;
// Some sizes from real world testing, which don't match the standard video sizes exactly
const TILE_SIZE_1_L = { width: 180, height: 135 };
const TILE_SIZE_3_P = { width: 379, height: 542 };
const TILE_SIZE_4_L = { width: 957, height: 542 };
const TILE_SIZE_1_L = 180 / 135;
const TILE_SIZE_3_P = 379 / 542;
const TILE_SIZE_4_L = 957 / 542;
// This is the size of an iPhone Xr in portrait mode
const TILE_SIZE_5_P = { width: 414, height: 896 };
const TILE_SIZE_5_P = 414 / 896;
export function invertSize(size: { width: number; height: number }): {
width: number;
height: number;
} {
return {
width: size.height,
height: size.width,
};
function inverse(ratio: number): number {
return 1 / ratio;
}
test.each([
{
videoSize: VIDEO_480_L,
tileSize: TILE_SIZE_1_L,
videoAspectRatio: VIDEO_480_L,
tileAspectRatio: TILE_SIZE_1_L,
expected: "cover",
},
{
videoSize: invertSize(VIDEO_480_L),
tileSize: TILE_SIZE_1_L,
videoAspectRatio: inverse(VIDEO_480_L),
tileAspectRatio: TILE_SIZE_1_L,
expected: "contain",
},
{
videoSize: VIDEO_720_L,
tileSize: TILE_SIZE_4_L,
videoAspectRatio: VIDEO_720_L,
tileAspectRatio: TILE_SIZE_4_L,
expected: "cover",
},
{
videoSize: invertSize(VIDEO_720_L),
tileSize: TILE_SIZE_4_L,
videoAspectRatio: inverse(VIDEO_720_L),
tileAspectRatio: TILE_SIZE_4_L,
expected: "contain",
},
{
videoSize: invertSize(VIDEO_1080_L),
tileSize: TILE_SIZE_3_P,
videoAspectRatio: inverse(VIDEO_1080_L),
tileAspectRatio: TILE_SIZE_3_P,
expected: "cover",
},
{
videoSize: VIDEO_1080_L,
tileSize: TILE_SIZE_5_P,
videoAspectRatio: VIDEO_1080_L,
tileAspectRatio: TILE_SIZE_5_P,
expected: "contain",
},
{
videoSize: invertSize(VIDEO_1080_L),
tileSize: TILE_SIZE_5_P,
videoAspectRatio: inverse(VIDEO_1080_L),
tileAspectRatio: TILE_SIZE_5_P,
expected: "cover",
},
{
// square video
videoSize: { width: 400, height: 400 },
tileSize: VIDEO_480_L,
videoAspectRatio: 400 / 400,
tileAspectRatio: VIDEO_480_L,
expected: "contain",
},
{
// Should default to cover if the initial size is 0:0.
// Or else it will cause a flash of "contain" mode until the real size is loaded, which can be jarring.
videoSize: VIDEO_480_L,
tileSize: { width: 0, height: 0 },
videoAspectRatio: VIDEO_480_L,
tileAspectRatio: 0 / 0,
expected: "cover",
},
{
videoSize: { width: 0, height: 0 },
tileSize: VIDEO_480_L,
videoAspectRatio: 0 / 0,
tileAspectRatio: VIDEO_480_L,
expected: "cover",
},
])(
"videoFit$ returns $expected when videoSize is $videoSize and tileSize is $tileSize",
({ videoSize, tileSize, expected }) => {
const scope = new ObservableScope();
const videoSize$ = constant(videoSize);
const tileSize$ = constant(tileSize);
const fit = videoFit$(scope, videoSize$, tileSize$);
expect(fit.value).toBe(expected);
},
"videoFit$ returns $expected when videoAspectRatio is $videoAspectRatio and tileAspectRatio is $tileAspectRatio",
({ videoAspectRatio, tileAspectRatio, expected }) =>
expect(autoVideoFit(videoAspectRatio, tileAspectRatio)).toBe(expected),
);
describe("extracting video size from participant stats", () => {
function createMockRtpStats(
isInbound: boolean,
props: Partial<RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats> = {},
): RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats {
const baseStats = {
id: "mock-stats-id",
timestamp: Date.now(),
type: isInbound ? "inbound-rtp" : "outbound-rtp",
kind: "video",
...props,
};
return baseStats as RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
}
test("get stats for local user", async () => {
const localParticipant = mockLocalParticipant({
identity: "@local:example.org:AAAAAA",
});
const mockReport: RTCStatsReport = new Map([
[
"OT01V639885149",
createMockRtpStats(false, {
frameWidth: 1280,
frameHeight: 720,
}),
],
]);
const track = {
source: Track.Source.Camera,
getRTCStatsReport: vi
.fn()
.mockImplementation(async () => Promise.resolve(mockReport)),
} as Partial<LocalTrack> as LocalTrack;
// Set up the prototype chain (there is an instanceof check in getRTCStatsReport)
Object.setPrototypeOf(track, LocalTrack.prototype);
localParticipant.getTrackPublication = vi
.fn()
.mockImplementation((source: Track.Source) => {
if (source === Track.Source.Camera) {
return {
track,
} as unknown as LocalTrackPublication;
} else {
return undefined;
}
});
const videoDimensions$ = videoSizeFromParticipant$(
constant(localParticipant),
);
const publishedDimensions: { width: number; height: number }[] = [];
videoDimensions$.subscribe((dimensions) => {
if (dimensions) publishedDimensions.push(dimensions);
});
await flushPromises();
const dimension = publishedDimensions.pop();
expect(dimension).toEqual({ width: 1280, height: 720 });
});
test("get stats for remote user", async () => {
// vi.useFakeTimers()
const remoteParticipant = mockRemoteParticipant({
identity: "@bob:example.org:AAAAAA",
});
const mockReport: RTCStatsReport = new Map([
[
"OT01V639885149",
createMockRtpStats(true, {
frameWidth: 480,
frameHeight: 640,
}),
],
]);
const track = {
source: Track.Source.Camera,
getRTCStatsReport: vi
.fn()
.mockImplementation(async () => Promise.resolve(mockReport)),
} as Partial<LocalTrack> as LocalTrack;
// Set up the prototype chain (there is an instanceof check in getRTCStatsReport)
Object.setPrototypeOf(track, LocalTrack.prototype);
remoteParticipant.getTrackPublication = vi
.fn()
.mockImplementation((source: Track.Source) => {
if (source === Track.Source.Camera) {
return {
track,
} as unknown as RemoteTrackPublication;
} else {
return undefined;
}
});
const videoDimensions$ = videoSizeFromParticipant$(
constant(remoteParticipant),
);
const publishedDimensions: { width: number; height: number }[] = [];
videoDimensions$.subscribe((dimensions) => {
if (dimensions) publishedDimensions.push(dimensions);
});
await flushPromises();
const dimension = publishedDimensions.pop();
expect(dimension).toEqual({ width: 480, height: 640 });
});
});

View File

@@ -5,107 +5,27 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { combineLatest, map, type Observable, of, switchMap } from "rxjs";
import {
type LocalParticipant,
type RemoteParticipant,
Track,
} from "livekit-client";
import { type ObservableScope } from "../state/ObservableScope.ts";
import { type Behavior } from "../state/Behavior.ts";
import {
observeInboundRtpStreamStats$,
observeOutboundRtpStreamStats$,
} from "../state/media/observeRtpStreamStats";
type Size = {
width: number;
height: number;
};
/**
* Computes the appropriate video fit mode ("cover" or "contain") based on the aspect ratios of the video and the tile.
* - If the video and tile have the same orientation (both landscape or both portrait), we use "cover" to fill the tile, even if it means cropping.
* - If the video and tile have different orientations, we use "contain" to ensure the entire video is visible, even if it means letterboxing (black bars).
* @param scope - the ObservableScope to create the Behavior in
* @param videoSize$ - an Observable of the video size (width and height) or undefined if the size is not yet known (no data yet received).
* @param tileSize$ - an Observable of the tile size (width and height) or undefined if the size is not yet known (not yet rendered).
*/
export function videoFit$(
scope: ObservableScope,
videoSize$: Observable<Size | undefined>,
tileSize$: Observable<Size | undefined>,
): Behavior<"cover" | "contain"> {
const fit$ = combineLatest([videoSize$, tileSize$]).pipe(
map(([videoSize, tileSize]) => {
if (!videoSize || !tileSize) {
// If we don't have the sizes, default to cover to avoid black bars.
// This is a reasonable default as it will ensure the video fills the tile, even if it means cropping.
return "cover";
}
if (
videoSize.width === 0 ||
videoSize.height === 0 ||
tileSize.width === 0 ||
tileSize.height === 0
) {
// If we have invalid sizes (e.g. width or height is 0), default to cover to avoid black bars.
return "cover";
}
const videoAspectRatio = videoSize.width / videoSize.height;
const tileAspectRatio = tileSize.width / tileSize.height;
export function autoVideoFit(
videoAspectRatio: number,
tileAspectRatio: number,
): "cover" | "contain" {
if (Number.isNaN(videoAspectRatio) || Number.isNaN(tileAspectRatio)) {
// If we have invalid sizes (e.g. useMeasure returns 0×0 on an initial render),
// default to cover to avoid black bars.
return "cover";
}
// If video is landscape (ratio > 1) and tile is portrait (ratio < 1) or vice versa,
// we want to use "contain" (fit) mode to avoid excessive cropping
const videoIsLandscape = videoAspectRatio > 1;
const tileIsLandscape = tileAspectRatio > 1;
// If video is landscape (ratio > 1) and tile is portrait (ratio < 1) or vice versa,
// we want to use "contain" (fit) mode to avoid excessive cropping
const videoIsLandscape = videoAspectRatio > 1;
const tileIsLandscape = tileAspectRatio > 1;
// If the orientations are the same, use the cover mode (Preserves the aspect ratio, and the image fills the container.)
// If they're not the same orientation, use the contain mode (Preserves the aspect ratio, but the image is letterboxed - black bars- to fit within the container.)
return videoIsLandscape === tileIsLandscape ? "cover" : "contain";
}),
);
return scope.behavior(fit$, "cover");
}
/**
* Helper function to get the video size from a participant.
* It observes the participant's video track stats and extracts the frame width and height.
* @param participant$ - an Observable of a LocalParticipant or RemoteParticipant, or null if no participant is selected.
* @returns an Observable of the video size (width and height) or undefined if the size cannot be determined.
*/
export function videoSizeFromParticipant$(
participant$: Observable<LocalParticipant | RemoteParticipant | null>,
): Observable<{ width: number; height: number } | undefined> {
return participant$
.pipe(
// If we have a participant, observe their video track stats. If not, return undefined.
switchMap((p) => {
if (!p) return of(undefined);
if (p.isLocal) {
return observeOutboundRtpStreamStats$(p, Track.Source.Camera);
} else {
return observeInboundRtpStreamStats$(p, Track.Source.Camera);
}
}),
)
.pipe(
// Extract the frame width and height from the stats. If we don't have valid stats, return undefined.
map((stats) => {
if (!stats) return undefined;
if (
// For video tracks, frameWidth and frameHeight should be numbers. If they're not, we can't determine the size.
typeof stats.frameWidth !== "number" ||
typeof stats.frameHeight !== "number"
) {
return undefined;
}
return {
width: stats.frameWidth,
height: stats.frameHeight,
};
}),
);
// If the orientations are the same, use the cover mode (Preserves the aspect ratio, and the image fills the container.)
// If they're not the same orientation, use the contain mode (Preserves the aspect ratio, but the image is letterboxed - black bars- to fit within the container.)
return videoIsLandscape === tileIsLandscape ? "cover" : "contain";
}