From 1de8d93b4bdb5090f4112aec0af49736b4e00820 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 25 Feb 2026 15:47:25 +0100 Subject: [PATCH 01/30] feat: video auto fit based on video stream size --- src/state/MediaViewModel.test.ts | 15 -- src/state/MediaViewModel.ts | 63 ++++++-- src/tile/GridTile.tsx | 41 ++--- src/tile/SpotlightTile.tsx | 22 ++- src/utils/videoFit.test.ts | 251 +++++++++++++++++++++++++++++++ src/utils/videoFit.ts | 94 ++++++++++++ 6 files changed, 441 insertions(+), 45 deletions(-) create mode 100644 src/utils/videoFit.test.ts create mode 100644 src/utils/videoFit.ts diff --git a/src/state/MediaViewModel.test.ts b/src/state/MediaViewModel.test.ts index 92868216..a7bbb571 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/MediaViewModel.test.ts @@ -92,21 +92,6 @@ test("control a participant's volume", () => { }); }); -test("toggle fit/contain for a participant's video", () => { - const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); - withTestScheduler(({ expectObservable, schedule }) => { - schedule("-ab|", { - a: () => vm.toggleFitContain(), - b: () => vm.toggleFitContain(), - }); - expectObservable(vm.cropVideo$).toBe("abc", { - a: true, - b: false, - c: true, - }); - }); -}); - test("local media remembers whether it should always be shown", () => { const vm1 = createLocalMedia( rtcMembership, diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 3da69c46..57b0428a 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -43,6 +43,8 @@ import { switchMap, throttleTime, distinctUntilChanged, + concat, + take, } from "rxjs"; import { alwaysShowSelf } from "../settings/settings"; @@ -55,6 +57,7 @@ import { platform } from "../Platform"; import { type MediaDevices } from "./MediaDevices"; import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; +import { videoFit$, videoSizeFromParticipant$ } from "../utils/videoFit.ts"; export function observeTrackReference$( participant: Participant, @@ -67,6 +70,10 @@ export function observeTrackReference$( ); } +/** + * Helper function to observe the RTC stats for a given participant and track source. + * It polls the stats every second and emits the latest stats object. + */ export function observeRtpStreamStats$( participant: Participant, source: Track.Source, @@ -76,7 +83,9 @@ export function observeRtpStreamStats$( > { return combineLatest([ observeTrackReference$(participant, source), - interval(1000).pipe(startWith(0)), + // This is used also for detecting video orientation, + // and we want that to be more responsive than the connection stats, so we poll more frequently at the start. + concat(interval(300).pipe(take(3)), interval(1000)).pipe(startWith(0)), ]).pipe( switchMap(async ([trackReference]) => { const track = trackReference?.publication?.track; @@ -90,7 +99,6 @@ export function observeRtpStreamStats$( if (!report) { return undefined; } - for (const v of report.values()) { if (v.type === type) { return v; @@ -103,6 +111,13 @@ export function observeRtpStreamStats$( ); } +/** + * Helper function to observe the inbound RTP stats for a given participant and track source. + * To be used for remote participants' audio and video tracks. + * It polls the stats every second and emits the latest stats object. + * @param participant - The LiveKit participant whose track stats we want to observe. + * @param source - The source of the track (e.g. Track.Source.Camera or Track.Source.Microphone). + */ export function observeInboundRtpStreamStats$( participant: Participant, source: Track.Source, @@ -112,6 +127,13 @@ export function observeInboundRtpStreamStats$( ); } +/** + * Helper function to observe the outbound RTP stats for a given participant and track source. + * To be used for the local participant's audio and video tracks. + * It polls the stats every second and emits the latest stats object. + * @param participant - The LiveKit participant whose track stats we want to observe. + * @param source - The source of the track (e.g. Track.Source.Camera or Track.Source.Microphone). + */ export function observeOutboundRtpStreamStats$( participant: Participant, source: Track.Source, @@ -263,7 +285,6 @@ abstract class BaseMediaViewModel { protected readonly participant$: Observable< LocalParticipant | RemoteParticipant | null >, - encryptionSystem: EncryptionSystem, audioSource: AudioSource, videoSource: VideoSource, @@ -397,13 +418,12 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { return this._videoEnabled$; } - private readonly _cropVideo$ = new BehaviorSubject(true); /** - * Whether the tile video should be contained inside the tile or be cropped to fit. + * Whether the tile video should be contained inside the tile (video-fit contain) or be cropped to fit (video-fit cover). */ - public readonly cropVideo$: Behavior = this._cropVideo$; + public readonly videoFit$: Behavior<"cover" | "contain">; - public constructor( + protected constructor( scope: ObservableScope, id: string, userId: string, @@ -443,10 +463,12 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { this._videoEnabled$ = this.scope.behavior( media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), ); - } - public toggleFitContain(): void { - this._cropVideo$.next(!this._cropVideo$.value); + this.videoFit$ = videoFit$( + this.scope, + videoSizeFromParticipant$(participant$), + this.actualSize$, + ); } public get local(): boolean { @@ -456,9 +478,28 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { public abstract get audioStreamStats$(): Observable< RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined >; + public abstract get videoStreamStats$(): Observable< RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined >; + + private readonly _actualSize$ = new BehaviorSubject< + { width: number; height: number } | undefined + >(undefined); + public readonly actualSize$ = this._actualSize$.asObservable(); + + /** + * Set the actual dimensions of the html element. + * This can be used to determine the best video fit (fit to frame / keep ratio). + * @param width - The actual width of the html element displaying the video. + * @param height - The actual height of the html element displaying the video. + */ + public setActualDimensions(width: number, height: number): void { + this._actualSize$.next({ + width, + height, + }); + } } /** @@ -616,6 +657,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { // This private field is used to override the value from the superclass private __speaking$: Behavior; + public get speaking$(): Behavior { return this.__speaking$; } @@ -661,6 +703,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { // This private field is used to override the value from the superclass private __videoEnabled$: Behavior; + public get videoEnabled$(): Behavior { return this.__videoEnabled$; } diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 92262f05..ad158db1 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -11,6 +11,7 @@ import { type ReactNode, type Ref, useCallback, + useEffect, useRef, useState, } from "react"; @@ -26,7 +27,6 @@ import { VolumeOffIcon, VisibilityOnIcon, UserProfileIcon, - ExpandIcon, VolumeOffSolidIcon, SwitchCameraSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; @@ -37,6 +37,7 @@ import { Menu, } from "@vector-im/compound-web"; import { useObservableEagerState } from "observable-hooks"; +import useMeasure from "react-use-measure"; import styles from "./GridTile.module.css"; import { @@ -105,18 +106,26 @@ const UserMediaTile: FC = ({ const audioEnabled = useBehavior(vm.audioEnabled$); const videoEnabled = useBehavior(vm.videoEnabled$); const speaking = useBehavior(vm.speaking$); - const cropVideo = useBehavior(vm.cropVideo$); - const onSelectFitContain = useCallback( - (e: Event) => { - e.preventDefault(); - vm.toggleFitContain(); - }, - [vm], - ); + const videoFit = useBehavior(vm.videoFit$); + const rtcBackendIdentity = vm.rtcBackendIdentity; const handRaised = useBehavior(vm.handRaised$); const reaction = useBehavior(vm.reaction$); + // We need to keep track of the tile size. + // We use this to get the tile ratio, and compare it to the video ratio to decide + // whether to fit the video to frame or keep the ratio. + const [measureRef, bounds] = useMeasure(); + // There is already a ref being passed in, so we need to merge it with the measureRef. + const tileRef = useMergedRefs(ref, measureRef); + + // Whenever bounds change, inform the viewModel + useEffect(() => { + if (bounds.width > 0 && bounds.height > 0) { + vm.setActualDimensions(bounds.width, bounds.height); + } + }, [bounds.width, bounds.height, vm]); + const AudioIcon = locallyMuted ? VolumeOffSolidIcon : audioEnabled @@ -132,12 +141,10 @@ const UserMediaTile: FC = ({ const menu = ( <> {menuStart} - + {/* + No additional menu item (used to be the manual fit to frame. + Placeholder for future menu items that should be placed here. + */} {menuEnd} ); @@ -150,13 +157,13 @@ const UserMediaTile: FC = ({ const tile = ( = ({ vm, ...props }) => { - const cropVideo = useBehavior(vm.cropVideo$); + const videoFit = useBehavior(vm.videoFit$); const baseProps: SpotlightUserMediaItemBaseProps & RefAttributes = { - videoFit: cropVideo ? "cover" : "contain", + videoFit, ...props, }; @@ -147,7 +148,22 @@ const SpotlightItem: FC = ({ "aria-hidden": ariaHidden, }) => { const ourRef = useRef(null); - const ref = useMergedRefs(ourRef, theirRef); + + // We need to keep track of the tile size. + // We use this to get the tile ratio, and compare it to the video ratio to decide + // whether to fit the video to frame or keep the ratio. + const [measureRef, bounds] = useMeasure(); + + // Whenever bounds change, inform the viewModel + useEffect(() => { + if (bounds.width > 0 && bounds.height > 0) { + if (!(vm instanceof ScreenShareViewModel)) { + vm.setActualDimensions(bounds.width, bounds.height); + } + } + }, [bounds.width, bounds.height, vm]); + + const ref = useMergedRefs(ourRef, theirRef, measureRef); const focusUrl = useBehavior(vm.focusUrl$); const displayName = useBehavior(vm.displayName$); const mxcAvatarUrl = useBehavior(vm.mxcAvatarUrl$); diff --git a/src/utils/videoFit.test.ts b/src/utils/videoFit.test.ts new file mode 100644 index 00000000..9390e8d4 --- /dev/null +++ b/src/utils/videoFit.test.ts @@ -0,0 +1,251 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, test, vi } from "vitest"; +import { + LocalTrack, + type LocalTrackPublication, + type RemoteTrackPublication, + Track, +} from "livekit-client"; + +import { ObservableScope } from "../state/ObservableScope"; +import { videoFit$, videoSizeFromParticipant$ } from "./videoFit"; +import { constant } from "../state/Behavior"; +import { + flushPromises, + mockLocalParticipant, + mockRemoteParticipant, +} from "./test"; + +describe("videoFit$ defaults", () => { + test.each([ + { + videoSize: { width: 1920, height: 1080 }, + tileSize: undefined, + }, + { + videoSize: { width: 1080, height: 1920 }, + tileSize: undefined, + }, + { + videoSize: undefined, + tileSize: { width: 1920, height: 1080 }, + }, + { + videoSize: undefined, + tileSize: { width: 1080, height: 1920 }, + }, + ])( + "videoFit$ returns `cover` when videoSize is $videoSize and tileSize is $tileSize", + ({ videoSize, tileSize }) => { + const scope = new ObservableScope(); + const videoSize$ = constant(videoSize); + const tileSize$ = constant(tileSize); + + const fit = videoFit$(scope, videoSize$, tileSize$); + expect(fit.value).toBe("cover"); + }, + ); +}); + +const VIDEO_480_L = { width: 640, height: 480 }; +const VIDEO_720_L = { width: 1280, height: 720 }; +const VIDEO_1080_L = { width: 1920, height: 1080 }; + +// Some sizes from real world testing, which don't match the standard video sizes exactly +const TILE_SIZE_1_L = { width: 180, height: 135 }; +const TILE_SIZE_3_P = { width: 379, height: 542 }; +const TILE_SIZE_4_L = { width: 957, height: 542 }; +// This is the size of an iPhone Xr in portrait mode +const TILE_SIZE_5_P = { width: 414, height: 896 }; + +export function invertSize(size: { width: number; height: number }): { + width: number; + height: number; +} { + return { + width: size.height, + height: size.width, + }; +} + +test.each([ + { + videoSize: VIDEO_480_L, + tileSize: TILE_SIZE_1_L, + expected: "cover", + }, + { + videoSize: invertSize(VIDEO_480_L), + tileSize: TILE_SIZE_1_L, + expected: "contain", + }, + { + videoSize: VIDEO_720_L, + tileSize: TILE_SIZE_4_L, + expected: "cover", + }, + { + videoSize: invertSize(VIDEO_720_L), + tileSize: TILE_SIZE_4_L, + expected: "contain", + }, + { + videoSize: invertSize(VIDEO_1080_L), + tileSize: TILE_SIZE_3_P, + expected: "cover", + }, + { + videoSize: VIDEO_1080_L, + tileSize: TILE_SIZE_5_P, + expected: "contain", + }, + { + videoSize: invertSize(VIDEO_1080_L), + tileSize: TILE_SIZE_5_P, + expected: "cover", + }, + { + // square video + videoSize: { width: 400, height: 400 }, + tileSize: VIDEO_480_L, + expected: "contain", + }, +])( + "videoFit$ returns $expected when videoSize is $videoSize and tileSize is $tileSize", + ({ videoSize, tileSize, expected }) => { + const scope = new ObservableScope(); + const videoSize$ = constant(videoSize); + const tileSize$ = constant(tileSize); + + const fit = videoFit$(scope, videoSize$, tileSize$); + expect(fit.value).toBe(expected); + }, +); + +describe("extracting video size from participant stats", () => { + function createMockRtpStats( + isInbound: boolean, + props: Partial = {}, + ): RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats { + const baseStats = { + id: "mock-stats-id", + timestamp: Date.now(), + type: isInbound ? "inbound-rtp" : "outbound-rtp", + kind: "video", + ...props, + }; + + return baseStats as RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; + } + + test("get stats for local user", async () => { + const localParticipant = mockLocalParticipant({ + identity: "@local:example.org:AAAAAA", + }); + + const mockReport: RTCStatsReport = new Map([ + [ + "OT01V639885149", + createMockRtpStats(false, { + frameWidth: 1280, + frameHeight: 720, + }), + ], + ]); + + const track = { + source: Track.Source.Camera, + getRTCStatsReport: vi + .fn() + .mockImplementation(async () => Promise.resolve(mockReport)), + } as Partial as LocalTrack; + + // Set up the prototype chain (there is an instanceof check in getRTCStatsReport) + Object.setPrototypeOf(track, LocalTrack.prototype); + + localParticipant.getTrackPublication = vi + .fn() + .mockImplementation((source: Track.Source) => { + if (source === Track.Source.Camera) { + return { + track, + } as unknown as LocalTrackPublication; + } else { + return undefined; + } + }); + + const videoDimensions$ = videoSizeFromParticipant$( + constant(localParticipant), + ); + + const publishedDimensions: { width: number; height: number }[] = []; + videoDimensions$.subscribe((dimensions) => { + if (dimensions) publishedDimensions.push(dimensions); + }); + + await flushPromises(); + + const dimension = publishedDimensions.pop(); + expect(dimension).toEqual({ width: 1280, height: 720 }); + }); + + test("get stats for remote user", async () => { + // vi.useFakeTimers() + const remoteParticipant = mockRemoteParticipant({ + identity: "@bob:example.org:AAAAAA", + }); + + const mockReport: RTCStatsReport = new Map([ + [ + "OT01V639885149", + createMockRtpStats(true, { + frameWidth: 480, + frameHeight: 640, + }), + ], + ]); + + const track = { + source: Track.Source.Camera, + getRTCStatsReport: vi + .fn() + .mockImplementation(async () => Promise.resolve(mockReport)), + } as Partial as LocalTrack; + + // Set up the prototype chain (there is an instanceof check in getRTCStatsReport) + Object.setPrototypeOf(track, LocalTrack.prototype); + + remoteParticipant.getTrackPublication = vi + .fn() + .mockImplementation((source: Track.Source) => { + if (source === Track.Source.Camera) { + return { + track, + } as unknown as RemoteTrackPublication; + } else { + return undefined; + } + }); + + const videoDimensions$ = videoSizeFromParticipant$( + constant(remoteParticipant), + ); + + const publishedDimensions: { width: number; height: number }[] = []; + videoDimensions$.subscribe((dimensions) => { + if (dimensions) publishedDimensions.push(dimensions); + }); + + await flushPromises(); + + const dimension = publishedDimensions.pop(); + expect(dimension).toEqual({ width: 480, height: 640 }); + }); +}); diff --git a/src/utils/videoFit.ts b/src/utils/videoFit.ts new file mode 100644 index 00000000..fdd91be7 --- /dev/null +++ b/src/utils/videoFit.ts @@ -0,0 +1,94 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, map, type Observable, of, switchMap } from "rxjs"; +import { + type LocalParticipant, + type RemoteParticipant, + Track, +} from "livekit-client"; + +import { type ObservableScope } from "../state/ObservableScope.ts"; +import { type Behavior } from "../state/Behavior.ts"; +import { + observeInboundRtpStreamStats$, + observeOutboundRtpStreamStats$, +} from "../state/MediaViewModel.ts"; + +type Size = { + width: number; + height: number; +}; + +export function videoFit$( + scope: ObservableScope, + videoSize$: Observable, + tileSize$: Observable, +): Behavior<"cover" | "contain"> { + const fit$ = combineLatest([videoSize$, tileSize$]).pipe( + map(([videoSize, tileSize]) => { + if (!videoSize || !tileSize) { + // If we don't have the sizes, default to cover to avoid black bars. + // This is a reasonable default as it will ensure the video fills the tile, even if it means cropping. + return "cover"; + } + const videoAspectRatio = videoSize.width / videoSize.height; + const tileAspectRatio = tileSize.width / tileSize.height; + + // If video is landscape (ratio > 1) and tile is portrait (ratio < 1) or vice versa, + // we want to use "contain" (fit) mode to avoid excessive cropping + const videoIsLandscape = videoAspectRatio > 1; + const tileIsLandscape = tileAspectRatio > 1; + + // If the orientations are the same, use the cover mode (Preserves the aspect ratio, and the image fills the container.) + // If they're not the same orientation, use the contain mode (Preserves the aspect ratio, but the image is letterboxed - black bars- to fit within the container.) + return videoIsLandscape === tileIsLandscape ? "cover" : "contain"; + }), + ); + + return scope.behavior(fit$, "cover"); +} + +/** + * Helper function to get the video size from a participant. + * It observes the participant's video track stats and extracts the frame width and height. + * @param participant$ - an Observable of a LocalParticipant or RemoteParticipant, or null if no participant is selected. + * @returns an Observable of the video size (width and height) or undefined if the size cannot be determined. + */ +export function videoSizeFromParticipant$( + participant$: Observable, +): Observable<{ width: number; height: number } | undefined> { + return participant$ + .pipe( + // If we have a participant, observe their video track stats. If not, return undefined. + switchMap((p) => { + if (!p) return of(undefined); + if (p.isLocal) { + return observeOutboundRtpStreamStats$(p, Track.Source.Camera); + } else { + return observeInboundRtpStreamStats$(p, Track.Source.Camera); + } + }), + ) + .pipe( + // Extract the frame width and height from the stats. If we don't have valid stats, return undefined. + map((stats) => { + if (!stats) return undefined; + if ( + // For video tracks, frameWidth and frameHeight should be numbers. If they're not, we can't determine the size. + typeof stats.frameWidth !== "number" || + typeof stats.frameHeight !== "number" + ) { + return undefined; + } + return { + width: stats.frameWidth, + height: stats.frameHeight, + }; + }), + ); +} From 15aa67ebb996119a577ffa0907b2742c1208b1cd Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 26 Feb 2026 16:46:08 +0100 Subject: [PATCH 02/30] remove unused fit to frame translation --- locales/en/app.json | 1 - 1 file changed, 1 deletion(-) diff --git a/locales/en/app.json b/locales/en/app.json index 0b0ac7b4..9a85478f 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -250,7 +250,6 @@ "video_tile": { "always_show": "Always show", "camera_starting": "Video loading...", - "change_fit_contain": "Fit to frame", "collapse": "Collapse", "expand": "Expand", "mute_for_me": "Mute for me", From ae8b1f840f188d6d428cd5f360dae17238a711d1 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 26 Feb 2026 17:02:43 +0100 Subject: [PATCH 03/30] add missing mocking --- src/tile/GridTile.test.tsx | 18 +++++++++++++++++- src/tile/SpotlightTile.test.tsx | 18 +++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 9bc0efb2..060119ef 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { type RemoteTrackPublication } from "livekit-client"; -import { test, expect } from "vitest"; +import { test, expect, beforeAll } from "vitest"; import { render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; @@ -28,6 +28,22 @@ global.IntersectionObserver = class MockIntersectionObserver { public disconnect(): void {} } as unknown as typeof IntersectionObserver; +// Mock ResizeObserver as it is needed by the useMeasure hook used in the GridTile, but is not implemented in JSDOM. +// We just need to mock it with empty methods as we don't need to test its functionality here. +beforeAll(() => { + window.ResizeObserver = class ResizeObserver { + public observe(): void { + // do nothing + } + public unobserve(): void { + // do nothing + } + public disconnect(): void { + // do nothing + } + }; +}); + test("GridTile is accessible", async () => { const vm = createRemoteMedia( mockRtcMembership("@alice:example.org", "AAAA"), diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index 981c0369..441b74c4 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { test, expect, vi } from "vitest"; +import { test, expect, vi, beforeAll } from "vitest"; import { isInaccessible, render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import userEvent from "@testing-library/user-event"; @@ -27,6 +27,22 @@ global.IntersectionObserver = class MockIntersectionObserver { public unobserve(): void {} } as unknown as typeof IntersectionObserver; +// Mock ResizeObserver as it is needed by the useMeasure hook used in the SpotlightTile, but is not implemented in JSDOM. +// We just need to mock it with empty methods as we don't need to test its functionality here. +beforeAll(() => { + window.ResizeObserver = class ResizeObserver { + public observe(): void { + // do nothing + } + public unobserve(): void { + // do nothing + } + public disconnect(): void { + // do nothing + } + }; +}); + test("SpotlightTile is accessible", async () => { const vm1 = createRemoteMedia( mockRtcMembership("@alice:example.org", "AAAA"), From adc329a7e7538fd4e5545ce61d5c3dde2206ff66 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 2 Mar 2026 14:41:47 +0100 Subject: [PATCH 04/30] post merge fix --- src/tile/SpotlightTile.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index ba99f826..a0b1309b 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -183,7 +183,7 @@ const SpotlightItem: FC = ({ // Whenever bounds change, inform the viewModel useEffect(() => { if (bounds.width > 0 && bounds.height > 0) { - if (!(vm instanceof ScreenShareViewModel)) { + if (vm.type != "screen share") { vm.setActualDimensions(bounds.width, bounds.height); } } From 5165e95d82c8cb10f84883e35f2f20dc50a609cf Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 2 Mar 2026 15:38:43 +0100 Subject: [PATCH 05/30] fix: default to cover is size are 0 --- src/utils/videoFit.test.ts | 12 ++++++++++++ src/utils/videoFit.ts | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/src/utils/videoFit.test.ts b/src/utils/videoFit.test.ts index 9390e8d4..5068526b 100644 --- a/src/utils/videoFit.test.ts +++ b/src/utils/videoFit.test.ts @@ -116,6 +116,18 @@ test.each([ tileSize: VIDEO_480_L, expected: "contain", }, + { + // Should default to cover if the initial size is 0:0. + // Or else it will cause a flash of "contain" mode until the real size is loaded, which can be jarring. + videoSize: VIDEO_480_L, + tileSize: { width: 0, height: 0 }, + expected: "cover", + }, + { + videoSize: { width: 0, height: 0 }, + tileSize: VIDEO_480_L, + expected: "cover", + }, ])( "videoFit$ returns $expected when videoSize is $videoSize and tileSize is $tileSize", ({ videoSize, tileSize, expected }) => { diff --git a/src/utils/videoFit.ts b/src/utils/videoFit.ts index c7b18f03..5f2cc2ce 100644 --- a/src/utils/videoFit.ts +++ b/src/utils/videoFit.ts @@ -36,6 +36,15 @@ export function videoFit$( // This is a reasonable default as it will ensure the video fills the tile, even if it means cropping. return "cover"; } + if ( + videoSize.width === 0 || + videoSize.height === 0 || + tileSize.width === 0 || + tileSize.height === 0 + ) { + // If we have invalid sizes (e.g. width or height is 0), default to cover to avoid black bars. + return "cover"; + } const videoAspectRatio = videoSize.width / videoSize.height; const tileAspectRatio = tileSize.width / tileSize.height; From 2cf4a38c1743ad2895c5c4ebfba8e1b7d474c76e Mon Sep 17 00:00:00 2001 From: Jake Janicke Date: Tue, 3 Mar 2026 16:47:41 -0600 Subject: [PATCH 06/30] Add volume logic to RemoteScreenShareViewModel Signed-off-by: Jake Janicke --- src/state/media/RemoteScreenShareViewModel.ts | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/state/media/RemoteScreenShareViewModel.ts b/src/state/media/RemoteScreenShareViewModel.ts index eff6d9c1..78fc0494 100644 --- a/src/state/media/RemoteScreenShareViewModel.ts +++ b/src/state/media/RemoteScreenShareViewModel.ts @@ -6,8 +6,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type RemoteParticipant } from "livekit-client"; -import { map } from "rxjs"; +import { Track, type RemoteParticipant } from "livekit-client"; +import { map, of, switchMap } from "rxjs"; import { type Behavior } from "../Behavior"; import { @@ -16,13 +16,17 @@ import { createBaseScreenShare, } from "./ScreenShareViewModel"; import { type ObservableScope } from "../ObservableScope"; +import { createVolumeControls, type VolumeControls } from "../VolumeControls"; +import { observeTrackReference$ } from "../observeTrackReference"; -export interface RemoteScreenShareViewModel extends BaseScreenShareViewModel { +export interface RemoteScreenShareViewModel + extends BaseScreenShareViewModel, VolumeControls { local: false; /** * Whether this screen share's video should be displayed. */ videoEnabled$: Behavior; + audioEnabled$: Behavior; } export interface RemoteScreenShareInputs extends BaseScreenShareInputs { @@ -36,9 +40,30 @@ export function createRemoteScreenShare( ): RemoteScreenShareViewModel { return { ...createBaseScreenShare(scope, inputs), + ...createVolumeControls(scope, { + pretendToBeDisconnected$, + sink$: scope.behavior( + inputs.participant$.pipe( + map( + (p) => (volume) => + p?.setVolume(volume, Track.Source.ScreenShareAudio), + ), + ), + ), + }), local: false, videoEnabled$: scope.behavior( pretendToBeDisconnected$.pipe(map((disconnected) => !disconnected)), ), + audioEnabled$: scope.behavior( + inputs.participant$.pipe( + switchMap((p) => + p + ? observeTrackReference$(p, Track.Source.ScreenShareAudio) + : of(null), + ), + map(Boolean), + ), + ), }; } From 72520dbb3f8f9a08701c207d79047e38160f4e4a Mon Sep 17 00:00:00 2001 From: Jake Janicke Date: Wed, 4 Mar 2026 01:41:16 -0600 Subject: [PATCH 07/30] Add screen share volume slider UI Signed-off-by: Jake Janicke --- locales/en/app.json | 1 + src/tile/SpotlightTile.module.css | 26 +++++++++ src/tile/SpotlightTile.tsx | 94 +++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+) diff --git a/locales/en/app.json b/locales/en/app.json index 0b0ac7b4..6ec4859e 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -256,6 +256,7 @@ "mute_for_me": "Mute for me", "muted_for_me": "Muted for me", "volume": "Volume", + "screen_share_volume": "Screen share volume", "waiting_for_media": "Waiting for media..." } } diff --git a/src/tile/SpotlightTile.module.css b/src/tile/SpotlightTile.module.css index 622496d2..43c3d53c 100644 --- a/src/tile/SpotlightTile.module.css +++ b/src/tile/SpotlightTile.module.css @@ -108,6 +108,32 @@ Please see LICENSE in the repository root for full details. z-index: 1; } +.volumeSlider { + width: 100%; +} + +/* Disable the hover effect for the screen share volume menu button */ +.volumeMenuItem:hover { + background: transparent; + cursor: default; +} + +.volumeMenuItem { + gap: var(--cpd-space-3x); +} + +.menuMuteButton { + background: none; + border: none; + cursor: pointer; + display: flex; +} + +/* Make icons change color with the theme */ +.menuMuteButton > svg { + color: var(--cpd-color-icon-primary); +} + .expand > svg { display: block; color: var(--cpd-color-icon-primary); diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 75c69479..93a1fbd9 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -20,6 +20,8 @@ import { CollapseIcon, ChevronLeftIcon, ChevronRightIcon, + VolumeOffIcon, + VolumeOnIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import { animated } from "@react-spring/web"; import { type Observable, map } from "rxjs"; @@ -27,6 +29,7 @@ import { useObservableRef } from "observable-hooks"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; import { type TrackReferenceOrPlaceholder } from "@livekit/components-core"; +import { Menu, MenuItem } from "@vector-im/compound-web"; import FullScreenMaximiseIcon from "../icons/FullScreenMaximise.svg?react"; import FullScreenMinimiseIcon from "../icons/FullScreenMinimise.svg?react"; @@ -45,6 +48,8 @@ import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel"; import { type RemoteScreenShareViewModel } from "../state/media/RemoteScreenShareViewModel"; import { type MediaViewModel } from "../state/media/MediaViewModel"; +import { Slider } from "../Slider"; +import { constant } from "../state/Behavior"; interface SpotlightItemBaseProps { ref?: Ref; @@ -260,6 +265,33 @@ export const SpotlightTile: FC = ({ const visibleIndex = media.findIndex((vm) => vm.id === visibleId); const canGoBack = visibleIndex > 0; const canGoToNext = visibleIndex !== -1 && visibleIndex < media.length - 1; + const currentMedia = media[visibleIndex]; + // isScreenShare only needs to check "audioEnabled$" but I wanted to be more specific + // just in case more models are added in the future, since screen shares always have video + const isScreenShare = + currentMedia && + "audioEnabled$" in currentMedia && + "videoEnabled$" in currentMedia; + + const hasAudio$ = useBehavior( + isScreenShare && currentMedia?.audioEnabled$ + ? currentMedia.audioEnabled$ + : constant(false), + ); + const isLocalScreenShare = isScreenShare && currentMedia.local; + const screenShareLocallyMuted = useBehavior( + isScreenShare + ? (currentMedia as RemoteScreenShareViewModel).playbackMuted$ + : constant(false), + ); + const ScreenShareVolumeIcon = screenShareLocallyMuted + ? VolumeOffIcon + : VolumeOnIcon; + const screenShareVolume = useBehavior( + isScreenShare + ? (currentMedia as RemoteScreenShareViewModel).playbackVolume$ + : constant(0), + ); const isFullscreen = useCallback((): boolean => { const rootElement = document.body; @@ -328,6 +360,7 @@ export const SpotlightTile: FC = ({ }, [latestVisibleId, latestMedia, setScrollToId]); const ToggleExpandIcon = expanded ? CollapseIcon : ExpandIcon; + const [openVolumeMenu, setOpenVolumeMenu] = useState(false); return ( = ({ ))}
+ {/* + Show volume slider only when the tile is a screenshare, has audio, + is in spotlight mode, and isn't your own screen share. + */} + {isScreenShare && + hasAudio$ && + onToggleExpanded && + !isLocalScreenShare && ( + + + + } + > + + + + ( + currentMedia as RemoteScreenShareViewModel + ).adjustPlaybackVolume(v) + } + onValueCommit={() => + ( + currentMedia as RemoteScreenShareViewModel + ).commitPlaybackVolume() + } + /> + + + )} + } + > + + + + + + ) + ); +}; + interface Props { ref?: Ref; vm: SpotlightTileViewModel; @@ -263,37 +332,9 @@ export const SpotlightTile: FC = ({ const latestMedia = useLatest(media); const latestVisibleId = useLatest(visibleId); const visibleIndex = media.findIndex((vm) => vm.id === visibleId); + const visibleMedia = media.at(visibleIndex); const canGoBack = visibleIndex > 0; const canGoToNext = visibleIndex !== -1 && visibleIndex < media.length - 1; - const currentMedia = media[visibleIndex]; - // only "audioEnabled$" needs to be checked but I wanted to be more specific just in - // case more models are added in the future, since screen shares always have video - const currentScreenShare = - currentMedia && - "audioEnabled$" in currentMedia && - "videoEnabled$" in currentMedia - ? (currentMedia as RemoteScreenShareViewModel) - : null; - - const isScreenShare = currentScreenShare != null; - - const hasAudio$ = useBehavior( - currentScreenShare?.audioEnabled$ ?? constant(false), - ); - - const isLocalScreenShare = currentScreenShare?.local ?? false; - - const screenShareLocallyMuted = useBehavior( - currentScreenShare?.playbackMuted$ ?? constant(false), - ); - - const ScreenShareVolumeIcon = screenShareLocallyMuted - ? VolumeOffIcon - : VolumeOnIcon; - - const screenShareVolume = useBehavior( - currentScreenShare?.playbackVolume$ ?? constant(0), - ); const isFullscreen = useCallback((): boolean => { const rootElement = document.body; @@ -362,7 +403,6 @@ export const SpotlightTile: FC = ({ }, [latestVisibleId, latestMedia, setScrollToId]); const ToggleExpandIcon = expanded ? CollapseIcon : ExpandIcon; - const [openVolumeMenu, setOpenVolumeMenu] = useState(false); return ( = ({ /> ))}
-
- {/* - Show volume slider only when the tile is a screenshare, has audio, - is in spotlight mode, and isn't your own screen share. - */} - {isScreenShare && - hasAudio$ && - onToggleExpanded && - !isLocalScreenShare && ( - - - - } - > - - - - ( - currentMedia as RemoteScreenShareViewModel - ).adjustPlaybackVolume(v) - } - onValueCommit={() => - ( - currentMedia as RemoteScreenShareViewModel - ).commitPlaybackVolume() - } - /> - - - )} - +
+ {visibleMedia?.type === "screen share" && !visibleMedia.local && ( + + )} + {platform === "desktop" && ( + + )} {onToggleExpanded && (
@@ -760,12 +726,13 @@ export const InCallView: FC = ({ const allConnections = useBehavior(vm.allConnections$); return ( + // The onClick handler here exists to control the visibility of the footer, + // and the footer is also viewable by moving focus into it, so this is fine. + // eslint-disable-next-line jsx-a11y/no-static-element-interactions, jsx-a11y/click-events-have-key-events
diff --git a/src/room/LayoutToggle.tsx b/src/room/LayoutToggle.tsx index 6cddc95f..ca6aa467 100644 --- a/src/room/LayoutToggle.tsx +++ b/src/room/LayoutToggle.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type ChangeEvent, type FC, type TouchEvent, useCallback } from "react"; +import { type ChangeEvent, type FC, useCallback } from "react"; import { useTranslation } from "react-i18next"; import { Tooltip } from "@vector-im/compound-web"; import { @@ -22,15 +22,9 @@ interface Props { layout: Layout; setLayout: (layout: Layout) => void; className?: string; - onTouchEnd?: (e: TouchEvent) => void; } -export const LayoutToggle: FC = ({ - layout, - setLayout, - className, - onTouchEnd, -}) => { +export const LayoutToggle: FC = ({ layout, setLayout, className }) => { const { t } = useTranslation(); const onChange = useCallback( @@ -47,7 +41,6 @@ export const LayoutToggle: FC = ({ value="spotlight" checked={layout === "spotlight"} onChange={onChange} - onTouchEnd={onTouchEnd} /> @@ -58,7 +51,6 @@ export const LayoutToggle: FC = ({ value="grid" checked={layout === "grid"} onChange={onChange} - onTouchEnd={onTouchEnd} /> diff --git a/src/tile/SpotlightTile.module.css b/src/tile/SpotlightTile.module.css index 178d13f3..fe13f906 100644 --- a/src/tile/SpotlightTile.module.css +++ b/src/tile/SpotlightTile.module.css @@ -84,7 +84,6 @@ Please see LICENSE in the repository root for full details. .expand { appearance: none; cursor: pointer; - opacity: 0; padding: var(--cpd-space-2x); border: none; border-radius: var(--cpd-radius-pill-effect); @@ -148,17 +147,21 @@ Please see LICENSE in the repository root for full details. } } -.expand:active { +.expand:active, .expand[data-state="open"] { background: var(--cpd-color-gray-100); } @media (hover) { + .tile > div > button { + opacity: 0; + } .tile:hover > div > button { opacity: 1; } } -.tile:has(:focus-visible) > div > button { +.tile:has(:focus-visible) > div > button, +.tile > div:has([data-state="open"]) > button { opacity: 1; } From 885a523e9189aa41bb83045f36e7fe9b0bdafe9b Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 9 Mar 2026 10:44:22 +0100 Subject: [PATCH 23/30] Fix formatting --- src/tile/SpotlightTile.module.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tile/SpotlightTile.module.css b/src/tile/SpotlightTile.module.css index fe13f906..af0e0add 100644 --- a/src/tile/SpotlightTile.module.css +++ b/src/tile/SpotlightTile.module.css @@ -147,7 +147,8 @@ Please see LICENSE in the repository root for full details. } } -.expand:active, .expand[data-state="open"] { +.expand:active, +.expand[data-state="open"] { background: var(--cpd-color-gray-100); } From 4922249f41339c6deedc77465bbda3bbd3373a61 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 9 Mar 2026 13:00:43 +0100 Subject: [PATCH 24/30] Unpin Knip and LiveKit track processors These dependencies were held back on old versions because of issues that are now resolved (see 2fca7e37191e81f015426eb3756a19b19277031e and 44eb8acaeec990656e921d954694e459722acb48). --- package.json | 4 +- yarn.lock | 224 ++++++++++++++++++++++++++------------------------- 2 files changed, 117 insertions(+), 111 deletions(-) diff --git a/package.json b/package.json index 705b0f10..9ee0ad26 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "@livekit/components-core": "^0.12.0", "@livekit/components-react": "^2.0.0", "@livekit/protocol": "^1.42.2", - "@livekit/track-processors": "^0.6.0 || ^0.7.1", + "@livekit/track-processors": "^0.7.1", "@mediapipe/tasks-vision": "^0.10.18", "@playwright/test": "^1.57.0", "@radix-ui/react-dialog": "^1.0.4", @@ -101,7 +101,7 @@ "i18next-browser-languagedetector": "^8.0.0", "i18next-parser": "^9.1.0", "jsdom": "^26.0.0", - "knip": "5.82.1", + "knip": "^5.86.0", "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", diff --git a/yarn.lock b/yarn.lock index 4675d0e1..bd90ded3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3345,15 +3345,15 @@ __metadata: languageName: node linkType: hard -"@livekit/track-processors@npm:^0.6.0 || ^0.7.1": - version: 0.6.1 - resolution: "@livekit/track-processors@npm:0.6.1" +"@livekit/track-processors@npm:^0.7.1": + version: 0.7.2 + resolution: "@livekit/track-processors@npm:0.7.2" dependencies: "@mediapipe/tasks-vision": "npm:0.10.14" peerDependencies: "@types/dom-mediacapture-transform": ^0.1.9 livekit-client: ^1.12.0 || ^2.1.0 - checksum: 10c0/80f54663c7e13de299de9e2565b6cbd2ba74ea0a4a8adf8a366e8cfd0e19dedfb9d699899137f1a6133414f28779877eeb3200074c03893bc63aeb0d8c912a91 + checksum: 10c0/d5638942205ea05a507254f61157696881332b866ff538d1e93bd2267c31ce80c9a81dfc4f8c4b7c96910452b13511ee06be1ae7ab30c299b73fe04baf80a673 languageName: node linkType: hard @@ -3371,7 +3371,7 @@ __metadata: languageName: node linkType: hard -"@napi-rs/wasm-runtime@npm:^1.1.0": +"@napi-rs/wasm-runtime@npm:^1.1.1": version: 1.1.1 resolution: "@napi-rs/wasm-runtime@npm:1.1.1" dependencies: @@ -3551,144 +3551,144 @@ __metadata: languageName: node linkType: hard -"@oxc-resolver/binding-android-arm-eabi@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-android-arm-eabi@npm:11.16.2" +"@oxc-resolver/binding-android-arm-eabi@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-android-arm-eabi@npm:11.19.1" conditions: os=android & cpu=arm languageName: node linkType: hard -"@oxc-resolver/binding-android-arm64@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-android-arm64@npm:11.16.2" +"@oxc-resolver/binding-android-arm64@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-android-arm64@npm:11.19.1" conditions: os=android & cpu=arm64 languageName: node linkType: hard -"@oxc-resolver/binding-darwin-arm64@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-darwin-arm64@npm:11.16.2" +"@oxc-resolver/binding-darwin-arm64@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-darwin-arm64@npm:11.19.1" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@oxc-resolver/binding-darwin-x64@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-darwin-x64@npm:11.16.2" +"@oxc-resolver/binding-darwin-x64@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-darwin-x64@npm:11.19.1" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@oxc-resolver/binding-freebsd-x64@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-freebsd-x64@npm:11.16.2" +"@oxc-resolver/binding-freebsd-x64@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-freebsd-x64@npm:11.19.1" conditions: os=freebsd & cpu=x64 languageName: node linkType: hard -"@oxc-resolver/binding-linux-arm-gnueabihf@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-linux-arm-gnueabihf@npm:11.16.2" +"@oxc-resolver/binding-linux-arm-gnueabihf@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-linux-arm-gnueabihf@npm:11.19.1" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@oxc-resolver/binding-linux-arm-musleabihf@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-linux-arm-musleabihf@npm:11.16.2" +"@oxc-resolver/binding-linux-arm-musleabihf@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-linux-arm-musleabihf@npm:11.19.1" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@oxc-resolver/binding-linux-arm64-gnu@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-linux-arm64-gnu@npm:11.16.2" +"@oxc-resolver/binding-linux-arm64-gnu@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-linux-arm64-gnu@npm:11.19.1" conditions: os=linux & cpu=arm64 & libc=glibc languageName: node linkType: hard -"@oxc-resolver/binding-linux-arm64-musl@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-linux-arm64-musl@npm:11.16.2" +"@oxc-resolver/binding-linux-arm64-musl@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-linux-arm64-musl@npm:11.19.1" conditions: os=linux & cpu=arm64 & libc=musl languageName: node linkType: hard -"@oxc-resolver/binding-linux-ppc64-gnu@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-linux-ppc64-gnu@npm:11.16.2" +"@oxc-resolver/binding-linux-ppc64-gnu@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-linux-ppc64-gnu@npm:11.19.1" conditions: os=linux & cpu=ppc64 & libc=glibc languageName: node linkType: hard -"@oxc-resolver/binding-linux-riscv64-gnu@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-linux-riscv64-gnu@npm:11.16.2" +"@oxc-resolver/binding-linux-riscv64-gnu@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-linux-riscv64-gnu@npm:11.19.1" conditions: os=linux & cpu=riscv64 & libc=glibc languageName: node linkType: hard -"@oxc-resolver/binding-linux-riscv64-musl@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-linux-riscv64-musl@npm:11.16.2" +"@oxc-resolver/binding-linux-riscv64-musl@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-linux-riscv64-musl@npm:11.19.1" conditions: os=linux & cpu=riscv64 & libc=musl languageName: node linkType: hard -"@oxc-resolver/binding-linux-s390x-gnu@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-linux-s390x-gnu@npm:11.16.2" +"@oxc-resolver/binding-linux-s390x-gnu@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-linux-s390x-gnu@npm:11.19.1" conditions: os=linux & cpu=s390x & libc=glibc languageName: node linkType: hard -"@oxc-resolver/binding-linux-x64-gnu@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-linux-x64-gnu@npm:11.16.2" +"@oxc-resolver/binding-linux-x64-gnu@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-linux-x64-gnu@npm:11.19.1" conditions: os=linux & cpu=x64 & libc=glibc languageName: node linkType: hard -"@oxc-resolver/binding-linux-x64-musl@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-linux-x64-musl@npm:11.16.2" +"@oxc-resolver/binding-linux-x64-musl@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-linux-x64-musl@npm:11.19.1" conditions: os=linux & cpu=x64 & libc=musl languageName: node linkType: hard -"@oxc-resolver/binding-openharmony-arm64@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-openharmony-arm64@npm:11.16.2" +"@oxc-resolver/binding-openharmony-arm64@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-openharmony-arm64@npm:11.19.1" conditions: os=openharmony & cpu=arm64 languageName: node linkType: hard -"@oxc-resolver/binding-wasm32-wasi@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-wasm32-wasi@npm:11.16.2" +"@oxc-resolver/binding-wasm32-wasi@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-wasm32-wasi@npm:11.19.1" dependencies: - "@napi-rs/wasm-runtime": "npm:^1.1.0" + "@napi-rs/wasm-runtime": "npm:^1.1.1" conditions: cpu=wasm32 languageName: node linkType: hard -"@oxc-resolver/binding-win32-arm64-msvc@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-win32-arm64-msvc@npm:11.16.2" +"@oxc-resolver/binding-win32-arm64-msvc@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-win32-arm64-msvc@npm:11.19.1" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@oxc-resolver/binding-win32-ia32-msvc@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-win32-ia32-msvc@npm:11.16.2" +"@oxc-resolver/binding-win32-ia32-msvc@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-win32-ia32-msvc@npm:11.19.1" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@oxc-resolver/binding-win32-x64-msvc@npm:11.16.2": - version: 11.16.2 - resolution: "@oxc-resolver/binding-win32-x64-msvc@npm:11.16.2" +"@oxc-resolver/binding-win32-x64-msvc@npm:11.19.1": + version: 11.19.1 + resolution: "@oxc-resolver/binding-win32-x64-msvc@npm:11.19.1" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -8380,7 +8380,7 @@ __metadata: "@livekit/components-core": "npm:^0.12.0" "@livekit/components-react": "npm:^2.0.0" "@livekit/protocol": "npm:^1.42.2" - "@livekit/track-processors": "npm:^0.6.0 || ^0.7.1" + "@livekit/track-processors": "npm:^0.7.1" "@mediapipe/tasks-vision": "npm:^0.10.18" "@playwright/test": "npm:^1.57.0" "@radix-ui/react-dialog": "npm:^1.0.4" @@ -8433,7 +8433,7 @@ __metadata: i18next-browser-languagedetector: "npm:^8.0.0" i18next-parser: "npm:^9.1.0" jsdom: "npm:^26.0.0" - knip: "npm:5.82.1" + knip: "npm:^5.86.0" livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" @@ -11039,17 +11039,6 @@ __metadata: languageName: node linkType: hard -"js-yaml@npm:^4.1.1": - version: 4.1.1 - resolution: "js-yaml@npm:4.1.1" - dependencies: - argparse: "npm:^2.0.1" - bin: - js-yaml: bin/js-yaml.js - checksum: 10c0/561c7d7088c40a9bb53cc75becbfb1df6ae49b34b5e6e5a81744b14ae8667ec564ad2527709d1a6e7d5e5fa6d483aa0f373a50ad98d42fde368ec4a190d4fae7 - languageName: node - linkType: hard - "jsbn@npm:1.1.0": version: 1.1.0 resolution: "jsbn@npm:1.1.0" @@ -11232,21 +11221,22 @@ __metadata: languageName: node linkType: hard -"knip@npm:5.82.1": - version: 5.82.1 - resolution: "knip@npm:5.82.1" +"knip@npm:^5.86.0": + version: 5.86.0 + resolution: "knip@npm:5.86.0" dependencies: "@nodelib/fs.walk": "npm:^1.2.3" fast-glob: "npm:^3.3.3" formatly: "npm:^0.3.0" jiti: "npm:^2.6.0" - js-yaml: "npm:^4.1.1" minimist: "npm:^1.2.8" - oxc-resolver: "npm:^11.15.0" + oxc-resolver: "npm:^11.19.1" picocolors: "npm:^1.1.1" picomatch: "npm:^4.0.1" smol-toml: "npm:^1.5.2" strip-json-comments: "npm:5.0.3" + unbash: "npm:^2.2.0" + yaml: "npm:^2.8.2" zod: "npm:^4.1.11" peerDependencies: "@types/node": ">=18" @@ -11254,7 +11244,7 @@ __metadata: bin: knip: bin/knip.js knip-bun: bin/knip-bun.js - checksum: 10c0/c3bfe898fe3103bb6a59ee2ba4297f05ea4d2db474571db89ae199ebbd74eafa5061d05b3bc2c75e4ec2322ba7ffee44493c76132d3d8991fae66ba742b9ccb4 + checksum: 10c0/6905c3c2bd21b1f5d51bf83568d1eff67d9d74dd9547c428f810b0dbc3624225a0c41b8e8caccbb111df2db175933aa853345798a05f91f9344ce3aca26898ff languageName: node linkType: hard @@ -12173,30 +12163,30 @@ __metadata: languageName: node linkType: hard -"oxc-resolver@npm:^11.15.0": - version: 11.16.2 - resolution: "oxc-resolver@npm:11.16.2" +"oxc-resolver@npm:^11.19.1": + version: 11.19.1 + resolution: "oxc-resolver@npm:11.19.1" dependencies: - "@oxc-resolver/binding-android-arm-eabi": "npm:11.16.2" - "@oxc-resolver/binding-android-arm64": "npm:11.16.2" - "@oxc-resolver/binding-darwin-arm64": "npm:11.16.2" - "@oxc-resolver/binding-darwin-x64": "npm:11.16.2" - "@oxc-resolver/binding-freebsd-x64": "npm:11.16.2" - "@oxc-resolver/binding-linux-arm-gnueabihf": "npm:11.16.2" - "@oxc-resolver/binding-linux-arm-musleabihf": "npm:11.16.2" - "@oxc-resolver/binding-linux-arm64-gnu": "npm:11.16.2" - "@oxc-resolver/binding-linux-arm64-musl": "npm:11.16.2" - "@oxc-resolver/binding-linux-ppc64-gnu": "npm:11.16.2" - "@oxc-resolver/binding-linux-riscv64-gnu": "npm:11.16.2" - "@oxc-resolver/binding-linux-riscv64-musl": "npm:11.16.2" - "@oxc-resolver/binding-linux-s390x-gnu": "npm:11.16.2" - "@oxc-resolver/binding-linux-x64-gnu": "npm:11.16.2" - "@oxc-resolver/binding-linux-x64-musl": "npm:11.16.2" - "@oxc-resolver/binding-openharmony-arm64": "npm:11.16.2" - "@oxc-resolver/binding-wasm32-wasi": "npm:11.16.2" - "@oxc-resolver/binding-win32-arm64-msvc": "npm:11.16.2" - "@oxc-resolver/binding-win32-ia32-msvc": "npm:11.16.2" - "@oxc-resolver/binding-win32-x64-msvc": "npm:11.16.2" + "@oxc-resolver/binding-android-arm-eabi": "npm:11.19.1" + "@oxc-resolver/binding-android-arm64": "npm:11.19.1" + "@oxc-resolver/binding-darwin-arm64": "npm:11.19.1" + "@oxc-resolver/binding-darwin-x64": "npm:11.19.1" + "@oxc-resolver/binding-freebsd-x64": "npm:11.19.1" + "@oxc-resolver/binding-linux-arm-gnueabihf": "npm:11.19.1" + "@oxc-resolver/binding-linux-arm-musleabihf": "npm:11.19.1" + "@oxc-resolver/binding-linux-arm64-gnu": "npm:11.19.1" + "@oxc-resolver/binding-linux-arm64-musl": "npm:11.19.1" + "@oxc-resolver/binding-linux-ppc64-gnu": "npm:11.19.1" + "@oxc-resolver/binding-linux-riscv64-gnu": "npm:11.19.1" + "@oxc-resolver/binding-linux-riscv64-musl": "npm:11.19.1" + "@oxc-resolver/binding-linux-s390x-gnu": "npm:11.19.1" + "@oxc-resolver/binding-linux-x64-gnu": "npm:11.19.1" + "@oxc-resolver/binding-linux-x64-musl": "npm:11.19.1" + "@oxc-resolver/binding-openharmony-arm64": "npm:11.19.1" + "@oxc-resolver/binding-wasm32-wasi": "npm:11.19.1" + "@oxc-resolver/binding-win32-arm64-msvc": "npm:11.19.1" + "@oxc-resolver/binding-win32-ia32-msvc": "npm:11.19.1" + "@oxc-resolver/binding-win32-x64-msvc": "npm:11.19.1" dependenciesMeta: "@oxc-resolver/binding-android-arm-eabi": optional: true @@ -12238,7 +12228,7 @@ __metadata: optional: true "@oxc-resolver/binding-win32-x64-msvc": optional: true - checksum: 10c0/b20a0fea18fdf31dbaee51354ce7b987ba8f3e780c6c1de9034628033a69d0b3085f9596d9925797d9340bdf4b98cd72a258b0728d0d5e5de2b1748154921b42 + checksum: 10c0/8ac4eaffa9c0bcbb9f4f4a2b43786457ec5a68684d8776cb78b5a15ce3d1a79d3e67262aa3c635f98a0c1cd6cd56a31fcb05bffb9a286100056e4ab06b928833 languageName: node linkType: hard @@ -15153,6 +15143,13 @@ __metadata: languageName: node linkType: hard +"unbash@npm:^2.2.0": + version: 2.2.0 + resolution: "unbash@npm:2.2.0" + checksum: 10c0/f218a30e2b65147dba16fcea5d9cbfe5af9d9518e98083b9790b9884959c82c5c8f85e7feeea717430e2ea6b352a1d57ad98e90fe488638606de12c9254cbf35 + languageName: node + linkType: hard + "unbox-primitive@npm:^1.1.0": version: 1.1.0 resolution: "unbox-primitive@npm:1.1.0" @@ -16105,6 +16102,15 @@ __metadata: languageName: node linkType: hard +"yaml@npm:^2.8.2": + version: 2.8.2 + resolution: "yaml@npm:2.8.2" + bin: + yaml: bin.mjs + checksum: 10c0/703e4dc1e34b324aa66876d63618dcacb9ed49f7e7fe9b70f1e703645be8d640f68ab84f12b86df8ac960bac37acf5513e115de7c970940617ce0343c8c9cd96 + languageName: node + linkType: hard + "yargs-parser@npm:^18.1.2": version: 18.1.3 resolution: "yargs-parser@npm:18.1.3" From 4c7fad30c064bdba8d58f15683827625b4e49f1d Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 9 Mar 2026 13:11:16 +0100 Subject: [PATCH 25/30] Silence new Knip errors --- knip.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/knip.ts b/knip.ts index d23d42fe..3be3e653 100644 --- a/knip.ts +++ b/knip.ts @@ -34,6 +34,12 @@ export default { // then Knip will flag it as a false positive // https://github.com/webpro-nl/knip/issues/766 "@vector-im/compound-web", + // Yarn plugins are allowed to depend on packages provided by the Yarn + // runtime. These shouldn't be listed in package.json, because plugins + // should work before Yarn even installs dependencies for the first time. + // https://yarnpkg.com/advanced/plugin-tutorial#what-does-a-plugin-look-like + "@yarnpkg/core", + "@yarnpkg/parsers", "matrix-widget-api", ], ignoreExportsUsedInFile: true, From 8f418ce9fd5c1d2090b242adf7801368b11231bb Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 9 Mar 2026 13:59:39 +0100 Subject: [PATCH 26/30] fix cla link --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e68c0f42..787ddc73 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -36,4 +36,4 @@ Uncomment this markdown table below and edit the last line `|||`: - [ ] Pull request includes screenshots or videos if containing UI changes - [ ] Tests written for new code (and old code if feasible). - [ ] Linter and other CI checks pass. -- [ ] I have licensed the changes to Element by completing the [Contributor License Agreement (CLA)](https://cla-assistant.io/element-hq/element-web) +- [ ] I have licensed the changes to Element by completing the [Contributor License Agreement (CLA)](https://cla-assistant.io/element-hq/element-call) From ca3837f44ebdb3340340af1b95d8eae924f67667 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 9 Mar 2026 15:07:42 +0100 Subject: [PATCH 27/30] fix merge issue that added back a deprecated test --- src/state/media/MediaViewModel.test.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/state/media/MediaViewModel.test.ts b/src/state/media/MediaViewModel.test.ts index f64dd3ee..9d873ccb 100644 --- a/src/state/media/MediaViewModel.test.ts +++ b/src/state/media/MediaViewModel.test.ts @@ -160,21 +160,6 @@ test("control a participant's screen share volume", () => { }); }); -test("toggle fit/contain for a participant's video", () => { - const vm = mockRemoteMedia(rtcMembership, {}, mockRemoteParticipant({})); - withTestScheduler(({ expectObservable, schedule }) => { - schedule("-ab|", { - a: () => vm.toggleCropVideo(), - b: () => vm.toggleCropVideo(), - }); - expectObservable(vm.cropVideo$).toBe("abc", { - a: true, - b: false, - c: true, - }); - }); -}); - test("local media remembers whether it should always be shown", () => { const vm1 = mockLocalMedia( rtcMembership, From 3da762ab36faf06915053cdab9f91b12fd8a76c8 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 9 Mar 2026 17:49:04 +0100 Subject: [PATCH 28/30] fix: typo inverting with/height in PIP spotlight tile --- src/room/InCallView.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index aceb07cf..d8803b22 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -584,8 +584,8 @@ export const InCallView: FC = ({ vm={layout.spotlight} expanded onToggleExpanded={null} - targetWidth={gridBounds.height} - targetHeight={gridBounds.width} + targetWidth={gridBounds.width} + targetHeight={gridBounds.height} showIndicators={false} focusable={!contentObscured} aria-hidden={contentObscured} From 143b560b20e46ca87b7cd802198d6ec12e65009a Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 10 Mar 2026 09:05:05 +0100 Subject: [PATCH 29/30] Add a playwright test to ensure object fit is correct --- playwright/widget/huddle-call.test.ts | 2 +- playwright/widget/pip-call.test.ts | 74 +++++++++++++++++++++++++++ playwright/widget/test-helpers.ts | 12 +++++ 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 playwright/widget/pip-call.test.ts diff --git a/playwright/widget/huddle-call.test.ts b/playwright/widget/huddle-call.test.ts index b42c0ab2..d4ba0006 100644 --- a/playwright/widget/huddle-call.test.ts +++ b/playwright/widget/huddle-call.test.ts @@ -60,7 +60,7 @@ widgetTest("Create and join a group call", async ({ addUser, browserName }) => { // The only way to know if it is muted or not is to look at the data-kind attribute.. const videoButton = frame.getByTestId("incall_videomute"); await expect(videoButton).toBeVisible(); - // video should be off by default in a voice call + // video should be on await expect(videoButton).toHaveAttribute("aria-label", /^Stop video$/); } diff --git a/playwright/widget/pip-call.test.ts b/playwright/widget/pip-call.test.ts new file mode 100644 index 00000000..49ebec52 --- /dev/null +++ b/playwright/widget/pip-call.test.ts @@ -0,0 +1,74 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "@playwright/test"; + +import { widgetTest } from "../fixtures/widget-user.ts"; +import { HOST1, TestHelpers } from "./test-helpers.ts"; + +widgetTest("Put call in PIP", async ({ addUser, browserName }) => { + test.skip( + browserName === "firefox", + "The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled", + ); + + test.slow(); + + const valere = await addUser("Valere", HOST1); + const timo = await addUser("Timo", HOST1); + + const callRoom = "TeamRoom"; + await TestHelpers.createRoom(callRoom, valere.page, [timo.mxId]); + + await TestHelpers.createRoom("DoubleTask", valere.page); + + await TestHelpers.acceptRoomInvite(callRoom, timo.page); + + await TestHelpers.switchToRoomNamed(valere.page, callRoom); + + // Start the call as Valere + await TestHelpers.startCallInCurrentRoom(valere.page, false); + await expect( + valere.page.locator('iframe[title="Element Call"]'), + ).toBeVisible(); + + await TestHelpers.joinCallFromLobby(valere.page); + + await TestHelpers.joinCallInCurrentRoom(timo.page); + + { + const frame = timo.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + + const videoButton = frame.getByTestId("incall_videomute"); + await expect(videoButton).toBeVisible(); + // check that the video is on + await expect(videoButton).toHaveAttribute("aria-label", /^Stop video$/); + } + + // Switch to the other room, the call should go to PIP + await TestHelpers.switchToRoomNamed(valere.page, "DoubleTask"); + + // We should see the PIP overlay + await expect(valere.page.locator(".mx_WidgetPip_overlay")).toBeVisible(); + + { + // wait a bit so that the PIP has rendered the video + await valere.page.waitForTimeout(600); + + // Check for a bug where the video had the wrong fit in PIP + const frame = valere.page + .locator('iframe[title="Element Call"]') + .contentFrame(); + const videoElements = await frame.locator("video").all(); + expect(videoElements.length).toBe(1); + + const pipVideo = videoElements[0]; + await expect(pipVideo).toHaveCSS("object-fit", "cover"); + } +}); diff --git a/playwright/widget/test-helpers.ts b/playwright/widget/test-helpers.ts index 6fe4479b..4562ba5a 100644 --- a/playwright/widget/test-helpers.ts +++ b/playwright/widget/test-helpers.ts @@ -276,4 +276,16 @@ export class TestHelpers { }); } } + + /** + * Switches to a room in the room list by its name. + * @param page - The EW page + * @param roomName - The name of the room to switch to + */ + public static async switchToRoomNamed( + page: Page, + roomName: string, + ): Promise { + await page.getByRole("option", { name: `Open room ${roomName}` }).click(); + } } From c9557e91d5453a2085b5c62c034eb6aa5346bafc Mon Sep 17 00:00:00 2001 From: fkwp Date: Wed, 11 Mar 2026 13:06:20 +0100 Subject: [PATCH 30/30] fix: add id-token permission as its required by tailscale login (part 3) (#3793) * Push docker images to oci.element.io * prettier * add id-token permission as its required by tailscale login * pass secrets to reusable workflows --- .github/workflows/build.yaml | 1 + .github/workflows/pr-deploy.yaml | 1 + .github/workflows/publish.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4f9e80f2..32ce25c9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -51,6 +51,7 @@ jobs: packages: write id-token: write uses: ./.github/workflows/build-and-publish-docker.yaml + secrets: inherit with: artifact_run_id: ${{ github.run_id }} docker_tags: | diff --git a/.github/workflows/pr-deploy.yaml b/.github/workflows/pr-deploy.yaml index 62b37aca..138ab2b5 100644 --- a/.github/workflows/pr-deploy.yaml +++ b/.github/workflows/pr-deploy.yaml @@ -62,6 +62,7 @@ jobs: packages: write id-token: write uses: ./.github/workflows/build-and-publish-docker.yaml + secrets: inherit with: artifact_run_id: ${{ github.event.workflow_run.id || github.run_id }} docker_tags: | diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index ade91019..ceedf781 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -57,6 +57,7 @@ jobs: packages: write id-token: write uses: ./.github/workflows/build-and-publish-docker.yaml + secrets: inherit with: artifact_run_id: ${{ github.event.workflow_run.id || github.run_id }} docker_tags: |