From a6fc710d8ce170861a615ac717d1ac17f14363b0 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 1 Jun 2026 16:12:57 +0200 Subject: [PATCH 1/6] Add onPipMediaOrientationUpdate to controls api. --- src/controls.ts | 1 + src/state/CallViewModel/CallViewModel.ts | 25 ++++++++++++++++++++++++ src/state/media/UserMediaViewModel.ts | 13 ++++++++---- src/state/media/observeRtpStreamStats.ts | 4 +++- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/controls.ts b/src/controls.ts index 1ddb17049..1978946dc 100644 --- a/src/controls.ts +++ b/src/controls.ts @@ -12,6 +12,7 @@ export interface Controls { canEnterPip(): boolean; enablePip(): void; disablePip(): void; + onPipMediaOrientationUpdate?: (orientation: "landscape" | "portrait") => void; setAvailableAudioDevices(devices: OutputDevice[]): void; setAudioDevice(id: string): void; diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 2bbf6f4ec..bdeca679a 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -1191,6 +1191,31 @@ export function createCallViewModel$( })), ); + spotlight$ + .pipe( + switchMap((media) => { + let layout; + switch (media[0].type) { + case "user": + layout = media[0].videoOrientation$; + break; + case "ringing": + layout = of("landscape" as const); + break; + case "screen share": + layout = of("landscape" as const); + break; + } + return layout; + }), + tap((orientation) => { + logger.info("controls api pip orientation updated:", orientation); + window.controls.onPipMediaOrientationUpdate?.(orientation); + }), + scope.bind(), + ) + .subscribe(); + /** * The media to be used to produce a layout. */ diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts index a20c489e6..ea0331030 100644 --- a/src/state/media/UserMediaViewModel.ts +++ b/src/state/media/UserMediaViewModel.ts @@ -48,6 +48,7 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel { audioEnabled$: Behavior; videoEnabled$: Behavior; videoFit$: Behavior<"cover" | "contain">; + videoOrientation$: Behavior<"landscape" | "portrait">; toggleCropVideo: () => void; /** * The expected identity of the LiveKit participant. Exposed for debugging. @@ -104,6 +105,7 @@ export function createBaseUserMedia( { width: number; height: number } | undefined >(undefined); + const videoSize$ = videoSizeFromParticipant$(participant$); return { ...createMemberMedia(scope, { ...inputs, @@ -129,11 +131,14 @@ export function createBaseUserMedia( videoEnabled$: scope.behavior( media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), ), - videoFit$: videoFit$( - scope, - videoSizeFromParticipant$(participant$), - targetSize$, + videoOrientation$: scope.behavior( + videoSize$.pipe( + map((s) => (s ? s.width / s.height : 1)), + map((aspect) => (aspect > 1 ? "landscape" : "portrait")), + ), + "portrait", ), + videoFit$: videoFit$(scope, videoSize$, targetSize$), toggleCropVideo: () => toggleCropVideo$.next(), rtcBackendIdentity, handRaised$, diff --git a/src/state/media/observeRtpStreamStats.ts b/src/state/media/observeRtpStreamStats.ts index 63fb1a1b0..457b8f727 100644 --- a/src/state/media/observeRtpStreamStats.ts +++ b/src/state/media/observeRtpStreamStats.ts @@ -32,7 +32,9 @@ export function observeRtpStreamStats$( > { return combineLatest([ observeTrackReference$(participant, source), - interval(1000).pipe(startWith(0)), + // The update frequency is high because we use this value to update the PiP oreintation and the fit/fill video tile props based on that + // We want it to be responsive. For just the debug tools 1s would be sufficient. + interval(350).pipe(startWith(0)), ]).pipe( switchMap(async ([trackReference]) => { const track = trackReference?.publication?.track; From d6635b7818a0804fc8cf85d77c5de0c737c5eaf2 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 1 Jun 2026 16:13:03 +0200 Subject: [PATCH 2/6] add docs --- docs/controls.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/controls.md b/docs/controls.md index b97fe795d..7a2a1d96c 100644 --- a/docs/controls.md +++ b/docs/controls.md @@ -7,6 +7,9 @@ A few aspects of Element Call's interface can be controlled through a global API - `controls.canEnterPip(): boolean` Determines whether it's possible to enter picture-in-picture mode. - `controls.enablePip(): void` Puts the call interface into picture-in-picture mode. Throws if not in a call. - `controls.disablePip(): void` Takes the call interface out of picture-in-picture mode, restoring it to its natural display mode. Throws if not in a call. +- `controls.onPipMediaOrientationUpdate: ((orientation: "landscape"|"portrait") => void) | undefined` Callback called whenever the PiP media orientation changes. + The client should track this value to already initiate the pip in the right orientation. + It should update the orientation of the current Pip window when called. ## Audio devices From b86909af52c34307343642772482fb954c8586d7 Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 2 Jun 2026 10:56:34 +0200 Subject: [PATCH 3/6] temp add tests --- src/state/CallViewModel/CallViewModel.test.ts | 37 ++++++++++++++++++- src/state/CallViewModel/CallViewModel.ts | 9 ++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index fc25df481..1bf26d9bb 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { test, vi, onTestFinished, it, describe } from "vitest"; +import { test, vi, onTestFinished, it, describe, expect } from "vitest"; import { BehaviorSubject, combineLatest, @@ -899,6 +899,41 @@ describe.each([ }, ); + // TODO add media to lk mocks + test.skip("onPipMediaOrientationUpdate is called with the spotlight media orientation", () => { + // Set the spy before creating the view model so the initial call is captured + const onPipMediaOrientationUpdate = vi.fn(); + window.controls.onPipMediaOrientationUpdate = onPipMediaOrientationUpdate; + onTestFinished(() => { + window.controls.onPipMediaOrientationUpdate = undefined; + }); + + withTestScheduler(({ behavior }) => { + // Alice starts as a regular participant, then shares her screen, then stops + const aliceSharingInputMarbles = "nyn"; + + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember]), + sharingScreen: new Map([ + [aliceParticipant, behavior(aliceSharingInputMarbles, yesNo)], + ]), + }, + () => {}, + ); + }); + + // Should be called exactly 3 times: + // 1. Initially with "portrait" (Alice is in spotlight as a user, default portrait orientation) + // 2. With "landscape" when Alice starts screen sharing (screen shares always use landscape) + // 3. With "portrait" again when Alice stops screen sharing and returns to user tile + expect(onPipMediaOrientationUpdate).toHaveBeenCalledTimes(3); + expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(1, "portrait"); + expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(2, "landscape"); + expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(3, "portrait"); + }); + test("PiP tile in expanded spotlight layout switches speakers without layout shifts", () => { withTestScheduler(({ behavior, schedule, expectObservable }) => { // Switch to spotlight immediately diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index bdeca679a..bd89c7c58 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -1208,13 +1208,12 @@ export function createCallViewModel$( } return layout; }), - tap((orientation) => { - logger.info("controls api pip orientation updated:", orientation); - window.controls.onPipMediaOrientationUpdate?.(orientation); - }), scope.bind(), ) - .subscribe(); + .subscribe((orientation) => { + logger.info("controls api pip orientation updated:", orientation); + window.controls.onPipMediaOrientationUpdate?.(orientation); + }); /** * The media to be used to produce a layout. From b554ce4a87d693bca5c46d602ede2eba9ff4da74 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 3 Jun 2026 10:18:45 +0200 Subject: [PATCH 4/6] Update CallViewModel.test.ts --- src/state/CallViewModel/CallViewModel.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 1bf26d9bb..09f6c1b25 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -900,7 +900,7 @@ describe.each([ ); // TODO add media to lk mocks - test.skip("onPipMediaOrientationUpdate is called with the spotlight media orientation", () => { + test("onPipMediaOrientationUpdate is called with the spotlight media orientation", () => { // Set the spy before creating the view model so the initial call is captured const onPipMediaOrientationUpdate = vi.fn(); window.controls.onPipMediaOrientationUpdate = onPipMediaOrientationUpdate; From 308a0420760d960e6ce4cd88efb040becfbce9cd Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 3 Jun 2026 10:43:26 +0200 Subject: [PATCH 5/6] fix error with no media --- src/state/CallViewModel/CallViewModel.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index bd89c7c58..477e5f478 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -1195,9 +1195,11 @@ export function createCallViewModel$( .pipe( switchMap((media) => { let layout; - switch (media[0].type) { + const pipMedia = media[0]; + if (pipMedia === undefined) return of(undefined); + switch (pipMedia.type) { case "user": - layout = media[0].videoOrientation$; + layout = pipMedia.videoOrientation$; break; case "ringing": layout = of("landscape" as const); @@ -1211,6 +1213,7 @@ export function createCallViewModel$( scope.bind(), ) .subscribe((orientation) => { + if (orientation === undefined) return; logger.info("controls api pip orientation updated:", orientation); window.controls.onPipMediaOrientationUpdate?.(orientation); }); From 838d2f3c5d4ef38573d81258de0d7c89e65559a0 Mon Sep 17 00:00:00 2001 From: Timo <16718859+toger5@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:47:13 +0200 Subject: [PATCH 6/6] Update src/state/media/observeRtpStreamStats.ts Co-authored-by: Johannes Marbach --- src/state/media/observeRtpStreamStats.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state/media/observeRtpStreamStats.ts b/src/state/media/observeRtpStreamStats.ts index 457b8f727..44181f857 100644 --- a/src/state/media/observeRtpStreamStats.ts +++ b/src/state/media/observeRtpStreamStats.ts @@ -32,7 +32,7 @@ export function observeRtpStreamStats$( > { return combineLatest([ observeTrackReference$(participant, source), - // The update frequency is high because we use this value to update the PiP oreintation and the fit/fill video tile props based on that + // The update frequency is high because we use this value to update the PiP orientation and the fit/fill video tile props based on that // We want it to be responsive. For just the debug tools 1s would be sufficient. interval(350).pipe(startWith(0)), ]).pipe(