diff --git a/docs/controls.md b/docs/controls.md index b97fe795d..7a2a1d96c 100644 --- a/docs/controls.md +++ b/docs/controls.md @@ -7,6 +7,9 @@ A few aspects of Element Call's interface can be controlled through a global API - `controls.canEnterPip(): boolean` Determines whether it's possible to enter picture-in-picture mode. - `controls.enablePip(): void` Puts the call interface into picture-in-picture mode. Throws if not in a call. - `controls.disablePip(): void` Takes the call interface out of picture-in-picture mode, restoring it to its natural display mode. Throws if not in a call. +- `controls.onPipMediaOrientationUpdate: ((orientation: "landscape"|"portrait") => void) | undefined` Callback called whenever the PiP media orientation changes. + The client should track this value to already initiate the pip in the right orientation. + It should update the orientation of the current Pip window when called. ## Audio devices diff --git a/src/controls.ts b/src/controls.ts index 1ddb17049..1978946dc 100644 --- a/src/controls.ts +++ b/src/controls.ts @@ -12,6 +12,7 @@ export interface Controls { canEnterPip(): boolean; enablePip(): void; disablePip(): void; + onPipMediaOrientationUpdate?: (orientation: "landscape" | "portrait") => void; setAvailableAudioDevices(devices: OutputDevice[]): void; setAudioDevice(id: string): void; diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 9eb2787a4..7ad7cacbb 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { test, vi, onTestFinished, it, describe } from "vitest"; +import { test, vi, onTestFinished, it, describe, expect } from "vitest"; import { BehaviorSubject, combineLatest, @@ -899,6 +899,41 @@ describe.each([ }, ); + // TODO add media to lk mocks + test("onPipMediaOrientationUpdate is called with the spotlight media orientation", () => { + // Set the spy before creating the view model so the initial call is captured + const onPipMediaOrientationUpdate = vi.fn(); + window.controls.onPipMediaOrientationUpdate = onPipMediaOrientationUpdate; + onTestFinished(() => { + window.controls.onPipMediaOrientationUpdate = undefined; + }); + + withTestScheduler(({ behavior }) => { + // Alice starts as a regular participant, then shares her screen, then stops + const aliceSharingInputMarbles = "nyn"; + + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember]), + sharingScreen: new Map([ + [aliceParticipant, behavior(aliceSharingInputMarbles, yesNo)], + ]), + }, + () => {}, + ); + }); + + // Should be called exactly 3 times: + // 1. Initially with "portrait" (Alice is in spotlight as a user, default portrait orientation) + // 2. With "landscape" when Alice starts screen sharing (screen shares always use landscape) + // 3. With "portrait" again when Alice stops screen sharing and returns to user tile + expect(onPipMediaOrientationUpdate).toHaveBeenCalledTimes(3); + expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(1, "portrait"); + expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(2, "landscape"); + expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(3, "portrait"); + }); + test("PiP tile in expanded spotlight layout switches speakers without layout shifts", () => { withTestScheduler(({ behavior, schedule, expectObservable }) => { // Switch to spotlight immediately diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index aaf679505..f778e6dd5 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -1199,6 +1199,33 @@ export function createCallViewModel$( })), ); + spotlight$ + .pipe( + switchMap((media) => { + let layout; + const pipMedia = media[0]; + if (pipMedia === undefined) return of(undefined); + switch (pipMedia.type) { + case "user": + layout = pipMedia.videoOrientation$; + break; + case "ringing": + layout = of("landscape" as const); + break; + case "screen share": + layout = of("landscape" as const); + break; + } + return layout; + }), + scope.bind(), + ) + .subscribe((orientation) => { + if (orientation === undefined) return; + logger.info("controls api pip orientation updated:", orientation); + window.controls.onPipMediaOrientationUpdate?.(orientation); + }); + /** * The media to be used to produce a layout. */ diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts index a20c489e6..ea0331030 100644 --- a/src/state/media/UserMediaViewModel.ts +++ b/src/state/media/UserMediaViewModel.ts @@ -48,6 +48,7 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel { audioEnabled$: Behavior; videoEnabled$: Behavior; videoFit$: Behavior<"cover" | "contain">; + videoOrientation$: Behavior<"landscape" | "portrait">; toggleCropVideo: () => void; /** * The expected identity of the LiveKit participant. Exposed for debugging. @@ -104,6 +105,7 @@ export function createBaseUserMedia( { width: number; height: number } | undefined >(undefined); + const videoSize$ = videoSizeFromParticipant$(participant$); return { ...createMemberMedia(scope, { ...inputs, @@ -129,11 +131,14 @@ export function createBaseUserMedia( videoEnabled$: scope.behavior( media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)), ), - videoFit$: videoFit$( - scope, - videoSizeFromParticipant$(participant$), - targetSize$, + videoOrientation$: scope.behavior( + videoSize$.pipe( + map((s) => (s ? s.width / s.height : 1)), + map((aspect) => (aspect > 1 ? "landscape" : "portrait")), + ), + "portrait", ), + videoFit$: videoFit$(scope, videoSize$, targetSize$), toggleCropVideo: () => toggleCropVideo$.next(), rtcBackendIdentity, handRaised$, diff --git a/src/state/media/observeRtpStreamStats.ts b/src/state/media/observeRtpStreamStats.ts index 63fb1a1b0..44181f857 100644 --- a/src/state/media/observeRtpStreamStats.ts +++ b/src/state/media/observeRtpStreamStats.ts @@ -32,7 +32,9 @@ export function observeRtpStreamStats$( > { return combineLatest([ observeTrackReference$(participant, source), - interval(1000).pipe(startWith(0)), + // The update frequency is high because we use this value to update the PiP orientation and the fit/fill video tile props based on that + // We want it to be responsive. For just the debug tools 1s would be sufficient. + interval(350).pipe(startWith(0)), ]).pipe( switchMap(async ([trackReference]) => { const track = trackReference?.publication?.track;