Merge pull request #4004 from element-hq/toger5/controls-api-onPipMediaOrientation

Controls api `onPipMediaOrientation`

docs/controls.md

@@ -7,6 +7,9 @@ A few aspects of Element Call's interface can be controlled through a global API
 - `controls.canEnterPip(): boolean` Determines whether it's possible to enter picture-in-picture mode.
 - `controls.enablePip(): void` Puts the call interface into picture-in-picture mode. Throws if not in a call.
 - `controls.disablePip(): void` Takes the call interface out of picture-in-picture mode, restoring it to its natural display mode. Throws if not in a call.
+- `controls.onPipMediaOrientationUpdate: ((orientation: "landscape"|"portrait") => void) | undefined` Callback called whenever the PiP media orientation changes.
+  The client should track this value to already initiate the pip in the right orientation.
+  It should update the orientation of the current Pip window when called.
 
 ## Audio devices
 

src/controls.ts

@@ -12,6 +12,7 @@ export interface Controls {
   canEnterPip(): boolean;
   enablePip(): void;
   disablePip(): void;
+  onPipMediaOrientationUpdate?: (orientation: "landscape" | "portrait") => void;
 
   setAvailableAudioDevices(devices: OutputDevice[]): void;
   setAudioDevice(id: string): void;

src/state/CallViewModel/CallViewModel.test.ts

@@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { test, vi, onTestFinished, it, describe } from "vitest";
+import { test, vi, onTestFinished, it, describe, expect } from "vitest";
 import {
   BehaviorSubject,
   combineLatest,
@@ -899,6 +899,41 @@ describe.each([
     },
   );
 
+  // TODO add media to lk mocks
+  test("onPipMediaOrientationUpdate is called with the spotlight media orientation", () => {
+    // Set the spy before creating the view model so the initial call is captured
+    const onPipMediaOrientationUpdate = vi.fn();
+    window.controls.onPipMediaOrientationUpdate = onPipMediaOrientationUpdate;
+    onTestFinished(() => {
+      window.controls.onPipMediaOrientationUpdate = undefined;
+    });
+
+    withTestScheduler(({ behavior }) => {
+      // Alice starts as a regular participant, then shares her screen, then stops
+      const aliceSharingInputMarbles = "nyn";
+
+      withCallViewModel(
+        {
+          remoteParticipants$: constant([aliceParticipant]),
+          rtcMembers$: constant([localRtcMember, aliceRtcMember]),
+          sharingScreen: new Map([
+            [aliceParticipant, behavior(aliceSharingInputMarbles, yesNo)],
+          ]),
+        },
+        () => {},
+      );
+    });
+
+    // Should be called exactly 3 times:
+    // 1. Initially with "portrait" (Alice is in spotlight as a user, default portrait orientation)
+    // 2. With "landscape" when Alice starts screen sharing (screen shares always use landscape)
+    // 3. With "portrait" again when Alice stops screen sharing and returns to user tile
+    expect(onPipMediaOrientationUpdate).toHaveBeenCalledTimes(3);
+    expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(1, "portrait");
+    expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(2, "landscape");
+    expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(3, "portrait");
+  });
+
   test("PiP tile in expanded spotlight layout switches speakers without layout shifts", () => {
     withTestScheduler(({ behavior, schedule, expectObservable }) => {
       // Switch to spotlight immediately

src/state/CallViewModel/CallViewModel.ts

@@ -1199,6 +1199,33 @@ export function createCallViewModel$(
     })),
   );
 
+  spotlight$
+    .pipe(
+      switchMap((media) => {
+        let layout;
+        const pipMedia = media[0];
+        if (pipMedia === undefined) return of(undefined);
+        switch (pipMedia.type) {
+          case "user":
+            layout = pipMedia.videoOrientation$;
+            break;
+          case "ringing":
+            layout = of("landscape" as const);
+            break;
+          case "screen share":
+            layout = of("landscape" as const);
+            break;
+        }
+        return layout;
+      }),
+      scope.bind(),
+    )
+    .subscribe((orientation) => {
+      if (orientation === undefined) return;
+      logger.info("controls api pip orientation updated:", orientation);
+      window.controls.onPipMediaOrientationUpdate?.(orientation);
+    });
+
   /**
    * The media to be used to produce a layout.
    */

src/state/media/UserMediaViewModel.ts

@@ -48,6 +48,7 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel {
   audioEnabled$: Behavior<boolean>;
   videoEnabled$: Behavior<boolean>;
   videoFit$: Behavior<"cover" | "contain">;
+  videoOrientation$: Behavior<"landscape" | "portrait">;
   toggleCropVideo: () => void;
   /**
    * The expected identity of the LiveKit participant. Exposed for debugging.
@@ -104,6 +105,7 @@ export function createBaseUserMedia(
     { width: number; height: number } | undefined
   >(undefined);
 
+  const videoSize$ = videoSizeFromParticipant$(participant$);
   return {
     ...createMemberMedia(scope, {
       ...inputs,
@@ -129,11 +131,14 @@ export function createBaseUserMedia(
     videoEnabled$: scope.behavior(
       media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)),
     ),
-    videoFit$: videoFit$(
-      scope,
-      videoSizeFromParticipant$(participant$),
-      targetSize$,
+    videoOrientation$: scope.behavior(
+      videoSize$.pipe(
+        map((s) => (s ? s.width / s.height : 1)),
+        map((aspect) => (aspect > 1 ? "landscape" : "portrait")),
+      ),
+      "portrait",
     ),
+    videoFit$: videoFit$(scope, videoSize$, targetSize$),
     toggleCropVideo: () => toggleCropVideo$.next(),
     rtcBackendIdentity,
     handRaised$,

src/state/media/observeRtpStreamStats.ts

@@ -32,7 +32,9 @@ export function observeRtpStreamStats$(
 > {
   return combineLatest([
     observeTrackReference$(participant, source),
-    interval(1000).pipe(startWith(0)),
+    // The update frequency is high because we use this value to update the PiP orientation and the fit/fill video tile props based on that
+    // We want it to be responsive. For just the debug tools 1s would be sufficient.
+    interval(350).pipe(startWith(0)),
   ]).pipe(
     switchMap(async ([trackReference]) => {
       const track = trackReference?.publication?.track;