remove blur on mobile

Update Compound to v10.2.2

docs/controls.md

@@ -7,6 +7,9 @@ A few aspects of Element Call's interface can be controlled through a global API
 - `controls.canEnterPip(): boolean` Determines whether it's possible to enter picture-in-picture mode.
 - `controls.enablePip(): void` Puts the call interface into picture-in-picture mode. Throws if not in a call.
 - `controls.disablePip(): void` Takes the call interface out of picture-in-picture mode, restoring it to its natural display mode. Throws if not in a call.
+- `controls.onPipMediaOrientationUpdate: ((orientation: "landscape"|"portrait") => void) | undefined` Callback called whenever the PiP media orientation changes.
+  The client should track this value to already initiate the pip in the right orientation.
+  It should update the orientation of the current Pip window when called.
 
 ## Audio devices
 

pnpm-lock.yaml

@@ -143,10 +143,10 @@ importers:
         version: 10.3.1(react@19.2.6)
       '@vector-im/compound-design-tokens':
         specifier: ^10.0.0
-        version: 10.2.1(@types/react@19.2.15)(react@19.2.6)
+        version: 10.2.2(@types/react@19.2.15)(react@19.2.6)
       '@vector-im/compound-web':
         specifier: ^9.3.0
-        version: 9.4.1(@fontsource/inconsolata@5.2.8)(@fontsource/inter@5.2.8)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(@vector-im/compound-design-tokens@10.2.1(@types/react@19.2.15)(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
+        version: 9.4.1(@fontsource/inconsolata@5.2.8)(@fontsource/inter@5.2.8)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(@vector-im/compound-design-tokens@10.2.2(@types/react@19.2.15)(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
       '@vitejs/plugin-react':
         specifier: ^4.0.1
         version: 4.7.0(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(sass@1.100.0)(terser@5.46.1)(yaml@2.9.0))
@@ -239,7 +239,7 @@ importers:
         version: 1.9.2
       matrix-js-sdk:
         specifier: github:matrix-org/matrix-js-sdk#develop
-        version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/7675fb2e65c3b13910b745cd7e7067c5aad4e816
+        version: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/8c95727b6278fe7942c20d0b9485f984dd0694b7
       matrix-widget-api:
         specifier: ^1.16.1
         version: 1.17.0
@@ -3999,8 +3999,8 @@ packages:
     peerDependencies:
       react: '>= 16.8.0'
 
-  '@vector-im/compound-design-tokens@10.2.1':
-    resolution: {integrity: sha512-N8to81u7qVYRgQiMr8Fr1mM+s6ZHRsiGpXLFJiHTP4YDyc7vXW6MBUVuUaUGFcbmxq76lqPaZt2AsnD2barn6Q==}
+  '@vector-im/compound-design-tokens@10.2.2':
+    resolution: {integrity: sha512-kncYroDEwyvvk4K4E67ExP4RYp9NEineD1tB38KeJwTmOvcrPtGjWyBAyJgOGeRuc7G5edNLkWnAbvk5yZJ+5w==}
     peerDependencies:
       '@types/react': '*'
       react: ^17 || ^18 || ^19.0.0
@@ -5922,8 +5922,8 @@ packages:
   matrix-events-sdk@0.0.1:
     resolution: {integrity: sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==}
 
-  matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/7675fb2e65c3b13910b745cd7e7067c5aad4e816:
-    resolution: {tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/7675fb2e65c3b13910b745cd7e7067c5aad4e816}
+  matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/8c95727b6278fe7942c20d0b9485f984dd0694b7:
+    resolution: {tarball: https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/8c95727b6278fe7942c20d0b9485f984dd0694b7}
     version: 41.7.0
     engines: {node: '>=22.0.0'}
 
@@ -11231,12 +11231,12 @@ snapshots:
       '@use-gesture/core': 10.3.1
       react: 19.2.6
 
-  '@vector-im/compound-design-tokens@10.2.1(@types/react@19.2.15)(react@19.2.6)':
+  '@vector-im/compound-design-tokens@10.2.2(@types/react@19.2.15)(react@19.2.6)':
     optionalDependencies:
       '@types/react': 19.2.15
       react: 19.2.6
 
-  '@vector-im/compound-web@9.4.1(@fontsource/inconsolata@5.2.8)(@fontsource/inter@5.2.8)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(@vector-im/compound-design-tokens@10.2.1(@types/react@19.2.15)(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)':
+  '@vector-im/compound-web@9.4.1(@fontsource/inconsolata@5.2.8)(@fontsource/inter@5.2.8)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(@vector-im/compound-design-tokens@10.2.2(@types/react@19.2.15)(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)':
     dependencies:
       '@floating-ui/react': 0.27.19(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
       '@fontsource/inconsolata': 5.2.8
@@ -11247,7 +11247,7 @@ snapshots:
       '@radix-ui/react-progress': 1.1.9(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
       '@radix-ui/react-separator': 1.1.9(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
       '@radix-ui/react-slot': 1.2.5(@types/react@19.2.15)(react@19.2.6)
-      '@vector-im/compound-design-tokens': 10.2.1(@types/react@19.2.15)(react@19.2.6)
+      '@vector-im/compound-design-tokens': 10.2.2(@types/react@19.2.15)(react@19.2.6)
       classnames: 2.5.1
       react: 19.2.6
       vaul: 1.1.2(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
@@ -13457,7 +13457,7 @@ snapshots:
 
   matrix-events-sdk@0.0.1: {}
 
-  matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/7675fb2e65c3b13910b745cd7e7067c5aad4e816:
+  matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/8c95727b6278fe7942c20d0b9485f984dd0694b7:
     dependencies:
       '@babel/runtime': 7.29.2
       '@matrix-org/matrix-sdk-crypto-wasm': 18.3.1

src/controls.ts

@@ -12,6 +12,7 @@ export interface Controls {
   canEnterPip(): boolean;
   enablePip(): void;
   disablePip(): void;
+  onPipMediaOrientationUpdate?: (orientation: "landscape" | "portrait") => void;
 
   setAvailableAudioDevices(devices: OutputDevice[]): void;
   setAudioDevice(id: string): void;

src/livekit/TrackProcessorContext.tsx

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
 
 import {
   ProcessorWrapper,
-  supportsBackgroundProcessors,
+  supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk,
   type BackgroundOptions,
 } from "@livekit/track-processors";
 import {
@@ -29,6 +29,7 @@ import {
 import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
 import { type Behavior } from "../state/Behavior";
 import { type ObservableScope } from "../state/ObservableScope";
+import { platform } from "../Platform";
 
 //TODO-MULTI-SFU: This is not yet fully there.
 // it is a combination of exposing observable and react hooks.
@@ -106,6 +107,10 @@ interface Props {
   children: JSX.Element;
 }
 
+function supportsBackgroundProcessors(): boolean {
+  return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop";
+}
+
 export const ProcessorProvider: FC<Props> = ({ children }) => {
   // The setting the user wants to have
   const [blurActivated] = useSetting(backgroundBlurSettings);

src/settings/rageshake.ts

@@ -149,7 +149,7 @@ class IndexedDBLogStore {
    * @return Resolves when the store is ready.
    */
   public async connect(): Promise<void> {
-    const req = this.indexedDB.open("logs");
+    const req = this.indexedDB.open("logs-element-call");
     return new Promise((resolve, reject) => {
       req.onsuccess = (): void => {
         this.db = req.result;

src/state/CallViewModel/CallViewModel.test.ts

@@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { test, vi, onTestFinished, it, describe } from "vitest";
+import { test, vi, onTestFinished, it, describe, expect } from "vitest";
 import {
   BehaviorSubject,
   combineLatest,
@@ -899,6 +899,41 @@ describe.each([
     },
   );
 
+  // TODO add media to lk mocks
+  test("onPipMediaOrientationUpdate is called with the spotlight media orientation", () => {
+    // Set the spy before creating the view model so the initial call is captured
+    const onPipMediaOrientationUpdate = vi.fn();
+    window.controls.onPipMediaOrientationUpdate = onPipMediaOrientationUpdate;
+    onTestFinished(() => {
+      window.controls.onPipMediaOrientationUpdate = undefined;
+    });
+
+    withTestScheduler(({ behavior }) => {
+      // Alice starts as a regular participant, then shares her screen, then stops
+      const aliceSharingInputMarbles = "nyn";
+
+      withCallViewModel(
+        {
+          remoteParticipants$: constant([aliceParticipant]),
+          rtcMembers$: constant([localRtcMember, aliceRtcMember]),
+          sharingScreen: new Map([
+            [aliceParticipant, behavior(aliceSharingInputMarbles, yesNo)],
+          ]),
+        },
+        () => {},
+      );
+    });
+
+    // Should be called exactly 3 times:
+    // 1. Initially with "portrait" (Alice is in spotlight as a user, default portrait orientation)
+    // 2. With "landscape" when Alice starts screen sharing (screen shares always use landscape)
+    // 3. With "portrait" again when Alice stops screen sharing and returns to user tile
+    expect(onPipMediaOrientationUpdate).toHaveBeenCalledTimes(3);
+    expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(1, "portrait");
+    expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(2, "landscape");
+    expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(3, "portrait");
+  });
+
   test("PiP tile in expanded spotlight layout switches speakers without layout shifts", () => {
     withTestScheduler(({ behavior, schedule, expectObservable }) => {
       // Switch to spotlight immediately

src/state/CallViewModel/CallViewModel.ts

@@ -766,11 +766,13 @@ export function createCallViewModel$(
             pretendToBeDisconnected$: localMembership.reconnecting$,
             displayName$: scope.behavior(
               matrixMemberMetadataStore
-                .createDisplayNameBehavior$(userId)
+                .createDisplayNameBehavior$(scope, userId)
                 .pipe(map((name) => name ?? userId)),
             ),
-            mxcAvatarUrl$:
-              matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
+            mxcAvatarUrl$: matrixMemberMetadataStore.createAvatarUrlBehavior$(
+              scope,
+              userId,
+            ),
             handRaised$: scope.behavior(
               handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)),
             ),
@@ -811,8 +813,10 @@ export function createCallViewModel$(
                 map((members) => members.get(userId)?.rawDisplayName || userId),
               ),
             ),
-            mxcAvatarUrl$:
-              matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
+            mxcAvatarUrl$: matrixMemberMetadataStore.createAvatarUrlBehavior$(
+              scope,
+              userId,
+            ),
             pickupState$,
             muteStates,
           }),
@@ -1199,6 +1203,33 @@ export function createCallViewModel$(
     })),
   );
 
+  spotlight$
+    .pipe(
+      switchMap((media) => {
+        let layout;
+        const pipMedia = media[0];
+        if (pipMedia === undefined) return of(undefined);
+        switch (pipMedia.type) {
+          case "user":
+            layout = pipMedia.videoOrientation$;
+            break;
+          case "ringing":
+            layout = of("landscape" as const);
+            break;
+          case "screen share":
+            layout = of("landscape" as const);
+            break;
+        }
+        return layout;
+      }),
+      scope.bind(),
+    )
+    .subscribe((orientation) => {
+      if (orientation === undefined) return;
+      logger.info("controls api pip orientation updated:", orientation);
+      window.controls.onPipMediaOrientationUpdate?.(orientation);
+    });
+
   /**
    * The media to be used to produce a layout.
    */

src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts

@@ -105,7 +105,7 @@ describe("MatrixMemberMetadata", () => {
     }
 
     it("should show our own user if present in rtc session and room", () => {
-      withTestScheduler(({ behavior, expectObservable }) => {
+      withTestScheduler(({ scope, behavior, expectObservable }) => {
         fakeMemberWith({
           userId: "@local:example.com",
           rawDisplayName: "it's a me",
@@ -118,8 +118,10 @@ describe("MatrixMemberMetadata", () => {
           memberships$,
           createRoomMembers$(testScope, mockMatrixRoom),
         );
-        const dn$ =
-          metadataStore.createDisplayNameBehavior$("@local:example.com");
+        const dn$ = metadataStore.createDisplayNameBehavior$(
+          scope,
+          "@local:example.com",
+        );
 
         expectObservable(dn$).toBe("a", {
           a: "it's a me",
@@ -146,7 +148,7 @@ describe("MatrixMemberMetadata", () => {
     it("should get displayName for users", () => {
       setUpBasicRoom();
 
-      withTestScheduler(({ behavior, expectObservable }) => {
+      withTestScheduler(({ scope, behavior, expectObservable }) => {
         const memberships$ = behavior("a", {
           a: [
             mockRtcMembership("@alice:example.com", "DEVICE1"),
@@ -158,8 +160,10 @@ describe("MatrixMemberMetadata", () => {
           memberships$,
           createRoomMembers$(testScope, mockMatrixRoom),
         );
-        const aliceDispName$ =
-          metadataStore.createDisplayNameBehavior$("@alice:example.com");
+        const aliceDispName$ = metadataStore.createDisplayNameBehavior$(
+          scope,
+          "@alice:example.com",
+        );
 
         expectObservable(aliceDispName$).toBe("a", {
           a: "Alice",
@@ -322,7 +326,7 @@ describe("MatrixMemberMetadata", () => {
     });
 
     it("should track individual member id with createDisplayNameBehavior", () => {
-      withTestScheduler(({ behavior, schedule, expectObservable }) => {
+      withTestScheduler(({ scope, behavior, schedule, expectObservable }) => {
         setUpBasicRoom();
         const BOB = "@bob:example.com";
         const CARL = "@carl:example.com";
@@ -356,8 +360,8 @@ describe("MatrixMemberMetadata", () => {
           createRoomMembers$(testScope, mockMatrixRoom),
         );
 
-        const bob$ = metadataStore.createDisplayNameBehavior$(BOB);
-        const carl$ = metadataStore.createDisplayNameBehavior$(CARL);
+        const bob$ = metadataStore.createDisplayNameBehavior$(scope, BOB);
+        const carl$ = metadataStore.createDisplayNameBehavior$(scope, CARL);
 
         expectObservable(bob$).toBe("abc-", {
           a: undefined,
@@ -378,7 +382,7 @@ describe("MatrixMemberMetadata", () => {
     });
 
     it("should disambiguate users with invisible characters", () => {
-      withTestScheduler(({ behavior, expectObservable }) => {
+      withTestScheduler(({ scope, behavior, expectObservable }) => {
         const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
         const bobZeroWidthSpaceRtcMember = mockRtcMembership(
           "@bob2:example.org",
@@ -411,12 +415,18 @@ describe("MatrixMemberMetadata", () => {
           createRoomMembers$(testScope, mockMatrixRoom),
         );
 
-        const bob$ =
-          metadataStore.createDisplayNameBehavior$("@bob:example.org");
-        const bob2$ =
-          metadataStore.createDisplayNameBehavior$("@bob2:example.org");
-        const carol$ =
-          metadataStore.createDisplayNameBehavior$("@carol:example.org");
+        const bob$ = metadataStore.createDisplayNameBehavior$(
+          scope,
+          "@bob:example.org",
+        );
+        const bob2$ = metadataStore.createDisplayNameBehavior$(
+          scope,
+          "@bob2:example.org",
+        );
+        const carol$ = metadataStore.createDisplayNameBehavior$(
+          scope,
+          "@carol:example.org",
+        );
         expectObservable(bob$).toBe("ab", {
           a: "Bob",
           b: "Bob (@bob:example.org)",
@@ -517,7 +527,7 @@ describe("MatrixMemberMetadata", () => {
     }
 
     it("should use avatar url from room members", () => {
-      withTestScheduler(({ behavior, expectObservable }) => {
+      withTestScheduler(({ scope, behavior, expectObservable }) => {
         fakeMemberWith({
           userId: "@local:example.com",
         });
@@ -536,11 +546,15 @@ describe("MatrixMemberMetadata", () => {
           memberships$,
           createRoomMembers$(testScope, mockMatrixRoom),
         );
-        const local$ =
-          metadataStore.createAvatarUrlBehavior$("@local:example.com");
+        const local$ = metadataStore.createAvatarUrlBehavior$(
+          scope,
+          "@local:example.com",
+        );
 
-        const alice$ =
-          metadataStore.createAvatarUrlBehavior$("@alice:example.com");
+        const alice$ = metadataStore.createAvatarUrlBehavior$(
+          scope,
+          "@alice:example.com",
+        );
 
         expectObservable(local$).toBe("a", {
           a: "mxc://example.com/@local:example.com",
@@ -558,7 +572,7 @@ describe("MatrixMemberMetadata", () => {
     });
 
     it("should update on avatar change and user join/leave", () => {
-      withTestScheduler(({ behavior, schedule, expectObservable }) => {
+      withTestScheduler(({ scope, behavior, schedule, expectObservable }) => {
         fakeMemberWith({ userId: "@carl:example.com" });
         fakeMemberWith({ userId: "@bob:example.com" });
         const memberships$ = behavior("ab-d", {
@@ -585,9 +599,14 @@ describe("MatrixMemberMetadata", () => {
           },
         });
 
-        const bob$ = metadataStore.createAvatarUrlBehavior$("@bob:example.com");
-        const carl$ =
-          metadataStore.createAvatarUrlBehavior$("@carl:example.com");
+        const bob$ = metadataStore.createAvatarUrlBehavior$(
+          scope,
+          "@bob:example.com",
+        );
+        const carl$ = metadataStore.createAvatarUrlBehavior$(
+          scope,
+          "@carl:example.com",
+        );
         expectObservable(bob$).toBe("a---", {
           a: "mxc://example.com/@bob:example.com",
         });

src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts

@@ -115,8 +115,14 @@ export const createMatrixMemberMetadata$ = (
   memberships$: Behavior<Pick<CallMembership, "userId">[]>,
   roomMembers$: Behavior<RoomMemberMap>,
 ): {
-  createDisplayNameBehavior$: (userId: string) => Behavior<string | undefined>;
-  createAvatarUrlBehavior$: (userId: string) => Behavior<string | undefined>;
+  createDisplayNameBehavior$: (
+    scope: ObservableScope,
+    userId: string,
+  ) => Behavior<string | undefined>;
+  createAvatarUrlBehavior$: (
+    scope: ObservableScope,
+    userId: string,
+  ) => Behavior<string | undefined>;
   displaynameMap$: Behavior<Map<string, string>>;
   avatarMap$: Behavior<Map<string, string | undefined>>;
 } => {
@@ -136,13 +142,13 @@ export const createMatrixMemberMetadata$ = (
     ),
   );
   return {
-    createDisplayNameBehavior$: (userId: string) =>
+    createDisplayNameBehavior$: (scope: ObservableScope, userId: string) =>
       scope.behavior(
         displaynameMap$.pipe(
           map((displaynameMap) => displaynameMap.get(userId)),
         ),
       ),
-    createAvatarUrlBehavior$: (userId: string) =>
+    createAvatarUrlBehavior$: (scope: ObservableScope, userId: string) =>
       scope.behavior(
         roomMembers$.pipe(
           map((roomMembers) => roomMembers.get(userId)?.getMxcAvatarUrl()),

src/state/media/UserMediaViewModel.ts

@@ -48,6 +48,7 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel {
   audioEnabled$: Behavior<boolean>;
   videoEnabled$: Behavior<boolean>;
   videoFit$: Behavior<"cover" | "contain">;
+  videoOrientation$: Behavior<"landscape" | "portrait">;
   toggleCropVideo: () => void;
   /**
    * The expected identity of the LiveKit participant. Exposed for debugging.
@@ -104,6 +105,7 @@ export function createBaseUserMedia(
     { width: number; height: number } | undefined
   >(undefined);
 
+  const videoSize$ = videoSizeFromParticipant$(participant$);
   return {
     ...createMemberMedia(scope, {
       ...inputs,
@@ -129,11 +131,14 @@ export function createBaseUserMedia(
     videoEnabled$: scope.behavior(
       media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)),
     ),
-    videoFit$: videoFit$(
-      scope,
-      videoSizeFromParticipant$(participant$),
-      targetSize$,
+    videoOrientation$: scope.behavior(
+      videoSize$.pipe(
+        map((s) => (s ? s.width / s.height : 1)),
+        map((aspect) => (aspect > 1 ? "landscape" : "portrait")),
+      ),
+      "portrait",
     ),
+    videoFit$: videoFit$(scope, videoSize$, targetSize$),
     toggleCropVideo: () => toggleCropVideo$.next(),
     rtcBackendIdentity,
     handRaised$,

src/state/media/observeRtpStreamStats.ts

@@ -32,7 +32,9 @@ export function observeRtpStreamStats$(
 > {
   return combineLatest([
     observeTrackReference$(participant, source),
-    interval(1000).pipe(startWith(0)),
+    // The update frequency is high because we use this value to update the PiP orientation and the fit/fill video tile props based on that
+    // We want it to be responsive. For just the debug tools 1s would be sufficient.
+    interval(350).pipe(startWith(0)),
   ]).pipe(
     switchMap(async ([trackReference]) => {
       const track = trackReference?.publication?.track;