Fix the interactivity of buttons while reconnecting or in earpiece mode (#3486)

* Fix the interactivity of buttons while reconnecting or in earpiece mode

When we're in one of these modes, we need to ensure that everything above the overlay (the header and footer buttons) is interactive, while everything obscured by the overlay (the media tiles) is non-interactive and removed from the accessibility tree. It's not a very easy task to trap focus *outside* an element, so the best solution I could come up with is to set tabindex="-1" manually on all interactive elements belonging to the media tiles.

* Write a Playwright test for reconnecting

* fix lints

Signed-off-by: Timo K <toger5@hotmail.de>

* fix test

Signed-off-by: Timo K <toger5@hotmail.de>

* enable http2 for matrx-rtc host to allow the jwt service to talk to the SFU

* remove rate limit for delayed events

* more time to connect to livekit SFU

* Due to a Firefox issue we set the start anchor for the tab test to the Mute microphone button

* adapt to most recent Element Web version

* Use the "End call" button as proofe for a started call

* Currrenty disabled due to recent Element Web
- not indicating the number of participants
- bypassing Lobby

* linting

* disable 'can only interact with header and footer while reconnecting' for firefox

---------

Signed-off-by: Timo K <toger5@hotmail.de>
Co-authored-by: Timo <16718859+toger5@users.noreply.github.com>
Co-authored-by: Timo K <toger5@hotmail.de>
Co-authored-by: fkwp <github-fkwp@w4ve.de>

backend/dev_nginx.conf

@@ -59,6 +59,8 @@ server {
     ssl_certificate         /root/ssl/cert.pem;
     ssl_certificate_key     /root/ssl/key.pem;
 
+    http2 on;
+
 
     location ^~ /livekit/jwt/ {
 

backend/playwright_homeserver.yaml

@@ -44,6 +44,10 @@ rc_message:
   per_second: 10000
   burst_count: 10000
 
+rc_delayed_event_mgmt:
+  per_second: 10000
+  burst_count: 10000
+
 rc_login:
   address:
     per_second: 10000

playwright/fixtures/widget-user.ts

@@ -159,8 +159,8 @@ export const widgetTest = test.extend<MyFixtures>({
     } = await registerUser(browser, userB);
 
     // Invite the second user
-    await ewPage1.getByRole("button", { name: "Add room" }).click();
-    await ewPage1.getByText("New room").click();
+    await ewPage1.getByRole("button", { name: "Add", exact: true }).click();
+    await ewPage1.getByRole("menuitem", { name: "New Room" }).click();
     await ewPage1.getByRole("textbox", { name: "Name" }).fill("Welcome Room");
     await ewPage1.getByRole("button", { name: "Create room" }).click();
     await expect(ewPage1.getByText("You created this room.")).toBeVisible();
@@ -184,9 +184,9 @@ export const widgetTest = test.extend<MyFixtures>({
 
     // Accept the invite
     await expect(
-      ewPage2.getByRole("treeitem", { name: "Welcome Room" }),
+      ewPage2.getByRole("option", { name: "Welcome Room" }),
     ).toBeVisible();
-    await ewPage2.getByRole("treeitem", { name: "Welcome Room" }).click();
+    await ewPage2.getByRole("option", { name: "Welcome Room" }).click();
     await ewPage2.getByRole("button", { name: "Accept" }).click();
     await expect(
       ewPage2.getByRole("main").getByRole("heading", { name: "Welcome Room" }),

playwright/reconnect.spec.ts

@@ -0,0 +1,60 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { expect, test } from "@playwright/test";
+
+// Skip test for Firefox, due to page.keyboard.press("Tab") not reliable on headless mode
+test.skip(
+  ({ browserName }) => browserName === "firefox",
+  'This test is not working on firefox, page.keyboard.press("Tab") not reliable in headless mode',
+);
+
+test("can only interact with header and footer while reconnecting", async ({
+  page,
+}) => {
+  await page.goto("/");
+  await page.getByTestId("home_callName").click();
+  await page.getByTestId("home_callName").fill("Test call");
+  await page.getByTestId("home_displayName").click();
+  await page.getByTestId("home_displayName").fill("Test user");
+  // If we do not call fastForward here, we end up with Date.now() returning an actual timestamp
+  // but once we call `await page.clock.fastForward(20000);` later this will reset Date.now() to 0
+  // and we will never get into probablyDisconnected state?
+  await page.clock.fastForward(10);
+  await page.getByTestId("home_go").click();
+
+  await expect(page.locator("video")).toBeVisible();
+  await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
+
+  // Join the call
+  await page.getByTestId("lobby_joinCall").click();
+
+  // The media tile for the local user should become visible
+  await new Promise((resolve) => setTimeout(resolve, 1500));
+  await expect(page.getByTestId("name_tag")).toContainText("Test user");
+
+  // Now disconnect from the internet
+  await page.route("https://synapse.m.localhost/**/*", async (route) => {
+    await new Promise((resolve) => setTimeout(resolve, 10000));
+    await route.continue();
+  });
+  await page.clock.fastForward(20000);
+
+  await expect(
+    page.getByRole("dialog", { name: "Reconnecting…" }),
+  ).toBeVisible();
+
+  // Tab order should jump directly from header to footer, skipping media tiles
+  await page.getByRole("button", { name: "Mute microphone" }).focus();
+  await expect(
+    page.getByRole("button", { name: "Mute microphone" }),
+  ).toBeFocused();
+  await page.keyboard.press("Tab");
+  await expect(page.getByRole("button", { name: "Stop video" })).toBeFocused();
+  // Most critically, we should be able to press the hangup button
+  await page.getByRole("button", { name: "End call" }).click();
+});

playwright/widget/simple-create.spec.ts

@@ -49,7 +49,10 @@ widgetTest("Start a new call as widget", async ({ asWidget, browserName }) => {
 
   // Check the join indicator on the room list
   await expect(
-    brooks.page.locator("div").filter({ hasText: /^Joined • 1$/ }),
+    brooks.page
+      .locator('iframe[title="Element Call"]')
+      .contentFrame()
+      .getByRole("button", { name: "End call" }),
   ).toBeVisible();
 
   // Join from the other side
@@ -59,26 +62,28 @@ widgetTest("Start a new call as widget", async ({ asWidget, browserName }) => {
   ).toBeVisible();
   await whistler.page.getByRole("button", { name: "Join" }).click();
 
-  await expect(
-    whistler.page
-      .locator('iframe[title="Element Call"]')
-      .contentFrame()
-      .getByTestId("lobby_joinCall"),
-  ).toBeVisible();
+  // Currently disabled due to recent Element Web is bypassing Lobby
+  // await expect(
+  //   whistler.page
+  //     .locator('iframe[title="Element Call"]')
+  //     .contentFrame()
+  //     .getByTestId("lobby_joinCall"),
+  // ).toBeVisible();
+  //
+  // await whistler.page
+  //   .locator('iframe[title="Element Call"]')
+  //   .contentFrame()
+  //   .getByTestId("lobby_joinCall")
+  //   .click();
 
-  await whistler.page
-    .locator('iframe[title="Element Call"]')
-    .contentFrame()
-    .getByTestId("lobby_joinCall")
-    .click();
+  // Currrenty disabled due to recent Element Web not indicating the number of participants
+  // await expect(
+  //   whistler.page.locator("div").filter({ hasText: /^Joined • 2$/ }),
+  // ).toBeVisible();
 
-  await expect(
-    whistler.page.locator("div").filter({ hasText: /^Joined • 2$/ }),
-  ).toBeVisible();
-
-  await expect(
-    brooks.page.locator("div").filter({ hasText: /^Joined • 2$/ }),
-  ).toBeVisible();
+  // await expect(
+  //   brooks.page.locator("div").filter({ hasText: /^Joined • 2$/ }),
+  // ).toBeVisible();
 
   // Whistler leaves
   await whistler.page.waitForTimeout(1000);

src/Overlay.module.css

@@ -35,6 +35,8 @@ Please see LICENSE in the repository root for full details.
 
 .bg.animate[data-state="closed"] {
   animation: fade-out 130ms;
+  opacity: 0;
+  pointer-events: none;
 }
 
 .overlay {

src/Toast.tsx

@@ -46,11 +46,11 @@ interface Props {
    */
   Icon?: ComponentType<SVGAttributes<SVGElement>>;
   /**
-   * Whether the toast should be portaled into the root of the document (rather
-   * than rendered in-place within the component tree).
+   * Whether the toast should be modal, making it fill the screen (by portalling
+   * it into the root of the document) and trap focus until dismissed.
    * @default true
    */
-  portal?: boolean;
+  modal?: boolean;
 }
 
 /**
@@ -62,7 +62,7 @@ export const Toast: FC<Props> = ({
   autoDismiss,
   children,
   Icon,
-  portal = true,
+  modal = true,
 }) => {
   const onOpenChange = useCallback(
     (open: boolean) => {
@@ -103,8 +103,8 @@ export const Toast: FC<Props> = ({
   );
 
   return (
-    <DialogRoot open={open} onOpenChange={onOpenChange}>
-      {portal ? <DialogPortal>{content}</DialogPortal> : content}
+    <DialogRoot open={open} onOpenChange={onOpenChange} modal={modal}>
+      {modal ? <DialogPortal>{content}</DialogPortal> : content}
     </DialogRoot>
   );
 };

src/reactions/RaisedHandIndicator.tsx

@@ -22,11 +22,13 @@ export function RaisedHandIndicator({
   miniature,
   showTimer,
   onClick,
+  tabIndex,
 }: {
   raisedHandTime?: Date;
   miniature?: boolean;
   showTimer?: boolean;
   onClick?: () => void;
+  tabIndex?: number;
 }): ReactNode {
   const { t } = useTranslation();
   const [raisedHandDuration, setRaisedHandDuration] = useState("");
@@ -94,6 +96,7 @@ export function RaisedHandIndicator({
           background: "none",
         }}
         onClick={clickCallback}
+        tabIndex={tabIndex}
       >
         {content}
       </button>

src/room/EarpieceOverlay.module.css

@@ -34,7 +34,6 @@
 
 .overlay[data-show="false"] {
   animation: fade-out 130ms forwards;
-  content-visibility: hidden;
   pointer-events: none;
 }
 

src/room/EarpieceOverlay.tsx

@@ -20,7 +20,7 @@ interface Props {
 export const EarpieceOverlay: FC<Props> = ({ show, onBackToVideoPressed }) => {
   const { t } = useTranslation();
   return (
-    <div className={styles.overlay} data-show={show}>
+    <div className={styles.overlay} data-show={show} aria-hidden={!show}>
       <BigIcon className={styles.icon}>
         <VoiceCallIcon aria-hidden />
       </BigIcon>

src/room/InCallView.tsx

@@ -54,7 +54,6 @@ import { type HeaderStyle, useUrlParams } from "../UrlParams";
 import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
 import { ElementWidgetActions, widget } from "../widget";
 import styles from "./InCallView.module.css";
-import overlayStyles from "../Overlay.module.css";
 import { GridTile } from "../tile/GridTile";
 import { type OTelGroupCallMembership } from "../otel/OTelGroupCallMembership";
 import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
@@ -119,6 +118,7 @@ import { EarpieceOverlay } from "./EarpieceOverlay.tsx";
 import { useAppBarHidden, useAppBarSecondaryButton } from "../AppBar.tsx";
 import { useBehavior } from "../useBehavior.ts";
 import { Toast } from "../Toast.tsx";
+import overlayStyles from "../Overlay.module.css";
 import { Avatar, Size as AvatarSize } from "../Avatar";
 import waitingStyles from "./WaitingForJoin.module.css";
 import { prefetchSounds } from "../soundUtils";
@@ -641,6 +641,38 @@ export const InCallView: FC<InCallViewProps> = ({
     }
   }
 
+  // The reconnecting toast cannot be dismissed
+  const onDismissReconnectingToast = useCallback(() => {}, []);
+  // We need to use a non-modal toast to avoid trapping focus within the toast.
+  // However, a non-modal toast will not render any background overlay on its
+  // own, so we must render one manually.
+  const reconnectingToast = (
+    <>
+      <div
+        className={classNames(overlayStyles.bg, overlayStyles.animate)}
+        data-state={reconnecting ? "open" : "closed"}
+      />
+      <Toast
+        onDismiss={onDismissReconnectingToast}
+        open={reconnecting}
+        modal={false}
+      >
+        {t("common.reconnecting")}
+      </Toast>
+    </>
+  );
+
+  const earpieceOverlay = (
+    <EarpieceOverlay
+      show={earpieceMode && !reconnecting}
+      onBackToVideoPressed={audioOutputSwitcher?.switch}
+    />
+  );
+
+  // If the reconnecting toast or earpiece overlay obscures the media tiles, we
+  // need to remove them from the accessibility tree and block focus.
+  const contentObscured = reconnecting || earpieceMode;
+
   const Tile = useMemo(
     () =>
       function Tile({
@@ -670,6 +702,7 @@ export const InCallView: FC<InCallViewProps> = ({
             className={classNames(className, styles.tile)}
             style={style}
             showSpeakingIndicators={showSpeakingIndicatorsValue}
+            focusable={!contentObscured}
           />
         ) : (
           <SpotlightTile
@@ -680,12 +713,13 @@ export const InCallView: FC<InCallViewProps> = ({
             targetWidth={targetWidth}
             targetHeight={targetHeight}
             showIndicators={showSpotlightIndicatorsValue}
+            focusable={!contentObscured}
             className={classNames(className, styles.tile)}
             style={style}
           />
         );
       },
-    [vm, openProfile],
+    [vm, openProfile, contentObscured],
   );
 
   const layouts = useMemo(() => {
@@ -714,6 +748,8 @@ export const InCallView: FC<InCallViewProps> = ({
           targetWidth={gridBounds.height}
           targetHeight={gridBounds.width}
           showIndicators={false}
+          focusable={!contentObscured}
+          aria-hidden={contentObscured}
         />
       );
     }
@@ -731,6 +767,7 @@ export const InCallView: FC<InCallViewProps> = ({
         model={layout}
         Layout={layers.fixed}
         Tile={Tile}
+        aria-hidden={contentObscured}
       />
     );
     const scrollingGrid = (
@@ -740,6 +777,7 @@ export const InCallView: FC<InCallViewProps> = ({
         model={layout}
         Layout={layers.scrolling}
         Tile={Tile}
+        aria-hidden={contentObscured}
       />
     );
     // The grid tiles go *under* the spotlight in the portrait layout, but
@@ -869,9 +907,6 @@ export const InCallView: FC<InCallViewProps> = ({
     </div>
   );
 
-  // The reconnecting toast cannot be dismissed
-  const onDismissReconnectingToast = useCallback(() => {}, []);
-
   return (
     <div
       className={styles.inRoom}
@@ -899,17 +934,8 @@ export const InCallView: FC<InCallViewProps> = ({
       {renderContent()}
       <CallEventAudioRenderer vm={vm} muted={muteAllAudio} />
       <ReactionsAudioRenderer vm={vm} muted={muteAllAudio} />
-      <Toast
-        onDismiss={onDismissReconnectingToast}
-        open={reconnecting}
-        portal={false}
-      >
-        {t("common.reconnecting")}
-      </Toast>
-      <EarpieceOverlay
-        show={earpieceMode && !reconnecting}
-        onBackToVideoPressed={audioOutputSwitcher?.switch}
-      />
+      {reconnectingToast}
+      {earpieceOverlay}
       <ReactionsOverlay vm={vm} />
       {waitingOverlay}
       {footer}

src/room/__snapshots__/InCallView.test.tsx.snap

@@ -111,6 +111,11 @@ exports[`InCallView > rendering > renders 1`] = `
       <div />
     </div>
     <div
+      class="bg animate"
+      data-state="closed"
+    />
+    <div
+      aria-hidden="true"
       class="overlay"
       data-show="false"
     >

src/tile/GridTile.test.tsx

@@ -64,6 +64,7 @@ test("GridTile is accessible", async () => {
             targetWidth={300}
             targetHeight={200}
             showSpeakingIndicators
+            focusable={true}
           />
         </ReactionsSenderProvider>,
       );

src/tile/GridTile.tsx

@@ -60,6 +60,7 @@ interface TileProps {
   targetHeight: number;
   displayName: string;
   showSpeakingIndicators: boolean;
+  focusable: boolean;
 }
 
 interface UserMediaTileProps extends TileProps {
@@ -81,6 +82,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
   menuEnd,
   className,
   displayName,
+  focusable,
   ...props
 }) => {
   const { toggleRaisedHand } = useReactionsSender();
@@ -162,6 +164,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
         />
       }
       displayName={displayName}
+      focusable={focusable}
       primaryButton={
         primaryButton ?? (
           <Menu
@@ -169,7 +172,10 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
             onOpenChange={setMenuOpen}
             title={displayName}
             trigger={
-              <button aria-label={t("common.options")}>
+              <button
+                aria-label={t("common.options")}
+                tabIndex={focusable ? undefined : -1}
+              >
                 <OverflowHorizontalIcon aria-hidden width={20} height={20} />
               </button>
             }
@@ -208,6 +214,7 @@ const LocalUserMediaTile: FC<LocalUserMediaTileProps> = ({
   ref,
   vm,
   onOpenProfile,
+  focusable,
   ...props
 }) => {
   const { t } = useTranslation();
@@ -236,6 +243,7 @@ const LocalUserMediaTile: FC<LocalUserMediaTileProps> = ({
             className={styles.switchCamera}
             aria-label={t("switch_camera")}
             onClick={switchCamera}
+            tabIndex={focusable ? undefined : -1}
           >
             <SwitchCameraSolidIcon aria-hidden width={20} height={20} />
           </button>
@@ -258,6 +266,7 @@ const LocalUserMediaTile: FC<LocalUserMediaTileProps> = ({
           />
         )
       }
+      focusable={focusable}
       {...props}
     />
   );
@@ -337,6 +346,7 @@ interface GridTileProps {
   className?: string;
   style?: ComponentProps<typeof animated.div>["style"];
   showSpeakingIndicators: boolean;
+  focusable: boolean;
 }
 
 export const GridTile: FC<GridTileProps> = ({

src/tile/MediaView.test.tsx

@@ -47,6 +47,7 @@ describe("MediaView", () => {
     video: trackReference,
     member: undefined,
     localParticipant: false,
+    focusable: true,
   };
 
   test("is accessible", async () => {

src/tile/MediaView.tsx

@@ -38,6 +38,7 @@ interface Props extends ComponentProps<typeof animated.div> {
   encryptionStatus: EncryptionStatus;
   nameTagLeadingIcon?: ReactNode;
   displayName: string;
+  focusable: boolean;
   primaryButton?: ReactNode;
   raisedHandTime?: Date;
   currentReaction?: ReactionOption;
@@ -61,6 +62,7 @@ export const MediaView: FC<Props> = ({
   unencryptedWarning,
   nameTagLeadingIcon,
   displayName,
+  focusable,
   primaryButton,
   encryptionStatus,
   raisedHandTime,
@@ -114,6 +116,7 @@ export const MediaView: FC<Props> = ({
             miniature={avatarSize < 96}
             showTimer={handRaiseTimerVisible}
             onClick={raisedHandOnClick}
+            tabIndex={focusable ? undefined : -1}
           />
           {currentReaction && (
             <ReactionIndicator
@@ -164,6 +167,7 @@ export const MediaView: FC<Props> = ({
               label={t("common.unencrypted")}
               placement="bottom"
               isTriggerInteractive={false}
+              nonInteractiveTriggerTabIndex={focusable ? undefined : -1}
             >
               <ErrorSolidIcon
                 width={20}

src/tile/SpotlightTile.test.tsx

@@ -59,6 +59,7 @@ test("SpotlightTile is accessible", async () => {
               expanded={false}
               onToggleExpanded={toggleExpanded}
               showIndicators
+              focusable={true}
             />,
           );
 

src/tile/SpotlightTile.tsx

@@ -59,6 +59,7 @@ interface SpotlightItemBaseProps {
   unencryptedWarning: boolean;
   encryptionStatus: EncryptionStatus;
   displayName: string;
+  focusable: boolean;
   "aria-hidden"?: boolean;
   localParticipant: boolean;
 }
@@ -112,6 +113,7 @@ interface SpotlightItemProps {
   vm: MediaViewModel;
   targetWidth: number;
   targetHeight: number;
+  focusable: boolean;
   intersectionObserver$: Observable<IntersectionObserver>;
   /**
    * Whether this item should act as a scroll snapping point.
@@ -125,6 +127,7 @@ const SpotlightItem: FC<SpotlightItemProps> = ({
   vm,
   targetWidth,
   targetHeight,
+  focusable,
   intersectionObserver$,
   snap,
   "aria-hidden": ariaHidden,
@@ -163,6 +166,7 @@ const SpotlightItem: FC<SpotlightItemProps> = ({
     member: vm.member,
     unencryptedWarning,
     displayName,
+    focusable,
     encryptionStatus,
     "aria-hidden": ariaHidden,
     localParticipant: vm.local,
@@ -185,6 +189,7 @@ interface Props {
   targetWidth: number;
   targetHeight: number;
   showIndicators: boolean;
+  focusable: boolean;
   className?: string;
   style?: ComponentProps<typeof animated.div>["style"];
 }
@@ -197,6 +202,7 @@ export const SpotlightTile: FC<Props> = ({
   targetWidth,
   targetHeight,
   showIndicators,
+  focusable = true,
   className,
   style,
 }) => {
@@ -293,6 +299,7 @@ export const SpotlightTile: FC<Props> = ({
           className={classNames(styles.advance, styles.back)}
           aria-label={t("common.back")}
           onClick={onBackClick}
+          tabIndex={focusable ? undefined : -1}
         >
           <ChevronLeftIcon aria-hidden width={24} height={24} />
         </button>
@@ -304,6 +311,7 @@ export const SpotlightTile: FC<Props> = ({
             vm={vm}
             targetWidth={targetWidth}
             targetHeight={targetHeight}
+            focusable={focusable}
             intersectionObserver$={intersectionObserver$}
             // This is how we get the container to scroll to the right media
             // when the previous/next buttons are clicked: we temporarily
@@ -319,6 +327,7 @@ export const SpotlightTile: FC<Props> = ({
           className={classNames(styles.expand)}
           aria-label={"maximise"}
           onClick={onToggleFullscreen}
+          tabIndex={focusable ? undefined : -1}
         >
           <FullScreenIcon aria-hidden width={20} height={20} />
         </button>
@@ -330,6 +339,7 @@ export const SpotlightTile: FC<Props> = ({
               expanded ? t("video_tile.collapse") : t("video_tile.expand")
             }
             onClick={onToggleExpanded}
+            tabIndex={focusable ? undefined : -1}
           >
             <ToggleExpandIcon aria-hidden width={20} height={20} />
           </button>
@@ -341,6 +351,7 @@ export const SpotlightTile: FC<Props> = ({
           className={classNames(styles.advance, styles.next)}
           aria-label={t("common.next")}
           onClick={onNextClick}
+          tabIndex={focusable ? undefined : -1}
         >
           <ChevronRightIcon aria-hidden width={24} height={24} />
         </button>