Merge branch 'livekit' into fkwp/delegation_of_delayed_events

playwright/widget/simple-create.spec.ts

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
 import { expect, test } from "@playwright/test";
 
 import { widgetTest } from "../fixtures/widget-user.ts";
+import { TestHelpers } from "./test-helpers.ts";
 
 // Skip test, including Fixtures
 widgetTest.skip(
@@ -20,19 +21,7 @@ widgetTest("Start a new call as widget", async ({ asWidget, browserName }) => {
 
   const { brooks, whistler } = asWidget;
 
-  await expect(
-    brooks.page.getByRole("button", { name: "Video call" }),
-  ).toBeVisible();
-  await brooks.page.getByRole("button", { name: "Video call" }).click();
-
-  await expect(
-    brooks.page.getByRole("menuitem", { name: "Legacy Call" }),
-  ).toBeVisible();
-  await expect(
-    brooks.page.getByRole("menuitem", { name: "Element Call" }),
-  ).toBeVisible();
-
-  await brooks.page.getByRole("menuitem", { name: "Element Call" }).click();
+  await TestHelpers.startCallInCurrentRoom(brooks.page, false);
 
   await expect(
     brooks.page
@@ -56,11 +45,7 @@ widgetTest("Start a new call as widget", async ({ asWidget, browserName }) => {
   ).toBeVisible();
 
   // Join from the other side
-  await expect(whistler.page.getByText("Video call started")).toBeVisible();
-  await expect(
-    whistler.page.getByRole("button", { name: "Join" }),
-  ).toBeVisible();
-  await whistler.page.getByRole("button", { name: "Join" }).click();
+  await TestHelpers.joinCallInCurrentRoom(whistler.page);
 
   // Currently disabled due to recent Element Web is bypassing Lobby
   // await expect(

playwright/widget/test-helpers.ts

@@ -34,6 +34,9 @@ export class TestHelpers {
     ).toBeVisible();
 
     await page.getByRole("menuitem", { name: "Element Call" }).click();
+
+    // TODO: Remove as soon as web merges https://github.com/element-hq/element-web/pull/32755
+    await this.dismissFileDialogPermissionIfNeeded(page);
   }
 
   public static async joinCallFromLobby(page: Page): Promise<void> {
@@ -60,6 +63,9 @@ export class TestHelpers {
     await expect(page.getByText(label)).toBeVisible();
     await expect(page.getByRole("button", { name: "Join" })).toBeVisible();
     await page.getByRole("button", { name: "Join" }).click();
+
+    // TODO: Remove as soon as web merges https://github.com/element-hq/element-web/pull/32755
+    await this.dismissFileDialogPermissionIfNeeded(page);
   }
 
   /**
@@ -235,9 +241,30 @@ export class TestHelpers {
   ): Promise<void> {
     await page.getByRole("button", { name: "Video call" }).click();
     await page.getByRole("menuitem", { name: "Element Call" }).click();
+
+    // TODO: Remove as soon as web merges https://github.com/element-hq/element-web/pull/32755
+    await this.dismissFileDialogPermissionIfNeeded(page);
+
     await TestHelpers.setEmbeddedElementCallRtcMode(page, mode);
     await page.getByRole("button", { name: "Close lobby" }).click();
   }
+
+  // TODO: Remove as soon as web merges https://github.com/element-hq/element-web/pull/32755
+  public static async dismissFileDialogPermissionIfNeeded(
+    page: Page,
+  ): Promise<void> {
+    const dialogHeading = page.getByRole("heading", {
+      name: "Approve widget permissions",
+    });
+
+    try {
+      await expect(dialogHeading).toBeVisible({ timeout: 3000 });
+      await page.getByRole("button", { name: "Approve" }).click();
+    } catch {
+      // Dialog did not appear, that's fine
+    }
+  }
+
   /**
    * Goes to the settings to set the RTC mode.
    * then closes the settings modal.

playwright/widget/voice-call-dm.spec.ts

@@ -45,6 +45,8 @@ widgetTest(
     await expect(whistler.page.getByText("Incoming voice call")).toBeVisible();
     await whistler.page.getByRole("button", { name: "Accept" }).click();
 
+    await TestHelpers.dismissFileDialogPermissionIfNeeded(whistler.page);
+
     await expect(
       whistler.page.locator('iframe[title="Element Call"]'),
     ).toBeVisible();
@@ -138,6 +140,8 @@ widgetTest(
     await expect(whistler.page.getByText("Incoming video call")).toBeVisible();
     await whistler.page.getByRole("button", { name: "Accept" }).click();
 
+    await TestHelpers.dismissFileDialogPermissionIfNeeded(whistler.page);
+
     await expect(
       whistler.page.locator('iframe[title="Element Call"]'),
     ).toBeVisible();

src/Avatar.test.tsx

@@ -12,7 +12,7 @@ import { type FC, type PropsWithChildren } from "react";
 import { type WidgetApi } from "matrix-widget-api";
 
 import { ClientContextProvider } from "./ClientContext";
-import { Avatar } from "./Avatar";
+import { Avatar, getAvatarFromWidgetAPI } from "./Avatar";
 import { mockMatrixRoomMember, mockRtcMembership } from "./utils/test";
 import { widget } from "./widget";
 
@@ -178,3 +178,36 @@ test("should attempt to use widget API if running as a widget", async () => {
 
   expect(widget!.api.downloadFile).toBeCalledWith(expectedMXCUrl);
 });
+
+test("Supports download files as base64", async () => {
+  const expectedMXCUrl = "mxc://example.org/alice-avatar";
+  const expectedBase64 =
+    "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAADIElEQVR4nAAQA+/8ApxhEfFNuwna" +
+    "+DO1pFMx5YDg6gb8p1WFkbFSox9H6r5c8jp1gxlHXrDfA/oQFi4A0gTXH9YBNgwRm12xO68QP6lv" +
+    "ZLKH9qW1VM6kz6zA3T1Ui8J+Xbnh2BZ7oXDe/2gajzoA6j1JGotpz99xO+T2NR634Nhx3zhuera/" +
+    "UdrpMLdEpwWXLnSqZRasGsrl93FjdTwRBMaqsx6vJksnPOmV9ttbXFIOb0XDGPbVythSC2n7P/bS" +
+    "Zv0U0QqbBLk/5Wu1werYzAHiz11Bj8bEylQ92Pxvo+PwF6/KbGnIHTvGZkFzDkMnqz3g7Pw3NOSP" +
+    "oV+qfyJuSI0AeZmrPejFQ8kzBSDWO8D7lr4+6ePRBRmZtKCf+fNjSCOyb5jqwhBnD2cycbJtQQbR" +
+    "A4qdPG2ONfTPeQgi96+zT7grBI0JwvgFBceJdLJd4BX1VQIyY+j7OYueNWqEpf8iYgMj78I95eRt" +
+    "nfPLwlxhVns84iL4Yvw8jDrB9vQi8ktpsdJOMiDwKrBGD3q56COD2oIA96CCBgiro4tkvkumZSAc" +
+    "ZKXRLsziUFGytWJLaPjwnzXv2hicPy6k9AXsF3QkysOZAkB3m9XPpixhq9b0OKqV/zZx3L79o6wZ" +
+    "Dr40J7sj7f+ARd545CP01r5omHt94tbnjgA46HsM2OhP+qQ882LN+Bhscq2WSHGSHT4J9MQcsWZP" +
+    "2+N2LdPy61MN4/1++BJHmDcDLQBUEwLvjZp1fRfzxV7yirwIiOA7Vr8z+1yvS/pSkfUzkjswybOd" +
+    "M5i0I8Q69MTXAKxqtR0/tyGkfCmHfupGASp/SAT9J8f3aQV+gDbpva592v4w8Cv5EMm7CzZPwThF" +
+    "kgTChNPts7F03ccxpblfIz0EiAON1DKk71rX07BvDlLHY1ItPuqZ7hjy19jrAgl+QqEE1btHVA5R" +
+    "uAnRXpEWc6rjARlJY5G1wbMk12rrqpr8rhR3YpFgLgOx4BtQ0D/hGe7KANSGBMQojmObId0asCmd" +
+    "XzmnQI9P8QnwsO9vtqZlgIoU4g+f2/G8Q3/nVMX7dujniwEAAP//KmiQs7P8MeIAAAAASUVORK5C" +
+    "YII=";
+  const mockWidgetAPI = {
+    downloadFile: vi.fn().mockImplementation(async (contentUri) => {
+      if (contentUri !== expectedMXCUrl) {
+        return Promise.reject(new Error("Unexpected content URI"));
+      }
+      return { file: expectedBase64 };
+    }),
+  } as unknown as WidgetApi;
+
+  const blob = await getAvatarFromWidgetAPI(mockWidgetAPI, expectedMXCUrl);
+
+  expect(blob).toBeInstanceOf(Blob);
+});

src/Avatar.tsx

@@ -173,7 +173,8 @@ async function getAvatarFromServer(
   return blob;
 }
 
-async function getAvatarFromWidgetAPI(
+// export for testing
+export async function getAvatarFromWidgetAPI(
   api: WidgetApi,
   src: string,
 ): Promise<Blob> {
@@ -181,9 +182,14 @@ async function getAvatarFromWidgetAPI(
   const file = response.file;
 
   // element-web sends a Blob, and the MSC4039 is considering changing the spec to strictly Blob, so only handling that
-  if (!(file instanceof Blob)) {
-    throw new Error("Downloaded file is not a Blob");
+  if (file instanceof Blob) {
+    return file;
+  } else if (typeof file === "string") {
+    // it is a base64 string
+    const bytes = Uint8Array.from(atob(file), (c) => c.charCodeAt(0));
+    return new Blob([bytes]);
   }
-
-  return file;
+  throw new Error(
+    "Downloaded file format is not supported: " + typeof file + "",
+  );
 }

src/livekit/openIDSFU.ts

@@ -137,6 +137,7 @@ export async function getSFUConfigWithOpenID(
       );
       logger?.info(`Got JWT from call's active focus URL.`);
     } catch (e) {
+      logger?.debug(`Failed fetching jwt with matrix 2.0 endpoint:`, e);
       if (e instanceof NotSupportedError) {
         logger?.warn(
           `Failed fetching jwt with matrix 2.0 endpoint (retry with legacy) Not supported`,

src/state/CallViewModel/localMember/LocalTransport.test.ts

@@ -60,6 +60,7 @@ describe("LocalTransport", () => {
       client: {
         // eslint-disable-next-line @typescript-eslint/naming-convention
         _unstable_getRTCTransports: async () => Promise.resolve([]),
+        getAccessToken: vi.fn().mockReturnValue("access_token"),
         getDomain: () => "",
         baseUrl: "example.org",
         // These won't be called in this error path but satisfy the type
@@ -102,6 +103,7 @@ describe("LocalTransport", () => {
         baseUrl: "https://lk.example.org",
         // Use empty domain to skip .well-known and use config directly
         getDomain: () => "",
+        getAccessToken: vi.fn().mockReturnValue("access_token"),
         // eslint-disable-next-line @typescript-eslint/naming-convention
         _unstable_getRTCTransports: async () => Promise.resolve([]),
         getOpenIdToken: vi.fn(),
@@ -149,6 +151,7 @@ describe("LocalTransport", () => {
         getOpenIdToken: vi.fn(),
         getDeviceId: vi.fn(),
         baseUrl: "https://lk.example.org",
+        getAccessToken: vi.fn().mockReturnValue("access_token"),
       },
       ownMembershipIdentity: ownMemberMock,
       forceJwtEndpoint: JwtEndpointVersion.Legacy,
@@ -217,6 +220,7 @@ describe("LocalTransport", () => {
           getDomain: () => "",
           // eslint-disable-next-line @typescript-eslint/naming-convention
           _unstable_getRTCTransports: async () => Promise.resolve([]),
+          getAccessToken: vi.fn().mockReturnValue("access_token"),
           getOpenIdToken: vi.fn(),
           getDeviceId: vi.fn(),
           baseUrl: "https://lk.example.org",
@@ -273,6 +277,7 @@ describe("LocalTransport", () => {
           // eslint-disable-next-line @typescript-eslint/naming-convention
           _unstable_getRTCTransports: async () =>
             Promise.resolve([aliceTransport]),
+          getAccessToken: vi.fn().mockReturnValue("access_token"),
           getOpenIdToken: vi.fn(),
           getDeviceId: vi.fn(),
           baseUrl: "https://lk.example.org",
@@ -323,6 +328,7 @@ describe("LocalTransport", () => {
           getDomain: vi.fn().mockReturnValue(""),
           // eslint-disable-next-line @typescript-eslint/naming-convention
           _unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
+          getAccessToken: vi.fn().mockReturnValue("access_token"),
           getOpenIdToken: vi.fn(),
           getDeviceId: vi.fn(),
         },
@@ -410,6 +416,42 @@ describe("LocalTransport", () => {
       });
     });
 
+    it("Should not call _unstable_getRTCTransports in widget mode but use well-known", async () => {
+      mockConfig({
+        livekit: { livekit_service_url: "https://do-not-use.lk.example.org" },
+      });
+
+      localTransportOpts.client.getDomain.mockReturnValue("example.org");
+
+      fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
+        "org.matrix.msc4143.rtc_foci": [
+          {
+            type: "livekit",
+            livekit_service_url: "https://use-me.jwt.call.example.org",
+          },
+        ],
+      });
+
+      localTransportOpts.client.getAccessToken.mockReturnValue(null);
+      const { advertised$, active$ } =
+        createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(advertised$.value).toBe(null);
+      expect(active$.value).toBe(null);
+      await flushPromises();
+
+      expect(
+        localTransportOpts.client._unstable_getRTCTransports,
+      ).not.toHaveBeenCalled();
+
+      const expectedTransport = {
+        type: "livekit",
+        livekit_service_url: "https://use-me.jwt.call.example.org",
+      };
+
+      expect(advertised$.value).toStrictEqual(expectedTransport);
+    });
+
     it("fails fast if the openID request fails for backend config", async () => {
       localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
         { type: "livekit", livekit_service_url: "https://lk.example.org" },
@@ -481,6 +523,7 @@ describe("LocalTransport", () => {
           baseUrl: "https://example.org",
           // eslint-disable-next-line @typescript-eslint/naming-convention
           _unstable_getRTCTransports: async () => Promise.resolve([]),
+          getAccessToken: vi.fn().mockReturnValue("access_token"),
           // These won't be called in this error path but satisfy the type
           getOpenIdToken: vi.fn(),
           getDeviceId: vi.fn(),

src/state/CallViewModel/localMember/LocalTransport.ts

@@ -56,7 +56,7 @@ interface Props {
   memberships$: Behavior<Epoch<CallMembership[]>>;
   client: Pick<
     MatrixClient,
-    "getDomain" | "baseUrl" | "_unstable_getRTCTransports"
+    "getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken"
   > &
     OpenIDClientParts;
   // Used by the jwt service to create the livekit room and compute the livekit alias.
@@ -314,7 +314,7 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
 async function makeTransport(
   client: Pick<
     MatrixClient,
-    "getDomain" | "baseUrl" | "_unstable_getRTCTransports"
+    "getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken"
   > &
     OpenIDClientParts,
   membership: CallMembershipIdentityParts,
@@ -371,11 +371,18 @@ async function makeTransport(
     for (const potentialTransport of transports) {
       if (isLivekitTransportConfig(potentialTransport)) {
         try {
+          logger.info(
+            `makeTransport: check transport authentication for "${potentialTransport.livekit_service_url}"`,
+          );
           // This will call the jwt/sfu/get endpoint to pre create the livekit room.
           return await doOpenIdAndJWTFromUrl(
             potentialTransport.livekit_service_url,
           );
         } catch (ex) {
+          logger.debug(
+            `makeTransport: Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`,
+            ex,
+          );
           // Explictly throw these
           if (ex instanceof FailToGetOpenIdToken) {
             throw ex;
@@ -383,24 +390,34 @@ async function makeTransport(
           if (ex instanceof NoMatrix2AuthorizationService) {
             throw ex;
           }
-          logger.debug(
-            `Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`,
-            ex,
-          );
         }
+      } else {
+        logger.info(
+          `makeTransport: "${potentialTransport.livekit_service_url}" is not a valid livekit transport  as SFU`,
+        );
       }
     }
     return null;
   }
 
   // MSC4143: Attempt to fetch transports from backend.
-  if ("_unstable_getRTCTransports" in client) {
+  // TODO: Workaround for an issue in the js-sdk RoomWidgetClient that
+  // is not yet implementing _unstable_getRTCTransports properly (via widget API new action).
+  // For now we just skip this call if we are in a widget.
+  // In widget mode the client is a `RoomWidgetClient` which has no access token (it is using the widget API).
+  // Could be removed once the js-sdk is fixed (https://github.com/matrix-org/matrix-js-sdk/issues/5245)
+  const isSPA = !!client.getAccessToken();
+  if (isSPA && "_unstable_getRTCTransports" in client) {
+    logger.info(
+      "makeTransport: First try to use getRTCTransports end point ...",
+    );
     try {
+      // TODO This should also check for server support?
       const transportList = await client._unstable_getRTCTransports();
       const selectedTransport = await getFirstUsableTransport(transportList);
       if (selectedTransport) {
         logger.info(
-          "Using backend-configured (client.getRTCTransports) SFU",
+          "makeTransport: ...Using backend-configured (client.getRTCTransports) SFU",
           selectedTransport,
         );
         return selectedTransport;
@@ -424,6 +441,10 @@ async function makeTransport(
     }
   }
 
+  logger.info(
+    `makeTransport: Trying to get transports from .well-known/matrix/client on domain ${client.getDomain()} ...`,
+  );
+
   // Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available.
   const domain = client.getDomain();
   if (domain) {
@@ -441,6 +462,10 @@ async function makeTransport(
     }
   }
 
+  logger.info(
+    `makeTransport: No valid transport found via backend or .well-known, falling back to config if available.`,
+  );
+
   // CONFIG: Least prioritized; Load from config file
   const urlFromConf = Config.get().livekit?.livekit_service_url;
   if (urlFromConf) {