diff --git a/playwright/widget/simple-create.spec.ts b/playwright/widget/simple-create.spec.ts index 8c889892..4686e99d 100644 --- a/playwright/widget/simple-create.spec.ts +++ b/playwright/widget/simple-create.spec.ts @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { expect, test } from "@playwright/test"; import { widgetTest } from "../fixtures/widget-user.ts"; +import { TestHelpers } from "./test-helpers.ts"; // Skip test, including Fixtures widgetTest.skip( @@ -20,19 +21,7 @@ widgetTest("Start a new call as widget", async ({ asWidget, browserName }) => { const { brooks, whistler } = asWidget; - await expect( - brooks.page.getByRole("button", { name: "Video call" }), - ).toBeVisible(); - await brooks.page.getByRole("button", { name: "Video call" }).click(); - - await expect( - brooks.page.getByRole("menuitem", { name: "Legacy Call" }), - ).toBeVisible(); - await expect( - brooks.page.getByRole("menuitem", { name: "Element Call" }), - ).toBeVisible(); - - await brooks.page.getByRole("menuitem", { name: "Element Call" }).click(); + await TestHelpers.startCallInCurrentRoom(brooks.page, false); await expect( brooks.page @@ -56,11 +45,7 @@ widgetTest("Start a new call as widget", async ({ asWidget, browserName }) => { ).toBeVisible(); // Join from the other side - await expect(whistler.page.getByText("Video call started")).toBeVisible(); - await expect( - whistler.page.getByRole("button", { name: "Join" }), - ).toBeVisible(); - await whistler.page.getByRole("button", { name: "Join" }).click(); + await TestHelpers.joinCallInCurrentRoom(whistler.page); // Currently disabled due to recent Element Web is bypassing Lobby // await expect( diff --git a/playwright/widget/test-helpers.ts b/playwright/widget/test-helpers.ts index 630681bd..31247e1a 100644 --- a/playwright/widget/test-helpers.ts +++ b/playwright/widget/test-helpers.ts @@ -34,6 +34,9 @@ export class TestHelpers { ).toBeVisible(); await page.getByRole("menuitem", { name: "Element Call" }).click(); + + // TODO: Remove as soon as web merges https://github.com/element-hq/element-web/pull/32755 + await this.dismissFileDialogPermissionIfNeeded(page); } public static async joinCallFromLobby(page: Page): Promise { @@ -60,6 +63,9 @@ export class TestHelpers { await expect(page.getByText(label)).toBeVisible(); await expect(page.getByRole("button", { name: "Join" })).toBeVisible(); await page.getByRole("button", { name: "Join" }).click(); + + // TODO: Remove as soon as web merges https://github.com/element-hq/element-web/pull/32755 + await this.dismissFileDialogPermissionIfNeeded(page); } /** @@ -235,9 +241,30 @@ export class TestHelpers { ): Promise { await page.getByRole("button", { name: "Video call" }).click(); await page.getByRole("menuitem", { name: "Element Call" }).click(); + + // TODO: Remove as soon as web merges https://github.com/element-hq/element-web/pull/32755 + await this.dismissFileDialogPermissionIfNeeded(page); + await TestHelpers.setEmbeddedElementCallRtcMode(page, mode); await page.getByRole("button", { name: "Close lobby" }).click(); } + + // TODO: Remove as soon as web merges https://github.com/element-hq/element-web/pull/32755 + public static async dismissFileDialogPermissionIfNeeded( + page: Page, + ): Promise { + const dialogHeading = page.getByRole("heading", { + name: "Approve widget permissions", + }); + + try { + await expect(dialogHeading).toBeVisible({ timeout: 3000 }); + await page.getByRole("button", { name: "Approve" }).click(); + } catch { + // Dialog did not appear, that's fine + } + } + /** * Goes to the settings to set the RTC mode. * then closes the settings modal. diff --git a/playwright/widget/voice-call-dm.spec.ts b/playwright/widget/voice-call-dm.spec.ts index a4e6255b..f5152e1f 100644 --- a/playwright/widget/voice-call-dm.spec.ts +++ b/playwright/widget/voice-call-dm.spec.ts @@ -45,6 +45,8 @@ widgetTest( await expect(whistler.page.getByText("Incoming voice call")).toBeVisible(); await whistler.page.getByRole("button", { name: "Accept" }).click(); + await TestHelpers.dismissFileDialogPermissionIfNeeded(whistler.page); + await expect( whistler.page.locator('iframe[title="Element Call"]'), ).toBeVisible(); @@ -138,6 +140,8 @@ widgetTest( await expect(whistler.page.getByText("Incoming video call")).toBeVisible(); await whistler.page.getByRole("button", { name: "Accept" }).click(); + await TestHelpers.dismissFileDialogPermissionIfNeeded(whistler.page); + await expect( whistler.page.locator('iframe[title="Element Call"]'), ).toBeVisible(); diff --git a/src/Avatar.test.tsx b/src/Avatar.test.tsx index 25d2c42b..1e32de0e 100644 --- a/src/Avatar.test.tsx +++ b/src/Avatar.test.tsx @@ -12,7 +12,7 @@ import { type FC, type PropsWithChildren } from "react"; import { type WidgetApi } from "matrix-widget-api"; import { ClientContextProvider } from "./ClientContext"; -import { Avatar } from "./Avatar"; +import { Avatar, getAvatarFromWidgetAPI } from "./Avatar"; import { mockMatrixRoomMember, mockRtcMembership } from "./utils/test"; import { widget } from "./widget"; @@ -178,3 +178,36 @@ test("should attempt to use widget API if running as a widget", async () => { expect(widget!.api.downloadFile).toBeCalledWith(expectedMXCUrl); }); + +test("Supports download files as base64", async () => { + const expectedMXCUrl = "mxc://example.org/alice-avatar"; + const expectedBase64 = + "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAADIElEQVR4nAAQA+/8ApxhEfFNuwna" + + "+DO1pFMx5YDg6gb8p1WFkbFSox9H6r5c8jp1gxlHXrDfA/oQFi4A0gTXH9YBNgwRm12xO68QP6lv" + + "ZLKH9qW1VM6kz6zA3T1Ui8J+Xbnh2BZ7oXDe/2gajzoA6j1JGotpz99xO+T2NR634Nhx3zhuera/" + + "UdrpMLdEpwWXLnSqZRasGsrl93FjdTwRBMaqsx6vJksnPOmV9ttbXFIOb0XDGPbVythSC2n7P/bS" + + "Zv0U0QqbBLk/5Wu1werYzAHiz11Bj8bEylQ92Pxvo+PwF6/KbGnIHTvGZkFzDkMnqz3g7Pw3NOSP" + + "oV+qfyJuSI0AeZmrPejFQ8kzBSDWO8D7lr4+6ePRBRmZtKCf+fNjSCOyb5jqwhBnD2cycbJtQQbR" + + "A4qdPG2ONfTPeQgi96+zT7grBI0JwvgFBceJdLJd4BX1VQIyY+j7OYueNWqEpf8iYgMj78I95eRt" + + "nfPLwlxhVns84iL4Yvw8jDrB9vQi8ktpsdJOMiDwKrBGD3q56COD2oIA96CCBgiro4tkvkumZSAc" + + "ZKXRLsziUFGytWJLaPjwnzXv2hicPy6k9AXsF3QkysOZAkB3m9XPpixhq9b0OKqV/zZx3L79o6wZ" + + "Dr40J7sj7f+ARd545CP01r5omHt94tbnjgA46HsM2OhP+qQ882LN+Bhscq2WSHGSHT4J9MQcsWZP" + + "2+N2LdPy61MN4/1++BJHmDcDLQBUEwLvjZp1fRfzxV7yirwIiOA7Vr8z+1yvS/pSkfUzkjswybOd" + + "M5i0I8Q69MTXAKxqtR0/tyGkfCmHfupGASp/SAT9J8f3aQV+gDbpva592v4w8Cv5EMm7CzZPwThF" + + "kgTChNPts7F03ccxpblfIz0EiAON1DKk71rX07BvDlLHY1ItPuqZ7hjy19jrAgl+QqEE1btHVA5R" + + "uAnRXpEWc6rjARlJY5G1wbMk12rrqpr8rhR3YpFgLgOx4BtQ0D/hGe7KANSGBMQojmObId0asCmd" + + "XzmnQI9P8QnwsO9vtqZlgIoU4g+f2/G8Q3/nVMX7dujniwEAAP//KmiQs7P8MeIAAAAASUVORK5C" + + "YII="; + const mockWidgetAPI = { + downloadFile: vi.fn().mockImplementation(async (contentUri) => { + if (contentUri !== expectedMXCUrl) { + return Promise.reject(new Error("Unexpected content URI")); + } + return { file: expectedBase64 }; + }), + } as unknown as WidgetApi; + + const blob = await getAvatarFromWidgetAPI(mockWidgetAPI, expectedMXCUrl); + + expect(blob).toBeInstanceOf(Blob); +}); diff --git a/src/Avatar.tsx b/src/Avatar.tsx index d0cb243c..99940540 100644 --- a/src/Avatar.tsx +++ b/src/Avatar.tsx @@ -173,7 +173,8 @@ async function getAvatarFromServer( return blob; } -async function getAvatarFromWidgetAPI( +// export for testing +export async function getAvatarFromWidgetAPI( api: WidgetApi, src: string, ): Promise { @@ -181,9 +182,14 @@ async function getAvatarFromWidgetAPI( const file = response.file; // element-web sends a Blob, and the MSC4039 is considering changing the spec to strictly Blob, so only handling that - if (!(file instanceof Blob)) { - throw new Error("Downloaded file is not a Blob"); + if (file instanceof Blob) { + return file; + } else if (typeof file === "string") { + // it is a base64 string + const bytes = Uint8Array.from(atob(file), (c) => c.charCodeAt(0)); + return new Blob([bytes]); } - - return file; + throw new Error( + "Downloaded file format is not supported: " + typeof file + "", + ); } diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 8360cdc7..d3756e6c 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -137,6 +137,7 @@ export async function getSFUConfigWithOpenID( ); logger?.info(`Got JWT from call's active focus URL.`); } catch (e) { + logger?.debug(`Failed fetching jwt with matrix 2.0 endpoint:`, e); if (e instanceof NotSupportedError) { logger?.warn( `Failed fetching jwt with matrix 2.0 endpoint (retry with legacy) Not supported`, diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 8454b09a..f6c00c7f 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -60,6 +60,7 @@ describe("LocalTransport", () => { client: { // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: async () => Promise.resolve([]), + getAccessToken: vi.fn().mockReturnValue("access_token"), getDomain: () => "", baseUrl: "example.org", // These won't be called in this error path but satisfy the type @@ -102,6 +103,7 @@ describe("LocalTransport", () => { baseUrl: "https://lk.example.org", // Use empty domain to skip .well-known and use config directly getDomain: () => "", + getAccessToken: vi.fn().mockReturnValue("access_token"), // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: async () => Promise.resolve([]), getOpenIdToken: vi.fn(), @@ -149,6 +151,7 @@ describe("LocalTransport", () => { getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), baseUrl: "https://lk.example.org", + getAccessToken: vi.fn().mockReturnValue("access_token"), }, ownMembershipIdentity: ownMemberMock, forceJwtEndpoint: JwtEndpointVersion.Legacy, @@ -217,6 +220,7 @@ describe("LocalTransport", () => { getDomain: () => "", // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: async () => Promise.resolve([]), + getAccessToken: vi.fn().mockReturnValue("access_token"), getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), baseUrl: "https://lk.example.org", @@ -273,6 +277,7 @@ describe("LocalTransport", () => { // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: async () => Promise.resolve([aliceTransport]), + getAccessToken: vi.fn().mockReturnValue("access_token"), getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), baseUrl: "https://lk.example.org", @@ -323,6 +328,7 @@ describe("LocalTransport", () => { getDomain: vi.fn().mockReturnValue(""), // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: vi.fn().mockResolvedValue([]), + getAccessToken: vi.fn().mockReturnValue("access_token"), getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), }, @@ -410,6 +416,42 @@ describe("LocalTransport", () => { }); }); + it("Should not call _unstable_getRTCTransports in widget mode but use well-known", async () => { + mockConfig({ + livekit: { livekit_service_url: "https://do-not-use.lk.example.org" }, + }); + + localTransportOpts.client.getDomain.mockReturnValue("example.org"); + + fetchMock.getOnce("https://example.org/.well-known/matrix/client", { + "org.matrix.msc4143.rtc_foci": [ + { + type: "livekit", + livekit_service_url: "https://use-me.jwt.call.example.org", + }, + ], + }); + + localTransportOpts.client.getAccessToken.mockReturnValue(null); + const { advertised$, active$ } = + createLocalTransport$(localTransportOpts); + openIdResolver.resolve?.(openIdResponse); + expect(advertised$.value).toBe(null); + expect(active$.value).toBe(null); + await flushPromises(); + + expect( + localTransportOpts.client._unstable_getRTCTransports, + ).not.toHaveBeenCalled(); + + const expectedTransport = { + type: "livekit", + livekit_service_url: "https://use-me.jwt.call.example.org", + }; + + expect(advertised$.value).toStrictEqual(expectedTransport); + }); + it("fails fast if the openID request fails for backend config", async () => { localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([ { type: "livekit", livekit_service_url: "https://lk.example.org" }, @@ -481,6 +523,7 @@ describe("LocalTransport", () => { baseUrl: "https://example.org", // eslint-disable-next-line @typescript-eslint/naming-convention _unstable_getRTCTransports: async () => Promise.resolve([]), + getAccessToken: vi.fn().mockReturnValue("access_token"), // These won't be called in this error path but satisfy the type getOpenIdToken: vi.fn(), getDeviceId: vi.fn(), diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 0b566ba0..da4fe1dc 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -56,7 +56,7 @@ interface Props { memberships$: Behavior>; client: Pick< MatrixClient, - "getDomain" | "baseUrl" | "_unstable_getRTCTransports" + "getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken" > & OpenIDClientParts; // Used by the jwt service to create the livekit room and compute the livekit alias. @@ -314,7 +314,7 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; async function makeTransport( client: Pick< MatrixClient, - "getDomain" | "baseUrl" | "_unstable_getRTCTransports" + "getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken" > & OpenIDClientParts, membership: CallMembershipIdentityParts, @@ -371,11 +371,18 @@ async function makeTransport( for (const potentialTransport of transports) { if (isLivekitTransportConfig(potentialTransport)) { try { + logger.info( + `makeTransport: check transport authentication for "${potentialTransport.livekit_service_url}"`, + ); // This will call the jwt/sfu/get endpoint to pre create the livekit room. return await doOpenIdAndJWTFromUrl( potentialTransport.livekit_service_url, ); } catch (ex) { + logger.debug( + `makeTransport: Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`, + ex, + ); // Explictly throw these if (ex instanceof FailToGetOpenIdToken) { throw ex; @@ -383,24 +390,34 @@ async function makeTransport( if (ex instanceof NoMatrix2AuthorizationService) { throw ex; } - logger.debug( - `Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`, - ex, - ); } + } else { + logger.info( + `makeTransport: "${potentialTransport.livekit_service_url}" is not a valid livekit transport as SFU`, + ); } } return null; } // MSC4143: Attempt to fetch transports from backend. - if ("_unstable_getRTCTransports" in client) { + // TODO: Workaround for an issue in the js-sdk RoomWidgetClient that + // is not yet implementing _unstable_getRTCTransports properly (via widget API new action). + // For now we just skip this call if we are in a widget. + // In widget mode the client is a `RoomWidgetClient` which has no access token (it is using the widget API). + // Could be removed once the js-sdk is fixed (https://github.com/matrix-org/matrix-js-sdk/issues/5245) + const isSPA = !!client.getAccessToken(); + if (isSPA && "_unstable_getRTCTransports" in client) { + logger.info( + "makeTransport: First try to use getRTCTransports end point ...", + ); try { + // TODO This should also check for server support? const transportList = await client._unstable_getRTCTransports(); const selectedTransport = await getFirstUsableTransport(transportList); if (selectedTransport) { logger.info( - "Using backend-configured (client.getRTCTransports) SFU", + "makeTransport: ...Using backend-configured (client.getRTCTransports) SFU", selectedTransport, ); return selectedTransport; @@ -424,6 +441,10 @@ async function makeTransport( } } + logger.info( + `makeTransport: Trying to get transports from .well-known/matrix/client on domain ${client.getDomain()} ...`, + ); + // Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available. const domain = client.getDomain(); if (domain) { @@ -441,6 +462,10 @@ async function makeTransport( } } + logger.info( + `makeTransport: No valid transport found via backend or .well-known, falling back to config if available.`, + ); + // CONFIG: Least prioritized; Load from config file const urlFromConf = Config.get().livekit?.livekit_service_url; if (urlFromConf) {