sdk implementation

playwright/sdk/harness.ts

@@ -5,7 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { expect, type Page } from "@playwright/test";
+import {
+  type Browser,
+  expect,
+  type Locator,
+  type Page,
+} from "@playwright/test";
 
 import { SynapseAdmin } from "../utils/synapse-admin.ts";
 
@@ -18,35 +23,74 @@ export const SDK_HARNESS_URL = "https://localhost:3002";
 const HOMESERVER_URL = "https://synapse.m.localhost";
 const PASSWORD = "foobarbaz1!";
 
+/** A registered user, with the session the registration opened. */
+export interface User {
+  username: string;
+  displayName: string;
+  userId: string;
+  deviceId: string;
+  accessToken: string;
+}
+
 /**
  * Registers two users through the Synapse admin API and has the first create a
- * public room, so that the second can join it by id without an invite.
+ * public room, so that the second can join it by id without an invite. The
+ * registration logs each user in, which spares the harness a `/login` call
+ * that the homeserver rate-limits.
  */
 export async function createUsersAndRoom(
   name: string,
-): Promise<{ usernames: [string, string]; roomId: string }> {
+): Promise<{ users: [User, User]; roomId: string }> {
   const admin = SynapseAdmin.forHomeserver(HOMESERVER_URL);
-  const usernames: [string, string] = [
-    `${name}_a_${Date.now()}`,
-    `${name}_b_${Date.now()}`,
-  ];
-  const [{ access_token: accessToken }] = await Promise.all(
-    usernames.map(async (username, index) =>
-      admin.registerUser(username, PASSWORD, `${name} ${"AB"[index]}`),
-    ),
-  );
+  const users = (await Promise.all(
+    ["a", "b"].map(async (letter, index): Promise<User> => {
+      const username = `${name}_${letter}_${Date.now()}`;
+      const displayName = `${name} ${"AB"[index]}`;
+      const registration = await admin.registerUser(
+        username,
+        PASSWORD,
+        displayName,
+      );
+      return {
+        username,
+        displayName,
+        userId: registration.user_id,
+        deviceId: registration.device_id,
+        accessToken: registration.access_token,
+      };
+    }),
+  )) as [User, User];
 
   const response = await fetch(
     `${HOMESERVER_URL}/_matrix/client/v3/createRoom`,
     {
       method: "POST",
       headers: {
-        Authorization: `Bearer ${accessToken}`,
+        Authorization: `Bearer ${users[0].accessToken}`,
         "Content-Type": "application/json",
       },
       body: JSON.stringify({
         name: `${name}'s session`,
         preset: "public_chat",
+        // Per-participant media keys travel in encrypted to-device messages,
+        // which only reach devices the sender's crypto tracks, and it tracks
+        // the members of encrypted rooms
+        initial_state: [
+          {
+            type: "m.room.encryption",
+            state_key: "",
+            content: { algorithm: "m.megolm.v1.aes-sha2" },
+          },
+        ],
+        // Every member has to be allowed to write its own membership, which
+        // a public room does not grant by default; the same levels Element
+        // Call gives a room it creates for a call
+        power_level_content_override: {
+          state_default: 0,
+          events_default: 0,
+          users_default: 0,
+          events: { "org.matrix.msc3401.call.member": 0 },
+        },
       }),
     },
   );
@@ -56,33 +100,68 @@ export async function createUsersAndRoom(
     );
   const { room_id: roomId } = (await response.json()) as { room_id: string };
 
-  return { usernames, roomId };
+  return { users, roomId };
 }
 
 /**
  * Opens the harness signed in as the given user and waits until the SDK
- * reports the session as joined. The status line carries the SDK's error if
- * it does not get there, so the failure says why.
+ * reports the session as connected. The status line carries the SDK's error
+ * if it does not get there, so the failure says why.
  */
 export async function startHarness(
   page: Page,
-  username: string,
+  user: User,
   roomId: string,
 ): Promise<void> {
   const query = new URLSearchParams({
     homeserver: HOMESERVER_URL,
-    username,
-    password: PASSWORD,
+    accessToken: user.accessToken,
+    userId: user.userId,
+    deviceId: user.deviceId,
     room: roomId,
   });
   await page.goto(`${SDK_HARNESS_URL}/?${query.toString()}`);
   await page.getByRole("button", { name: "Start" }).click();
+  await waitForConnected(page);
+}
 
-  // A login, a crypto setup and an initial sync happen first. An error is
-  // final, so it is not worth waiting out the timeout for "Joined" after one.
+/**
+ * A login, a crypto setup and an initial sync happen before the session can
+ * connect. An error is final, so it is not worth waiting out the timeout for
+ * "connected" after one.
+ */
+export async function waitForConnected(page: Page): Promise<void> {
   const status = page.getByTestId("status");
-  await expect(status).toHaveText(/^(Joined|Error)/, { timeout: 120_000 });
+  await expect(status).toHaveText(/^(connected|Error)/, { timeout: 120_000 });
   const text = await status.textContent();
-  if (text !== "Joined")
-    throw new Error(`The harness did not join the session: ${text}`);
+  if (text !== "connected")
+    throw new Error(`The harness did not connect to the session: ${text}`);
+}
+
+/**
+ * A context of its own per user, since a browser profile holds one login. No
+ * permissions to grant: each browser is launched with fake media that is
+ * handed out without asking (see playwright.config.ts).
+ */
+export async function newPage(browser: Browser): Promise<Page> {
+  const context = await browser.newContext({ ignoreHTTPSErrors: true });
+  return context.newPage();
+}
+
+/** Two users in one session, each on a page of their own. */
+export async function startPair(
+  browser: Browser,
+  name: string,
+): Promise<{ pages: [Page, Page]; users: [User, User]; roomId: string }> {
+  const { users, roomId } = await createUsersAndRoom(name);
+  const pages = await Promise.all([newPage(browser), newPage(browser)]);
+  await Promise.all(
+    pages.map(async (page, i) => startHarness(page, users[i], roomId)),
+  );
+  return { pages: pages as [Page, Page], users, roomId };
+}
+
+/** The tile a page shows for a user. */
+export function tileOf(page: Page, user: User): Locator {
+  return page.locator(`[data-testid="member"][data-user-id="${user.userId}"]`);
 }

playwright/sdk/lifecycle.spec.ts

@@ -0,0 +1,67 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { expect, test } from "@playwright/test";
+
+import { startPair, tileOf, waitForConnected } from "./harness.ts";
+
+/**
+ * Members coming and going: a member that leaves disappears for the peer, one
+ * that comes back is shown again, and the harness is left in the state its
+ * status line claims.
+ */
+
+test.describe.configure({ timeout: 300_000 });
+
+test("a member that leaves disappears for the peer", async ({ browser }) => {
+  const {
+    pages: [leaver, stayer],
+    users: [leaving, staying],
+  } = await startPair(browser, "sdkleave");
+  await expect(stayer.getByTestId("member")).toHaveCount(2, {
+    timeout: 60_000,
+  });
+
+  await leaver.getByRole("button", { name: "Leave" }).click();
+  await expect(leaver.getByTestId("status")).toHaveText("Left");
+  await expect(leaver.getByTestId("member")).toHaveCount(0);
+
+  await expect(tileOf(stayer, leaving)).toHaveCount(0, { timeout: 60_000 });
+  await expect(tileOf(stayer, staying)).toHaveCount(1);
+  await expect(stayer.getByTestId("status")).toHaveText("connected");
+});
+
+test("a member that reloads is shown again with its media", async ({
+  browser,
+}) => {
+  const {
+    pages: [reloader, watcher],
+    users: [reloading],
+  } = await startPair(browser, "sdkreload");
+  await expect(watcher.getByTestId("member")).toHaveCount(2, {
+    timeout: 60_000,
+  });
+
+  // The page comes back on the same device, so the new membership replaces
+  // the old one rather than sitting next to it
+  await reloader.reload();
+  await reloader.getByRole("button", { name: "Start" }).click();
+  await waitForConnected(reloader);
+
+  await expect(tileOf(watcher, reloading)).toHaveCount(1, {
+    timeout: 120_000,
+  });
+  const video = tileOf(watcher, reloading).locator("video");
+  await expect
+    .poll(async () => video.evaluate((v: HTMLVideoElement) => v.videoWidth), {
+      timeout: 60_000,
+    })
+    .toBeGreaterThan(0);
+  await expect(reloader.getByTestId("member")).toHaveCount(2, {
+    timeout: 60_000,
+  });
+});

playwright/sdk/media.spec.ts

@@ -0,0 +1,158 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { expect, type Locator, type Page, test } from "@playwright/test";
+
+import { startPair, tileOf, type User } from "./harness.ts";
+
+/**
+ * What a member's media looks like from the other side: it plays, it is
+ * encrypted, it follows the mute switches, and the SFU sends only as much as
+ * the tile on screen can show. Each test reads the labels the harness puts on
+ * its media elements, which come straight from the SDK's track behaviors.
+ *
+ * One session serves every test here: logging two browsers in and connecting
+ * them takes most of a minute, and none of the tests leaves the session in a
+ * state the next cannot start from.
+ */
+
+test.describe.configure({ mode: "serial", timeout: 300_000 });
+
+let pages: [Page, Page];
+let users: [User, User];
+
+test.beforeAll(async ({ browser }) => {
+  ({ pages, users } = await startPair(browser, "sdkmedia"));
+  for (const page of pages)
+    await expect(page.getByTestId("member")).toHaveCount(2, {
+      timeout: 60_000,
+    });
+});
+
+test.afterAll(async () => {
+  for (const page of pages ?? []) await page.context().close();
+});
+
+test("the peer's camera and microphone play", async () => {
+  const peer = remoteTile(1, 0);
+  const video = peer.locator("video");
+  await expect
+    .poll(async () => video.evaluate((v: HTMLVideoElement) => v.videoWidth), {
+      timeout: 60_000,
+    })
+    .toBeGreaterThan(0);
+  await expect(video).toHaveJSProperty("paused", false);
+  const audio = peer.locator("audio");
+  await expect
+    .poll(async () =>
+      audio.evaluate((a: HTMLAudioElement) => a.srcObject !== null),
+    )
+    .toBe(true);
+  await expect(audio).toHaveJSProperty("paused", false);
+});
+
+test("the peer's tracks are encrypted", async () => {
+  const peer = remoteTile(1, 0);
+  await expect(peer.locator("video")).toHaveAttribute("data-encrypted", "true");
+  await expect(peer.locator("audio")).toHaveAttribute("data-encrypted", "true");
+});
+
+test("muting the microphone and the camera is seen by the peer", async () => {
+  const [page] = pages;
+  const peer = remoteTile(1, 0);
+  const video = peer.locator("video");
+  const audio = peer.locator("audio");
+  await expect(video).toHaveAttribute("data-muted", "false");
+  await expect(audio).toHaveAttribute("data-muted", "false");
+
+  await page.getByRole("button", { name: "Microphone" }).click();
+  await expect(audio).toHaveAttribute("data-muted", "true", {
+    timeout: 20_000,
+  });
+  await expect(video).toHaveAttribute("data-muted", "false");
+
+  await page.getByRole("button", { name: "Camera" }).click();
+  await expect(video).toHaveAttribute("data-muted", "true", {
+    timeout: 20_000,
+  });
+
+  await page.getByRole("button", { name: "Microphone" }).click();
+  await page.getByRole("button", { name: "Camera" }).click();
+  await expect(audio).toHaveAttribute("data-muted", "false", {
+    timeout: 20_000,
+  });
+  await expect(video).toHaveAttribute("data-muted", "false", {
+    timeout: 20_000,
+  });
+});
+
+test("a larger tile receives a higher resolution", async () => {
+  const peer = remoteTile(1, 0);
+  const video = peer.locator("video");
+  // The tile starts small enough for the lowest simulcast layer
+  const small = await receivedWidth(video, (width) => width > 0);
+  await resize(peer, 1280);
+  const large = await receivedWidth(video, (width) => width > small);
+  expect(large).toBeGreaterThan(small);
+  await resize(peer, 240);
+  await receivedWidth(video, (width) => width < large);
+});
+
+test("a hidden tile stops receiving video, and a shown one resumes", async () => {
+  const peer = remoteTile(1, 0);
+  const video = peer.locator("video");
+  await expect(frames(video)).resolves.toBeGreaterThan(0);
+
+  await peer.evaluate((tile) => (tile.style.display = "none"));
+  await expect
+    .poll(async () => framesStill(video), { timeout: 60_000 })
+    .toBe(true);
+
+  await peer.evaluate((tile) => (tile.style.display = ""));
+  await expect
+    .poll(async () => !(await framesStill(video)), { timeout: 60_000 })
+    .toBe(true);
+});
+
+/** The tile on one page for the user of the other page. */
+function remoteTile(viewer: 0 | 1, shown: 0 | 1): Locator {
+  return tileOf(pages[viewer], users[shown]);
+}
+
+async function resize(tile: Locator, width: number): Promise<void> {
+  await tile.evaluate((t, w) => (t.style.width = `${w}px`), width);
+}
+
+/** The width of the video as decoded, once it satisfies the condition. */
+async function receivedWidth(
+  video: Locator,
+  until: (width: number) => boolean,
+): Promise<number> {
+  // Switching simulcast layers takes the SFU a few seconds
+  let width = 0;
+  await expect
+    .poll(
+      async () => {
+        width = await video.evaluate((v: HTMLVideoElement) => v.videoWidth);
+        return until(width);
+      },
+      { timeout: 60_000 },
+    )
+    .toBe(true);
+  return width;
+}
+
+async function frames(video: Locator): Promise<number> {
+  return Number(await video.getAttribute("data-frames"));
+}
+
+/** Whether no frame arrived over a couple of seconds. */
+async function framesStill(video: Locator): Promise<boolean> {
+  const before = await frames(video);
+  await new Promise((resolve) => setTimeout(resolve, 2500));
+  return (await frames(video)) === before;
+}

playwright/sdk/smoke.spec.ts

@@ -5,17 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { type Browser, expect, type Page, test } from "@playwright/test";
+import { expect, test } from "@playwright/test";
 
-import { createUsersAndRoom, startHarness } from "./harness.ts";
+import { startPair, tileOf } from "./harness.ts";
 
 /**
  * The MatrixRTC SDK driven through its harness in `sdk/dev`: no Element Call
  * on the page, only the SDK's public API.
- *
- * This is the smoke test the implementation is built against. It fails until
- * `createRtcSession` does something, and it fails on the status line first, so
- * the SDK's own error is what the report shows.
  */
 
 // Two browsers each log in, set up crypto and sync before anything is on
@@ -23,34 +19,14 @@ import { createUsersAndRoom, startHarness } from "./harness.ts";
 test.describe.configure({ timeout: 300_000 });
 
 test("two browsers see each other in one session", async ({ browser }) => {
-  const { usernames, roomId } = await createUsersAndRoom("sdksmoke");
-  const [pageA, pageB] = await Promise.all(
-    usernames.map(async () => newPage(browser)),
-  );
-
-  await Promise.all([
-    startHarness(pageA, usernames[0], roomId),
-    startHarness(pageB, usernames[1], roomId),
-  ]);
+  const { pages, users } = await startPair(browser, "sdksmoke");
 
   // Each page shows itself and the other, each tile named by its user
-  for (const page of [pageA, pageB]) {
+  for (const page of pages) {
     await expect(page.getByTestId("member")).toHaveCount(2, {
       timeout: 60_000,
     });
-    for (const username of usernames)
-      await expect(
-        page.getByTestId("member").filter({ hasText: username }),
-      ).toHaveCount(1);
+    for (const user of users)
+      await expect(tileOf(page, user)).toContainText(user.displayName);
   }
 });
-
-/**
- * A context of its own per user, since a browser profile holds one login. No
- * permissions to grant: each browser is launched with fake media that is
- * handed out without asking (see playwright.config.ts).
- */
-async function newPage(browser: Browser): Promise<Page> {
-  const context = await browser.newContext({ ignoreHTTPSErrors: true });
-  return context.newPage();
-}