From e39fe3bc7f6b35c5c70665b8527cde3b3d3c4153 Mon Sep 17 00:00:00 2001 From: fkwp Date: Mon, 10 Mar 2025 20:55:59 +0100 Subject: [PATCH 01/11] add information about rc_delayed_event_mgmt --- docs/self-hosting.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 26644ed6..2548becf 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -46,10 +46,15 @@ experimental_features: max_event_delay_duration: 24h rc_message: - # This needs to match at least the heart-beat frequency plus a bit of headroom - # Currently the heart-beat is every 5 seconds which translates into a rate of 0.2s + # This needs to match at least e2ee key sharing frequency plus a bit of headroom + # Note key sharing events are bursty per_second: 0.5 burst_count: 30 + # This needs to match at least the heart-beat frequency plus a bit of headroom + # Currently the heart-beat is every 5 seconds which translates into a rate of 0.2s + rc_delayed_event_mgmt: + per_second: 1 + burst_count: 20 ``` ### MatrixRTC Backend From f9ced9f10dd5f971e6bd4b59e58bc09d422b3f48 Mon Sep 17 00:00:00 2001 From: fkwp Date: Mon, 10 Mar 2025 20:58:12 +0100 Subject: [PATCH 02/11] typo --- docs/self-hosting.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 2548becf..94569338 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -52,9 +52,9 @@ rc_message: burst_count: 30 # This needs to match at least the heart-beat frequency plus a bit of headroom # Currently the heart-beat is every 5 seconds which translates into a rate of 0.2s - rc_delayed_event_mgmt: - per_second: 1 - burst_count: 20 +rc_delayed_event_mgmt: + per_second: 1 + burst_count: 20 ``` ### MatrixRTC Backend From 687dafab3ecef61fdad1763287640cd2488d5918 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 13 Mar 2025 14:44:04 +0100 Subject: [PATCH 03/11] First integration tests with playwright --- .github/workflows/playwright.yml | 30 +++++++++++++ .gitignore | 6 +++ README.md | 39 +++++++++++++++++ package.json | 5 ++- playwright.config.ts | 74 ++++++++++++++++++++++++++++++++ playwright/create-call.spec.ts | 59 +++++++++++++++++++++++++ playwright/landing.spec.ts | 30 +++++++++++++ yarn.lock | 21 +++++++++ 8 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/playwright.yml create mode 100644 playwright.config.ts create mode 100644 playwright/create-call.spec.ts create mode 100644 playwright/landing.spec.ts diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 00000000..381d8bef --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,30 @@ +name: Playwright Tests +on: + pull_request: {} + push: + branches: [livekit, full-mesh] +jobs: + test: + timeout-minutes: 5mn + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: lts/* + - name: Install dependencies + run: npm install -g yarn && yarn + - name: Install Playwright Browsers + run: yarn playwright install --with-deps + - name: Run backend components + run: | + docker-compose -f dev-backend-docker-compose.yml up -d + docker ps + - name: Run Playwright tests + run: yarn playwright test + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: playwright-report + path: playwright-report/ + retention-days: 3 diff --git a/.gitignore b/.gitignore index 7b2cd2c2..938fe508 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,9 @@ public/config.json backend/synapse_tmp/* /coverage yarn-error.log + +# Playwright +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ diff --git a/README.md b/README.md index acadb9c0..4880c9bf 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,45 @@ yarn backend # podman-compose -f dev-backend-docker-compose.yml up ``` +### Playwright tests + +Our Playwright tests run automatically as part of our CI along with our other tests, +on every pull request. + +You may need to follow instructions to set up your development environment for running +Playwright by following and +. + +However the Playwright tests are run, an element-call instance must be running on +https://localhost:3000 (this is configured in `playwright.config.ts`) - this is what will +be tested. + +The local backend environment should be running for the test to work: +`yarn backend` + +There are a few different ways to run the tests yourself. The simplest is to run: + +```shell +yarn run test:playwright +``` + +This will run the Playwright tests once, non-interactively. + +There is a more user-friendly way to run the tests in interactive mode: + +```shell +yarn run test:playwright:open +``` + +The easiest way to develop new test is to use the codegen feature of Playwright: + +```shell +npx playwright codegen +``` + +This will record your action and write the test code for you. Use the tool bar to test visibility, text content, +clicking.. + ### Test Coverage diff --git a/package.json b/package.json index cacd969b..2f087f60 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,9 @@ "i18n:check": "i18next --fail-on-warnings --fail-on-update", "test": "vitest", "test:coverage": "vitest --coverage", - "backend": "docker-compose -f dev-backend-docker-compose.yml up" + "backend": "docker-compose -f dev-backend-docker-compose.yml up", + "test:playwright": "playwright test", + "test:playwright:open": "yarn test:playwright --ui" }, "devDependencies": { "@babel/core": "^7.16.5", @@ -43,6 +45,7 @@ "@opentelemetry/sdk-trace-base": "^1.25.1", "@opentelemetry/sdk-trace-web": "^1.9.1", "@opentelemetry/semantic-conventions": "^1.25.1", + "@playwright/test": "^1.51.0", "@radix-ui/react-dialog": "^1.0.4", "@radix-ui/react-slider": "^1.1.2", "@radix-ui/react-visually-hidden": "^1.0.3", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 00000000..a4e7e06e --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,74 @@ +import { defineConfig, devices } from "@playwright/test"; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// import dotenv from 'dotenv'; +// import path from 'path'; +// dotenv.config({ path: path.resolve(__dirname, '.env') }); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: "./playwright", + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: "html", + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: "https://localhost:3000", + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: "on-first-retry", + }, + /* Configure projects for major browsers */ + projects: [ + { + name: "chromium", + use: { + ...devices["Desktop Chrome"], + permissions: ["microphone", "camera"], + launchOptions: { + args: [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + "--mute-audio", + ], + }, + }, + }, + + { + name: "firefox", + use: { + ...devices["Desktop Firefox"], + launchOptions: { + firefoxUserPrefs: { + "permissions.default.microphone": 1, + "permissions.default.camera": 1, + }, + }, + }, + }, + + // No safari for now, until I find a solution to fix `Not allowed to request resource` due to calling + // clear http to the homeserver + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'yarn run backend', + // url: 'http://synapse.localhost:8008', + // reuseExistingServer: !process.env.CI, + // }, +}); diff --git a/playwright/create-call.spec.ts b/playwright/create-call.spec.ts new file mode 100644 index 00000000..8c2395cb --- /dev/null +++ b/playwright/create-call.spec.ts @@ -0,0 +1,59 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "@playwright/test"; + +test("Start a new call then leave and show the feedback screen", async ({ + page, +}) => { + await page.goto("/"); + + await page.getByTestId("home_callName").click(); + await page.getByTestId("home_callName").fill("HelloCall"); + await page.getByTestId("home_displayName").click(); + await page.getByTestId("home_displayName").fill("John Doe"); + await page.getByTestId("home_go").click(); + + await expect(page.locator("video")).toBeVisible(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + + // Check the button toolbar + // await expect(page.getByRole('button', { name: 'Mute microphone' })).toBeVisible(); + // await expect(page.getByRole('button', { name: 'Stop video' })).toBeVisible(); + await expect(page.getByRole("button", { name: "Settings" })).toBeVisible(); + await expect(page.getByRole("button", { name: "End call" })).toBeVisible(); + + // Join the call + await page.getByTestId("lobby_joinCall").click(); + + // Ensure that the call is connected + await page + .locator("div") + .filter({ hasText: /^HelloCall$/ }) + .click(); + // Check the number of participants + await expect(page.locator("div").filter({ hasText: /^1$/ })).toBeVisible(); + // The tooltip with the name should be visible + await expect(page.getByTestId("name_tag")).toContainText("John Doe"); + + // leave the call + await page.getByTestId("incall_leave").click(); + await expect(page.getByRole("heading")).toContainText( + "John Doe, your call has ended. How did it go?", + ); + await expect(page.getByRole("main")).toContainText( + "We'd love to hear your feedback so we can improve your experience.", + ); + + // Strange that the submit button test-id is home_go + await expect(page.getByTestId("home_go")).toBeVisible(); + await expect(page.getByTestId("home_go")).toContainText("Submit"); + + await expect( + page.getByRole("link", { name: "Not now, return to home screen" }), + ).toBeVisible(); +}); diff --git a/playwright/landing.spec.ts b/playwright/landing.spec.ts new file mode 100644 index 00000000..b22a037e --- /dev/null +++ b/playwright/landing.spec.ts @@ -0,0 +1,30 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { test, expect } from "@playwright/test"; + +test("has title", async ({ page }) => { + await page.goto("/"); + + await expect(page).toHaveTitle(/Element Call/); +}); + +test("Landing page", async ({ page }) => { + await page.goto("/"); + + // There should be a login button in the header + await expect(page.getByRole("link", { name: "Log In" })).toBeVisible(); + + await expect( + page.getByRole("heading", { name: "Start new call" }), + ).toBeVisible(); + + await expect(page.getByTestId("home_callName")).toBeVisible(); + await expect(page.getByTestId("home_displayName")).toBeVisible(); + + await expect(page.getByTestId("home_go")).toBeVisible(); +}); diff --git a/yarn.lock b/yarn.lock index c6b9c56b..61d70d96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2137,6 +2137,13 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== +"@playwright/test@^1.51.0": + version "1.51.0" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.51.0.tgz#8d5c8400b465a0bfdbcf993e390ceecb903ea6d2" + integrity sha512-dJ0dMbZeHhI+wb77+ljx/FeC8VBP6j/rj9OAojO08JI80wTZy6vRk9KvHKiDCUh4iMpEiseMgqRBIeW+eKX6RA== + dependencies: + playwright "1.51.0" + "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" @@ -6995,6 +7002,20 @@ picomatch@^4.0.1, picomatch@^4.0.2: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-4.0.2.tgz#77c742931e8f3b8820946c76cd0c1f13730d1dab" integrity sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg== +playwright-core@1.51.0: + version "1.51.0" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.51.0.tgz#bb23ea6bb6298242d088ae5e966ffcf8dc9827e8" + integrity sha512-x47yPE3Zwhlil7wlNU/iktF7t2r/URR3VLbH6EknJd/04Qc/PSJ0EY3CMXipmglLG+zyRxW6HNo2EGbKLHPWMg== + +playwright@1.51.0: + version "1.51.0" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.51.0.tgz#9ba154497ba62bc6dc199c58ee19295eb35a4707" + integrity sha512-442pTfGM0xxfCYxuBa/Pu6B2OqxqqaYq39JS8QDMGThUvIOCd6s0ANDog3uwA0cHavVlnTQzGCN7Id2YekDSXA== + dependencies: + playwright-core "1.51.0" + optionalDependencies: + fsevents "2.3.2" + pluralize@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1" From 6eb1a029377dd06e2f36045f40b82f0b58f10ca6 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 13 Mar 2025 15:20:40 +0100 Subject: [PATCH 04/11] fix: vitest --coverage was running playwright tests --- vitest.config.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vitest.config.js b/vitest.config.js index 68fef5be..a6c3107f 100644 --- a/vitest.config.js +++ b/vitest.config.js @@ -13,6 +13,7 @@ export default defineConfig((configEnv) => }, }, setupFiles: ["src/vitest.setup.ts"], + include: ["src/**/*.test.ts", "src/**/*.test.tsx"], coverage: { reporter: ["html", "json"], include: ["src/"], @@ -21,6 +22,7 @@ export default defineConfig((configEnv) => "src/utils/test.ts", "src/utils/test-viewmodel.ts", "src/utils/test-fixtures.ts", + "playwright/**", ], }, }, From 102aaa80e252321964d3cefef60bd6af2daa3914 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 13 Mar 2025 15:23:01 +0100 Subject: [PATCH 05/11] typo in playwright.yml timeout --- .github/workflows/playwright.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 381d8bef..cb87d9e5 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -5,7 +5,7 @@ on: branches: [livekit, full-mesh] jobs: test: - timeout-minutes: 5mn + timeout-minutes: 10 runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From a7af470549b88fa7748f4d04663c255de6245202 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 13 Mar 2025 15:33:35 +0100 Subject: [PATCH 06/11] yml: try docker compose instead of dash compose --- .github/workflows/playwright.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index cb87d9e5..8049b7ef 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -18,7 +18,7 @@ jobs: run: yarn playwright install --with-deps - name: Run backend components run: | - docker-compose -f dev-backend-docker-compose.yml up -d + docker compose -f dev-backend-docker-compose.yml up -d docker ps - name: Run Playwright tests run: yarn playwright test From 4efc32f8e2febace796c64a1d7dcedcd7fc14ac5 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 13 Mar 2025 15:37:54 +0100 Subject: [PATCH 07/11] fix: local element-call not running --- playwright.config.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index a4e7e06e..accb1b5b 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -66,9 +66,9 @@ export default defineConfig({ ], /* Run your local dev server before starting the tests */ - // webServer: { - // command: 'yarn run backend', - // url: 'http://synapse.localhost:8008', - // reuseExistingServer: !process.env.CI, - // }, + webServer: { + command: "yarn dev", + url: "https://localhost:3000", + reuseExistingServer: !process.env.CI, + }, }); From 45d3a5e332676995b4126831483ac5cfcbc6f43c Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 13 Mar 2025 15:48:31 +0100 Subject: [PATCH 08/11] playwright add ignoreHTTPSErrors option --- playwright.config.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playwright.config.ts b/playwright.config.ts index accb1b5b..1d3db581 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -38,6 +38,7 @@ export default defineConfig({ use: { ...devices["Desktop Chrome"], permissions: ["microphone", "camera"], + ignoreHTTPSErrors: true, launchOptions: { args: [ "--use-fake-ui-for-media-stream", @@ -52,6 +53,7 @@ export default defineConfig({ name: "firefox", use: { ...devices["Desktop Firefox"], + ignoreHTTPSErrors: true, launchOptions: { firefoxUserPrefs: { "permissions.default.microphone": 1, @@ -70,5 +72,6 @@ export default defineConfig({ command: "yarn dev", url: "https://localhost:3000", reuseExistingServer: !process.env.CI, + ignoreHTTPSErrors: true, }, }); From 4c477bc4b12e49b309fb95393697b6115d062e66 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 13 Mar 2025 16:31:38 +0100 Subject: [PATCH 09/11] fix: github action, missing config file for dev env --- .github/workflows/playwright.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 8049b7ef..97798480 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -20,6 +20,8 @@ jobs: run: | docker compose -f dev-backend-docker-compose.yml up -d docker ps + - name: Copy config file + run: cp config/config.devenv.json public/config.json - name: Run Playwright tests run: yarn playwright test - uses: actions/upload-artifact@v4 From 9dc515f45d8c5d77b5ba1c15ab834914fcc1b6d4 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 13 Mar 2025 17:16:48 +0100 Subject: [PATCH 10/11] Fixup: bad copy in the text --- README.md | 19 +++++++++++++++++++ playwright/create-call.spec.ts | 6 +----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4880c9bf..75e6b0df 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,25 @@ npx playwright codegen This will record your action and write the test code for you. Use the tool bar to test visibility, text content, clicking.. +##### Investigate a failed test from the CI + +In the failed action page, click on the failed job, then scroll down to the `upload-artifact` step. +You will find a link to download the zip report, as per: + +``` +Artifact playwright-report has been successfully uploaded! Final size is 1360358 bytes. Artifact ID is 2746265841 +Artifact download URL: https://github.com/element-hq/element-call/actions/runs/13837660687/artifacts/2746265841 +``` + +Unzip the report then use this command to open the report in your browser: + +```shell +npx playwright show-report ~/Downloads/playwright-report/ +``` + +Under the failed test there is a small icon looking like "3 columns" (next to test name file name), +click on it to see the live screenshots/console output. + ### Test Coverage diff --git a/playwright/create-call.spec.ts b/playwright/create-call.spec.ts index 8c2395cb..759cd2db 100644 --- a/playwright/create-call.spec.ts +++ b/playwright/create-call.spec.ts @@ -46,13 +46,9 @@ test("Start a new call then leave and show the feedback screen", async ({ "John Doe, your call has ended. How did it go?", ); await expect(page.getByRole("main")).toContainText( - "We'd love to hear your feedback so we can improve your experience.", + "Why not finish by setting up a password to keep your account?", ); - // Strange that the submit button test-id is home_go - await expect(page.getByTestId("home_go")).toBeVisible(); - await expect(page.getByTestId("home_go")).toContainText("Submit"); - await expect( page.getByRole("link", { name: "Not now, return to home screen" }), ).toBeVisible(); From b9700b998602938c75c6e7b53458725da52c49a0 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 14 Mar 2025 14:50:47 +0100 Subject: [PATCH 11/11] code review --- .github/workflows/playwright.yml | 5 +++-- README.md | 2 +- playwright.config.ts | 15 +++++++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 97798480..ddd23bbb 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -11,9 +11,10 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: lts/* + cache: "yarn" + node-version-file: ".node-version" - name: Install dependencies - run: npm install -g yarn && yarn + run: yarn install --frozen-lockfile - name: Install Playwright Browsers run: yarn playwright install --with-deps - name: Run backend components diff --git a/README.md b/README.md index 75e6b0df..70f46106 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ npx playwright codegen ``` This will record your action and write the test code for you. Use the tool bar to test visibility, text content, -clicking.. +clicking. ##### Investigate a failed test from the CI diff --git a/playwright.config.ts b/playwright.config.ts index 1d3db581..0724a43a 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,12 +1,11 @@ -import { defineConfig, devices } from "@playwright/test"; +/* +Copyright 2025 New Vector Ltd. -/** - * Read environment variables from file. - * https://github.com/motdotla/dotenv - */ -// import dotenv from 'dotenv'; -// import path from 'path'; -// dotenv.config({ path: path.resolve(__dirname, '.env') }); +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { defineConfig, devices } from "@playwright/test"; /** * See https://playwright.dev/docs/test-configuration.