From 4f3dc9af75c08b6fd80528c0551c6432e0b9a10f Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Mon, 9 Nov 2026 12:52:06 +0100 Subject: [PATCH] Shard playwright tests --- .github/workflows/test.yaml | 65 +++++++++++++++++++++++++++++++++++-- playwright.config.ts | 46 ++++++++++++++------------ 2 files changed, 88 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5e2703f67..a5bcb0351 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -71,10 +71,27 @@ jobs: flags: unittests fail_ci_if_error: true playwright: - name: Run end-to-end tests + name: Run end-to-end tests (${{ matrix.part }}) if: ${{ !inputs.disable_e2e_tests }} timeout-minutes: 60 runs-on: ubuntu-latest + # We manually shard the tests across runners. Playwright's own shards cut + # the list of tests into equal counts in file order, which would leave the + # widget suite, the slowest tests by far, together at the end. + strategy: + fail-fast: false + matrix: + include: + - part: chromium-1 + args: --project=chromium --shard=1/2 + - part: chromium-2 + args: --project=chromium --shard=2/2 + - part: widget-1 + args: --project=widget --shard=1/2 + - part: widget-2 + args: --project=widget --shard=2/2 + - part: firefox-mobile + args: --project=firefox --project=mobile steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -113,11 +130,55 @@ jobs: - name: Run Playwright tests env: USE_DOCKER: 1 - run: pnpm exec playwright test + ARGS: ${{ matrix.args }} + # Unquoted on purpose: the part's arguments are several words + run: pnpm exec playwright test $ARGS - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ !cancelled() }} + with: + name: blob-report-${{ matrix.part }} + path: blob-report + if-no-files-found: error + # Only read by the merge below. + retention-days: 1 + + e2e_report: + # Combines the blobs into a single HTML report and blocks PRs on failure. + name: Run end-to-end tests + needs: playwright + if: ${{ always() && !inputs.disable_e2e_tests }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + # Inside a called workflow, github.repository is the caller so we need to make sure to still check out the EC repo here. + repository: ${{ inputs.matrix-js-sdk-sha && 'element-hq/element-call' || github.repository }} + persist-credentials: false + - name: Enable Corepack + run: corepack enable + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + cache: "pnpm" + node-version-file: ".node-version" + - name: Install dependencies + run: scripts/layered.sh --ignore-pnpmfile + env: + PR_HEAD_REPO_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} + JS_SDK_GITHUB_BASE_REF: ${{ inputs.matrix-js-sdk-sha }} + - name: Download the parts' reports + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: blob-report-* + path: all-blob-reports + merge-multiple: true + - name: Merge into one HTML report + run: pnpm exec playwright merge-reports --reporter html ./all-blob-reports + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: html-report path: playwright-report if-no-files-found: error retention-days: 4 + - name: Fail on a failed part + if: ${{ needs.playwright.result != 'success' }} + run: exit 1 diff --git a/playwright.config.ts b/playwright.config.ts index 2de044194..5129731a9 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -26,6 +26,19 @@ process.env.NODE_EXTRA_CA_CERTS = join( "backend/dev_tls_local-ca.crt", ); +const desktopChrome = { + ...devices["Desktop Chrome"], + permissions: ["clipboard-write", "clipboard-read", "microphone", "camera"], + ignoreHTTPSErrors: true, + launchOptions: { + args: [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + "--mute-audio", + ], + }, +}; + /** * See https://playwright.dev/docs/test-configuration. */ @@ -43,11 +56,11 @@ export default defineConfig({ * Reporter to use. See https://playwright.dev/docs/test-reporters * * On CI, `list` prints a line per test as it finishes, which keeps the job - * log moving; `github` annotates failures on the PR; `html` is uploaded as - * an artifact and must never try to open a browser there. + * log moving; `github` annotates failures on the PR; the `blob` format is + * uploaded by sharded CI runs and combined into a HTML report by the workflow. */ reporter: process.env.CI - ? [["list"], ["github"], ["html", { open: "never" }]] + ? [["list"], ["github"], ["blob"]] : [["list"], ["html"]], /* Stop a broken run early on CI instead of waiting out every retry. */ maxFailures: process.env.CI ? 5 : undefined, @@ -65,24 +78,15 @@ export default defineConfig({ projects: [ { name: "chromium", - testIgnore: "**/mobile/**", - use: { - ...devices["Desktop Chrome"], - permissions: [ - "clipboard-write", - "clipboard-read", - "microphone", - "camera", - ], - ignoreHTTPSErrors: true, - launchOptions: { - args: [ - "--use-fake-ui-for-media-stream", - "--use-fake-device-for-media-stream", - "--mute-audio", - ], - }, - }, + testIgnore: ["**/mobile/**", "**/widget/**"], + use: desktopChrome, + }, + { + // Element Call as a widget in Element Web, on Chromium. This uses its + // own project so that it can be sharded on CI. + name: "widget", + testMatch: "**/widget/**/*.@(spec|test).ts", + use: desktopChrome, }, { name: "firefox",