From 21a2cc093259ebe4b70c0475f25197e8998bd5e2 Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Wed, 21 Oct 2026 19:41:54 -0700 Subject: [PATCH] ci(viewerjs): run browser suites with workers and shards Frontend coverage runs as three shards (PM_TEST_WORKERS=3 each) that upload their raw V8 files as coverage-raw-N; a merge job downloads them, fails unless all three arrived with raw files and no clashing names, then runs coverage:merge. Firefox and WebKit page workflows run as two shards each with three workers. The required viewerjs context keeps its name and now needs the shards, the merge and the browser shards, failing unless every one succeeded. --- .github/workflows/viewerjs-check.yml | 127 ++++++++++++++--- TESTING.md | 23 ++-- tests/vendor/ci-workflow.test.mjs | 197 +++++++++++++++++++++++++++ 3 files changed, 317 insertions(+), 30 deletions(-) create mode 100644 tests/vendor/ci-workflow.test.mjs diff --git a/.github/workflows/viewerjs-check.yml b/.github/workflows/viewerjs-check.yml index 41ddf1b..a009e1d 100644 --- a/.github/workflows/viewerjs-check.yml +++ b/.github/workflows/viewerjs-check.yml @@ -16,9 +16,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# Timeouts are the measured job time plus about 50%, so a hung run stops early. +# Before sharding (main 2cecfcf, one worker): frontend coverage 32m10s, page workflows firefox 18m59s, +# webkit 19m26s. Sharded with 3 workers per job: estimates pending the first CI run (coverage shard ~10m, +# merge ~5m, workflow shard ~8m). Re-measure when a job passes its estimate. jobs: frontend-coverage: - name: frontend coverage (chromium and touch) + name: frontend coverage (shard ${{ matrix.shard }}/3) runs-on: ubuntu-latest # Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the # same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json; @@ -26,12 +30,15 @@ jobs: container: image: mcr.microsoft.com/playwright:v1.64.1-noble options: --user 1001 - # This job runs every page workflow twice (chromium, then touch) on one worker with V8 coverage on, so it - # takes about twice a single-browser job. Measured 2026-10-20: whole job 18m26s-24m57s (main 5ded3cf - # 22m15s; PR head 6359999 was cancelled at the old 25-minute limit while still passing), coverage step - # alone 20m55s-22m17s+, against ~12-15 minutes for the firefox/webkit jobs. 40 minutes leaves headroom - # for suite growth while still stopping a hung run; re-measure if a run passes 30 minutes. - timeout-minutes: 40 + timeout-minutes: 15 + # Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test + # server, leave one CPU for the servers and the runner. + env: + PM_TEST_WORKERS: 3 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] steps: - uses: actions/checkout@v7 # The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache @@ -51,23 +58,92 @@ jobs: echo "$venv/bin" >> "$GITHUB_PATH" - run: python --version && node --version && node --version | grep -q '^v24\.' - run: npm ci --prefix tools/vendor --ignore-scripts - - run: npm run --prefix tools/vendor check - - run: npm run --prefix tools/vendor test - - name: Frontend unit, viewer, workflow, and coverage checks - run: npm run --prefix tools/vendor test:coverage - - name: Upload frontend coverage and Chromium traces + # Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests + # run on the lightest shard instead. + - if: matrix.shard == 3 + run: npm run --prefix tools/vendor check + - if: matrix.shard == 3 + run: npm run --prefix tools/vendor test + - name: Page workflows with coverage, shard ${{ matrix.shard }}/3 + run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3 + - name: Upload raw coverage + uses: actions/upload-artifact@v7 + with: + name: coverage-raw-${{ matrix.shard }} + path: coverage/js/raw/ + if-no-files-found: error + retention-days: 3 + - name: Upload Chromium and touch traces + if: always() + uses: actions/upload-artifact@v7 + with: + name: frontend-traces-shard-${{ matrix.shard }} + path: tools/vendor/test-results/ + retention-days: 14 + + frontend-coverage-merge: + name: frontend coverage (merge) + needs: frontend-coverage + runs-on: ubuntu-latest + # Same image as the shards: the raw files name sources by the container's workspace path. + container: + image: mcr.microsoft.com/playwright:v1.64.1-noble + options: --user 1001 + timeout-minutes: 8 + steps: + - uses: actions/checkout@v7 + - run: node --version && node --version | grep -q '^v24\.' + - run: npm ci --prefix tools/vendor --ignore-scripts + # Each artifact lands in its own coverage-raw-N directory, so the next step can count them. + - uses: actions/download-artifact@v8 + with: + pattern: coverage-raw-* + path: ${{ runner.temp }}/coverage-raw + # coverage:merge only checks that raw files exist; merging two of three shards would report + # partial coverage, so a missing or empty shard artifact fails here. + - name: Verify every shard's raw coverage arrived + shell: bash + env: + STAGED_RAW: ${{ runner.temp }}/coverage-raw + EXPECTED_SHARDS: 3 + run: | + found=0 + for shard in $(seq 1 "$EXPECTED_SHARDS"); do + directory="$STAGED_RAW/coverage-raw-$shard" + count=0 + if [ -d "$directory" ]; then count=$(find "$directory" -maxdepth 1 -type f -name '*.json' | wc -l); fi + echo "coverage-raw-$shard: $count raw files" + if [ "$count" -gt 0 ]; then found=$((found + 1)); fi + done + echo "shard artifacts with raw coverage: $found of $EXPECTED_SHARDS" + if [ "$found" -ne "$EXPECTED_SHARDS" ]; then + echo "::error::Expected raw coverage from $EXPECTED_SHARDS shards, got $found of $EXPECTED_SHARDS" + exit 1 + fi + staged=$(find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' | wc -l) + mkdir -p coverage/js/raw + # cp -n never overwrites; a clash shows up as fewer merged files than staged ones. + find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' -exec cp -n -t coverage/js/raw {} + || true + merged=$(find coverage/js/raw -maxdepth 1 -type f -name '*.json' | wc -l) + if [ "$merged" -ne "$staged" ]; then + echo "::error::Shards wrote clashing raw file names: $staged staged, $merged merged" + exit 1 + fi + echo "merged $merged raw files" + - name: Frontend coverage report and baseline check + run: npm run --prefix tools/vendor coverage:merge + - name: Upload frontend coverage report if: always() uses: actions/upload-artifact@v7 with: name: frontend-test-reports path: | - tools/vendor/test-results/ coverage/js/ !coverage/js/raw/** retention-days: 14 browser-workflows: - name: page workflows (${{ matrix.browser }}) + name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2) runs-on: ubuntu-latest # Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the # same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json; @@ -75,11 +151,16 @@ jobs: container: image: mcr.microsoft.com/playwright:v1.64.1-noble options: --user 1001 - timeout-minutes: 25 + timeout-minutes: 12 + # Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test + # server, leave one CPU for the servers and the runner. + env: + PM_TEST_WORKERS: 3 strategy: fail-fast: false matrix: browser: [firefox, webkit] + shard: [1, 2] steps: - uses: actions/checkout@v7 # The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache @@ -99,21 +180,21 @@ jobs: echo "$venv/bin" >> "$GITHUB_PATH" - run: python --version && node --version && node --version | grep -q '^v24\.' - run: npm ci --prefix tools/vendor --ignore-scripts - - name: Page workflows in ${{ matrix.browser }} - run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} + - name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2 + run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2 - name: Upload browser traces if: always() uses: actions/upload-artifact@v7 with: - name: browser-traces-${{ matrix.browser }} + name: browser-traces-${{ matrix.browser }}-shard-${{ matrix.shard }} path: tools/vendor/test-results/ retention-days: 14 # Branch protection requires the exact context "viewerjs". This job keeps that - # name and fails unless every frontend job above succeeded, including when one - # was cancelled or skipped. + # name and fails unless every frontend job above succeeded (every shard, the + # coverage merge and every browser shard), including when one was cancelled or skipped. viewerjs: - needs: [frontend-coverage, browser-workflows] + needs: [frontend-coverage, frontend-coverage-merge, browser-workflows] if: always() runs-on: ubuntu-latest timeout-minutes: 5 @@ -121,8 +202,10 @@ jobs: - name: Require every frontend job to succeed env: COVERAGE_RESULT: ${{ needs.frontend-coverage.result }} + MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }} WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }} run: | echo "frontend-coverage: $COVERAGE_RESULT" + echo "frontend-coverage-merge: $MERGE_RESULT" echo "browser-workflows: $WORKFLOWS_RESULT" - test "$COVERAGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success + test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success diff --git a/TESTING.md b/TESTING.md index 5948a2b..3d84a81 100644 --- a/TESTING.md +++ b/TESTING.md @@ -118,16 +118,23 @@ second without a browser and fails, naming the files, when a measured `web/js` file has no baseline entry or the baseline lists a file that no longer exists. It uses the gate's own `discoverSources`, and `npm test` runs the same check, so run it before pushing a new or deleted frontend file instead of -waiting for the `frontend coverage` job. +waiting for the `frontend coverage` jobs. The `viewerjs` check runs on every PR, including backend-only changes. It is an -aggregate job that passes only when the `frontend coverage (chromium and touch)` -job and both `page workflows (firefox)` and `page workflows (webkit)` jobs -succeed; the browsers run on separate runners in parallel. GitHub artifacts -named `frontend-test-reports` contain HTML coverage, a JSON summary, and the -Chromium/touch failure screenshots/traces; `browser-traces-firefox` and -`browser-traces-webkit` hold the other browsers' traces. Open traces with -Playwright's `show-trace` tool. +aggregate job that passes only when every frontend job succeeds: the three +`frontend coverage (shard N/3)` jobs, `frontend coverage (merge)`, and the four +`page workflows (firefox|webkit, shard N/2)` jobs. A failed, cancelled or skipped +job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public +runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3` +(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins +and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3 +days). The merge job downloads them, fails unless all three arrived with raw +files and without clashing names, then runs `coverage:merge` (report plus +baseline check). `tests/vendor/ci-workflow.test.mjs` checks this shape. GitHub +artifacts named `frontend-test-reports` contain the merged HTML coverage and +JSON summary; `frontend-traces-shard-N` hold the Chromium/touch failure +screenshots/traces and `browser-traces--shard-N` the other browsers' +traces. Open traces with Playwright's `show-trace` tool. The real-host suite covers both Classic and Nodes 2.0 startup, switching both ways, actual database prompt selection, execution-text serialization, workflow diff --git a/tests/vendor/ci-workflow.test.mjs b/tests/vendor/ci-workflow.test.mjs new file mode 100644 index 0000000..848d3a1 --- /dev/null +++ b/tests/vendor/ci-workflow.test.mjs @@ -0,0 +1,197 @@ +// Structural checks on .github/workflows/viewerjs-check.yml. The tooling has no YAML parser, so these +// read the workflow's fixed two-space layout: jobs at two spaces, their keys at four, steps at six. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises'; +import { readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const workflow = readFileSync(new URL('../../.github/workflows/viewerjs-check.yml', import.meta.url), 'utf8'); +const PINNED_IMAGE = /^\s+image: mcr\.microsoft\.com\/playwright:v\d+\.\d+\.\d+-noble$/m; + +function jobs(text) { + const lines = text.split('\n'); + const start = lines.indexOf('jobs:'); + assert.notEqual(start, -1, 'workflow has a jobs section'); + const found = new Map(); + let current = null; + for (const line of lines.slice(start + 1)) { + const header = /^ {2}([\w-]+):\s*$/.exec(line); + if (header) { + current = header[1]; + found.set(current, []); + } else if (current && (line.startsWith(' ') || line.trim() === '' || /^ {2}#/.test(line))) { + found.get(current).push(line); + } else if (current && /^\S/.test(line)) { + break; + } + } + return new Map([...found].map(([id, body]) => [id, body.join('\n')])); +} + +function steps(jobText) { + const parts = jobText.split(/^(?= {6}- )/m); + return parts.slice(1); +} + +function runScript(stepText) { + const lines = stepText.split('\n'); + const index = lines.findIndex(line => /^ {8}run: \|\s*$/.test(line)); + assert.notEqual(index, -1, 'step has a block run script'); + const body = []; + for (const line of lines.slice(index + 1)) { + if (line.trim() !== '' && !line.startsWith(' ')) break; + body.push(line.slice(10)); + } + return body.join('\n'); +} + +function needsOf(jobText) { + const match = /^ {4}needs: \[([^\]]*)\]\s*$/m.exec(jobText) ?? /^ {4}needs: ([\w-]+)\s*$/m.exec(jobText); + assert.ok(match, 'job declares needs'); + return match[1].split(',').map(value => value.trim()).filter(Boolean); +} + +function matrixList(jobText, key) { + const match = new RegExp(`^ {8}${key}: \\[([^\\]]*)\\]\\s*$`, 'm').exec(jobText); + assert.ok(match, `job has a ${key} matrix`); + return match[1].split(',').map(value => value.trim()); +} + +const allJobs = jobs(workflow); +const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\/vendor test:(workflows|coverage)/.test(body)) + .map(([id]) => id); +const mergeJob = () => allJobs.get('frontend-coverage-merge'); +const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step)); + +test('viewerjs keeps its name and needs every frontend job', () => { + const viewerjs = allJobs.get('viewerjs'); + assert.ok(viewerjs, 'the required context is the job id viewerjs'); + assert.doesNotMatch(viewerjs, /^ {4}name:/m, 'no display name may replace the context "viewerjs"'); + const needs = needsOf(viewerjs); + const others = [...allJobs.keys()].filter(id => id !== 'viewerjs'); + assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow'); + assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']); + assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped'); + const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step))); + for (const id of needs) { + const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs); + assert.ok(variable, `${id}'s result reaches the check`); + assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`); + } +}); + +test('viewerjs fails unless every needed job succeeded', () => { + const viewerjs = allJobs.get('viewerjs'); + const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step))); + const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]); + const run = env => { + try { + execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' }); + return true; + } catch { + return false; + } + }; + const allSuccess = Object.fromEntries(variables.map(name => [name, 'success'])); + assert.equal(run(allSuccess), true); + for (const name of variables) { + for (const result of ['failure', 'cancelled', 'skipped']) { + assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`); + } + } +}); + +test('coverage runs as three shards that each upload their raw coverage', () => { + const coverage = allJobs.get('frontend-coverage'); + assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']); + assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m); + assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/); + const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step)); + assert.ok(upload, 'each shard uploads coverage-raw-N'); + assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m); + assert.match(upload, /^ {10}retention-days: 3$/m); + assert.match(upload, /^ {10}if-no-files-found: error$/m); +}); + +test('merge job downloads every shard and runs the merge-only report and check', () => { + const merge = mergeJob(); + assert.ok(merge, 'frontend-coverage-merge exists'); + assert.deepEqual(needsOf(merge), ['frontend-coverage']); + assert.match(merge, /^ {4}name: frontend coverage \(merge\)$/m); + const download = steps(merge).find(step => /uses: actions\/download-artifact@/.test(step)); + assert.ok(download, 'merge downloads the shard artifacts'); + assert.match(download, /^ {10}pattern: coverage-raw-\*$/m); + assert.doesNotMatch(download, /merge-multiple: true/, 'each artifact keeps its own directory so the check can count them'); + assert.ok(verifyStep(), 'merge has an explicit artifact count check'); + assert.match(verifyStep(), new RegExp(`^ {10}EXPECTED_SHARDS: ${matrixList(allJobs.get('frontend-coverage'), 'shard').length}$`, 'm'), + 'the expected count matches the shard matrix'); + const order = steps(merge).map(step => (/name: (.*)/.exec(step) ?? [, step.trim().split('\n')[0]])[1]); + const verifyAt = order.findIndex(name => /Verify every shard/.test(name)); + const mergeAt = steps(merge).findIndex(step => /npm run --prefix tools\/vendor coverage:merge/.test(step)); + assert.ok(verifyAt >= 0 && mergeAt > verifyAt, 'the count check runs before coverage:merge'); + const report = steps(merge).find(step => /name: frontend-test-reports$/m.test(step)); + assert.ok(report, 'merge uploads the HTML report as frontend-test-reports'); + assert.match(report, /!coverage\/js\/raw\/\*\*/); +}); + +async function stagedShards(root, shards) { + const staged = join(root, 'staged'); + for (const [shard, files] of shards) { + const directory = join(staged, `coverage-raw-${shard}`); + await mkdir(directory, { recursive: true }); + for (const file of files) await writeFile(join(directory, file), '{"result":[]}'); + } + await mkdir(staged, { recursive: true }); + return staged; +} + +async function verify(t, shards) { + const root = await mkdtemp(join(tmpdir(), 'pm-ci-merge-')); + t.after(() => rm(root, { recursive: true, force: true })); + const staged = await stagedShards(root, shards); + try { + execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', runScript(verifyStep())], { + cwd: root, stdio: 'pipe', + env: { PATH: process.env.PATH, STAGED_RAW: staged, EXPECTED_SHARDS: '3' }, + }); + } catch (error) { + return { ok: false, output: `${error.stdout}${error.stderr}` }; + } + return { ok: true, raw: (await readdir(join(root, 'coverage/js/raw'))).sort() }; +} + +test('merge refuses when a shard\'s raw coverage is missing', async t => { + const missing = await verify(t, [[1, ['a.json']], [3, ['c.json']]]); + assert.equal(missing.ok, false); + assert.match(missing.output, /2 of 3/); + const empty = await verify(t, [[1, ['a.json']], [2, []], [3, ['c.json']]]); + assert.equal(empty.ok, false, 'an artifact without raw files counts as missing'); + const collision = await verify(t, [[1, ['a.json']], [2, ['a.json']], [3, ['c.json']]]); + assert.equal(collision.ok, false, 'two shards writing the same file name must not silently drop one'); + const complete = await verify(t, [[1, ['a.json', 'node.json']], [2, ['b.json']], [3, ['c.json']]]); + assert.deepEqual(complete, { ok: true, raw: ['a.json', 'b.json', 'c.json', 'node.json'] }); +}); + +test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () => { + assert.deepEqual([...browserJobIds].sort(), ['browser-workflows', 'frontend-coverage']); + for (const id of browserJobIds) { + const body = allJobs.get(id); + assert.match(body, /^ {6}PM_TEST_WORKERS: 3$/m, `${id} sets PM_TEST_WORKERS for the whole job`); + assert.match(body, PINNED_IMAGE, `${id} keeps the pinned Playwright image`); + assert.match(body, /PIP_SHA256: [0-9a-f]{64}/, `${id} keeps the pinned pip bootstrap`); + assert.match(body, /^ {4}timeout-minutes: \d+$/m); + } + assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards'); +}); + +test('page workflows run each browser in two shards', () => { + const workflows = allJobs.get('browser-workflows'); + assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']); + assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']); + assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/); + assert.match(workflows, /^ {6}fail-fast: false$/m); + assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/); +});