diff --git a/.github/workflows/viewerjs-check.yml b/.github/workflows/viewerjs-check.yml index 41ddf1b..a009e1d 100644 --- a/.github/workflows/viewerjs-check.yml +++ b/.github/workflows/viewerjs-check.yml @@ -16,9 +16,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# Timeouts are the measured job time plus about 50%, so a hung run stops early. +# Before sharding (main 2cecfcf, one worker): frontend coverage 32m10s, page workflows firefox 18m59s, +# webkit 19m26s. Sharded with 3 workers per job: estimates pending the first CI run (coverage shard ~10m, +# merge ~5m, workflow shard ~8m). Re-measure when a job passes its estimate. jobs: frontend-coverage: - name: frontend coverage (chromium and touch) + name: frontend coverage (shard ${{ matrix.shard }}/3) runs-on: ubuntu-latest # Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the # same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json; @@ -26,12 +30,15 @@ jobs: container: image: mcr.microsoft.com/playwright:v1.64.1-noble options: --user 1001 - # This job runs every page workflow twice (chromium, then touch) on one worker with V8 coverage on, so it - # takes about twice a single-browser job. Measured 2026-10-20: whole job 18m26s-24m57s (main 5ded3cf - # 22m15s; PR head 6359999 was cancelled at the old 25-minute limit while still passing), coverage step - # alone 20m55s-22m17s+, against ~12-15 minutes for the firefox/webkit jobs. 40 minutes leaves headroom - # for suite growth while still stopping a hung run; re-measure if a run passes 30 minutes. - timeout-minutes: 40 + timeout-minutes: 15 + # Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test + # server, leave one CPU for the servers and the runner. + env: + PM_TEST_WORKERS: 3 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] steps: - uses: actions/checkout@v7 # The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache @@ -51,23 +58,92 @@ jobs: echo "$venv/bin" >> "$GITHUB_PATH" - run: python --version && node --version && node --version | grep -q '^v24\.' - run: npm ci --prefix tools/vendor --ignore-scripts - - run: npm run --prefix tools/vendor check - - run: npm run --prefix tools/vendor test - - name: Frontend unit, viewer, workflow, and coverage checks - run: npm run --prefix tools/vendor test:coverage - - name: Upload frontend coverage and Chromium traces + # Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests + # run on the lightest shard instead. + - if: matrix.shard == 3 + run: npm run --prefix tools/vendor check + - if: matrix.shard == 3 + run: npm run --prefix tools/vendor test + - name: Page workflows with coverage, shard ${{ matrix.shard }}/3 + run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3 + - name: Upload raw coverage + uses: actions/upload-artifact@v7 + with: + name: coverage-raw-${{ matrix.shard }} + path: coverage/js/raw/ + if-no-files-found: error + retention-days: 3 + - name: Upload Chromium and touch traces + if: always() + uses: actions/upload-artifact@v7 + with: + name: frontend-traces-shard-${{ matrix.shard }} + path: tools/vendor/test-results/ + retention-days: 14 + + frontend-coverage-merge: + name: frontend coverage (merge) + needs: frontend-coverage + runs-on: ubuntu-latest + # Same image as the shards: the raw files name sources by the container's workspace path. + container: + image: mcr.microsoft.com/playwright:v1.64.1-noble + options: --user 1001 + timeout-minutes: 8 + steps: + - uses: actions/checkout@v7 + - run: node --version && node --version | grep -q '^v24\.' + - run: npm ci --prefix tools/vendor --ignore-scripts + # Each artifact lands in its own coverage-raw-N directory, so the next step can count them. + - uses: actions/download-artifact@v8 + with: + pattern: coverage-raw-* + path: ${{ runner.temp }}/coverage-raw + # coverage:merge only checks that raw files exist; merging two of three shards would report + # partial coverage, so a missing or empty shard artifact fails here. + - name: Verify every shard's raw coverage arrived + shell: bash + env: + STAGED_RAW: ${{ runner.temp }}/coverage-raw + EXPECTED_SHARDS: 3 + run: | + found=0 + for shard in $(seq 1 "$EXPECTED_SHARDS"); do + directory="$STAGED_RAW/coverage-raw-$shard" + count=0 + if [ -d "$directory" ]; then count=$(find "$directory" -maxdepth 1 -type f -name '*.json' | wc -l); fi + echo "coverage-raw-$shard: $count raw files" + if [ "$count" -gt 0 ]; then found=$((found + 1)); fi + done + echo "shard artifacts with raw coverage: $found of $EXPECTED_SHARDS" + if [ "$found" -ne "$EXPECTED_SHARDS" ]; then + echo "::error::Expected raw coverage from $EXPECTED_SHARDS shards, got $found of $EXPECTED_SHARDS" + exit 1 + fi + staged=$(find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' | wc -l) + mkdir -p coverage/js/raw + # cp -n never overwrites; a clash shows up as fewer merged files than staged ones. + find "$STAGED_RAW" -mindepth 2 -maxdepth 2 -type f -name '*.json' -exec cp -n -t coverage/js/raw {} + || true + merged=$(find coverage/js/raw -maxdepth 1 -type f -name '*.json' | wc -l) + if [ "$merged" -ne "$staged" ]; then + echo "::error::Shards wrote clashing raw file names: $staged staged, $merged merged" + exit 1 + fi + echo "merged $merged raw files" + - name: Frontend coverage report and baseline check + run: npm run --prefix tools/vendor coverage:merge + - name: Upload frontend coverage report if: always() uses: actions/upload-artifact@v7 with: name: frontend-test-reports path: | - tools/vendor/test-results/ coverage/js/ !coverage/js/raw/** retention-days: 14 browser-workflows: - name: page workflows (${{ matrix.browser }}) + name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2) runs-on: ubuntu-latest # Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the # same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json; @@ -75,11 +151,16 @@ jobs: container: image: mcr.microsoft.com/playwright:v1.64.1-noble options: --user 1001 - timeout-minutes: 25 + timeout-minutes: 12 + # Public-repo ubuntu-latest runners have 4 vCPUs: three Playwright workers, each with its own test + # server, leave one CPU for the servers and the runner. + env: + PM_TEST_WORKERS: 3 strategy: fail-fast: false matrix: browser: [firefox, webkit] + shard: [1, 2] steps: - uses: actions/checkout@v7 # The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache @@ -99,21 +180,21 @@ jobs: echo "$venv/bin" >> "$GITHUB_PATH" - run: python --version && node --version && node --version | grep -q '^v24\.' - run: npm ci --prefix tools/vendor --ignore-scripts - - name: Page workflows in ${{ matrix.browser }} - run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} + - name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2 + run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2 - name: Upload browser traces if: always() uses: actions/upload-artifact@v7 with: - name: browser-traces-${{ matrix.browser }} + name: browser-traces-${{ matrix.browser }}-shard-${{ matrix.shard }} path: tools/vendor/test-results/ retention-days: 14 # Branch protection requires the exact context "viewerjs". This job keeps that - # name and fails unless every frontend job above succeeded, including when one - # was cancelled or skipped. + # name and fails unless every frontend job above succeeded (every shard, the + # coverage merge and every browser shard), including when one was cancelled or skipped. viewerjs: - needs: [frontend-coverage, browser-workflows] + needs: [frontend-coverage, frontend-coverage-merge, browser-workflows] if: always() runs-on: ubuntu-latest timeout-minutes: 5 @@ -121,8 +202,10 @@ jobs: - name: Require every frontend job to succeed env: COVERAGE_RESULT: ${{ needs.frontend-coverage.result }} + MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }} WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }} run: | echo "frontend-coverage: $COVERAGE_RESULT" + echo "frontend-coverage-merge: $MERGE_RESULT" echo "browser-workflows: $WORKFLOWS_RESULT" - test "$COVERAGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success + test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success diff --git a/TESTING.md b/TESTING.md index 5948a2b..3d84a81 100644 --- a/TESTING.md +++ b/TESTING.md @@ -118,16 +118,23 @@ second without a browser and fails, naming the files, when a measured `web/js` file has no baseline entry or the baseline lists a file that no longer exists. It uses the gate's own `discoverSources`, and `npm test` runs the same check, so run it before pushing a new or deleted frontend file instead of -waiting for the `frontend coverage` job. +waiting for the `frontend coverage` jobs. The `viewerjs` check runs on every PR, including backend-only changes. It is an -aggregate job that passes only when the `frontend coverage (chromium and touch)` -job and both `page workflows (firefox)` and `page workflows (webkit)` jobs -succeed; the browsers run on separate runners in parallel. GitHub artifacts -named `frontend-test-reports` contain HTML coverage, a JSON summary, and the -Chromium/touch failure screenshots/traces; `browser-traces-firefox` and -`browser-traces-webkit` hold the other browsers' traces. Open traces with -Playwright's `show-trace` tool. +aggregate job that passes only when every frontend job succeeds: the three +`frontend coverage (shard N/3)` jobs, `frontend coverage (merge)`, and the four +`page workflows (firefox|webkit, shard N/2)` jobs. A failed, cancelled or skipped +job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public +runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3` +(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins +and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3 +days). The merge job downloads them, fails unless all three arrived with raw +files and without clashing names, then runs `coverage:merge` (report plus +baseline check). `tests/vendor/ci-workflow.test.mjs` checks this shape. GitHub +artifacts named `frontend-test-reports` contain the merged HTML coverage and +JSON summary; `frontend-traces-shard-N` hold the Chromium/touch failure +screenshots/traces and `browser-traces--shard-N` the other browsers' +traces. Open traces with Playwright's `show-trace` tool. The real-host suite covers both Classic and Nodes 2.0 startup, switching both ways, actual database prompt selection, execution-text serialization, workflow diff --git a/tests/vendor/ci-workflow.test.mjs b/tests/vendor/ci-workflow.test.mjs new file mode 100644 index 0000000..848d3a1 --- /dev/null +++ b/tests/vendor/ci-workflow.test.mjs @@ -0,0 +1,197 @@ +// Structural checks on .github/workflows/viewerjs-check.yml. The tooling has no YAML parser, so these +// read the workflow's fixed two-space layout: jobs at two spaces, their keys at four, steps at six. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises'; +import { readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const workflow = readFileSync(new URL('../../.github/workflows/viewerjs-check.yml', import.meta.url), 'utf8'); +const PINNED_IMAGE = /^\s+image: mcr\.microsoft\.com\/playwright:v\d+\.\d+\.\d+-noble$/m; + +function jobs(text) { + const lines = text.split('\n'); + const start = lines.indexOf('jobs:'); + assert.notEqual(start, -1, 'workflow has a jobs section'); + const found = new Map(); + let current = null; + for (const line of lines.slice(start + 1)) { + const header = /^ {2}([\w-]+):\s*$/.exec(line); + if (header) { + current = header[1]; + found.set(current, []); + } else if (current && (line.startsWith(' ') || line.trim() === '' || /^ {2}#/.test(line))) { + found.get(current).push(line); + } else if (current && /^\S/.test(line)) { + break; + } + } + return new Map([...found].map(([id, body]) => [id, body.join('\n')])); +} + +function steps(jobText) { + const parts = jobText.split(/^(?= {6}- )/m); + return parts.slice(1); +} + +function runScript(stepText) { + const lines = stepText.split('\n'); + const index = lines.findIndex(line => /^ {8}run: \|\s*$/.test(line)); + assert.notEqual(index, -1, 'step has a block run script'); + const body = []; + for (const line of lines.slice(index + 1)) { + if (line.trim() !== '' && !line.startsWith(' ')) break; + body.push(line.slice(10)); + } + return body.join('\n'); +} + +function needsOf(jobText) { + const match = /^ {4}needs: \[([^\]]*)\]\s*$/m.exec(jobText) ?? /^ {4}needs: ([\w-]+)\s*$/m.exec(jobText); + assert.ok(match, 'job declares needs'); + return match[1].split(',').map(value => value.trim()).filter(Boolean); +} + +function matrixList(jobText, key) { + const match = new RegExp(`^ {8}${key}: \\[([^\\]]*)\\]\\s*$`, 'm').exec(jobText); + assert.ok(match, `job has a ${key} matrix`); + return match[1].split(',').map(value => value.trim()); +} + +const allJobs = jobs(workflow); +const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\/vendor test:(workflows|coverage)/.test(body)) + .map(([id]) => id); +const mergeJob = () => allJobs.get('frontend-coverage-merge'); +const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step)); + +test('viewerjs keeps its name and needs every frontend job', () => { + const viewerjs = allJobs.get('viewerjs'); + assert.ok(viewerjs, 'the required context is the job id viewerjs'); + assert.doesNotMatch(viewerjs, /^ {4}name:/m, 'no display name may replace the context "viewerjs"'); + const needs = needsOf(viewerjs); + const others = [...allJobs.keys()].filter(id => id !== 'viewerjs'); + assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow'); + assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']); + assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped'); + const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step))); + for (const id of needs) { + const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs); + assert.ok(variable, `${id}'s result reaches the check`); + assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`); + } +}); + +test('viewerjs fails unless every needed job succeeded', () => { + const viewerjs = allJobs.get('viewerjs'); + const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step))); + const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]); + const run = env => { + try { + execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' }); + return true; + } catch { + return false; + } + }; + const allSuccess = Object.fromEntries(variables.map(name => [name, 'success'])); + assert.equal(run(allSuccess), true); + for (const name of variables) { + for (const result of ['failure', 'cancelled', 'skipped']) { + assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`); + } + } +}); + +test('coverage runs as three shards that each upload their raw coverage', () => { + const coverage = allJobs.get('frontend-coverage'); + assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']); + assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m); + assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/); + const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step)); + assert.ok(upload, 'each shard uploads coverage-raw-N'); + assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m); + assert.match(upload, /^ {10}retention-days: 3$/m); + assert.match(upload, /^ {10}if-no-files-found: error$/m); +}); + +test('merge job downloads every shard and runs the merge-only report and check', () => { + const merge = mergeJob(); + assert.ok(merge, 'frontend-coverage-merge exists'); + assert.deepEqual(needsOf(merge), ['frontend-coverage']); + assert.match(merge, /^ {4}name: frontend coverage \(merge\)$/m); + const download = steps(merge).find(step => /uses: actions\/download-artifact@/.test(step)); + assert.ok(download, 'merge downloads the shard artifacts'); + assert.match(download, /^ {10}pattern: coverage-raw-\*$/m); + assert.doesNotMatch(download, /merge-multiple: true/, 'each artifact keeps its own directory so the check can count them'); + assert.ok(verifyStep(), 'merge has an explicit artifact count check'); + assert.match(verifyStep(), new RegExp(`^ {10}EXPECTED_SHARDS: ${matrixList(allJobs.get('frontend-coverage'), 'shard').length}$`, 'm'), + 'the expected count matches the shard matrix'); + const order = steps(merge).map(step => (/name: (.*)/.exec(step) ?? [, step.trim().split('\n')[0]])[1]); + const verifyAt = order.findIndex(name => /Verify every shard/.test(name)); + const mergeAt = steps(merge).findIndex(step => /npm run --prefix tools\/vendor coverage:merge/.test(step)); + assert.ok(verifyAt >= 0 && mergeAt > verifyAt, 'the count check runs before coverage:merge'); + const report = steps(merge).find(step => /name: frontend-test-reports$/m.test(step)); + assert.ok(report, 'merge uploads the HTML report as frontend-test-reports'); + assert.match(report, /!coverage\/js\/raw\/\*\*/); +}); + +async function stagedShards(root, shards) { + const staged = join(root, 'staged'); + for (const [shard, files] of shards) { + const directory = join(staged, `coverage-raw-${shard}`); + await mkdir(directory, { recursive: true }); + for (const file of files) await writeFile(join(directory, file), '{"result":[]}'); + } + await mkdir(staged, { recursive: true }); + return staged; +} + +async function verify(t, shards) { + const root = await mkdtemp(join(tmpdir(), 'pm-ci-merge-')); + t.after(() => rm(root, { recursive: true, force: true })); + const staged = await stagedShards(root, shards); + try { + execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', runScript(verifyStep())], { + cwd: root, stdio: 'pipe', + env: { PATH: process.env.PATH, STAGED_RAW: staged, EXPECTED_SHARDS: '3' }, + }); + } catch (error) { + return { ok: false, output: `${error.stdout}${error.stderr}` }; + } + return { ok: true, raw: (await readdir(join(root, 'coverage/js/raw'))).sort() }; +} + +test('merge refuses when a shard\'s raw coverage is missing', async t => { + const missing = await verify(t, [[1, ['a.json']], [3, ['c.json']]]); + assert.equal(missing.ok, false); + assert.match(missing.output, /2 of 3/); + const empty = await verify(t, [[1, ['a.json']], [2, []], [3, ['c.json']]]); + assert.equal(empty.ok, false, 'an artifact without raw files counts as missing'); + const collision = await verify(t, [[1, ['a.json']], [2, ['a.json']], [3, ['c.json']]]); + assert.equal(collision.ok, false, 'two shards writing the same file name must not silently drop one'); + const complete = await verify(t, [[1, ['a.json', 'node.json']], [2, ['b.json']], [3, ['c.json']]]); + assert.deepEqual(complete, { ok: true, raw: ['a.json', 'b.json', 'c.json', 'node.json'] }); +}); + +test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () => { + assert.deepEqual([...browserJobIds].sort(), ['browser-workflows', 'frontend-coverage']); + for (const id of browserJobIds) { + const body = allJobs.get(id); + assert.match(body, /^ {6}PM_TEST_WORKERS: 3$/m, `${id} sets PM_TEST_WORKERS for the whole job`); + assert.match(body, PINNED_IMAGE, `${id} keeps the pinned Playwright image`); + assert.match(body, /PIP_SHA256: [0-9a-f]{64}/, `${id} keeps the pinned pip bootstrap`); + assert.match(body, /^ {4}timeout-minutes: \d+$/m); + } + assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards'); +}); + +test('page workflows run each browser in two shards', () => { + const workflows = allJobs.get('browser-workflows'); + assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']); + assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']); + assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/); + assert.match(workflows, /^ {6}fail-fast: false$/m); + assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/); +});