Vito Sansevero
ed809f7fa4
Final review fixes for the per-worker test servers (#274): - A failed test gets the stderr its server wrote during that test attached as `server-stderr`, so an aiohttp traceback behind a 500 reaches the CI report. The launcher keeps the newest 256 KB with a stream position per test (stderrMark/stderrSince), and the fixture attaches it only when the test did not end as expected. - The port announcement counts only as a complete line; a chunk ending mid-number was read as a shorter port. - server.py --exit-on-stdin-eof shuts down gracefully (database closed, temp root removed) when stdin closes. The launcher holds that pipe, so a SIGKILLed worker leaves no server behind, on every OS. - settled() gives up after 600 frames with the observation stats instead of hanging to the test timeout. - The Classic text-sync test waits for the page's 50 ms timer to install serializeValue instead of sleeping 60 ms. With that timer run 100 ms late (a scratch copy), the old wait failed 20/20 and the poll passed 20/20; the spec passed 240/240 at 8 workers x10 and 240/240 at 1 CPU with 4 workers. - The workflow test checks each npm cache key names its job's pinned image tag. Every new unit test failed before its change (missing export, the old regex timing out on the split announcement, an unknown server flag, a hung settle, a mismatched cache key).
224 lines
12 KiB
JavaScript
224 lines
12 KiB
JavaScript
// Structural checks on .github/workflows/viewerjs-check.yml. The tooling has no YAML parser, so these
|
|
// read the workflow's fixed two-space layout: jobs at two spaces, their keys at four, steps at six.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises';
|
|
import { readFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
|
|
const workflow = readFileSync(new URL('../../.github/workflows/viewerjs-check.yml', import.meta.url), 'utf8');
|
|
const PINNED_IMAGE = /^\s+image: mcr\.microsoft\.com\/playwright:v\d+\.\d+\.\d+-noble$/m;
|
|
|
|
function jobs(text) {
|
|
const lines = text.split('\n');
|
|
const start = lines.indexOf('jobs:');
|
|
assert.notEqual(start, -1, 'workflow has a jobs section');
|
|
const found = new Map();
|
|
let current = null;
|
|
for (const line of lines.slice(start + 1)) {
|
|
const header = /^ {2}([\w-]+):\s*$/.exec(line);
|
|
if (header) {
|
|
current = header[1];
|
|
found.set(current, []);
|
|
} else if (current && (line.startsWith(' ') || line.trim() === '' || /^ {2}#/.test(line))) {
|
|
found.get(current).push(line);
|
|
} else if (current && /^\S/.test(line)) {
|
|
break;
|
|
}
|
|
}
|
|
return new Map([...found].map(([id, body]) => [id, body.join('\n')]));
|
|
}
|
|
|
|
function steps(jobText) {
|
|
const parts = jobText.split(/^(?= {6}- )/m);
|
|
return parts.slice(1);
|
|
}
|
|
|
|
function runScript(stepText) {
|
|
const lines = stepText.split('\n');
|
|
const index = lines.findIndex(line => /^ {8}run: \|\s*$/.test(line));
|
|
assert.notEqual(index, -1, 'step has a block run script');
|
|
const body = [];
|
|
for (const line of lines.slice(index + 1)) {
|
|
if (line.trim() !== '' && !line.startsWith(' ')) break;
|
|
body.push(line.slice(10));
|
|
}
|
|
return body.join('\n');
|
|
}
|
|
|
|
function needsOf(jobText) {
|
|
const match = /^ {4}needs: \[([^\]]*)\]\s*$/m.exec(jobText) ?? /^ {4}needs: ([\w-]+)\s*$/m.exec(jobText);
|
|
assert.ok(match, 'job declares needs');
|
|
return match[1].split(',').map(value => value.trim()).filter(Boolean);
|
|
}
|
|
|
|
function matrixList(jobText, key) {
|
|
const match = new RegExp(`^ {8}${key}: \\[([^\\]]*)\\]\\s*$`, 'm').exec(jobText);
|
|
assert.ok(match, `job has a ${key} matrix`);
|
|
return match[1].split(',').map(value => value.trim());
|
|
}
|
|
|
|
const allJobs = jobs(workflow);
|
|
const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\/vendor test:(workflows|coverage)/.test(body))
|
|
.map(([id]) => id);
|
|
const mergeJob = () => allJobs.get('frontend-coverage-merge');
|
|
const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step));
|
|
|
|
test('viewerjs keeps its name and needs every frontend job', () => {
|
|
const viewerjs = allJobs.get('viewerjs');
|
|
assert.ok(viewerjs, 'the required context is the job id viewerjs');
|
|
assert.doesNotMatch(viewerjs, /^ {4}name:/m, 'no display name may replace the context "viewerjs"');
|
|
const needs = needsOf(viewerjs);
|
|
const others = [...allJobs.keys()].filter(id => id !== 'viewerjs');
|
|
assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow');
|
|
assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
|
assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped');
|
|
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
|
for (const id of needs) {
|
|
const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs);
|
|
assert.ok(variable, `${id}'s result reaches the check`);
|
|
assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`);
|
|
}
|
|
});
|
|
|
|
test('viewerjs fails unless every needed job succeeded', () => {
|
|
const viewerjs = allJobs.get('viewerjs');
|
|
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
|
const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]);
|
|
const run = env => {
|
|
try {
|
|
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' });
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
};
|
|
const allSuccess = Object.fromEntries(variables.map(name => [name, 'success']));
|
|
assert.equal(run(allSuccess), true);
|
|
for (const name of variables) {
|
|
for (const result of ['failure', 'cancelled', 'skipped']) {
|
|
assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`);
|
|
}
|
|
}
|
|
});
|
|
|
|
test('coverage runs as three shards that each upload their raw coverage', () => {
|
|
const coverage = allJobs.get('frontend-coverage');
|
|
assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']);
|
|
assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m);
|
|
assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/);
|
|
const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step));
|
|
assert.ok(upload, 'each shard uploads coverage-raw-N');
|
|
assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m);
|
|
assert.match(upload, /^ {10}retention-days: 3$/m);
|
|
assert.match(upload, /^ {10}if-no-files-found: error$/m);
|
|
});
|
|
|
|
test('merge job downloads every shard and runs the merge-only report and check', () => {
|
|
const merge = mergeJob();
|
|
assert.ok(merge, 'frontend-coverage-merge exists');
|
|
assert.deepEqual(needsOf(merge), ['frontend-coverage']);
|
|
assert.match(merge, /^ {4}name: frontend coverage \(merge\)$/m);
|
|
const download = steps(merge).find(step => /uses: actions\/download-artifact@/.test(step));
|
|
assert.ok(download, 'merge downloads the shard artifacts');
|
|
assert.match(download, /^ {10}pattern: coverage-raw-\*$/m);
|
|
assert.doesNotMatch(download, /merge-multiple: true/, 'each artifact keeps its own directory so the check can count them');
|
|
assert.ok(verifyStep(), 'merge has an explicit artifact count check');
|
|
assert.match(verifyStep(), new RegExp(`^ {10}EXPECTED_SHARDS: ${matrixList(allJobs.get('frontend-coverage'), 'shard').length}$`, 'm'),
|
|
'the expected count matches the shard matrix');
|
|
const order = steps(merge).map(step => (/name: (.*)/.exec(step) ?? [, step.trim().split('\n')[0]])[1]);
|
|
const verifyAt = order.findIndex(name => /Verify every shard/.test(name));
|
|
const mergeAt = steps(merge).findIndex(step => /npm run --prefix tools\/vendor coverage:merge/.test(step));
|
|
assert.ok(verifyAt >= 0 && mergeAt > verifyAt, 'the count check runs before coverage:merge');
|
|
const report = steps(merge).find(step => /name: frontend-test-reports$/m.test(step));
|
|
assert.ok(report, 'merge uploads the HTML report as frontend-test-reports');
|
|
assert.match(report, /!coverage\/js\/raw\/\*\*/);
|
|
});
|
|
|
|
async function stagedShards(root, shards) {
|
|
const staged = join(root, 'staged');
|
|
for (const [shard, files] of shards) {
|
|
const directory = join(staged, `coverage-raw-${shard}`);
|
|
await mkdir(directory, { recursive: true });
|
|
for (const file of files) await writeFile(join(directory, file), '{"result":[]}');
|
|
}
|
|
await mkdir(staged, { recursive: true });
|
|
return staged;
|
|
}
|
|
|
|
async function verify(t, shards) {
|
|
const root = await mkdtemp(join(tmpdir(), 'pm-ci-merge-'));
|
|
t.after(() => rm(root, { recursive: true, force: true }));
|
|
const staged = await stagedShards(root, shards);
|
|
try {
|
|
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', runScript(verifyStep())], {
|
|
cwd: root, stdio: 'pipe',
|
|
env: { PATH: process.env.PATH, STAGED_RAW: staged, EXPECTED_SHARDS: '3' },
|
|
});
|
|
} catch (error) {
|
|
return { ok: false, output: `${error.stdout}${error.stderr}` };
|
|
}
|
|
return { ok: true, raw: (await readdir(join(root, 'coverage/js/raw'))).sort() };
|
|
}
|
|
|
|
test('merge refuses when a shard\'s raw coverage is missing', async t => {
|
|
const missing = await verify(t, [[1, ['a.json']], [3, ['c.json']]]);
|
|
assert.equal(missing.ok, false);
|
|
assert.match(missing.output, /2 of 3/);
|
|
const empty = await verify(t, [[1, ['a.json']], [2, []], [3, ['c.json']]]);
|
|
assert.equal(empty.ok, false, 'an artifact without raw files counts as missing');
|
|
const collision = await verify(t, [[1, ['a.json']], [2, ['a.json']], [3, ['c.json']]]);
|
|
assert.equal(collision.ok, false, 'two shards writing the same file name must not silently drop one');
|
|
const complete = await verify(t, [[1, ['a.json', 'node.json']], [2, ['b.json']], [3, ['c.json']]]);
|
|
assert.deepEqual(complete, { ok: true, raw: ['a.json', 'b.json', 'c.json', 'node.json'] });
|
|
});
|
|
|
|
test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () => {
|
|
assert.deepEqual([...browserJobIds].sort(), ['browser-workflows', 'frontend-coverage']);
|
|
for (const id of browserJobIds) {
|
|
const body = allJobs.get(id);
|
|
assert.match(body, /^ {6}PM_TEST_WORKERS: 3$/m, `${id} sets PM_TEST_WORKERS for the whole job`);
|
|
assert.match(body, PINNED_IMAGE, `${id} keeps the pinned Playwright image`);
|
|
assert.match(body, /PIP_SHA256: [0-9a-f]{64}/, `${id} keeps the pinned pip bootstrap`);
|
|
assert.match(body, /^ {4}timeout-minutes: \d+$/m);
|
|
}
|
|
assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards');
|
|
});
|
|
|
|
test('page workflows run each browser in two shards', () => {
|
|
const workflows = allJobs.get('browser-workflows');
|
|
assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']);
|
|
assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']);
|
|
assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/);
|
|
assert.match(workflows, /^ {6}fail-fast: false$/m);
|
|
assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/);
|
|
});
|
|
|
|
test('every npm ci restores the npm download cache keyed on the vendor lockfile first', () => {
|
|
const installing = [...allJobs].filter(([, body]) => /npm ci --prefix tools\/vendor/.test(body));
|
|
assert.deepEqual(installing.map(([id]) => id).sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
|
for (const [id, body] of installing) {
|
|
const jobSteps = steps(body);
|
|
const cacheAt = jobSteps.findIndex(step => /uses: actions\/cache@v\d+/.test(step));
|
|
const installAt = jobSteps.findIndex(step => /npm ci --prefix tools\/vendor/.test(step));
|
|
assert.ok(cacheAt >= 0 && cacheAt < installAt, `${id} restores the npm cache before npm ci`);
|
|
assert.match(jobSteps[cacheAt], /^ {10}path: ~\/\.npm$/m, `${id} caches npm's download cache, not node_modules`);
|
|
assert.match(jobSteps[cacheAt], /hashFiles\('tools\/vendor\/package-lock\.json'\)/, `${id} keys the cache on the lockfile`);
|
|
assert.match(jobSteps[installAt], /npm ci --prefix tools\/vendor --ignore-scripts/, `${id} still installs without scripts`);
|
|
// The key names the job's own pinned image, so an image bump starts a fresh cache.
|
|
const image = /^ {6}image: mcr\.microsoft\.com\/playwright:(v\d+\.\d+\.\d+)-noble$/m.exec(body)?.[1];
|
|
assert.ok(image, `${id} runs in the pinned Playwright image`);
|
|
assert.match(jobSteps[cacheAt], new RegExp(`key: npm-\\$\\{\\{ runner\\.os \\}\\}-playwright-${image.replaceAll('.', '\\.')}-\\$\\{\\{ hashFiles`),
|
|
`${id} keys the cache on its image ${image}`);
|
|
}
|
|
});
|
|
|
|
test('browser job timeouts keep a real margin over measured runs', () => {
|
|
for (const id of ['frontend-coverage', 'browser-workflows']) {
|
|
const minutes = Number(/^ {4}timeout-minutes: (\d+)$/m.exec(allJobs.get(id))[1]);
|
|
assert.ok(minutes >= 20, `${id} allows at least 20 minutes, since runner speed varies`);
|
|
}
|
|
});
|