Vito Sansevero
323023d59c
Every job in test.yml and code-quality.yml gets timeout-minutes = ceil(1.5 x slowest of three green main runs) + 3, min 5, with the runs and times written above jobs:. The matrix and the coverage gate run python -m tests.run_suite. ci-workflow.test.mjs checks every job in the three CI workflows has a limit and that the matrix and coverage use the runner. Job names are unchanged.
250 lines
13 KiB
JavaScript
250 lines
13 KiB
JavaScript
// Structural checks on .github/workflows/viewerjs-check.yml, plus the time limits of every CI workflow.
|
|
// The tooling has no YAML parser, so these read the workflow's fixed two-space layout: jobs at two
|
|
// spaces, their keys at four, steps at six.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises';
|
|
import { readFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
|
|
const workflow = readFileSync(new URL('../../.github/workflows/viewerjs-check.yml', import.meta.url), 'utf8');
|
|
const PINNED_IMAGE = /^\s+image: mcr\.microsoft\.com\/playwright:v\d+\.\d+\.\d+-noble$/m;
|
|
|
|
function jobs(text) {
|
|
const lines = text.split('\n');
|
|
const start = lines.indexOf('jobs:');
|
|
assert.notEqual(start, -1, 'workflow has a jobs section');
|
|
const found = new Map();
|
|
let current = null;
|
|
for (const line of lines.slice(start + 1)) {
|
|
const header = /^ {2}([\w-]+):\s*$/.exec(line);
|
|
if (header) {
|
|
current = header[1];
|
|
found.set(current, []);
|
|
} else if (current && (line.startsWith(' ') || line.trim() === '' || /^ {2}#/.test(line))) {
|
|
found.get(current).push(line);
|
|
} else if (current && /^\S/.test(line)) {
|
|
break;
|
|
}
|
|
}
|
|
return new Map([...found].map(([id, body]) => [id, body.join('\n')]));
|
|
}
|
|
|
|
function steps(jobText) {
|
|
const parts = jobText.split(/^(?= {6}- )/m);
|
|
return parts.slice(1);
|
|
}
|
|
|
|
function runScript(stepText) {
|
|
const lines = stepText.split('\n');
|
|
const index = lines.findIndex(line => /^ {8}run: \|\s*$/.test(line));
|
|
assert.notEqual(index, -1, 'step has a block run script');
|
|
const body = [];
|
|
for (const line of lines.slice(index + 1)) {
|
|
if (line.trim() !== '' && !line.startsWith(' ')) break;
|
|
body.push(line.slice(10));
|
|
}
|
|
return body.join('\n');
|
|
}
|
|
|
|
function needsOf(jobText) {
|
|
const match = /^ {4}needs: \[([^\]]*)\]\s*$/m.exec(jobText) ?? /^ {4}needs: ([\w-]+)\s*$/m.exec(jobText);
|
|
assert.ok(match, 'job declares needs');
|
|
return match[1].split(',').map(value => value.trim()).filter(Boolean);
|
|
}
|
|
|
|
function matrixList(jobText, key) {
|
|
const match = new RegExp(`^ {8}${key}: \\[([^\\]]*)\\]\\s*$`, 'm').exec(jobText);
|
|
assert.ok(match, `job has a ${key} matrix`);
|
|
return match[1].split(',').map(value => value.trim());
|
|
}
|
|
|
|
const allJobs = jobs(workflow);
|
|
const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\/vendor test:(workflows|coverage)/.test(body))
|
|
.map(([id]) => id);
|
|
const mergeJob = () => allJobs.get('frontend-coverage-merge');
|
|
const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step));
|
|
|
|
test('viewerjs keeps its name and needs every frontend job', () => {
|
|
const viewerjs = allJobs.get('viewerjs');
|
|
assert.ok(viewerjs, 'the required context is the job id viewerjs');
|
|
assert.doesNotMatch(viewerjs, /^ {4}name:/m, 'no display name may replace the context "viewerjs"');
|
|
const needs = needsOf(viewerjs);
|
|
const others = [...allJobs.keys()].filter(id => id !== 'viewerjs');
|
|
assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow');
|
|
assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
|
assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped');
|
|
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
|
for (const id of needs) {
|
|
const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs);
|
|
assert.ok(variable, `${id}'s result reaches the check`);
|
|
assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`);
|
|
}
|
|
});
|
|
|
|
test('viewerjs fails unless every needed job succeeded', () => {
|
|
const viewerjs = allJobs.get('viewerjs');
|
|
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
|
const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]);
|
|
const run = env => {
|
|
try {
|
|
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' });
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
};
|
|
const allSuccess = Object.fromEntries(variables.map(name => [name, 'success']));
|
|
assert.equal(run(allSuccess), true);
|
|
for (const name of variables) {
|
|
for (const result of ['failure', 'cancelled', 'skipped']) {
|
|
assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`);
|
|
}
|
|
}
|
|
});
|
|
|
|
test('coverage runs as three shards that each upload their raw coverage', () => {
|
|
const coverage = allJobs.get('frontend-coverage');
|
|
assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']);
|
|
assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m);
|
|
assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/);
|
|
const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step));
|
|
assert.ok(upload, 'each shard uploads coverage-raw-N');
|
|
assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m);
|
|
assert.match(upload, /^ {10}retention-days: 3$/m);
|
|
assert.match(upload, /^ {10}if-no-files-found: error$/m);
|
|
});
|
|
|
|
test('merge job downloads every shard and runs the merge-only report and check', () => {
|
|
const merge = mergeJob();
|
|
assert.ok(merge, 'frontend-coverage-merge exists');
|
|
assert.deepEqual(needsOf(merge), ['frontend-coverage']);
|
|
assert.match(merge, /^ {4}name: frontend coverage \(merge\)$/m);
|
|
const download = steps(merge).find(step => /uses: actions\/download-artifact@/.test(step));
|
|
assert.ok(download, 'merge downloads the shard artifacts');
|
|
assert.match(download, /^ {10}pattern: coverage-raw-\*$/m);
|
|
assert.doesNotMatch(download, /merge-multiple: true/, 'each artifact keeps its own directory so the check can count them');
|
|
assert.ok(verifyStep(), 'merge has an explicit artifact count check');
|
|
assert.match(verifyStep(), new RegExp(`^ {10}EXPECTED_SHARDS: ${matrixList(allJobs.get('frontend-coverage'), 'shard').length}$`, 'm'),
|
|
'the expected count matches the shard matrix');
|
|
const order = steps(merge).map(step => (/name: (.*)/.exec(step) ?? [, step.trim().split('\n')[0]])[1]);
|
|
const verifyAt = order.findIndex(name => /Verify every shard/.test(name));
|
|
const mergeAt = steps(merge).findIndex(step => /npm run --prefix tools\/vendor coverage:merge/.test(step));
|
|
assert.ok(verifyAt >= 0 && mergeAt > verifyAt, 'the count check runs before coverage:merge');
|
|
const report = steps(merge).find(step => /name: frontend-test-reports$/m.test(step));
|
|
assert.ok(report, 'merge uploads the HTML report as frontend-test-reports');
|
|
assert.match(report, /!coverage\/js\/raw\/\*\*/);
|
|
});
|
|
|
|
async function stagedShards(root, shards) {
|
|
const staged = join(root, 'staged');
|
|
for (const [shard, files] of shards) {
|
|
const directory = join(staged, `coverage-raw-${shard}`);
|
|
await mkdir(directory, { recursive: true });
|
|
for (const file of files) await writeFile(join(directory, file), '{"result":[]}');
|
|
}
|
|
await mkdir(staged, { recursive: true });
|
|
return staged;
|
|
}
|
|
|
|
async function verify(t, shards) {
|
|
const root = await mkdtemp(join(tmpdir(), 'pm-ci-merge-'));
|
|
t.after(() => rm(root, { recursive: true, force: true }));
|
|
const staged = await stagedShards(root, shards);
|
|
try {
|
|
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', runScript(verifyStep())], {
|
|
cwd: root, stdio: 'pipe',
|
|
env: { PATH: process.env.PATH, STAGED_RAW: staged, EXPECTED_SHARDS: '3' },
|
|
});
|
|
} catch (error) {
|
|
return { ok: false, output: `${error.stdout}${error.stderr}` };
|
|
}
|
|
return { ok: true, raw: (await readdir(join(root, 'coverage/js/raw'))).sort() };
|
|
}
|
|
|
|
test('merge refuses when a shard\'s raw coverage is missing', async t => {
|
|
const missing = await verify(t, [[1, ['a.json']], [3, ['c.json']]]);
|
|
assert.equal(missing.ok, false);
|
|
assert.match(missing.output, /2 of 3/);
|
|
const empty = await verify(t, [[1, ['a.json']], [2, []], [3, ['c.json']]]);
|
|
assert.equal(empty.ok, false, 'an artifact without raw files counts as missing');
|
|
const collision = await verify(t, [[1, ['a.json']], [2, ['a.json']], [3, ['c.json']]]);
|
|
assert.equal(collision.ok, false, 'two shards writing the same file name must not silently drop one');
|
|
const complete = await verify(t, [[1, ['a.json', 'node.json']], [2, ['b.json']], [3, ['c.json']]]);
|
|
assert.deepEqual(complete, { ok: true, raw: ['a.json', 'b.json', 'c.json', 'node.json'] });
|
|
});
|
|
|
|
test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () => {
|
|
assert.deepEqual([...browserJobIds].sort(), ['browser-workflows', 'frontend-coverage']);
|
|
for (const id of browserJobIds) {
|
|
const body = allJobs.get(id);
|
|
assert.match(body, /^ {6}PM_TEST_WORKERS: 3$/m, `${id} sets PM_TEST_WORKERS for the whole job`);
|
|
assert.match(body, PINNED_IMAGE, `${id} keeps the pinned Playwright image`);
|
|
assert.match(body, /PIP_SHA256: [0-9a-f]{64}/, `${id} keeps the pinned pip bootstrap`);
|
|
assert.match(body, /^ {4}timeout-minutes: \d+$/m);
|
|
}
|
|
assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards');
|
|
});
|
|
|
|
test('page workflows run each browser in two shards', () => {
|
|
const workflows = allJobs.get('browser-workflows');
|
|
assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']);
|
|
assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']);
|
|
assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/);
|
|
assert.match(workflows, /^ {6}fail-fast: false$/m);
|
|
assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/);
|
|
});
|
|
|
|
test('every npm ci restores the npm download cache keyed on the vendor lockfile first', () => {
|
|
const installing = [...allJobs].filter(([, body]) => /npm ci --prefix tools\/vendor/.test(body));
|
|
assert.deepEqual(installing.map(([id]) => id).sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
|
for (const [id, body] of installing) {
|
|
const jobSteps = steps(body);
|
|
const cacheAt = jobSteps.findIndex(step => /uses: actions\/cache@v\d+/.test(step));
|
|
const installAt = jobSteps.findIndex(step => /npm ci --prefix tools\/vendor/.test(step));
|
|
assert.ok(cacheAt >= 0 && cacheAt < installAt, `${id} restores the npm cache before npm ci`);
|
|
assert.match(jobSteps[cacheAt], /^ {10}path: ~\/\.npm$/m, `${id} caches npm's download cache, not node_modules`);
|
|
assert.match(jobSteps[cacheAt], /hashFiles\('tools\/vendor\/package-lock\.json'\)/, `${id} keys the cache on the lockfile`);
|
|
assert.match(jobSteps[installAt], /npm ci --prefix tools\/vendor --ignore-scripts/, `${id} still installs without scripts`);
|
|
// The key names the job's own pinned image, so an image bump starts a fresh cache.
|
|
const image = /^ {6}image: mcr\.microsoft\.com\/playwright:(v\d+\.\d+\.\d+)-noble$/m.exec(body)?.[1];
|
|
assert.ok(image, `${id} runs in the pinned Playwright image`);
|
|
assert.match(jobSteps[cacheAt], new RegExp(`key: npm-\\$\\{\\{ runner\\.os \\}\\}-playwright-${image.replaceAll('.', '\\.')}-\\$\\{\\{ hashFiles`),
|
|
`${id} keys the cache on its image ${image}`);
|
|
}
|
|
});
|
|
|
|
test('browser job timeouts keep a real margin over measured runs', () => {
|
|
for (const id of ['frontend-coverage', 'browser-workflows']) {
|
|
const minutes = Number(/^ {4}timeout-minutes: (\d+)$/m.exec(allJobs.get(id))[1]);
|
|
assert.ok(minutes >= 20, `${id} allows at least 20 minutes, since runner speed varies`);
|
|
}
|
|
});
|
|
|
|
// Every job is bounded (#350): a hang ends at a measured limit instead of GitHub's 6-hour default.
|
|
const BOUNDED_WORKFLOWS = ['test.yml', 'code-quality.yml', 'viewerjs-check.yml'];
|
|
const readWorkflow = name => readFileSync(new URL(`../../.github/workflows/${name}`, import.meta.url), 'utf8');
|
|
|
|
test('every job in the CI workflows has a numeric timeout-minutes', () => {
|
|
for (const name of BOUNDED_WORKFLOWS) {
|
|
const found = jobs(readWorkflow(name));
|
|
assert.ok(found.size > 0, `${name} has jobs`);
|
|
for (const [id, body] of found) {
|
|
const limit = /^ {4}timeout-minutes: (\d+)\s*$/m.exec(body);
|
|
assert.ok(limit, `${name} job ${id} sets a numeric timeout-minutes`);
|
|
assert.ok(Number(limit[1]) >= 5, `${name} job ${id} allows at least 5 minutes`);
|
|
}
|
|
}
|
|
});
|
|
|
|
test('the Python matrix and coverage run the suite through the watchdog runner', () => {
|
|
const found = jobs(readWorkflow('test.yml'));
|
|
assert.match(found.get('test'), /^ {8}run: python -m tests\.run_suite -v$/m);
|
|
assert.match(found.get('coverage'), /^ {10}python -m coverage run -m tests\.run_suite$/m);
|
|
for (const id of ['test', 'coverage']) {
|
|
assert.doesNotMatch(found.get(id), /unittest discover/, `${id} no longer runs the bare unittest discover`);
|
|
}
|
|
});
|