ci: run the full matrix once per PR, in the merge queue (#420)
Pull requests now run a fast subset: Python 3.12 on ubuntu-latest, the coverage gate, black/flake8, frontend modules, bandit and the Chromium/touch frontend coverage shards plus merge. The merge queue, pushes to main and manual runs keep the full matrix: Linux/Windows/macOS x Python 3.12-3.14, real ComfyUI compatibility and the Firefox/WebKit page workflows. A dropped matrix leg never reports a check, so the new required "all tests" gate requires every test.yml job to succeed outside pull requests and real ComfyUI to be skipped only on pull requests; viewerjs does the same for the browser shards. branch-protection.json drops the eight legs a pull request no longer runs and adds "all tests".
This commit is contained in:
Vito Sansevero
Vendored
+105
-24
@@ -67,6 +67,45 @@ const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\
|
||||
const mergeJob = () => allJobs.get('frontend-coverage-merge');
|
||||
const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step));
|
||||
|
||||
// Jobs that run only in the merge queue, on main and on demand (#420): skipped on pull_request.
|
||||
const QUEUE_ONLY_IF = /^ {4}if: github\.event_name != 'pull_request'$/m;
|
||||
const EVENTS = ['pull_request', 'merge_group', 'push', 'workflow_dispatch'];
|
||||
|
||||
function gateVariables(jobText) {
|
||||
return new Map([...jobText.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.([\w-]+)\.result \}\}$/gm)]
|
||||
.map(match => [match[2], match[1]]));
|
||||
}
|
||||
|
||||
function runGate(jobText, env) {
|
||||
const script = runScript(steps(jobText).find(step => /run: \|/.test(step)));
|
||||
try {
|
||||
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// A gate passes only when every always-on job succeeded and every queue-only job is skipped on a
|
||||
// pull request and succeeded on every other event. Anything else fails it.
|
||||
function assertGateBehaviour(jobText, queueOnly) {
|
||||
assert.match(jobText, /^ {10}EVENT_NAME: \$\{\{ github\.event_name \}\}$/m, 'the gate knows the event');
|
||||
const variables = gateVariables(jobText);
|
||||
for (const id of queueOnly) assert.ok(variables.has(id), `${id} reaches the gate`);
|
||||
for (const event of EVENTS) {
|
||||
const expected = id => (queueOnly.includes(id) && event === 'pull_request' ? 'skipped' : 'success');
|
||||
const green = Object.fromEntries([...variables].map(([id, name]) => [name, expected(id)]));
|
||||
assert.equal(runGate(jobText, { ...green, EVENT_NAME: event }), true, `${event}: the expected results pass`);
|
||||
for (const [id, name] of variables) {
|
||||
for (const result of ['success', 'failure', 'cancelled', 'skipped']) {
|
||||
if (result === expected(id)) continue;
|
||||
assert.equal(runGate(jobText, { ...green, EVENT_NAME: event, [name]: result }), false,
|
||||
`${event}: ${id}=${result} must fail the gate`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test('viewerjs keeps its name and needs every frontend job', () => {
|
||||
const viewerjs = allJobs.get('viewerjs');
|
||||
assert.ok(viewerjs, 'the required context is the job id viewerjs');
|
||||
@@ -76,32 +115,17 @@ test('viewerjs keeps its name and needs every frontend job', () => {
|
||||
assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow');
|
||||
assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
|
||||
assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped');
|
||||
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
||||
for (const id of needs) {
|
||||
const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs);
|
||||
assert.ok(variable, `${id}'s result reaches the check`);
|
||||
assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`);
|
||||
}
|
||||
assert.deepEqual([...gateVariables(viewerjs).keys()].sort(), [...needs].sort(), 'every needed result reaches the check');
|
||||
});
|
||||
|
||||
test('viewerjs fails unless every needed job succeeded', () => {
|
||||
const viewerjs = allJobs.get('viewerjs');
|
||||
const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
|
||||
const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]);
|
||||
const run = env => {
|
||||
try {
|
||||
execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const allSuccess = Object.fromEntries(variables.map(name => [name, 'success']));
|
||||
assert.equal(run(allSuccess), true);
|
||||
for (const name of variables) {
|
||||
for (const result of ['failure', 'cancelled', 'skipped']) {
|
||||
assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`);
|
||||
}
|
||||
test('viewerjs requires the browser shards everywhere but on a pull request', () => {
|
||||
assertGateBehaviour(allJobs.get('viewerjs'), ['browser-workflows']);
|
||||
});
|
||||
|
||||
test('Firefox/WebKit page workflows run only outside pull requests; coverage runs on every event', () => {
|
||||
assert.match(allJobs.get('browser-workflows'), QUEUE_ONLY_IF);
|
||||
for (const id of ['frontend-coverage', 'frontend-coverage-merge']) {
|
||||
assert.doesNotMatch(allJobs.get(id), /^ {4}if:/m, `${id} runs on every event`);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -276,3 +300,60 @@ test('the bandit pre-commit hook fails at the severity CI enforces and passes ev
|
||||
assert.doesNotMatch(hook, /true|exit-zero|bash -c/, 'nothing masks the exit code or drops a file');
|
||||
assert.doesNotMatch(hook, /pass_filenames: false/, 'the hook scans the staged files');
|
||||
});
|
||||
|
||||
// Event split (#420): pull requests run a fast subset, the merge queue and main run everything.
|
||||
const testJobs = () => jobs(readWorkflow('test.yml'));
|
||||
|
||||
test('the Python matrix is one leg on a pull request and the full matrix on every other event', () => {
|
||||
const body = testJobs().get('test');
|
||||
const lists = key => {
|
||||
const match = new RegExp(`^ {8}${key}: \\$\\{\\{ github\\.event_name == 'pull_request' && fromJSON\\('(\\[[^\\]]*\\])'\\) \\|\\| fromJSON\\('(\\[[^\\]]*\\])'\\) \\}\\}$`, 'm').exec(body);
|
||||
assert.ok(match, `${key} is chosen by event`);
|
||||
return { pr: JSON.parse(match[1]), other: JSON.parse(match[2]) };
|
||||
};
|
||||
assert.deepEqual(lists('os'), { pr: ['ubuntu-latest'], other: ['ubuntu-latest', 'windows-latest', 'macos-latest'] });
|
||||
assert.deepEqual(lists('python-version'), { pr: ['3.12'], other: ['3.12', '3.13', '3.14'] });
|
||||
assert.match(body, /^ {6}fail-fast: false$/m);
|
||||
assert.match(body, /^ {4}name: py\$\{\{ matrix\.python-version \}\} on \$\{\{ matrix\.os \}\}$/m);
|
||||
});
|
||||
|
||||
test('real ComfyUI runs only outside pull requests; the other test.yml checks run on every event', () => {
|
||||
const found = testJobs();
|
||||
assert.match(found.get('comfyui'), QUEUE_ONLY_IF);
|
||||
for (const id of ['test', 'coverage', 'lint', 'js-test']) {
|
||||
assert.doesNotMatch(found.get(id), /^ {4}if:/m, `${id} runs on every event`);
|
||||
}
|
||||
});
|
||||
|
||||
test('all tests needs every gating test.yml job and requires the full set outside pull requests', () => {
|
||||
const found = testJobs();
|
||||
const gate = found.get('all-tests');
|
||||
assert.ok(gate, 'the aggregate gate exists');
|
||||
assert.match(gate, /^ {4}name: all tests$/m, 'the required context is "all tests"');
|
||||
assert.match(gate, /^ {4}if: always\(\)$/m);
|
||||
const gating = [...found.keys()].filter(id => !['all-tests', 'perf-budget'].includes(id));
|
||||
assert.deepEqual([...needsOf(gate)].sort(), gating.sort(), 'the gate needs every job but the non-gating perf budget');
|
||||
assert.deepEqual([...gateVariables(gate).keys()].sort(), gating.sort(), 'every needed result reaches the gate');
|
||||
assertGateBehaviour(gate, ['comfyui']);
|
||||
});
|
||||
|
||||
test('the CI workflows run on pull_request, merge_group and push without path filters', () => {
|
||||
for (const name of BOUNDED_WORKFLOWS) {
|
||||
const text = readWorkflow(name);
|
||||
const triggers = text.slice(text.indexOf('\non:'), text.indexOf('\n', text.indexOf('\non:') + 1) + 400);
|
||||
for (const event of ['merge_group', 'pull_request', 'push']) {
|
||||
assert.match(triggers, new RegExp(`^ {2}${event}:`, 'm'), `${name} runs on ${event}`);
|
||||
}
|
||||
assert.doesNotMatch(text, /^ {4}paths(-ignore)?:/m, `${name} has no path filter that would leave a required check pending`);
|
||||
}
|
||||
});
|
||||
|
||||
test('every required check reports on a pull request and the gates carry the full matrix', () => {
|
||||
const contexts = JSON.parse(readRepoFile('.github/branch-protection.json')).required_status_checks.contexts;
|
||||
assert.deepEqual([...contexts].sort(), [
|
||||
'all tests', 'bandit (high severity)', 'black and flake8', 'coverage gate (85%)', 'frontend modules',
|
||||
'py3.12 on ubuntu-latest', 'real ComfyUI compatibility', 'viewerjs',
|
||||
].sort());
|
||||
assert.equal(contexts.filter(context => /^py/.test(context)).length, 1,
|
||||
'a dropped matrix leg never reports on a pull request, so it cannot be required');
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user