ci(coverage): shard and merge modes for the frontend coverage run
frontend-coverage.mjs now plans its steps from the command line. No arguments still runs everything, then reports and checks. --shard I/N runs one slice of the chromium+touch workflows into coverage/js/raw without wiping it or reporting; shard 1 also runs the node units and the viewer suite. --merge-only reports and checks over raw coverage the shards wrote, with --update-baseline as before, and refuses an empty or missing raw dir. npm scripts: test:coverage:shard and coverage:merge.
This commit is contained in:
Vito Sansevero
+93
@@ -0,0 +1,93 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { plan, runCoverage } from '../../tools/vendor/frontend-coverage.mjs';
|
||||
|
||||
const steps = argv => plan(argv).map(({ step, ...options }) => (Object.keys(options).length ? { step, ...options } : step));
|
||||
|
||||
function tempPaths(t) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'pm-coverage-cli-'));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const reportRoot = join(root, 'coverage/js');
|
||||
return { root, paths: { repositoryRoot: root, tools: join(root, 'tools'), reportRoot, raw: join(reportRoot, 'raw'), sourceRoot: join(root, 'web/js') } };
|
||||
}
|
||||
|
||||
function recorder() {
|
||||
const calls = [];
|
||||
const run = (args, env = {}) => { calls.push({ args, env }); };
|
||||
return { calls, run };
|
||||
}
|
||||
|
||||
test('no arguments plans the unchanged full run', () => {
|
||||
assert.deepEqual(steps([]), [
|
||||
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: false },
|
||||
]);
|
||||
assert.deepEqual(steps(['--update-baseline']), [
|
||||
'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: true },
|
||||
]);
|
||||
});
|
||||
|
||||
test('--shard 2/3 plans only workflows with --shard 2/3 and no report', () => {
|
||||
assert.deepEqual(steps(['--shard', '2/3']), ['prepare', { step: 'workflows', shard: '2/3' }]);
|
||||
assert.deepEqual(steps(['--shard=3/3']), ['prepare', { step: 'workflows', shard: '3/3' }]);
|
||||
});
|
||||
|
||||
test('--shard 1/N also runs the node units and the viewer suite', () => {
|
||||
assert.deepEqual(steps(['--shard', '1/3']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/3' }]);
|
||||
assert.deepEqual(steps(['--shard', '1/1']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/1' }]);
|
||||
});
|
||||
|
||||
test('--merge-only plans the report and the check over existing raw coverage', () => {
|
||||
assert.deepEqual(steps(['--merge-only']), ['require-raw', 'report', { step: 'check', updateBaseline: false }]);
|
||||
assert.deepEqual(steps(['--merge-only', '--update-baseline']), ['require-raw', 'report', { step: 'check', updateBaseline: true }]);
|
||||
});
|
||||
|
||||
test('invalid shard and mode combinations are refused', () => {
|
||||
for (const argv of [['--shard'], ['--shard', '0/3'], ['--shard', '4/3'], ['--shard', '2'], ['--shard', 'a/b'], ['--shard', '1/0'], ['--shard', '1.5/3']]) {
|
||||
assert.throws(() => plan(argv), /--shard needs I\/N/, argv.join(' '));
|
||||
}
|
||||
assert.throws(() => plan(['--shard', '1/2', '--merge-only']), /either --shard or --merge-only/);
|
||||
assert.throws(() => plan(['--shard', '1/2', '--update-baseline']), /--update-baseline .* --merge-only or a full run/);
|
||||
assert.throws(() => plan(['--shard', '1/2', '--shard', '2/2']), /--shard once/);
|
||||
assert.throws(() => plan(['--report']), /Unknown argument: --report/);
|
||||
});
|
||||
|
||||
test('--merge-only refuses an empty raw dir', async t => {
|
||||
const { paths } = tempPaths(t);
|
||||
const { calls, run } = recorder();
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage .*coverage\/js\/raw.* run the --shard steps first/i);
|
||||
mkdirSync(paths.raw, { recursive: true });
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
|
||||
writeFileSync(join(paths.raw, 'notes.txt'), 'not coverage');
|
||||
await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i);
|
||||
assert.deepEqual(calls, [], 'nothing is reported or checked over partial or empty coverage');
|
||||
});
|
||||
|
||||
test('a shard keeps raw coverage other shards wrote and passes its shard to Playwright', async t => {
|
||||
const { paths } = tempPaths(t);
|
||||
mkdirSync(paths.raw, { recursive: true });
|
||||
const earlier = join(paths.raw, 'browser-earlier-shard.json');
|
||||
writeFileSync(earlier, '{"result":[]}');
|
||||
const { calls, run } = recorder();
|
||||
await runCoverage({ argv: ['--shard', '2/3'], run, paths });
|
||||
assert.ok(existsSync(earlier), 'raw files from another shard survive');
|
||||
assert.equal(calls.length, 1, 'shard 2 runs only the workflows');
|
||||
const [{ args, env }] = calls;
|
||||
assert.deepEqual(args.slice(-6), ['--project', 'chromium', '--project', 'touch', '--shard', '2/3']);
|
||||
assert.equal(env.PM_JS_COVERAGE, paths.raw);
|
||||
});
|
||||
|
||||
test('shard 1 runs the node units into the shared raw dir, then the viewer suite, then its workflows', async t => {
|
||||
const { root, paths } = tempPaths(t);
|
||||
mkdirSync(join(root, 'tests/js'), { recursive: true });
|
||||
writeFileSync(join(root, 'tests/js/a.test.js'), '');
|
||||
const { calls, run } = recorder();
|
||||
await runCoverage({ argv: ['--shard', '1/3'], run, paths });
|
||||
assert.equal(calls.length, 3);
|
||||
assert.deepEqual(calls[0].args, ['--test', 'tests/js/a.test.js']);
|
||||
assert.equal(calls[0].env.NODE_V8_COVERAGE, paths.raw);
|
||||
assert.ok(calls[1].args.includes(join(paths.tools, 'playwright.config.mjs')));
|
||||
assert.deepEqual(calls[2].args.slice(-2), ['--shard', '1/3']);
|
||||
});
|
||||
Vendored
+99
-20
@@ -1,4 +1,4 @@
|
||||
import { readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { existsSync, readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join, relative } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
@@ -7,33 +7,69 @@ import { repositoryRoot } from './vendor-viewerjs.mjs';
|
||||
import { discoverSources, checkCoverage } from '@comfyassets/toolkit/testing/coverage';
|
||||
export { discoverSources, checkCoverage };
|
||||
const metrics = ['lines', 'branches', 'functions', 'statements'];
|
||||
const SHARD = /^([1-9]\d*)\/([1-9]\d*)$/;
|
||||
const USAGE = 'usage: frontend-coverage.mjs [--update-baseline] | --shard I/N | --merge-only [--update-baseline]';
|
||||
|
||||
function run(args, env = {}) {
|
||||
function runNode(args, env = {}) {
|
||||
const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, stdio: 'inherit',
|
||||
env: { ...process.env, ...env } });
|
||||
if (result.status !== 0) throw new Error(`Command failed (${result.status}): ${args.join(' ')}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const tools = dirname(fileURLToPath(import.meta.url));
|
||||
function defaultPaths() {
|
||||
const reportRoot = join(repositoryRoot, 'coverage/js');
|
||||
const raw = join(reportRoot, 'raw');
|
||||
const sourceRoot = join(repositoryRoot, 'web/js');
|
||||
return { repositoryRoot, tools: dirname(fileURLToPath(import.meta.url)), reportRoot,
|
||||
raw: join(reportRoot, 'raw'), sourceRoot: join(repositoryRoot, 'web/js') };
|
||||
}
|
||||
|
||||
function parseShard(value) {
|
||||
const match = SHARD.exec(value ?? '');
|
||||
if (!match || Number(match[1]) > Number(match[2])) {
|
||||
throw new Error(`--shard needs I/N with 1 <= I <= N, got ${value ?? 'nothing'}\n${USAGE}`);
|
||||
}
|
||||
return { index: Number(match[1]), total: Number(match[2]), value };
|
||||
}
|
||||
|
||||
// Turn the command line into the ordered steps of one coverage run:
|
||||
// - no mode: the full run (clean, node units, viewer suite, chromium+touch
|
||||
// workflows, report, check), exactly as before shards existed;
|
||||
// - --shard I/N: one slice of the workflows into coverage/js/raw, without wiping
|
||||
// it or reporting; shard 1 also runs the node units and the viewer suite;
|
||||
// - --merge-only: report and check over raw coverage the shards already wrote.
|
||||
export function plan(argv) {
|
||||
let shard = null;
|
||||
let mergeOnly = false;
|
||||
let updateBaseline = false;
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--update-baseline') updateBaseline = true;
|
||||
else if (arg === '--merge-only') mergeOnly = true;
|
||||
else if (arg === '--shard' || arg.startsWith('--shard=')) {
|
||||
if (shard) throw new Error(`Pass --shard once\n${USAGE}`);
|
||||
shard = parseShard(arg === '--shard' ? argv[(index += 1)] : arg.slice('--shard='.length));
|
||||
} else throw new Error(`Unknown argument: ${arg}\n${USAGE}`);
|
||||
}
|
||||
if (shard && mergeOnly) throw new Error(`Pass either --shard or --merge-only\n${USAGE}`);
|
||||
if (shard && updateBaseline) throw new Error(`--update-baseline needs --merge-only or a full run, not a shard\n${USAGE}`);
|
||||
const check = { step: 'check', updateBaseline };
|
||||
if (mergeOnly) return [{ step: 'require-raw' }, { step: 'report' }, check];
|
||||
if (shard) {
|
||||
const first = shard.index === 1 ? [{ step: 'node-units' }, { step: 'viewer' }] : [];
|
||||
return [{ step: 'prepare' }, ...first, { step: 'workflows', shard: shard.value }];
|
||||
}
|
||||
return [{ step: 'clean' }, { step: 'node-units' }, { step: 'viewer' }, { step: 'workflows', shard: null },
|
||||
{ step: 'report' }, check];
|
||||
}
|
||||
|
||||
function rawCoverageFiles(raw) {
|
||||
return existsSync(raw) ? readdirSync(raw).filter(file => file.endsWith('.json')) : [];
|
||||
}
|
||||
|
||||
function checkSummary({ tools, reportRoot, sourceRoot }, updateBaseline) {
|
||||
const files = discoverSources(sourceRoot);
|
||||
rmSync(reportRoot, { recursive: true, force: true });
|
||||
mkdirSync(raw, { recursive: true });
|
||||
const tests = readdirSync(join(repositoryRoot, 'tests/js')).filter(file => file.endsWith('.test.js')).map(file => `tests/js/${file}`);
|
||||
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
|
||||
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
|
||||
'--project', 'chromium', '--project', 'touch'], { PM_JS_COVERAGE: raw });
|
||||
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
|
||||
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
|
||||
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
|
||||
const summary = JSON.parse(readFileSync(join(reportRoot, 'coverage-summary.json'), 'utf8'));
|
||||
const baselinePath = join(tools, 'frontend-coverage-baseline.json');
|
||||
if (process.argv.includes('--update-baseline')) {
|
||||
if (updateBaseline) {
|
||||
const previous = JSON.parse(readFileSync(baselinePath, 'utf8'));
|
||||
const failures = checkCoverage(summary, previous, sourceRoot, files.filter(file => previous[file]));
|
||||
if (failures.length) throw new Error(`Baseline updates cannot lower coverage:\n${failures.join('\n')}`);
|
||||
@@ -53,8 +89,51 @@ async function main() {
|
||||
const failures = checkCoverage(summary, baseline, sourceRoot, files);
|
||||
if (failures.length) throw new Error(`Frontend coverage regressed:\n${failures.join('\n')}`);
|
||||
}
|
||||
console.log(`Frontend coverage passed for ${files.length} files. Report: ${relative(repositoryRoot, reportRoot)}/index.html`);
|
||||
return files.length;
|
||||
}
|
||||
|
||||
export async function runCoverage({ argv = process.argv.slice(2), run = runNode, paths = defaultPaths() } = {}) {
|
||||
const { repositoryRoot: root, tools, reportRoot, raw, sourceRoot } = paths;
|
||||
const playwright = join(tools, 'node_modules/@playwright/test/cli.js');
|
||||
const actions = {
|
||||
clean() {
|
||||
rmSync(reportRoot, { recursive: true, force: true });
|
||||
mkdirSync(raw, { recursive: true });
|
||||
},
|
||||
prepare() {
|
||||
mkdirSync(raw, { recursive: true });
|
||||
},
|
||||
'require-raw'() {
|
||||
if (!rawCoverageFiles(raw).length) {
|
||||
throw new Error(`No raw coverage in ${relative(root, raw) || raw}: run the --shard steps first, `
|
||||
+ 'or put every shard\'s raw files there');
|
||||
}
|
||||
},
|
||||
'node-units'() {
|
||||
const tests = readdirSync(join(root, 'tests/js')).filter(file => file.endsWith('.test.js'))
|
||||
.map(file => `tests/js/${file}`);
|
||||
run(['--test', ...tests], { NODE_V8_COVERAGE: raw });
|
||||
},
|
||||
viewer() {
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]);
|
||||
},
|
||||
workflows({ shard }) {
|
||||
run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'),
|
||||
'--project', 'chromium', '--project', 'touch', ...(shard ? ['--shard', shard] : [])], { PM_JS_COVERAGE: raw });
|
||||
},
|
||||
report() {
|
||||
run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot,
|
||||
'--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot,
|
||||
'--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']);
|
||||
},
|
||||
check({ updateBaseline }) {
|
||||
const count = checkSummary(paths, updateBaseline);
|
||||
console.log(`Frontend coverage passed for ${count} files. Report: ${relative(root, reportRoot)}/index.html`);
|
||||
},
|
||||
};
|
||||
for (const { step, ...options } of plan(argv)) await actions[step](options);
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
runCoverage().catch(error => { console.error(error.message); process.exitCode = 1; });
|
||||
}
|
||||
|
||||
Vendored
+2
@@ -15,6 +15,8 @@
|
||||
"test:comfyui": "playwright test --config comfyui.config.mjs",
|
||||
"test:executions": "playwright test --config playwright.execution.config.mjs",
|
||||
"test:coverage": "node frontend-coverage.mjs",
|
||||
"test:coverage:shard": "node frontend-coverage.mjs --shard",
|
||||
"coverage:merge": "node frontend-coverage.mjs --merge-only",
|
||||
"coverage:baseline-check": "node coverage-baseline-check.mjs",
|
||||
"coverage:update": "node frontend-coverage.mjs --update-baseline",
|
||||
"test:compat": "playwright test --config pack-compat.config.mjs"
|
||||
|
||||
Reference in New Issue
Block a user