From 3ebf5c48cf8b85b958648309051750df5721c2af Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Wed, 21 Oct 2026 19:34:08 -0700 Subject: [PATCH] ci(coverage): shard and merge modes for the frontend coverage run frontend-coverage.mjs now plans its steps from the command line. No arguments still runs everything, then reports and checks. --shard I/N runs one slice of the chromium+touch workflows into coverage/js/raw without wiping it or reporting; shard 1 also runs the node units and the viewer suite. --merge-only reports and checks over raw coverage the shards wrote, with --update-baseline as before, and refuses an empty or missing raw dir. npm scripts: test:coverage:shard and coverage:merge. --- tests/vendor/frontend-coverage-cli.test.mjs | 93 +++++++++++++++ tools/vendor/frontend-coverage.mjs | 119 ++++++++++++++++---- tools/vendor/package.json | 2 + 3 files changed, 194 insertions(+), 20 deletions(-) create mode 100644 tests/vendor/frontend-coverage-cli.test.mjs diff --git a/tests/vendor/frontend-coverage-cli.test.mjs b/tests/vendor/frontend-coverage-cli.test.mjs new file mode 100644 index 00000000..8243181b --- /dev/null +++ b/tests/vendor/frontend-coverage-cli.test.mjs @@ -0,0 +1,93 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { plan, runCoverage } from '../../tools/vendor/frontend-coverage.mjs'; + +const steps = argv => plan(argv).map(({ step, ...options }) => (Object.keys(options).length ? { step, ...options } : step)); + +function tempPaths(t) { + const root = mkdtempSync(join(tmpdir(), 'pm-coverage-cli-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const reportRoot = join(root, 'coverage/js'); + return { root, paths: { repositoryRoot: root, tools: join(root, 'tools'), reportRoot, raw: join(reportRoot, 'raw'), sourceRoot: join(root, 'web/js') } }; +} + +function recorder() { + const calls = []; + const run = (args, env = {}) => { calls.push({ args, env }); }; + return { calls, run }; +} + +test('no arguments plans the unchanged full run', () => { + assert.deepEqual(steps([]), [ + 'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: false }, + ]); + assert.deepEqual(steps(['--update-baseline']), [ + 'clean', 'node-units', 'viewer', { step: 'workflows', shard: null }, 'report', { step: 'check', updateBaseline: true }, + ]); +}); + +test('--shard 2/3 plans only workflows with --shard 2/3 and no report', () => { + assert.deepEqual(steps(['--shard', '2/3']), ['prepare', { step: 'workflows', shard: '2/3' }]); + assert.deepEqual(steps(['--shard=3/3']), ['prepare', { step: 'workflows', shard: '3/3' }]); +}); + +test('--shard 1/N also runs the node units and the viewer suite', () => { + assert.deepEqual(steps(['--shard', '1/3']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/3' }]); + assert.deepEqual(steps(['--shard', '1/1']), ['prepare', 'node-units', 'viewer', { step: 'workflows', shard: '1/1' }]); +}); + +test('--merge-only plans the report and the check over existing raw coverage', () => { + assert.deepEqual(steps(['--merge-only']), ['require-raw', 'report', { step: 'check', updateBaseline: false }]); + assert.deepEqual(steps(['--merge-only', '--update-baseline']), ['require-raw', 'report', { step: 'check', updateBaseline: true }]); +}); + +test('invalid shard and mode combinations are refused', () => { + for (const argv of [['--shard'], ['--shard', '0/3'], ['--shard', '4/3'], ['--shard', '2'], ['--shard', 'a/b'], ['--shard', '1/0'], ['--shard', '1.5/3']]) { + assert.throws(() => plan(argv), /--shard needs I\/N/, argv.join(' ')); + } + assert.throws(() => plan(['--shard', '1/2', '--merge-only']), /either --shard or --merge-only/); + assert.throws(() => plan(['--shard', '1/2', '--update-baseline']), /--update-baseline .* --merge-only or a full run/); + assert.throws(() => plan(['--shard', '1/2', '--shard', '2/2']), /--shard once/); + assert.throws(() => plan(['--report']), /Unknown argument: --report/); +}); + +test('--merge-only refuses an empty raw dir', async t => { + const { paths } = tempPaths(t); + const { calls, run } = recorder(); + await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage .*coverage\/js\/raw.* run the --shard steps first/i); + mkdirSync(paths.raw, { recursive: true }); + await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i); + writeFileSync(join(paths.raw, 'notes.txt'), 'not coverage'); + await assert.rejects(runCoverage({ argv: ['--merge-only'], run, paths }), /no raw coverage/i); + assert.deepEqual(calls, [], 'nothing is reported or checked over partial or empty coverage'); +}); + +test('a shard keeps raw coverage other shards wrote and passes its shard to Playwright', async t => { + const { paths } = tempPaths(t); + mkdirSync(paths.raw, { recursive: true }); + const earlier = join(paths.raw, 'browser-earlier-shard.json'); + writeFileSync(earlier, '{"result":[]}'); + const { calls, run } = recorder(); + await runCoverage({ argv: ['--shard', '2/3'], run, paths }); + assert.ok(existsSync(earlier), 'raw files from another shard survive'); + assert.equal(calls.length, 1, 'shard 2 runs only the workflows'); + const [{ args, env }] = calls; + assert.deepEqual(args.slice(-6), ['--project', 'chromium', '--project', 'touch', '--shard', '2/3']); + assert.equal(env.PM_JS_COVERAGE, paths.raw); +}); + +test('shard 1 runs the node units into the shared raw dir, then the viewer suite, then its workflows', async t => { + const { root, paths } = tempPaths(t); + mkdirSync(join(root, 'tests/js'), { recursive: true }); + writeFileSync(join(root, 'tests/js/a.test.js'), ''); + const { calls, run } = recorder(); + await runCoverage({ argv: ['--shard', '1/3'], run, paths }); + assert.equal(calls.length, 3); + assert.deepEqual(calls[0].args, ['--test', 'tests/js/a.test.js']); + assert.equal(calls[0].env.NODE_V8_COVERAGE, paths.raw); + assert.ok(calls[1].args.includes(join(paths.tools, 'playwright.config.mjs'))); + assert.deepEqual(calls[2].args.slice(-2), ['--shard', '1/3']); +}); diff --git a/tools/vendor/frontend-coverage.mjs b/tools/vendor/frontend-coverage.mjs index ec9af1da..30de4152 100644 --- a/tools/vendor/frontend-coverage.mjs +++ b/tools/vendor/frontend-coverage.mjs @@ -1,4 +1,4 @@ -import { readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, readFileSync, readdirSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { dirname, join, relative } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { spawnSync } from 'node:child_process'; @@ -7,33 +7,69 @@ import { repositoryRoot } from './vendor-viewerjs.mjs'; import { discoverSources, checkCoverage } from '@comfyassets/toolkit/testing/coverage'; export { discoverSources, checkCoverage }; const metrics = ['lines', 'branches', 'functions', 'statements']; +const SHARD = /^([1-9]\d*)\/([1-9]\d*)$/; +const USAGE = 'usage: frontend-coverage.mjs [--update-baseline] | --shard I/N | --merge-only [--update-baseline]'; -function run(args, env = {}) { +function runNode(args, env = {}) { const result = spawnSync(process.execPath, args, { cwd: repositoryRoot, stdio: 'inherit', env: { ...process.env, ...env } }); if (result.status !== 0) throw new Error(`Command failed (${result.status}): ${args.join(' ')}`); } -async function main() { - const tools = dirname(fileURLToPath(import.meta.url)); +function defaultPaths() { const reportRoot = join(repositoryRoot, 'coverage/js'); - const raw = join(reportRoot, 'raw'); - const sourceRoot = join(repositoryRoot, 'web/js'); + return { repositoryRoot, tools: dirname(fileURLToPath(import.meta.url)), reportRoot, + raw: join(reportRoot, 'raw'), sourceRoot: join(repositoryRoot, 'web/js') }; +} + +function parseShard(value) { + const match = SHARD.exec(value ?? ''); + if (!match || Number(match[1]) > Number(match[2])) { + throw new Error(`--shard needs I/N with 1 <= I <= N, got ${value ?? 'nothing'}\n${USAGE}`); + } + return { index: Number(match[1]), total: Number(match[2]), value }; +} + +// Turn the command line into the ordered steps of one coverage run: +// - no mode: the full run (clean, node units, viewer suite, chromium+touch +// workflows, report, check), exactly as before shards existed; +// - --shard I/N: one slice of the workflows into coverage/js/raw, without wiping +// it or reporting; shard 1 also runs the node units and the viewer suite; +// - --merge-only: report and check over raw coverage the shards already wrote. +export function plan(argv) { + let shard = null; + let mergeOnly = false; + let updateBaseline = false; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--update-baseline') updateBaseline = true; + else if (arg === '--merge-only') mergeOnly = true; + else if (arg === '--shard' || arg.startsWith('--shard=')) { + if (shard) throw new Error(`Pass --shard once\n${USAGE}`); + shard = parseShard(arg === '--shard' ? argv[(index += 1)] : arg.slice('--shard='.length)); + } else throw new Error(`Unknown argument: ${arg}\n${USAGE}`); + } + if (shard && mergeOnly) throw new Error(`Pass either --shard or --merge-only\n${USAGE}`); + if (shard && updateBaseline) throw new Error(`--update-baseline needs --merge-only or a full run, not a shard\n${USAGE}`); + const check = { step: 'check', updateBaseline }; + if (mergeOnly) return [{ step: 'require-raw' }, { step: 'report' }, check]; + if (shard) { + const first = shard.index === 1 ? [{ step: 'node-units' }, { step: 'viewer' }] : []; + return [{ step: 'prepare' }, ...first, { step: 'workflows', shard: shard.value }]; + } + return [{ step: 'clean' }, { step: 'node-units' }, { step: 'viewer' }, { step: 'workflows', shard: null }, + { step: 'report' }, check]; +} + +function rawCoverageFiles(raw) { + return existsSync(raw) ? readdirSync(raw).filter(file => file.endsWith('.json')) : []; +} + +function checkSummary({ tools, reportRoot, sourceRoot }, updateBaseline) { const files = discoverSources(sourceRoot); - rmSync(reportRoot, { recursive: true, force: true }); - mkdirSync(raw, { recursive: true }); - const tests = readdirSync(join(repositoryRoot, 'tests/js')).filter(file => file.endsWith('.test.js')).map(file => `tests/js/${file}`); - run(['--test', ...tests], { NODE_V8_COVERAGE: raw }); - const playwright = join(tools, 'node_modules/@playwright/test/cli.js'); - run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]); - run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'), - '--project', 'chromium', '--project', 'touch'], { PM_JS_COVERAGE: raw }); - run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot, - '--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot, - '--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']); const summary = JSON.parse(readFileSync(join(reportRoot, 'coverage-summary.json'), 'utf8')); const baselinePath = join(tools, 'frontend-coverage-baseline.json'); - if (process.argv.includes('--update-baseline')) { + if (updateBaseline) { const previous = JSON.parse(readFileSync(baselinePath, 'utf8')); const failures = checkCoverage(summary, previous, sourceRoot, files.filter(file => previous[file])); if (failures.length) throw new Error(`Baseline updates cannot lower coverage:\n${failures.join('\n')}`); @@ -53,8 +89,51 @@ async function main() { const failures = checkCoverage(summary, baseline, sourceRoot, files); if (failures.length) throw new Error(`Frontend coverage regressed:\n${failures.join('\n')}`); } - console.log(`Frontend coverage passed for ${files.length} files. Report: ${relative(repositoryRoot, reportRoot)}/index.html`); + return files.length; } + +export async function runCoverage({ argv = process.argv.slice(2), run = runNode, paths = defaultPaths() } = {}) { + const { repositoryRoot: root, tools, reportRoot, raw, sourceRoot } = paths; + const playwright = join(tools, 'node_modules/@playwright/test/cli.js'); + const actions = { + clean() { + rmSync(reportRoot, { recursive: true, force: true }); + mkdirSync(raw, { recursive: true }); + }, + prepare() { + mkdirSync(raw, { recursive: true }); + }, + 'require-raw'() { + if (!rawCoverageFiles(raw).length) { + throw new Error(`No raw coverage in ${relative(root, raw) || raw}: run the --shard steps first, ` + + 'or put every shard\'s raw files there'); + } + }, + 'node-units'() { + const tests = readdirSync(join(root, 'tests/js')).filter(file => file.endsWith('.test.js')) + .map(file => `tests/js/${file}`); + run(['--test', ...tests], { NODE_V8_COVERAGE: raw }); + }, + viewer() { + run([playwright, 'test', '--config', join(tools, 'playwright.config.mjs')]); + }, + workflows({ shard }) { + run([playwright, 'test', '--config', join(tools, 'playwright.workflows.config.mjs'), + '--project', 'chromium', '--project', 'touch', ...(shard ? ['--shard', shard] : [])], { PM_JS_COVERAGE: raw }); + }, + report() { + run([join(tools, 'node_modules/c8/bin/c8.js'), 'report', '--all', '--src', sourceRoot, + '--include', 'web/js/**/*.js', '--include', 'web/js/**/*.mjs', '--temp-directory', raw, '--reports-dir', reportRoot, + '--reporter', 'text', '--reporter', 'json-summary', '--reporter', 'html']); + }, + check({ updateBaseline }) { + const count = checkSummary(paths, updateBaseline); + console.log(`Frontend coverage passed for ${count} files. Report: ${relative(root, reportRoot)}/index.html`); + }, + }; + for (const { step, ...options } of plan(argv)) await actions[step](options); +} + if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch(error => { console.error(error.message); process.exitCode = 1; }); + runCoverage().catch(error => { console.error(error.message); process.exitCode = 1; }); } diff --git a/tools/vendor/package.json b/tools/vendor/package.json index c64b9034..167fb59c 100644 --- a/tools/vendor/package.json +++ b/tools/vendor/package.json @@ -15,6 +15,8 @@ "test:comfyui": "playwright test --config comfyui.config.mjs", "test:executions": "playwright test --config playwright.execution.config.mjs", "test:coverage": "node frontend-coverage.mjs", + "test:coverage:shard": "node frontend-coverage.mjs --shard", + "coverage:merge": "node frontend-coverage.mjs --merge-only", "coverage:baseline-check": "node coverage-baseline-check.mjs", "coverage:update": "node frontend-coverage.mjs --update-baseline", "test:compat": "playwright test --config pack-compat.config.mjs"