ComfyAssets-ComfyUI_PromptManager/tests/vendor/ci-workflow.test.mjs

// Structural checks on .github/workflows/viewerjs-check.yml, plus the time limits of every CI workflow.
// The tooling has no YAML parser, so these read the workflow's fixed two-space layout: jobs at two
// spaces, their keys at four, steps at six.
import test from 'node:test';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises';
import { readFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';

const workflow = readFileSync(new URL('../../.github/workflows/viewerjs-check.yml', import.meta.url), 'utf8');
const PINNED_IMAGE = /^\s+image: mcr\.microsoft\.com\/playwright:v\d+\.\d+\.\d+-noble$/m;

function jobs(text) {
    const lines = text.split('\n');
    const start = lines.indexOf('jobs:');
    assert.notEqual(start, -1, 'workflow has a jobs section');
    const found = new Map();
    let current = null;
    for (const line of lines.slice(start + 1)) {
        const header = /^ {2}([\w-]+):\s*$/.exec(line);
        if (header) {
            current = header[1];
            found.set(current, []);
        } else if (current && (line.startsWith('    ') || line.trim() === '' || /^ {2}#/.test(line))) {
            found.get(current).push(line);
        } else if (current && /^\S/.test(line)) {
            break;
        }
    }
    return new Map([...found].map(([id, body]) => [id, body.join('\n')]));
}

function steps(jobText) {
    const parts = jobText.split(/^(?= {6}- )/m);
    return parts.slice(1);
}

function runScript(stepText) {
    const lines = stepText.split('\n');
    const index = lines.findIndex(line => /^ {8}run: \|\s*$/.test(line));
    assert.notEqual(index, -1, 'step has a block run script');
    const body = [];
    for (const line of lines.slice(index + 1)) {
        if (line.trim() !== '' && !line.startsWith('          ')) break;
        body.push(line.slice(10));
    }
    return body.join('\n');
}

function needsOf(jobText) {
    const match = /^ {4}needs: \[([^\]]*)\]\s*$/m.exec(jobText) ?? /^ {4}needs: ([\w-]+)\s*$/m.exec(jobText);
    assert.ok(match, 'job declares needs');
    return match[1].split(',').map(value => value.trim()).filter(Boolean);
}

function matrixList(jobText, key) {
    const match = new RegExp(`^ {8}${key}: \\[([^\\]]*)\\]\\s*$`, 'm').exec(jobText);
    assert.ok(match, `job has a ${key} matrix`);
    return match[1].split(',').map(value => value.trim());
}

const allJobs = jobs(workflow);
const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\/vendor test:(workflows|coverage)/.test(body))
    .map(([id]) => id);
const mergeJob = () => allJobs.get('frontend-coverage-merge');
const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step));

test('viewerjs keeps its name and needs every frontend job', () => {
    const viewerjs = allJobs.get('viewerjs');
    assert.ok(viewerjs, 'the required context is the job id viewerjs');
    assert.doesNotMatch(viewerjs, /^ {4}name:/m, 'no display name may replace the context "viewerjs"');
    const needs = needsOf(viewerjs);
    const others = [...allJobs.keys()].filter(id => id !== 'viewerjs');
    assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow');
    assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
    assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped');
    const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
    for (const id of needs) {
        const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs);
        assert.ok(variable, `${id}'s result reaches the check`);
        assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`);
    }
});

test('viewerjs fails unless every needed job succeeded', () => {
    const viewerjs = allJobs.get('viewerjs');
    const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step)));
    const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]);
    const run = env => {
        try {
            execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' });
            return true;
        } catch {
            return false;
        }
    };
    const allSuccess = Object.fromEntries(variables.map(name => [name, 'success']));
    assert.equal(run(allSuccess), true);
    for (const name of variables) {
        for (const result of ['failure', 'cancelled', 'skipped']) {
            assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`);
        }
    }
});

test('coverage runs as three shards that each upload their raw coverage', () => {
    const coverage = allJobs.get('frontend-coverage');
    assert.deepEqual(matrixList(coverage, 'shard'), ['1', '2', '3']);
    assert.match(coverage, /^ {4}name: frontend coverage \(shard \$\{\{ matrix\.shard \}\}\/3\)$/m);
    assert.match(coverage, /npm run --prefix tools\/vendor test:coverage:shard -- \$\{\{ matrix\.shard \}\}\/3/);
    const upload = steps(coverage).find(step => /name: coverage-raw-\$\{\{ matrix\.shard \}\}/.test(step));
    assert.ok(upload, 'each shard uploads coverage-raw-N');
    assert.match(upload, /^ {10}path: coverage\/js\/raw\/?$/m);
    assert.match(upload, /^ {10}retention-days: 3$/m);
    assert.match(upload, /^ {10}if-no-files-found: error$/m);
});

test('merge job downloads every shard and runs the merge-only report and check', () => {
    const merge = mergeJob();
    assert.ok(merge, 'frontend-coverage-merge exists');
    assert.deepEqual(needsOf(merge), ['frontend-coverage']);
    assert.match(merge, /^ {4}name: frontend coverage \(merge\)$/m);
    const download = steps(merge).find(step => /uses: actions\/download-artifact@/.test(step));
    assert.ok(download, 'merge downloads the shard artifacts');
    assert.match(download, /^ {10}pattern: coverage-raw-\*$/m);
    assert.doesNotMatch(download, /merge-multiple: true/, 'each artifact keeps its own directory so the check can count them');
    assert.ok(verifyStep(), 'merge has an explicit artifact count check');
    assert.match(verifyStep(), new RegExp(`^ {10}EXPECTED_SHARDS: ${matrixList(allJobs.get('frontend-coverage'), 'shard').length}$`, 'm'),
        'the expected count matches the shard matrix');
    const order = steps(merge).map(step => (/name: (.*)/.exec(step) ?? [, step.trim().split('\n')[0]])[1]);
    const verifyAt = order.findIndex(name => /Verify every shard/.test(name));
    const mergeAt = steps(merge).findIndex(step => /npm run --prefix tools\/vendor coverage:merge/.test(step));
    assert.ok(verifyAt >= 0 && mergeAt > verifyAt, 'the count check runs before coverage:merge');
    const report = steps(merge).find(step => /name: frontend-test-reports$/m.test(step));
    assert.ok(report, 'merge uploads the HTML report as frontend-test-reports');
    assert.match(report, /!coverage\/js\/raw\/\*\*/);
});

async function stagedShards(root, shards) {
    const staged = join(root, 'staged');
    for (const [shard, files] of shards) {
        const directory = join(staged, `coverage-raw-${shard}`);
        await mkdir(directory, { recursive: true });
        for (const file of files) await writeFile(join(directory, file), '{"result":[]}');
    }
    await mkdir(staged, { recursive: true });
    return staged;
}

async function verify(t, shards) {
    const root = await mkdtemp(join(tmpdir(), 'pm-ci-merge-'));
    t.after(() => rm(root, { recursive: true, force: true }));
    const staged = await stagedShards(root, shards);
    try {
        execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', runScript(verifyStep())], {
            cwd: root, stdio: 'pipe',
            env: { PATH: process.env.PATH, STAGED_RAW: staged, EXPECTED_SHARDS: '3' },
        });
    } catch (error) {
        return { ok: false, output: `${error.stdout}${error.stderr}` };
    }
    return { ok: true, raw: (await readdir(join(root, 'coverage/js/raw'))).sort() };
}

test('merge refuses when a shard\'s raw coverage is missing', async t => {
    const missing = await verify(t, [[1, ['a.json']], [3, ['c.json']]]);
    assert.equal(missing.ok, false);
    assert.match(missing.output, /2 of 3/);
    const empty = await verify(t, [[1, ['a.json']], [2, []], [3, ['c.json']]]);
    assert.equal(empty.ok, false, 'an artifact without raw files counts as missing');
    const collision = await verify(t, [[1, ['a.json']], [2, ['a.json']], [3, ['c.json']]]);
    assert.equal(collision.ok, false, 'two shards writing the same file name must not silently drop one');
    const complete = await verify(t, [[1, ['a.json', 'node.json']], [2, ['b.json']], [3, ['c.json']]]);
    assert.deepEqual(complete, { ok: true, raw: ['a.json', 'b.json', 'c.json', 'node.json'] });
});

test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () => {
    assert.deepEqual([...browserJobIds].sort(), ['browser-workflows', 'frontend-coverage']);
    for (const id of browserJobIds) {
        const body = allJobs.get(id);
        assert.match(body, /^ {6}PM_TEST_WORKERS: 3$/m, `${id} sets PM_TEST_WORKERS for the whole job`);
        assert.match(body, PINNED_IMAGE, `${id} keeps the pinned Playwright image`);
        assert.match(body, /PIP_SHA256: [0-9a-f]{64}/, `${id} keeps the pinned pip bootstrap`);
        assert.match(body, /^ {4}timeout-minutes: \d+$/m);
    }
    assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards');
});

test('page workflows run each browser in two shards', () => {
    const workflows = allJobs.get('browser-workflows');
    assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']);
    assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']);
    assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/);
    assert.match(workflows, /^ {6}fail-fast: false$/m);
    assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/);
});

test('every npm ci restores the npm download cache keyed on the vendor lockfile first', () => {
    const installing = [...allJobs].filter(([, body]) => /npm ci --prefix tools\/vendor/.test(body));
    assert.deepEqual(installing.map(([id]) => id).sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']);
    for (const [id, body] of installing) {
        const jobSteps = steps(body);
        const cacheAt = jobSteps.findIndex(step => /uses: actions\/cache@v\d+/.test(step));
        const installAt = jobSteps.findIndex(step => /npm ci --prefix tools\/vendor/.test(step));
        assert.ok(cacheAt >= 0 && cacheAt < installAt, `${id} restores the npm cache before npm ci`);
        assert.match(jobSteps[cacheAt], /^ {10}path: ~\/\.npm$/m, `${id} caches npm's download cache, not node_modules`);
        assert.match(jobSteps[cacheAt], /hashFiles\('tools\/vendor\/package-lock\.json'\)/, `${id} keys the cache on the lockfile`);
        assert.match(jobSteps[installAt], /npm ci --prefix tools\/vendor --ignore-scripts/, `${id} still installs without scripts`);
        // The key names the job's own pinned image, so an image bump starts a fresh cache.
        const image = /^ {6}image: mcr\.microsoft\.com\/playwright:(v\d+\.\d+\.\d+)-noble$/m.exec(body)?.[1];
        assert.ok(image, `${id} runs in the pinned Playwright image`);
        assert.match(jobSteps[cacheAt], new RegExp(`key: npm-\\$\\{\\{ runner\\.os \\}\\}-playwright-${image.replaceAll('.', '\\.')}-\\$\\{\\{ hashFiles`),
            `${id} keys the cache on its image ${image}`);
    }
});

test('browser job timeouts keep a real margin over measured runs', () => {
    for (const id of ['frontend-coverage', 'browser-workflows']) {
        const minutes = Number(/^ {4}timeout-minutes: (\d+)$/m.exec(allJobs.get(id))[1]);
        assert.ok(minutes >= 20, `${id} allows at least 20 minutes, since runner speed varies`);
    }
});

// Every job is bounded (#350): a hang ends at a measured limit instead of GitHub's 6-hour default.
const BOUNDED_WORKFLOWS = ['test.yml', 'code-quality.yml', 'viewerjs-check.yml'];
const readWorkflow = name => readFileSync(new URL(`../../.github/workflows/${name}`, import.meta.url), 'utf8');

test('every job in the CI workflows has a numeric timeout-minutes', () => {
    for (const name of BOUNDED_WORKFLOWS) {
        const found = jobs(readWorkflow(name));
        assert.ok(found.size > 0, `${name} has jobs`);
        for (const [id, body] of found) {
            const limit = /^ {4}timeout-minutes: (\d+)\s*$/m.exec(body);
            assert.ok(limit, `${name} job ${id} sets a numeric timeout-minutes`);
            assert.ok(Number(limit[1]) >= 5, `${name} job ${id} allows at least 5 minutes`);
        }
    }
});

test('the Python matrix and coverage run the suite through the watchdog runner', () => {
    const found = jobs(readWorkflow('test.yml'));
    assert.match(found.get('test'), /^ {8}run: python -m tests\.run_suite -v$/m);
    assert.match(found.get('coverage'), /^ {10}python -m coverage run -m tests\.run_suite$/m);
    for (const id of ['test', 'coverage']) {
        assert.doesNotMatch(found.get(id), /unittest discover/, `${id} no longer runs the bare unittest discover`);
    }
});