From 323023d59cb79b2734bfae5877d19fbbfd3853f1 Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Mon, 26 Oct 2026 05:58:17 -0700 Subject: [PATCH] ci: bound every job and run the Python suite through the watchdog (#350) Every job in test.yml and code-quality.yml gets timeout-minutes = ceil(1.5 x slowest of three green main runs) + 3, min 5, with the runs and times written above jobs:. The matrix and the coverage gate run python -m tests.run_suite. ci-workflow.test.mjs checks every job in the three CI workflows has a limit and that the matrix and coverage use the runner. Job names are unchanged. --- .github/workflows/code-quality.yml | 3 +++ .github/workflows/test.yml | 22 +++++++++++++++++++--- tests/vendor/ci-workflow.test.mjs | 30 ++++++++++++++++++++++++++++-- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 135bee20..b50684f2 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -19,6 +19,9 @@ jobs: security: name: bandit (high severity) runs-on: ubuntu-latest + # ceil(1.5 x slowest of three green main runs) + 3, never under 5 (test.yml): runs 40257217665, + # 40254137380, 40240706583 took 23s / 16s / 23s -> 5 min. + timeout-minutes: 5 steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c0f018b2..2d032aca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,10 +19,22 @@ concurrency: env: PYTHONFAULTHANDLER: "1" +# Every job has a time limit: ceil(1.5 x the slowest of three recent green runs on main) + 3 +# minutes, never under 5. The 3 minutes leave room for the Python runner's watchdog (120 s, see +# tests/run_suite.py) to print its stack dump before GitHub cancels the job. A limit changes only +# with new measurements written here (TESTING.md "Time limits and hangs"). +# Measured 2026-10-26 on runs 40257217681 (3be0ada), 40240706568 (fb9134f), 40228167328 (f9835aa): +# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min +# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s) +# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min +# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min +# perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s, +# frontend modules 23s / 19s / 29s -> 5 min (the floor) jobs: test: name: py${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} + timeout-minutes: 27 strategy: fail-fast: false matrix: @@ -45,12 +57,12 @@ jobs: pip install -r requirements-dev.txt - name: Run tests - run: python -m unittest discover -s tests -v + run: python -m tests.run_suite -v comfyui: name: real ComfyUI compatibility runs-on: ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 24 steps: - uses: actions/checkout@v7 with: @@ -127,6 +139,7 @@ jobs: coverage: name: coverage gate (85%) runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v7 @@ -143,7 +156,7 @@ jobs: - name: Measure coverage run: | - python -m coverage run -m unittest discover -s tests + python -m coverage run -m tests.run_suite python -m coverage combine python -m coverage report --fail-under=85 @@ -154,6 +167,7 @@ jobs: name: perf budget if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest + timeout-minutes: 5 steps: - uses: actions/checkout@v7 @@ -176,6 +190,7 @@ jobs: lint: name: black and flake8 runs-on: ubuntu-latest + timeout-minutes: 5 steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v7 @@ -188,6 +203,7 @@ jobs: js-test: name: frontend modules runs-on: ubuntu-latest + timeout-minutes: 5 steps: - uses: actions/checkout@v7 diff --git a/tests/vendor/ci-workflow.test.mjs b/tests/vendor/ci-workflow.test.mjs index b2d8064a..f489fd64 100644 --- a/tests/vendor/ci-workflow.test.mjs +++ b/tests/vendor/ci-workflow.test.mjs @@ -1,5 +1,6 @@ -// Structural checks on .github/workflows/viewerjs-check.yml. The tooling has no YAML parser, so these -// read the workflow's fixed two-space layout: jobs at two spaces, their keys at four, steps at six. +// Structural checks on .github/workflows/viewerjs-check.yml, plus the time limits of every CI workflow. +// The tooling has no YAML parser, so these read the workflow's fixed two-space layout: jobs at two +// spaces, their keys at four, steps at six. import test from 'node:test'; import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; @@ -221,3 +222,28 @@ test('browser job timeouts keep a real margin over measured runs', () => { assert.ok(minutes >= 20, `${id} allows at least 20 minutes, since runner speed varies`); } }); + +// Every job is bounded (#350): a hang ends at a measured limit instead of GitHub's 6-hour default. +const BOUNDED_WORKFLOWS = ['test.yml', 'code-quality.yml', 'viewerjs-check.yml']; +const readWorkflow = name => readFileSync(new URL(`../../.github/workflows/${name}`, import.meta.url), 'utf8'); + +test('every job in the CI workflows has a numeric timeout-minutes', () => { + for (const name of BOUNDED_WORKFLOWS) { + const found = jobs(readWorkflow(name)); + assert.ok(found.size > 0, `${name} has jobs`); + for (const [id, body] of found) { + const limit = /^ {4}timeout-minutes: (\d+)\s*$/m.exec(body); + assert.ok(limit, `${name} job ${id} sets a numeric timeout-minutes`); + assert.ok(Number(limit[1]) >= 5, `${name} job ${id} allows at least 5 minutes`); + } + } +}); + +test('the Python matrix and coverage run the suite through the watchdog runner', () => { + const found = jobs(readWorkflow('test.yml')); + assert.match(found.get('test'), /^ {8}run: python -m tests\.run_suite -v$/m); + assert.match(found.get('coverage'), /^ {10}python -m coverage run -m tests\.run_suite$/m); + for (const id of ['test', 'coverage']) { + assert.doesNotMatch(found.get(id), /unittest discover/, `${id} no longer runs the bare unittest discover`); + } +});