diff --git a/.github/workflows/portability.yml b/.github/workflows/portability.yml index 43782e03..0a8767e3 100644 --- a/.github/workflows/portability.yml +++ b/.github/workflows/portability.yml @@ -21,15 +21,16 @@ env: PYTHONFAULTHANDLER: "1" # Time limit rule as in test.yml: ceil(1.5 x the slowest of three recent green runs on main) + 3 -# minutes, never under 5. Measured 2026-10-26 in test.yml on runs 40257217681 (3be0ada), -# 40240706568 (fb9134f), 40228167328 (f9835aa), before these legs moved here in #420: -# slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min -# (macos 6m14s-8m30s, windows 11m42s-15m54s) +# minutes, never under 5, timed as whole-job wall time. One matrix job, so the slowest leg of each +# run counts (tests/vendor/ci-workflow.test.mjs recomputes it, #493). +# Measured 2026-10-29 on main runs 40935337099 (dd247e3), 40938407252 (dd247e3), 40941240624 (dd247e3): +# slowest leg: 7m24s (py3.12 windows) / 7m31s (py3.13 windows) / 7m32s (py3.14 windows) -> 15 min +# (windows 4m56s-7m32s, macos 4m39s-6m14s) jobs: test: name: py${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} - timeout-minutes: 27 + timeout-minutes: 15 strategy: fail-fast: false matrix: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 85194d1a..50a7cdd2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,8 +28,8 @@ env: # py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min # (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s) # 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs. -# Windows and macOS moved to portability.yml (#420), which keeps this 27-minute limit; this -# Linux-only matrix keeps it too until it is re-measured. +# Windows and macOS moved to portability.yml (#420), which has its own measured limit (#493); +# this Linux-only matrix keeps 27 minutes until it is re-measured. # real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min # coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min # perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s, diff --git a/TESTING.md b/TESTING.md index 359d4afe..6c206af3 100644 --- a/TESTING.md +++ b/TESTING.md @@ -267,11 +267,13 @@ the process exits non-zero. use `\\localhost\`) or a default TLS context (it reads the Windows certificate store) costs seconds there; the slowest list shows it. -Every job in `test.yml`, `code-quality.yml` and `viewerjs-check.yml` has -`timeout-minutes` (`tests/vendor/ci-workflow.test.mjs` checks it). The rule is -ceil(1.5 x the slowest of three recent green runs on `main`) + 3 minutes, never -under 5; the three run ids, their times and the resulting limits are written -in a comment above `jobs:`. The 3 minutes leave room for one watchdog firing, so +Every job in `test.yml`, `code-quality.yml`, `viewerjs-check.yml` and +`portability.yml` has `timeout-minutes` (`tests/vendor/ci-workflow.test.mjs` +checks it). The rule is ceil(1.5 x the slowest of three recent green runs on +`main`) + 3 minutes, never under 5, timed as whole-job wall time; the three run +ids, their times and the resulting limits are written in a comment above +`jobs:`. For `portability.yml` the tooling test recomputes the limit from that +comment (#493); its one matrix job takes the slowest Windows or macOS leg. The 3 minutes leave room for one watchdog firing, so the dump prints before GitHub cancels the job. A job that hits its limit, or a watchdog dump, is a failure like any other: read the dump, find the blocking call, fix it. Never raise a limit or rerun to get green; a limit changes only diff --git a/tests/vendor/ci-workflow.test.mjs b/tests/vendor/ci-workflow.test.mjs index c146bb58..19bc5ae8 100644 --- a/tests/vendor/ci-workflow.test.mjs +++ b/tests/vendor/ci-workflow.test.mjs @@ -353,6 +353,26 @@ test('Windows and macOS run 3.12-3.14 on main, nightly and on demand, never on a assert.doesNotMatch(text, /cancel-in-progress: true|cancel-in-progress: \$/, 'no portability run is cancelled'); }); +// #493: the portability limit follows the rule in its comment: ceil(1.5 x the slowest of three +// green main runs) + 3 minutes, never under 5, with those runs and times written above jobs:. +// The times are whole-job wall times. That the runs were green and on main is the author's word. +test('the portability limit is computed from three measured green main runs', () => { + const text = readWorkflow('portability.yml'); + const comment = text.slice(0, text.indexOf('\njobs:')); + const runs = [...comment.matchAll(/^# Measured \d{4}-\d{2}-\d{2} on main runs (\d+) \([0-9a-f]{7}\), (\d+) \([0-9a-f]{7}\), (\d+) \([0-9a-f]{7}\):$/gm)]; + assert.equal(runs.length, 1, 'the comment names the three runs it measured, once'); + assert.equal(new Set(runs[0].slice(1)).size, 3, 'three different runs'); + const time = '(\\d+)m([0-5]\\d)s \\(([^)]+)\\)'; + const slowest = new RegExp(`^#\\s+slowest leg: ${time} / ${time} / ${time} -> (\\d+) min$`, 'm').exec(comment); + assert.ok(slowest, 'the comment gives the slowest leg of each run and the limit'); + const seconds = [1, 4, 7].map(i => Number(slowest[i]) * 60 + Number(slowest[i + 1])); + const expected = Math.max(5, Math.ceil(1.5 * Math.max(...seconds) / 60) + 3); + assert.equal(Number(slowest[10]), expected, 'the written limit follows the rule'); + assert.equal(comment.match(/-> \d+ min/g).length, 1, 'no older measurement stays beside it'); + const limit = Number(/^ {4}timeout-minutes: (\d+)\s*$/m.exec(jobs(text).get('test'))[1]); + assert.equal(limit, expected, 'the job uses the written limit'); +}); + test('real ComfyUI runs only outside pull requests; the other test.yml checks run on every event', () => { const found = testJobs(); assert.match(found.get('comfyui'), QUEUE_ONLY_IF);