From 0c93777972a8659cd4e84a56c2959742b5b28141 Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Wed, 28 Oct 2026 04:39:08 -0700 Subject: [PATCH] ci: move Windows and macOS out of the blocking path (#420) The maintainer deferred Windows and macOS to the end of the redesign. The merge queue now runs Linux on Python 3.12-3.14, real ComfyUI, the frontend coverage shards and the Firefox/WebKit page workflows; pull requests keep the Linux 3.12 subset. The six Windows/macOS legs move to the new portability.yml, which runs on push to main, nightly and on demand, keeps the 27-minute limit and is never a required check. Red runs there are fixed at their cause before the cutover (#334). --- .github/workflows/portability.yml | 55 +++++++++++++++++++++++++++++++ .github/workflows/test.yml | 9 +++-- TESTING.md | 31 ++++++++++------- tests/vendor/ci-workflow.test.mjs | 33 +++++++++++++------ 4 files changed, 103 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/portability.yml diff --git a/.github/workflows/portability.yml b/.github/workflows/portability.yml new file mode 100644 index 00000000..75122ab0 --- /dev/null +++ b/.github/workflows/portability.yml @@ -0,0 +1,55 @@ +name: Windows and macOS + +permissions: + contents: read + +# Non-blocking portability runs (#420): never on pull requests or in the merge queue, and no context +# here is a required check. A red run is fixed at its cause before the workspace cutover (#334). +on: + push: + branches: [main] + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +# Every run gets its own group (run_id), so no main commit's run is ever cancelled or dropped as pending. +concurrency: + group: ${{ github.workflow }}-${{ github.run_id }} + cancel-in-progress: false + +env: + PYTHONFAULTHANDLER: "1" + +# Time limit rule as in test.yml: ceil(1.5 x the slowest of three recent green runs on main) + 3 +# minutes, never under 5. Measured 2026-10-26 in test.yml on runs 40257217681 (3be0ada), +# 40240706568 (fb9134f), 40228167328 (f9835aa), before these legs moved here in #420: +# slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min +# (macos 6m14s-8m30s, windows 11m42s-15m54s) +jobs: + test: + name: py${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 27 + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + python-version: ["3.12", "3.13", "3.14"] + + steps: + - uses: actions/checkout@v7 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: requirements-dev.txt + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Run tests + run: python -m tests.run_suite -v diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b1fe8bf..85194d1a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,8 @@ env: # py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min # (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s) # 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs. +# Windows and macOS moved to portability.yml (#420), which keeps this 27-minute limit; this +# Linux-only matrix keeps it too until it is re-measured. # real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min # coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min # perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s, @@ -35,8 +37,9 @@ env: # all tests: a result check without setup, 5 min (the floor) # # Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend -# modules. The merge queue (merge_group), pushes to main and manual runs take the full matrix and the -# real ComfyUI job. "all tests" is the required gate that holds the queue to that full set. +# modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on +# Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set. +# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge. jobs: test: name: py${{ matrix.python-version }} on ${{ matrix.os }} @@ -45,7 +48,7 @@ jobs: strategy: fail-fast: false matrix: - os: ${{ github.event_name == 'pull_request' && fromJSON('["ubuntu-latest"]') || fromJSON('["ubuntu-latest", "windows-latest", "macos-latest"]') }} + os: [ubuntu-latest] python-version: ${{ github.event_name == 'pull_request' && fromJSON('["3.12"]') || fromJSON('["3.12", "3.13", "3.14"]') }} steps: diff --git a/TESTING.md b/TESTING.md index dca69a15..d5852b4f 100644 --- a/TESTING.md +++ b/TESTING.md @@ -273,11 +273,12 @@ gh run view --json jobs # shows a job that alr gh run view --job --log-failed # that job's failed log, at once ``` -Windows, macOS, Python 3.13/3.14, real ComfyUI and Firefox/WebKit failures -first show up in the merge queue, on the merge-group commit rather than the +Python 3.13/3.14, real ComfyUI and Firefox/WebKit failures first show up in the +merge queue, on the merge-group commit rather than the pushed one. Find that run with `gh run list --event merge_group --workflow test.yml` (or `viewerjs-check.yml`) and match its head branch -(`gh-readonly-queue/main/pr--...`). +(`gh-readonly-queue/main/pr--...`). Windows and macOS failures show up only +after the merge, in `gh run list --workflow portability.yml`. Start fixing as soon as a failed log shows the cause, but push the fix only after the run for that commit has finished: pull-request runs use `cancel-in-progress`, @@ -527,12 +528,11 @@ Before changing paths, SQLite, streams, restore or cleanup: open with `FirstWriterHold`) and give peers that must all succeed the ordinary busy timeout; cross-thread handoffs wait on events with generous ceilings, never a short wall clock (#242). -- Run native Windows CI early for these changes: pull requests run only Linux, so - start the full matrix on the branch with `gh workflow run test.yml --ref `. - That run reports the same check names on the PR head, and a red one can hold - the PR out of the queue until it is fixed. - Include a Windows impact and evidence note in the PR. All nine OS/Python - combinations (they run again in the merge queue) and every applicable +- Run native Windows CI early for these changes. Pull requests and the merge + queue run only Linux, so start Windows and macOS on the branch with + `gh workflow run portability.yml --ref `. + Include a Windows impact and evidence note in the PR. All six Windows/macOS + combinations in that run, every Linux leg and every applicable exact-head check must pass before merge. Portable `ntpath`/`PureWindowsPath` controls do not establish native filesystem, locking or cleanup success. @@ -544,13 +544,14 @@ what runs: | Job | `pull_request` | `merge_group`, push to `main`, `workflow_dispatch` | | --- | --- | --- | | `py3.12 on ubuntu-latest` | runs | runs | -| other Python/OS legs (3.12-3.14 x Linux/Windows/macOS) | not created | run | +| `py3.13 on ubuntu-latest`, `py3.14 on ubuntu-latest` | not created | run | | `real ComfyUI compatibility` | skipped | runs | | `coverage gate (85%)`, `black and flake8`, `frontend modules`, `bandit (high severity)` | run | run | | `frontend coverage (shard N/3)`, `frontend coverage (merge)` (vendor pins and tooling tests on shard 3) | run | run | | `page workflows (firefox\|webkit, shard N/2)` | skipped | run | | `all tests`, `viewerjs` (gates) | run | run | | `perf budget` | skipped | push and dispatch only (not in the queue) | +| `py3.12-3.14 on windows-latest/macos-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required | GitHub has one required set for pull requests and the queue. A job skipped by its `if:` reports Success. A dropped matrix leg never reports, so it would stay @@ -560,8 +561,14 @@ guarantee: to be skipped. - On every other event they require every job to succeed. -So a merge group cannot turn green unless all nine Python/OS legs, real ComfyUI -and the four browser shards passed. `tests/vendor/ci-workflow.test.mjs` pins the +So a merge group cannot turn green unless all three Linux Python legs, real +ComfyUI and the four browser shards passed. + +Windows and macOS (Python 3.12-3.14) are out of the blocking path for now +(maintainer, 2026-10-28). They run in `portability.yml` on every push to `main`, +nightly and on demand, and no context there is required. A red portability run +is still a failure: fix it at its cause, not by rerunning, and have every one +green before the workspace cutover (#334). `tests/vendor/ci-workflow.test.mjs` pins the event split and runs both gate scripts for each event. ## Required merge checks diff --git a/tests/vendor/ci-workflow.test.mjs b/tests/vendor/ci-workflow.test.mjs index 1407a4e3..d8ed881a 100644 --- a/tests/vendor/ci-workflow.test.mjs +++ b/tests/vendor/ci-workflow.test.mjs @@ -248,7 +248,8 @@ test('browser job timeouts keep a real margin over measured runs', () => { }); // Every job is bounded (#350): a hang ends at a measured limit instead of GitHub's 6-hour default. -const BOUNDED_WORKFLOWS = ['test.yml', 'code-quality.yml', 'viewerjs-check.yml']; +const GATING_WORKFLOWS = ['test.yml', 'code-quality.yml', 'viewerjs-check.yml']; +const BOUNDED_WORKFLOWS = [...GATING_WORKFLOWS, 'portability.yml']; const readWorkflow = name => readFileSync(new URL(`../../.github/workflows/${name}`, import.meta.url), 'utf8'); test('every job in the CI workflows has a numeric timeout-minutes', () => { @@ -304,19 +305,30 @@ test('the bandit pre-commit hook fails at the severity CI enforces and passes ev // Event split (#420): pull requests run a fast subset, the merge queue and main run everything. const testJobs = () => jobs(readWorkflow('test.yml')); -test('the Python matrix is one leg on a pull request and the full matrix on every other event', () => { +test('the Python matrix is Linux 3.12 on a pull request and Linux 3.12-3.14 on every other event', () => { const body = testJobs().get('test'); - const lists = key => { - const match = new RegExp(`^ {8}${key}: \\$\\{\\{ github\\.event_name == 'pull_request' && fromJSON\\('(\\[[^\\]]*\\])'\\) \\|\\| fromJSON\\('(\\[[^\\]]*\\])'\\) \\}\\}$`, 'm').exec(body); - assert.ok(match, `${key} is chosen by event`); - return { pr: JSON.parse(match[1]), other: JSON.parse(match[2]) }; - }; - assert.deepEqual(lists('os'), { pr: ['ubuntu-latest'], other: ['ubuntu-latest', 'windows-latest', 'macos-latest'] }); - assert.deepEqual(lists('python-version'), { pr: ['3.12'], other: ['3.12', '3.13', '3.14'] }); + assert.deepEqual(matrixList(body, 'os'), ['ubuntu-latest'], 'Windows and macOS never gate a merge'); + const match = /^ {8}python-version: \$\{\{ github\.event_name == 'pull_request' && fromJSON\('(\[[^\]]*\])'\) \|\| fromJSON\('(\[[^\]]*\])'\) \}\}$/m.exec(body); + assert.ok(match, 'python-version is chosen by event'); + assert.deepEqual({ pr: JSON.parse(match[1]), other: JSON.parse(match[2]) }, { pr: ['3.12'], other: ['3.12', '3.13', '3.14'] }); assert.match(body, /^ {6}fail-fast: false$/m); assert.match(body, /^ {4}name: py\$\{\{ matrix\.python-version \}\} on \$\{\{ matrix\.os \}\}$/m); }); +test('Windows and macOS run 3.12-3.14 on main, nightly and on demand, never on a PR or in the queue', () => { + const text = readWorkflow('portability.yml'); + const triggers = text.slice(text.indexOf('\non:'), text.indexOf('\nconcurrency:')); + for (const event of ['push', 'schedule', 'workflow_dispatch']) assert.match(triggers, new RegExp(`^ {2}${event}:`, 'm')); + for (const event of ['pull_request', 'merge_group']) assert.doesNotMatch(triggers, new RegExp(`^ {2}${event}`, 'm')); + assert.match(triggers, /^ {4}branches: \[main\]$/m, 'push runs only for main'); + const body = jobs(text).get('test'); + assert.deepEqual(matrixList(body, 'os'), ['windows-latest', 'macos-latest']); + assert.deepEqual(matrixList(body, 'python-version'), ['"3.12"', '"3.13"', '"3.14"']); + assert.match(body, /^ {6}fail-fast: false$/m); + assert.match(body, /^ {8}run: python -m tests\.run_suite -v$/m); + assert.doesNotMatch(text, /cancel-in-progress: true|cancel-in-progress: \$/, 'no portability run is cancelled'); +}); + test('real ComfyUI runs only outside pull requests; the other test.yml checks run on every event', () => { const found = testJobs(); assert.match(found.get('comfyui'), QUEUE_ONLY_IF); @@ -338,7 +350,7 @@ test('all tests needs every gating test.yml job and requires the full set outsid }); test('the CI workflows run on pull_request, merge_group and push without path filters', () => { - for (const name of BOUNDED_WORKFLOWS) { + for (const name of GATING_WORKFLOWS) { const text = readWorkflow(name); const triggers = text.slice(text.indexOf('\non:'), text.indexOf('\n', text.indexOf('\non:') + 1) + 400); for (const event of ['merge_group', 'pull_request', 'push']) { @@ -357,4 +369,5 @@ test('every required check reports on a pull request and the gates carry the ful ].sort()); assert.equal(contexts.filter(context => /^py/.test(context)).length, 1, 'a dropped matrix leg never reports on a pull request, so it cannot be required'); + assert.ok(!contexts.some(context => /windows|macos/.test(context)), 'Windows and macOS never gate a merge'); });