From 34dabe2cd2a40a7e9d27e00bd26e367bcba25bb4 Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Sat, 31 Oct 2026 10:49:10 -0700 Subject: [PATCH] ci: stop testing macOS in CI macOS is not a popular ComfyUI platform. Drop the macOS legs from the portability workflow and its native-trash proof; Windows keeps its Python 3.12-3.14 legs. The macOS code stays, and README calls its support best-effort. Closes #571 --- .github/workflows/portability.yml | 38 ++++--------------------------- .github/workflows/test.yml | 4 ++-- README.md | 4 ++-- TESTING.md | 23 ++++++++++--------- tests/vendor/ci-workflow.test.mjs | 23 ++++--------------- 5 files changed, 26 insertions(+), 66 deletions(-) diff --git a/.github/workflows/portability.yml b/.github/workflows/portability.yml index 417f1f03..7d73e16d 100644 --- a/.github/workflows/portability.yml +++ b/.github/workflows/portability.yml @@ -1,9 +1,9 @@ -name: Windows and macOS +name: Windows permissions: contents: read -# Non-blocking portability runs (#420): never on pull requests or in the merge queue, and no context +# Non-blocking Windows runs (#420; macOS dropped from CI in #571): never on pull requests or in the merge queue, and no context # here is a required check. A red run is fixed at its cause before the workspace cutover (#334). on: push: @@ -25,10 +25,7 @@ env: # run counts (tests/vendor/ci-workflow.test.mjs recomputes it, #493). # Measured 2026-10-29 on main runs 40935337099 (dd247e3), 40938407252 (dd247e3), 40941240624 (dd247e3): # slowest leg: 7m24s (py3.12 windows) / 7m31s (py3.13 windows) / 7m32s (py3.14 windows) -> 15 min -# (windows 4m56s-7m32s, macos 4m39s-6m14s) -# native-trash (#499) has no green main runs yet: its probe, run 40949370943 (2026-10-29), took -# about 1 minute per macOS leg, so ceil(1.5 x 2) + 3 gives it 6 minutes (step 4). Recalibrate -# after three green runs. +# (windows 4m56s-7m32s) # native-trash-linux (#543) has no green main runs yet: it installs the dev requirements and runs # one test module, about 1 minute by the same estimate, so ceil(1.5 x 2) + 3 gives it 6 minutes. # Recalibrate after three green runs. @@ -40,7 +37,7 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, macos-latest] + os: [windows-latest] python-version: ["3.12", "3.13", "3.14"] steps: @@ -60,7 +57,7 @@ jobs: # The runner's TEMP is on C:, the OS disk, where a flush took about 6 ms # and a test database's create-to-delete about 111 ms; on the D: work - # disk the same took 0.5 ms and 23 ms (macOS: 19 ms). Tests write their + # disk the same took 0.5 ms and 23 ms. Tests write their # databases under TEMP, so it moves to a folder on D: (measured in #487, # run 40915866444). The runner's TEMP was an 8.3 short name # (C:\Users\RUNNER~1\...), so the new one is spelled by its short name too @@ -83,31 +80,6 @@ jobs: PM_TEST_SLOWEST: "40" run: python -m tests.run_suite -v - # The real Trash on both macOS CPUs (#499 design ยง10.2). It runs apart from the suite, so a red - # suite never skips the native proof. - native-trash: - name: native trash on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - timeout-minutes: 6 - strategy: - fail-fast: false - matrix: - os: [macos-latest, macos-15-intel] - - steps: - - uses: actions/checkout@v7 - - - name: Set up Python - uses: actions/setup-python@v7 - with: - python-version: "3.12" - - - name: Native trash proof - timeout-minutes: 4 - env: - PM_NATIVE_TRASH: macos - run: python -m unittest tests.test_trash_macos_native tests.test_trash_macos -v - # The same tests as the fake mount table, on a real ext4 mount used as the output root (#543). A # runner can mount with sudo; the tests run as the normal user, so owner and mode checks are real. native-trash-linux: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 50a7cdd2..33a074b6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ env: # py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min # (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s) # 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs. -# Windows and macOS moved to portability.yml (#420), which has its own measured limit (#493); +# Windows moved to portability.yml (#420; macOS left CI in #571), which has its own measured limit (#493); # this Linux-only matrix keeps 27 minutes until it is re-measured. # real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min # coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min @@ -39,7 +39,7 @@ env: # Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend # modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on # Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set. -# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge. +# Windows runs in portability.yml (push to main, nightly, on demand) and never gate a merge. jobs: test: name: py${{ matrix.python-version }} on ${{ matrix.os }} diff --git a/README.md b/README.md index fbbf9f94..e71786e3 100644 --- a/README.md +++ b/README.md @@ -236,12 +236,12 @@ The documentation for those pages is in [README-legacy.md](README-legacy.md). | Layer | What runs | | --- | --- | -| Python | Over 200 test files on Linux, Windows and macOS with Python 3.12 to 3.14, with an 85% coverage gate | +| Python | Over 200 test files on Linux and Windows with Python 3.12 to 3.14, with an 85% coverage gate | | Frontend | About 100 unit test files, with coverage floors per file | | Browser | Over 60 Playwright spec files in Chromium, Firefox, WebKit and touch emulation | | Real ComfyUI | A pinned CPU ComfyUI, started in Classic and Nodes 2.0, running real workflows | -Linux runs on every pull request and in the merge queue. Windows and macOS run on every push to main and nightly. To run the tests yourself, see [TESTING.md](TESTING.md). +Linux runs on every pull request and in the merge queue. Windows runs on every push to main and nightly. macOS is not tested; it should work, but support is best-effort. To run the tests yourself, see [TESTING.md](TESTING.md). ## License diff --git a/TESTING.md b/TESTING.md index c345ef7a..53530495 100644 --- a/TESTING.md +++ b/TESTING.md @@ -261,7 +261,7 @@ the process exits non-zero. job fails anyway and the dump is what matters. - `PM_TEST_SLOWEST=N` (off by default) lists the N slowest tests after the summary, each timed from its `start:` line to the next one like the hang - limit. The Windows and macOS jobs (`portability.yml`) set it to 40, so every + limit. The Windows jobs (`portability.yml`) set it to 40, so every run shows where its time went (#487). The list prints only when the run ends: a job GitHub cancels or a watchdog exit prints none, so read the timestamps between `start:` lines instead. @@ -327,7 +327,7 @@ Python 3.13/3.14, real ComfyUI and Firefox/WebKit failures first show up in the merge queue, on the merge-group commit rather than the pushed one. Find that run with `gh run list --event merge_group --workflow test.yml` (or `viewerjs-check.yml`) and match its head branch -(`gh-readonly-queue/main/pr--...`). Windows and macOS failures show up only +(`gh-readonly-queue/main/pr--...`). Windows failures show up only after the merge, in `gh run list --workflow portability.yml`. Start fixing as soon as a failed log shows the cause, but push the fix only after @@ -450,7 +450,7 @@ measurement: `ubuntu-latest`, one Python, no coverage, only the wall-clock tests It runs on every push to `main` and on `workflow_dispatch`, never on pull requests, because shared-runner timing must not block a merge. Dispatch it on a branch to measure before merging. -Shared Windows and macOS runners do not run them; the structural tests cover +Shared Windows runners do not run them; the structural tests cover portability. Run it locally with `PROMPTMANAGER_PERF_BUDGET=1 python -m unittest tests.test_api_workspace_images.TestWorkspaceImages.test_filter_budget` and quote the printed p95 in the PR body. @@ -646,10 +646,10 @@ Before changing paths, SQLite, streams, restore or cleanup: ordinary busy timeout; cross-thread handoffs wait on events with generous ceilings, never a short wall clock (#242). - Run native Windows CI early for these changes. Pull requests and the merge - queue run only Linux, so start Windows and macOS on the branch with + queue run only Linux, so start Windows on the branch with `gh workflow run portability.yml --ref `. - Include a Windows impact and evidence note in the PR. All six Windows/macOS - combinations in that run, every Linux leg and every applicable + Include a Windows impact and evidence note in the PR. All three Windows + legs in that run, every Linux leg and every applicable exact-head check must pass before merge. Portable `ntpath`/`PureWindowsPath` controls do not establish native filesystem, locking or cleanup success. @@ -668,7 +668,7 @@ what runs: | `page workflows (firefox\|webkit, shard N/2)` | skipped | run | | `all tests`, `viewerjs` (gates) | run | run | | `perf budget` | skipped | push and dispatch only (not in the queue) | -| `py3.12-3.14 on windows-latest/macos-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required | +| `py3.12-3.14 on windows-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required | GitHub has one required set for pull requests and the queue. A job skipped by its `if:` reports Success. A dropped matrix leg never reports, so it would stay @@ -681,8 +681,9 @@ guarantee: So a merge group cannot turn green unless all three Linux Python legs, real ComfyUI and the four browser shards passed. -Windows and macOS (Python 3.12-3.14) are out of the blocking path for now -(maintainer, 2026-10-28). They run in `portability.yml` on every push to `main`, +Windows (Python 3.12-3.14) is out of the blocking path for now +(maintainer, 2026-10-28). macOS is not tested in CI (maintainer, 2026-10-31, #571); +its code stays and support is best-effort. They run in `portability.yml` on every push to `main`, nightly and on demand, and no context there is required. A red portability run is still a failure: fix it at its cause, not by rerunning, and have every one green before the workspace cutover (#334). `tests/vendor/ci-workflow.test.mjs` pins the @@ -2549,8 +2550,8 @@ files to the system trash. **No code path unlinks or overwrites a media file.** ("Moving files to the Recycle Bin isn't available on Windows: files larger than the bin's capacity would be deleted permanently."). Library-only removal works everywhere. Tests use `tests/fake_trash.py`; the browser test server installs it, so no test touches a real trash. The native proof, - `tests/test_trash_macos_native.py`, runs only with `PM_NATIVE_TRASH=macos` in - portability.yml's `native-trash` job (`macos-latest` and `macos-15-intel`): the startup + `tests/test_trash_macos_native.py`, runs only with `PM_NATIVE_TRASH=macos` on a Mac, by hand + (CI dropped its `native-trash` macOS job in #571; it last passed on `macos-latest` and `macos-15-intel`): the startup volume, a name clash, fresh APFS and FAT32 disk images, an image used as an output root, a read-only image and an unwritable folder (design and probe: `docs/egm/499-macos-trash/`, local). diff --git a/tests/vendor/ci-workflow.test.mjs b/tests/vendor/ci-workflow.test.mjs index ad81231f..fc835910 100644 --- a/tests/vendor/ci-workflow.test.mjs +++ b/tests/vendor/ci-workflow.test.mjs @@ -331,7 +331,7 @@ const testJobs = () => jobs(readWorkflow('test.yml')); test('the Python matrix is Linux 3.12 on a pull request and Linux 3.12-3.14 on every other event', () => { const body = testJobs().get('test'); - assert.deepEqual(matrixList(body, 'os'), ['ubuntu-latest'], 'Windows and macOS never gate a merge'); + assert.deepEqual(matrixList(body, 'os'), ['ubuntu-latest'], 'Windows never gates a merge'); const match = /^ {8}python-version: \$\{\{ github\.event_name == 'pull_request' && fromJSON\('(\[[^\]]*\])'\) \|\| fromJSON\('(\[[^\]]*\])'\) \}\}$/m.exec(body); assert.ok(match, 'python-version is chosen by event'); assert.deepEqual({ pr: JSON.parse(match[1]), other: JSON.parse(match[2]) }, { pr: ['3.12'], other: ['3.12', '3.13', '3.14'] }); @@ -339,34 +339,21 @@ test('the Python matrix is Linux 3.12 on a pull request and Linux 3.12-3.14 on e assert.match(body, /^ {4}name: py\$\{\{ matrix\.python-version \}\} on \$\{\{ matrix\.os \}\}$/m); }); -test('Windows and macOS run 3.12-3.14 on main, nightly and on demand, never on a PR or in the queue', () => { +test('Windows runs 3.12-3.14 on main, nightly and on demand, never on a PR or in the queue', () => { const text = readWorkflow('portability.yml'); const triggers = text.slice(text.indexOf('\non:'), text.indexOf('\nconcurrency:')); for (const event of ['push', 'schedule', 'workflow_dispatch']) assert.match(triggers, new RegExp(`^ {2}${event}:`, 'm')); for (const event of ['pull_request', 'merge_group']) assert.doesNotMatch(triggers, new RegExp(`^ {2}${event}`, 'm')); assert.match(triggers, /^ {4}branches: \[main\]$/m, 'push runs only for main'); const body = jobs(text).get('test'); - assert.deepEqual(matrixList(body, 'os'), ['windows-latest', 'macos-latest']); + assert.deepEqual(matrixList(body, 'os'), ['windows-latest'], 'macOS is not tested in CI (#571)'); + assert.doesNotMatch(text, /macos-/, 'no macOS runner is used'); assert.deepEqual(matrixList(body, 'python-version'), ['"3.12"', '"3.13"', '"3.14"']); assert.match(body, /^ {6}fail-fast: false$/m); assert.match(body, /^ {8}run: python -m tests\.run_suite -v$/m); assert.doesNotMatch(text, /cancel-in-progress: true|cancel-in-progress: \$/, 'no portability run is cancelled'); }); -// #499: the native trash proof runs on both macOS CPUs, apart from the suite so a red suite never skips it. -test('portability runs the native trash proof on arm64 and Intel macOS', () => { - const body = jobs(readWorkflow('portability.yml')).get('native-trash'); - assert.ok(body, 'portability.yml has a native-trash job'); - assert.deepEqual(matrixList(body, 'os'), ['macos-latest', 'macos-15-intel']); - assert.match(body, /^ {6}fail-fast: false$/m); - assert.doesNotMatch(body, /^ {4}needs:/m, 'it does not wait for the suite job'); - assert.match(body, /^ {10}PM_NATIVE_TRASH: macos$/m); - assert.match(body, /^ {8}run: python -m unittest tests\.test_trash_macos_native tests\.test_trash_macos -v$/m); - const jobMinutes = Number(/^ {4}timeout-minutes: (\d+)$/m.exec(body)[1]); - const stepMinutes = Number(/^ {8}timeout-minutes: (\d+)$/m.exec(body)?.[1]); - assert.ok(stepMinutes > 0 && jobMinutes - stepMinutes >= 2, 'the proof step stops before the job limit'); -}); - // #493: the portability limit follows the rule in its comment: ceil(1.5 x the slowest of three // green main runs) + 3 minutes, never under 5, with those runs and times written above jobs:. // The times are whole-job wall times. That the runs were green and on main is the author's word. @@ -427,5 +414,5 @@ test('every required check reports on a pull request and the gates carry the ful ].sort()); assert.equal(contexts.filter(context => /^py/.test(context)).length, 1, 'a dropped matrix leg never reports on a pull request, so it cannot be required'); - assert.ok(!contexts.some(context => /windows|macos/.test(context)), 'Windows and macOS never gate a merge'); + assert.ok(!contexts.some(context => /windows|macos/.test(context)), 'Windows never gates a merge'); });