diff --git a/.github/workflows/tier2.yml b/.github/workflows/tier2.yml index a8a362e..9b60c01 100644 --- a/.github/workflows/tier2.yml +++ b/.github/workflows/tier2.yml @@ -17,34 +17,92 @@ jobs: (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-m2')) # Self-hosted Mac registered by the maintainer. See docs/ci-m2.md - # for runner setup and required model paths. + # for runner setup and the COMFY_DIR / checkpoint prerequisites. runs-on: [self-hosted, macOS, ARM64, coreml] - timeout-minutes: 60 + timeout-minutes: 90 steps: - uses: actions/checkout@v4 - - name: uv sync - run: uv sync - - - name: Update ComfyUI to latest master (canary) - # Tier 2 deliberately tracks the moving host: every run resets the - # runner's ComfyUI checkout to origin/master and records the resolved - # SHA, so upstream API breakage surfaces here instead of in a user's - # install. (pyproject `requires-comfyui` stays the published-compat - # declaration; this lane is the early-warning canary, not the contract.) - # COMFY_DIR is exported by the self-hosted runner's .env (see docs/ci-m2.md). - # The node under test lives at $COMFY_DIR/custom_nodes/ComfyUI-CoreMLSuite, - # symlinked to $GITHUB_WORKSPACE at runner-setup time, so this reset does - # not touch the checkout being validated. + # Hybrid ComfyUI strategy (see docs/ci-m2.md "ComfyUI version under test"): + # - schedule (nightly) -> latest origin/master + ComfyUI's own + # requirements.txt (constrained). Canary for upstream API breakage. + # - PR label / dispatch -> the pinned requires-comfyui SHA + the frozen + # `comfy` uv group. Reproducible merge gate, immune to overnight drift. + - name: Resolve ComfyUI ref + mode run: | + if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then + echo "COMFY_MODE=latest" >> "$GITHUB_ENV" + echo "COMFY_REF=master" >> "$GITHUB_ENV" + else + PIN="$(sed -nE 's/^requires-comfyui *= *"==?([0-9a-f]+)".*/\1/p' pyproject.toml)" + if [ -z "$PIN" ]; then echo "could not parse requires-comfyui from pyproject.toml"; exit 1; fi + echo "COMFY_MODE=pinned" >> "$GITHUB_ENV" + echo "COMFY_REF=$PIN" >> "$GITHUB_ENV" + fi + + - name: Set up ComfyUI checkout + # COMFY_DIR is exported by the self-hosted runner's .env and MUST be a + # runner-owned ComfyUI clone (never your dev checkout — this step does + # git reset --hard and rewrites custom_nodes). Cloned on first run. + run: | + set -euo pipefail + if [ -z "${COMFY_DIR:-}" ]; then echo "COMFY_DIR unset (see docs/ci-m2.md)"; exit 1; fi + if [ ! -d "$COMFY_DIR/.git" ]; then + echo "cloning ComfyUI into $COMFY_DIR" + git clone https://github.com/comfyanonymous/ComfyUI.git "$COMFY_DIR" + fi git -C "$COMFY_DIR" fetch --quiet origin - git -C "$COMFY_DIR" checkout --quiet master \ - || git -C "$COMFY_DIR" checkout --quiet -b master origin/master - git -C "$COMFY_DIR" reset --hard --quiet origin/master + if [ "$COMFY_MODE" = "latest" ]; then + git -C "$COMFY_DIR" checkout --quiet -B master origin/master + git -C "$COMFY_DIR" reset --hard --quiet origin/master + else + git -C "$COMFY_DIR" checkout --quiet --force "$COMFY_REF" + fi COMFY_SHA="$(git -C "$COMFY_DIR" rev-parse HEAD)" echo "COMFY_SHA=$COMFY_SHA" >> "$GITHUB_ENV" - echo "Tier 2 tested against ComfyUI \`$COMFY_SHA\` (latest master)" \ - >> "$GITHUB_STEP_SUMMARY" + echo "Tier 2 mode=$COMFY_MODE, ComfyUI \`$COMFY_SHA\`" >> "$GITHUB_STEP_SUMMARY" + + # Point ComfyUI's custom-node loader at this checkout. Refresh the + # symlink only; refuse to clobber a real directory (guards against a + # COMFY_DIR that is accidentally a dev checkout). + NODE_LINK="$COMFY_DIR/custom_nodes/ComfyUI-CoreMLSuite" + if [ -e "$NODE_LINK" ] && [ ! -L "$NODE_LINK" ]; then + echo "ERROR: $NODE_LINK is a real directory, not a symlink." + echo "COMFY_DIR must be a runner-owned ComfyUI, not your dev checkout." + exit 1 + fi + mkdir -p "$COMFY_DIR/custom_nodes" + ln -sfn "$GITHUB_WORKSPACE" "$NODE_LINK" + + - name: Install dependencies + run: | + set -euo pipefail + if [ "$COMFY_MODE" = "latest" ]; then + # Node deps (our coremltools-9 toolchain), then ComfyUI's own + # requirements for the pulled SHA, capped by the toolchain ceiling. + uv sync + uv pip install -r "$COMFY_DIR/requirements.txt" \ + -c constraints/comfy-ceiling.txt + else + # Pinned gate: the frozen group mirrors the known-good pinned SHA. + uv sync --group comfy + fi + + - name: Convert UNet variants if missing + # Idempotent: converter.convert / compile_model skip when the .mlmodelc + # already exists, so this is a no-op once the runner cache is warm. + run: | + set -euo pipefail + for V in none 8 6 4; do + if [ "$V" = "none" ]; then SUF=""; else SUF="_q$V"; fi + OUT="$COMFY_DIR/models/unet/v1-5-pruned-emaonly_1x512x512_se${SUF}_unet.mlmodelc" + if [ -d "$OUT" ]; then echo "skip nbits=$V (exists)"; continue; fi + echo "converting nbits=$V" + # --no-sync: keep the deps we just installed (latest mode's + # ComfyUI requirements / pinned mode's --group comfy); a bare + # `uv run` would re-sync to the lock and drop them. + QUANT_NBITS="$V" uv run --no-sync python bench/scripts/convert_sd15.py + done - name: Start ComfyUI server (background) run: | @@ -52,7 +110,7 @@ jobs: nohup "$GITHUB_WORKSPACE/.venv/bin/python" main.py --port 8188 --cpu-vae > /tmp/comfyui-ci.log 2>&1 & # Poll the HTTP endpoint for readiness — robust to startup-banner # wording / colored-log changes in a floating-latest ComfyUI. - for _ in $(seq 1 60); do + for _ in $(seq 1 90); do if curl -sf -o /dev/null http://127.0.0.1:8188/system_stats; then echo "comfy ready (ComfyUI ${COMFY_SHA:-unknown})"; exit 0 fi @@ -61,21 +119,28 @@ jobs: echo "comfy failed to start"; tail -100 /tmp/comfyui-ci.log; exit 1 - name: Run Tier 2 (m2 marker) - run: uv run pytest -m m2 tests/ -v + run: uv run --no-sync pytest -m m2 tests/ -v - - name: Run bench harness + - name: Run bench harness (baseline UNet) run: | - uv run python bench/run.py \ + uv run --no-sync python bench/run.py \ --model "$COMFY_DIR/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc" \ --compute-units CPU_AND_NE CPU_AND_GPU \ --repeats 30 --assumed-steps 20 + - name: Run quantization tradeoff matrix (Phase 6) + # Benches none/8/6/4 on ANE and records size / ms-step / PSNR-vs-none + # into bench/results/quant_matrix_.{json,md}. + run: uv run --no-sync python bench/scripts/quant_matrix.py + - name: Upload bench results uses: actions/upload-artifact@v4 if: always() with: name: bench-results - path: bench/results/*.json + path: | + bench/results/*.json + bench/results/*.md - name: Stop ComfyUI server if: always() diff --git a/constraints/comfy-ceiling.txt b/constraints/comfy-ceiling.txt new file mode 100644 index 0000000..c582dec --- /dev/null +++ b/constraints/comfy-ceiling.txt @@ -0,0 +1,18 @@ +# Toolchain ceiling for installing a floating-latest ComfyUI's requirements.txt +# in the Tier 2 nightly canary (.github/workflows/tier2.yml, latest mode). +# +# ComfyUI's requirements.txt requests bare `torch`/`torchvision`/`torchaudio` +# and `numpy>=1.25.0`, which would float past the versions coremltools 9 / +# apple-ml-stable-diffusion have been validated against (see docs/deps.md). +# These constraints cap the resolution so the canary keeps testing the same +# toolchain the suite actually ships. +# +# If upstream ComfyUI ever hard-requires something beyond these bounds, the +# install FAILS — and that failure is the signal we want: it means the host +# outgrew the pinned toolchain and coremltools / ml-stable-diffusion need a +# deliberate bump (a Phase 5-style upgrade), not a silent float. +torch>=2.7,<2.8 +torchvision>=0.22,<0.23 +torchaudio>=2.7,<2.8 +numpy>=1.25,<2 +coremltools>=9,<10 diff --git a/docs/ci-m2.md b/docs/ci-m2.md index 8dcd86b..a5aaa81 100644 --- a/docs/ci-m2.md +++ b/docs/ci-m2.md @@ -7,13 +7,13 @@ half of the matrix has to live on real hardware. ## One-time runner setup -1. **Install dependencies on the Mac.** Python 3.11.x (matching the - `requires-python` pin), `uv`, `git`, plus the ComfyUI checkout at the - path the workflow expects (default: `$HOME/dev/ComfyUI`). The - workflow reads `COMFY_DIR` from the runner's env. +1. **Install dependencies on the Mac.** Python 3.12.x (matching the + `requires-python = ">=3.12,<3.13"` pin), `uv`, and `git`. The workflow + clones and manages the ComfyUI checkout itself (see step 3), so you do + not pre-install ComfyUI. ```bash - brew install python@3.11 uv git + brew install python@3.12 uv git ``` 2. **Register the runner.** From the repo Settings → Actions → Runners @@ -32,41 +32,31 @@ half of the matrix has to live on real hardware. ./svc.sh install && ./svc.sh start # run as a launchd service ``` -3. **Persist `COMFY_DIR` for the runner.** The workflow needs to know - where the ComfyUI checkout lives. Add it to the runner's `.env`: +3. **Persist `COMFY_DIR` for the runner.** Point it at a **dedicated, + runner-owned** ComfyUI directory — **not** your personal dev checkout. + The workflow does `git reset --hard` on it and rewrites its + `custom_nodes/ComfyUI-CoreMLSuite` symlink, so it must be disposable. + Add it to the runner's `.env`: ```bash - echo 'COMFY_DIR=/Users//dev/ComfyUI' >> ~/actions-runner/.env + echo 'COMFY_DIR=/Users//actions-runner/comfyui' >> ~/actions-runner/.env ``` -4. **Symlink the node under test into ComfyUI.** `actions/checkout` clones - the PR into `$GITHUB_WORKSPACE` (`~/actions-runner/_work//`), - but ComfyUI only loads custom nodes from `$COMFY_DIR/custom_nodes/`. - Without a link, Tier 2 would spin up the server against a *stale* copy of - the node instead of the checked-out PR. Point the load path at the - runner's workspace once (the workspace path is stable for a self-hosted - runner): + You don't have to clone ComfyUI yourself: the `Set up ComfyUI checkout` + step clones it on first run, checks out the right ref (latest or the + pinned SHA — see *ComfyUI version under test*), installs ComfyUI's deps, + and symlinks `$COMFY_DIR/custom_nodes/ComfyUI-CoreMLSuite` → + `$GITHUB_WORKSPACE` so the server always loads the checked-out PR. If + `COMFY_DIR` is a real node directory rather than a symlink, the step + fails loudly instead of deleting it. - ```bash - rm -rf "$COMFY_DIR/custom_nodes/ComfyUI-CoreMLSuite" - ln -s ~/actions-runner/_work/ComfyUI-CoreMLSuite/ComfyUI-CoreMLSuite \ - "$COMFY_DIR/custom_nodes/ComfyUI-CoreMLSuite" - ``` - - After this, `uv sync`, `pytest`, and the ComfyUI server all run against - the same tree. - -5. **Pre-convert the baseline SD1.5 model.** The bench step expects - `$COMFY_DIR/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc`. - Run the conversion once manually: - - ```bash - cd $GITHUB_WORKSPACE - uv run python bench/scripts/convert_sd15.py - ``` - - Re-runs of the same combination are a no-op; the converter skips when - the .mlmodelc already exists. +4. **Provide the SD1.5 checkpoint.** Drop + `v1-5-pruned-emaonly.safetensors` (~4 GB) into + `$COMFY_DIR/models/checkpoints/`. This is the one heavy artifact that + stays as a runner-local cache; everything derived from it (the + `.mlmodelc` UNet variants) is converted automatically by the + `Convert UNet variants if missing` step and cached across runs. Override + the filename with `CKPT_NAME` in the runner `.env` if needed. ## Triggers @@ -79,22 +69,37 @@ The Tier 2 workflow (`.github/workflows/tier2.yml`) runs: ## ComfyUI version under test -Every Tier 2 run resets `$COMFY_DIR` to **latest `origin/master`** before -starting the server (`Update ComfyUI to latest master` step). This is a -deliberate early-warning canary: the suite tracks a moving host, so upstream -API breakage should surface here — in CI — rather than in a user's install. -The resolved ComfyUI SHA is written to the job's step summary (and `COMFY_SHA` -in the env) so any failure says exactly which commit it was tested against. +Tier 2 runs a **hybrid** strategy, keyed on the trigger, so the suite tracks a +moving host without making PRs flaky to overnight upstream drift: -This is separate from `pyproject.toml`'s `requires-comfyui` pin, which is the -**published-compatibility declaration** for the Comfy registry, not the CI -target. Bump that pin deliberately once a newer ComfyUI is validated; do not -expect it to match the floating SHA Tier 2 reports. +| Trigger | ComfyUI ref | ComfyUI deps | +|---|---|---| +| **schedule** (nightly) | latest `origin/master` | its own `requirements.txt`, capped by `constraints/comfy-ceiling.txt` | +| **PR label `run-m2`** / **dispatch** | pinned `requires-comfyui` SHA | the frozen `comfy` uv group | -Because the step does `git reset --hard`, the runner's ComfyUI checkout must -not hold local commits you care about — treat it as disposable. The symlinked -`custom_nodes/ComfyUI-CoreMLSuite` lives outside that repo's tracked tree, so -the reset never touches the node under test. +- **Nightly = canary.** It pulls the latest ComfyUI and installs *ComfyUI's + own* dependency set. The frozen `comfy` uv group cannot track a moving host + by hand (a latest checkout needs e.g. `comfyui-frontend-package==1.44.19`, + `comfy_aimdo`, `alembic`, `blake3` that an older pin never listed), so latest + mode defers to upstream's `requirements.txt`. Upstream API or dependency + breakage surfaces here, in CI, instead of in a user's install. +- **PR / dispatch = reproducible gate.** It checks out the pinned + `requires-comfyui` SHA and uses the frozen `comfy` group — the known-good + combination — so a PR fails for its own reasons, not because ComfyUI moved. + +The resolved ComfyUI SHA and mode are written to the job step summary (and +`COMFY_SHA` in the env), so any failure names the exact commit it hit. + +**The toolchain ceiling is deliberate.** `constraints/comfy-ceiling.txt` caps +`torch<2.8` / `numpy<2` / `coremltools 9` while installing latest ComfyUI's +requirements. If upstream ever hard-requires something past those bounds the +nightly install *fails on purpose* — that is the signal that coremltools / +`ml-stable-diffusion` need a deliberate Phase-5-style bump, not a silent float +that would break the ANE path (see `docs/deps.md`). + +`pyproject.toml`'s `requires-comfyui` is both the PR-gate ref and the +published-compatibility declaration for the Comfy registry. Bump it once a +newer ComfyUI is validated (the nightly canary is what tells you it's safe). ## Artifacts