aszc-dev-ComfyUI-CoreMLSuite/.github/workflows/tier2.yml

name: Tier 2 — M2 / ANE (self-hosted)

on:
  pull_request:
    types: [labeled]
  schedule:
    # Nightly at 04:00 UTC (~05/06 in PL). Keeps the M2 path honest
    # without burning the runner on every PR.
    - cron: "0 4 * * *"
  workflow_dispatch:

jobs:
  m2:
    if: |
      github.event_name == 'schedule' ||
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'pull_request' &&
       contains(github.event.pull_request.labels.*.name, 'run-m2'))
    # Self-hosted Mac registered by the maintainer. See docs/ci-m2.md
    # for runner setup and the COMFY_DIR / checkpoint prerequisites.
    runs-on: [self-hosted, macOS, ARM64, coreml]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v4

      # Hybrid ComfyUI strategy (see docs/ci-m2.md "ComfyUI version under test"):
      #   - schedule (nightly)  -> latest origin/master + ComfyUI's own
      #     requirements.txt (constrained). Canary for upstream API breakage.
      #   - PR label / dispatch -> the pinned requires-comfyui SHA + the frozen
      #     `comfy` uv group. Reproducible merge gate, immune to overnight drift.
      - name: Resolve ComfyUI ref + mode
        run: |
          if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then
            echo "COMFY_MODE=latest" >> "$GITHUB_ENV"
            echo "COMFY_REF=master"  >> "$GITHUB_ENV"
          else
            PIN="$(sed -nE 's/^requires-comfyui *= *"==?([0-9a-f]+)".*/\1/p' pyproject.toml)"
            if [ -z "$PIN" ]; then echo "could not parse requires-comfyui from pyproject.toml"; exit 1; fi
            echo "COMFY_MODE=pinned" >> "$GITHUB_ENV"
            echo "COMFY_REF=$PIN"    >> "$GITHUB_ENV"
          fi

      - name: Set up ComfyUI checkout
        # COMFY_DIR is exported by the self-hosted runner's .env and MUST be a
        # runner-owned ComfyUI clone (never your dev checkout — this step does
        # git reset --hard and rewrites custom_nodes). Cloned on first run.
        run: |
          set -euo pipefail
          if [ -z "${COMFY_DIR:-}" ]; then echo "COMFY_DIR unset (see docs/ci-m2.md)"; exit 1; fi
          if [ ! -d "$COMFY_DIR/.git" ]; then
            echo "cloning ComfyUI into $COMFY_DIR"
            git clone https://github.com/comfyanonymous/ComfyUI.git "$COMFY_DIR"
          fi
          git -C "$COMFY_DIR" fetch --quiet origin
          if [ "$COMFY_MODE" = "latest" ]; then
            git -C "$COMFY_DIR" checkout --quiet -B master origin/master
            git -C "$COMFY_DIR" reset --hard --quiet origin/master
          else
            git -C "$COMFY_DIR" checkout --quiet --force "$COMFY_REF"
          fi
          COMFY_SHA="$(git -C "$COMFY_DIR" rev-parse HEAD)"
          echo "COMFY_SHA=$COMFY_SHA" >> "$GITHUB_ENV"
          echo "Tier 2 mode=$COMFY_MODE, ComfyUI \`$COMFY_SHA\`" >> "$GITHUB_STEP_SUMMARY"

          # Point ComfyUI's custom-node loader at this checkout. Refresh the
          # symlink only; refuse to clobber a real directory (guards against a
          # COMFY_DIR that is accidentally a dev checkout).
          NODE_LINK="$COMFY_DIR/custom_nodes/ComfyUI-CoreMLSuite"
          if [ -e "$NODE_LINK" ] && [ ! -L "$NODE_LINK" ]; then
            echo "ERROR: $NODE_LINK is a real directory, not a symlink."
            echo "COMFY_DIR must be a runner-owned ComfyUI, not your dev checkout."
            exit 1
          fi
          mkdir -p "$COMFY_DIR/custom_nodes"
          ln -sfn "$GITHUB_WORKSPACE" "$NODE_LINK"

      - name: Install dependencies
        run: |
          set -euo pipefail
          if [ "$COMFY_MODE" = "latest" ]; then
            # Node deps (our coremltools-9 toolchain), then ComfyUI's own
            # requirements for the pulled SHA, capped by the toolchain ceiling.
            uv sync
            uv pip install -r "$COMFY_DIR/requirements.txt" \
              -c constraints/comfy-ceiling.txt
          else
            # Pinned gate: the frozen group mirrors the known-good pinned SHA.
            uv sync --group comfy
          fi

      - name: Convert UNet variants if missing
        # Idempotent: converter.convert / compile_model skip when the .mlmodelc
        # already exists, so this is a no-op once the runner cache is warm.
        run: |
          set -euo pipefail
          for V in none 8 6 4; do
            if [ "$V" = "none" ]; then SUF=""; else SUF="_q$V"; fi
            OUT="$COMFY_DIR/models/unet/v1-5-pruned-emaonly_1x512x512_se${SUF}_unet.mlmodelc"
            if [ -d "$OUT" ]; then echo "skip nbits=$V (exists)"; continue; fi
            echo "converting nbits=$V"
            # --no-sync: keep the deps we just installed (latest mode's
            # ComfyUI requirements / pinned mode's --group comfy); a bare
            # `uv run` would re-sync to the lock and drop them.
            QUANT_NBITS="$V" uv run --no-sync python bench/scripts/convert_sd15.py
          done

      - name: Start ComfyUI server (background)
        run: |
          cd "$COMFY_DIR"
          nohup "$GITHUB_WORKSPACE/.venv/bin/python" main.py --port 8188 --cpu-vae > /tmp/comfyui-ci.log 2>&1 &
          # Poll the HTTP endpoint for readiness — robust to startup-banner
          # wording / colored-log changes in a floating-latest ComfyUI.
          for _ in $(seq 1 90); do
            if curl -sf -o /dev/null http://127.0.0.1:8188/system_stats; then
              echo "comfy ready (ComfyUI ${COMFY_SHA:-unknown})"; exit 0
            fi
            sleep 2
          done
          echo "comfy failed to start"; tail -100 /tmp/comfyui-ci.log; exit 1

      - name: Run Tier 2 (m2 marker)
        run: uv run --no-sync pytest -m m2 tests/ -v

      - name: Run bench harness (baseline UNet)
        run: |
          uv run --no-sync python bench/run.py \
            --model "$COMFY_DIR/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc" \
            --compute-units CPU_AND_NE CPU_AND_GPU \
            --repeats 30 --assumed-steps 20

      - name: Run quantization tradeoff matrix (Phase 6)
        # Benches none/8/6/4 on ANE and records size / ms-step / PSNR-vs-none
        # into bench/results/quant_matrix_<sha>.{json,md}.
        run: uv run --no-sync python bench/scripts/quant_matrix.py

      - name: Upload bench results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: bench-results
          path: |
            bench/results/*.json
            bench/results/*.md

      - name: Stop ComfyUI server
        if: always()
        run: pkill -f "main.py.*8188" || true