diff --git a/.github/workflows/tier0.yml b/.github/workflows/tier0.yml index 75dead2..f380824 100644 --- a/.github/workflows/tier0.yml +++ b/.github/workflows/tier0.yml @@ -23,8 +23,8 @@ jobs: - name: Install Tier 0 deps run: | python -m pip install --upgrade pip - # Pins mirror pyproject (Phase 1 baseline). Tier 0 only needs - # torch + numpy + pytest; everything else is Mac-only. + # Tier 0 only needs torch + numpy + pytest; everything else + # is Mac-only. python -m pip install \ "torch==2.0.1" "numpy<1.25" \ "pytest>=8" "pytest-xdist" diff --git a/.github/workflows/tier2.yml b/.github/workflows/tier2.yml index 0950dc7..98943f4 100644 --- a/.github/workflows/tier2.yml +++ b/.github/workflows/tier2.yml @@ -16,14 +16,14 @@ jobs: github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-m2')) - # Self-hosted Mac registered by the maintainer. See docs/ci-m2.md - # for runner setup and the COMFY_DIR / checkpoint prerequisites. + # Self-hosted Apple Silicon runner. Prerequisites: COMFY_DIR pointing at + # a runner-owned ComfyUI clone, plus a cached SD1.5 checkpoint. runs-on: [self-hosted, macOS, ARM64, coreml] timeout-minutes: 90 steps: - uses: actions/checkout@v4 - # Hybrid ComfyUI strategy (see docs/ci-m2.md "ComfyUI version under test"): + # Hybrid ComfyUI strategy: # - schedule (nightly) -> latest origin/master + ComfyUI's own # requirements.txt (constrained). Canary for upstream API breakage. # - PR label / dispatch -> the pinned requires-comfyui SHA + the frozen @@ -46,7 +46,7 @@ jobs: # git reset --hard and rewrites custom_nodes). Cloned on first run. run: | set -euo pipefail - if [ -z "${COMFY_DIR:-}" ]; then echo "COMFY_DIR unset (see docs/ci-m2.md)"; exit 1; fi + if [ -z "${COMFY_DIR:-}" ]; then echo "COMFY_DIR unset"; exit 1; fi # Init-in-place rather than `git clone`: COMFY_DIR may already hold the # cached checkpoint (models/checkpoints) or converted .mlmodelc, and # `git clone` refuses a non-empty target. init + fetch + `checkout -f` @@ -95,22 +95,6 @@ jobs: uv sync --group comfy fi - - name: Convert UNet variants if missing - # Idempotent: converter.convert / compile_model skip when the .mlmodelc - # already exists, so this is a no-op once the runner cache is warm. - run: | - set -euo pipefail - for V in none 8 6 4; do - if [ "$V" = "none" ]; then SUF=""; else SUF="_q$V"; fi - OUT="$COMFY_DIR/models/unet/v1-5-pruned-emaonly_1x512x512_se${SUF}_unet.mlmodelc" - if [ -d "$OUT" ]; then echo "skip nbits=$V (exists)"; continue; fi - echo "converting nbits=$V" - # --no-sync: keep the deps we just installed (latest mode's - # ComfyUI requirements / pinned mode's --group comfy); a bare - # `uv run` would re-sync to the lock and drop them. - QUANT_NBITS="$V" uv run --no-sync python bench/scripts/convert_sd15.py - done - - name: Start ComfyUI server (background) run: | cd "$COMFY_DIR" @@ -126,29 +110,11 @@ jobs: echo "comfy failed to start"; tail -100 /tmp/comfyui-ci.log; exit 1 - name: Run Tier 2 (m2 marker) + # The golden-image workflow drives the Core ML Converter node, so the + # UNet is converted on demand on the first run and reused from the + # runner-local cache afterwards. run: uv run --no-sync pytest -m m2 tests/ -v - - name: Run bench harness (baseline UNet) - run: | - uv run --no-sync python bench/run.py \ - --model "$COMFY_DIR/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc" \ - --compute-units CPU_AND_NE CPU_AND_GPU \ - --repeats 30 --assumed-steps 20 - - - name: Run quantization tradeoff matrix (Phase 6) - # Benches none/8/6/4 on ANE and records size / ms-step / PSNR-vs-none - # into bench/results/quant_matrix_.{json,md}. - run: uv run --no-sync python bench/scripts/quant_matrix.py - - - name: Upload bench results - uses: actions/upload-artifact@v4 - if: always() - with: - name: bench-results - path: | - bench/results/*.json - bench/results/*.md - - name: Stop ComfyUI server if: always() run: pkill -f "main.py.*8188" || true diff --git a/.gitignore b/.gitignore index b578bb9..b883889 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,4 @@ comfy_env/ coremlsuite-venv/ experiment_results/ test_results/ -bench/scripts/*.log tests/m2/_latest_generated.png diff --git a/Makefile b/Makefile deleted file mode 100644 index c1d51d8..0000000 --- a/Makefile +++ /dev/null @@ -1,88 +0,0 @@ -# ComfyUI-CoreMLSuite — tiered test/bench dispatcher (Phase 4). -# -# Tiers (see MODERNIZATION_SPEC.md): -# Tier 0 (unit): framework-free pure-logic tests, run anywhere in seconds. -# Tier 1 (smoke): macOS-ARM, no ANE, no full model — converts a synthetic -# micro-UNet to catch coremltools / ml-stable-diffusion -# API breakage in minutes. -# Tier 2 (m2): real ANE on Apple Silicon; integration + bench. -# -# COMFY_DIR defaults to the canonical custom-node layout (two dirs up from here). -# PY defaults to the project's uv-managed venv interpreter. - -COMFY_DIR ?= $(realpath $(CURDIR)/../..) -PY ?= $(CURDIR)/.venv/bin/python -PYTEST ?= $(PY) -m pytest - -UNAME_S := $(shell uname -s) -UNAME_M := $(shell uname -m) -IS_MACOS_ARM := $(filter Darwin,$(UNAME_S))$(filter arm64,$(UNAME_M)) - -.PHONY: help test-unit test-smoke test-m2 bench bench-rerun ci-tier0 ci-tier1 clean check-macos-arm - -help: - @echo "ComfyUI-CoreMLSuite — make targets" - @echo "" - @echo " test-unit Tier 0: pure-logic pytest, runs anywhere, seconds" - @echo " test-smoke Tier 1: synthetic micro-UNet ct.convert + load (macOS-ARM, minutes)" - @echo " test-m2 Tier 2: pytest -m m2 against a real Core ML UNet (Apple Silicon + ANE)" - @echo " bench Run bench/run.py against a converted .mlmodelc" - @echo "" - @echo "Vars: COMFY_DIR (default: $(COMFY_DIR)), PY (default: $(PY))" - -check-macos-arm: - @if [ -z "$(IS_MACOS_ARM)" ]; then \ - echo "this target requires macOS on Apple Silicon (got $(UNAME_S)/$(UNAME_M))"; \ - exit 2; \ - fi - -# Tier 0 — Linux-safe pure logic. Should not import comfy/coremltools. -test-unit: - $(PYTEST) -m unit tests/ - -# Tier 1 — macOS-ARM smoke. Converts a synthetic UNet through coremltools to -# catch API breakage without needing a real SD checkpoint or the ANE. -test-smoke: check-macos-arm - $(PYTEST) -m smoke tests/ - -# Tier 2 — full Apple Silicon path: integration + m2 golden + bench. -# Requires a converted .mlmodelc (see bench/scripts/convert_sd15.py). -test-m2: check-macos-arm - $(PYTEST) -m m2 tests/ - -# Bench harness. Override MODEL=/path/to/.mlmodelc for an explicit model. -MODEL ?= $(COMFY_DIR)/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc -COMPUTE_UNITS ?= CPU_AND_NE CPU_AND_GPU -REPEATS ?= 30 -ASSUMED_STEPS ?= 20 -bench: check-macos-arm - $(PY) bench/run.py \ - --model "$(MODEL)" \ - --compute-units $(COMPUTE_UNITS) \ - --repeats $(REPEATS) \ - --assumed-steps $(ASSUMED_STEPS) - -# Phase 6: quantization tradeoff matrix. Expects the {none, 8, 6, 4} -# variants to already exist on disk (run `make convert-quant` first or -# QUANT_NBITS=8 .venv/bin/python bench/scripts/convert_sd15.py). -bench-quant: check-macos-arm - $(PY) bench/scripts/quant_matrix.py - -convert-quant: check-macos-arm - @for n in 8 6 4; do \ - echo "=== converting nbits=$$n ==="; \ - QUANT_NBITS=$$n $(PY) bench/scripts/convert_sd15.py || exit 1; \ - done - -# What CI actually invokes — same as test-unit but echoes the env capture -# alongside so failed runs land with diagnostics. -ci-tier0: - @echo "## env (Tier 0)" && $(PY) --version && uv pip freeze --python "$(PY)" 2>/dev/null | head -50 || true - $(MAKE) test-unit - -ci-tier1: check-macos-arm - @echo "## env (Tier 1)" && $(PY) --version && uv pip freeze --python "$(PY)" 2>/dev/null | head -50 || true - $(MAKE) test-smoke - -clean: - rm -rf .pytest_cache tests/m2/_latest_generated.png pytestdebug.log diff --git a/README.md b/README.md index 7571fa1..1c48973 100644 --- a/README.md +++ b/README.md @@ -370,21 +370,21 @@ The models used in this workflow are available at the following links: ![sdxl](./assets/sdxl_conversion.png?raw=true) -## Quantization (Phase 6, opt-in) +## Quantization (opt-in) The `Core ML Converter` and `Core ML LCM Converter` nodes accept an optional `quantize_nbits` dropdown that runs k-means weight palettization (`coremltools.optimize.coreml.palettize_weights`) on the UNet before save. -Values: `none` (default — no quantization, identical to pre-Phase-6 +Values: `none` (default — no quantization, identical to unquantized behavior and filenames), `8`, `6`, `4`. The number is appended to the .mlpackage stem as `_q` so quantized and unquantized variants coexist on disk and in cache. ### SD1.5 1×512×512 SPLIT_EINSUM tradeoffs (M2 Pro, ANE) -Measured by `bench/scripts/quant_matrix.py` (20 forward passes, fixed -seed for the PSNR comparison): +Measured with 20 UNet forward passes at a fixed seed for the PSNR +comparison: | nbits | size (MB) | size vs none | fwd median (ms) | PSNR vs `none` (dB) | |---|---:|---:|---:|---:| @@ -408,7 +408,7 @@ PSNR is comfortably higher (the sampler averages over 20 steps). margin, `none` if you want bit-identical output for golden testing. The default stays `none` so existing workflows produce byte-for-byte -identical output — the Phase 6 anchor (`tests/m2/test_golden_image.py`) +identical output — the golden-image anchor (`tests/m2/test_golden_image.py`) verifies this on every Tier 2 run. ## Limitations diff --git a/bench/env/baseline-1e5791d.txt b/bench/env/baseline-1e5791d.txt deleted file mode 100644 index ad2819d..0000000 --- a/bench/env/baseline-1e5791d.txt +++ /dev/null @@ -1,134 +0,0 @@ -# Phase 1 baseline environment capture -timestamp_utc: 2026-05-24T14:36:51Z -repo_sha: 1e5791d1087b4dfb095e7761f05ac4e82c92cf7a -repo_branch: modernize/phase5-toolchain-bump -comfyui_dir: /Users/aszczepanski/dev/ComfyUI -comfyui_sha: ab5413351eee61f3d7f10c74e75286df0058bb18 -python_bin: /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/bin/python -python_version: Python 3.12.11 -platform_machine: arm64 -platform_uname: Darwin mac-2.home 25.1.0 Darwin Kernel Version 25.1.0: Mon Oct 20 19:33:00 PDT 2025; root:xnu-12377.41.6~2/RELEASE_ARM64_T6020 arm64 -macos_product: macOS -macos_version: 26.1 -macos_build: 25B78 - -## python-coreml-stable-diffusion (resolved) -Name: python-coreml-stable-diffusion -Version: 1.1.0 -Location: /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/lib/python3.12/site-packages -Requires: coremltools, diffusers, diffusionkit, huggingface-hub, invisible-watermark, matplotlib, numpy, pytest, safetensors, scikit-learn, scipy, torch, transformers -Required-by: - -## python-coreml-stable-diffusion installed git metadata -{"url":"https://github.com/apple/ml-stable-diffusion.git","vcs_info":{"vcs":"git","commit_id":"e5d960c41a6a4ab200b8db379194127607b1c590","requested_revision":"e5d960c41a6a4ab200b8db379194127607b1c590"}} - -## coremltools version -scikit-learn version 1.7.2 is not supported. Minimum required version: 0.17. Maximum required version: 1.5.1. Disabling scikit-learn conversion API. -Torch version 2.7.1 has not been tested with coremltools. You may run into unexpected errors. Torch 2.7.0 is the most recent version that has been tested. -coremltools 9.0 - -## torch version -torch 2.7.1 - -## numpy version -numpy 1.26.4 - -## freeze -accelerate==1.13.0 -aiohappyeyeballs==2.6.2 -aiohttp==3.13.5 -aiosignal==1.4.0 -annotated-types==0.7.0 -antlr4-python3-runtime==4.9.3 -argmaxtools==0.1.23 -attrs==26.1.0 -beartype==0.22.9 -cattrs==26.1.0 -certifi==2026.5.20 -cffi==2.0.0 -charset-normalizer==3.4.7 -click==8.4.1 -comfyui-frontend-package==1.14.6 -contourpy==1.3.2 -coremltools==9.0 -cycler==0.12.1 -diffusers==0.30.2 -diffusionkit==0.4.0 -einops==0.8.2 -filelock==3.29.0 -fonttools==4.63.0 -frozenlist==1.8.0 -fsspec==2026.4.0 -gitdb==4.0.12 -gitpython==3.1.50 -huggingface-hub==0.24.6 -idna==3.16 -importlib-metadata==9.0.0 -iniconfig==2.3.0 -invisible-watermark==0.2.0 -jaxtyping==0.3.9 -jinja2==3.1.6 -joblib==1.5.3 -kiwisolver==1.5.0 -kornia==0.8.3 -kornia-rs==0.1.14 -markupsafe==3.0.3 -matplotlib==3.10.9 -mlx==0.31.2 -mlx-metal==0.31.2 -mpmath==1.3.0 -multidict==6.7.1 -networkx==3.6.1 -numpy==1.26.4 -omegaconf==2.3.0 -opencv-python==4.11.0.86 -overrides==7.7.0 -packaging==26.2 -peft==0.13.2 -pillow==12.2.0 -platformdirs==4.9.6 -pluggy==1.6.0 -propcache==0.5.2 -protobuf==7.35.0 -psutil==7.2.2 -pyaml==26.2.1 -pycparser==3.0 -pydantic==2.13.4 -pydantic-core==2.46.4 -pygments==2.20.0 -pyparsing==3.3.2 -pytest==9.0.3 -python-coreml-stable-diffusion @ git+https://github.com/apple/ml-stable-diffusion.git@e5d960c41a6a4ab200b8db379194127607b1c590 -python-dateutil==2.9.0.post0 -pywavelets==1.8.0 -pyyaml==6.0.3 -regex==2026.5.9 -requests==2.34.2 -safetensors==0.7.0 -scikit-learn==1.7.2 -scipy==1.15.3 -sentencepiece==0.2.1 -sentry-sdk==2.60.0 -setuptools==81.0.0 -six==1.17.0 -smmap==5.0.3 -soundfile==0.13.1 -spandrel==0.4.2 -sympy==1.14.0 -tabulate==0.10.0 -threadpoolctl==3.6.0 -tokenizers==0.19.1 -torch==2.7.1 -torchaudio==2.11.0 -torchsde==0.2.6 -torchvision==0.22.1 -tqdm==4.67.3 -trampoline==0.1.2 -transformers==4.44.2 -typing-extensions==4.15.0 -typing-inspection==0.4.2 -urllib3==2.7.0 -wadler-lindig==0.1.7 -wandb==0.27.0 -yarl==1.24.2 -zipp==4.1.0 diff --git a/bench/env/baseline-ef2a18c.txt b/bench/env/baseline-ef2a18c.txt deleted file mode 100644 index f8bc2b9..0000000 --- a/bench/env/baseline-ef2a18c.txt +++ /dev/null @@ -1,129 +0,0 @@ -# Phase 1 baseline environment capture -timestamp_utc: 2026-05-22T13:06:37Z -repo_sha: ef2a18cff3d8b563ae053d27c1c5c233bd05d47f -repo_branch: modernize/phase1-freeze-baseline -comfyui_dir: /Users/aszczepanski/dev/ComfyUI -comfyui_sha: ab5413351eee61f3d7f10c74e75286df0058bb18 -python_bin: /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/bin/python -python_version: Python 3.11.14 -platform_machine: arm64 -platform_uname: Darwin mac-2.home 25.1.0 Darwin Kernel Version 25.1.0: Mon Oct 20 19:33:00 PDT 2025; root:xnu-12377.41.6~2/RELEASE_ARM64_T6020 arm64 -macos_product: macOS -macos_version: 26.1 -macos_build: 25B78 - -## python-coreml-stable-diffusion (resolved) -Name: python-coreml-stable-diffusion -Version: 1.1.0 -Location: /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/lib/python3.11/site-packages -Requires: coremltools, diffusers, diffusionkit, huggingface-hub, invisible-watermark, matplotlib, numpy, pytest, safetensors, scikit-learn, scipy, torch, transformers -Required-by: - -## python-coreml-stable-diffusion installed git metadata -{"url":"https://github.com/apple/ml-stable-diffusion.git","vcs_info":{"vcs":"git","commit_id":"e5d960c41a6a4ab200b8db379194127607b1c590","requested_revision":"e5d960c41a6a4ab200b8db379194127607b1c590"}} - -## coremltools version -scikit-learn version 1.7.2 is not supported. Minimum required version: 0.17. Maximum required version: 1.5.1. Disabling scikit-learn conversion API. -Minimum required torch version for importing coremltools.optimize.torch is 2.1.0. Got torch version 2.0.1. -coremltools 8.2 - -## torch version -torch 2.0.1 - -## numpy version -numpy 1.23.5 - -## freeze -accelerate==1.13.0 -aiohappyeyeballs==2.6.2 -aiohttp==3.13.5 -aiosignal==1.4.0 -annotated-types==0.7.0 -antlr4-python3-runtime==4.9.3 -argmaxtools==0.1.23 -attrs==26.1.0 -beartype==0.22.9 -cattrs==26.1.0 -certifi==2026.5.20 -charset-normalizer==3.4.7 -click==8.4.1 -comfyui-frontend-package==1.14.6 -contourpy==1.3.2 -coremltools==8.2 -cycler==0.12.1 -diffusers==0.30.2 -diffusionkit==0.4.0 -einops==0.8.2 -filelock==3.29.0 -fonttools==4.63.0 -frozenlist==1.8.0 -fsspec==2026.4.0 -gitdb==4.0.12 -gitpython==3.1.50 -huggingface-hub==0.24.6 -idna==3.16 -importlib-metadata==9.0.0 -iniconfig==2.3.0 -invisible-watermark==0.2.0 -jaxtyping==0.3.9 -jinja2==3.1.6 -joblib==1.5.3 -kiwisolver==1.5.0 -kornia==0.8.3 -kornia-rs==0.1.14 -markupsafe==3.0.3 -matplotlib==3.10.9 -mlx==0.31.2 -mlx-metal==0.31.2 -mpmath==1.3.0 -multidict==6.7.1 -networkx==3.6.1 -numpy==1.23.5 -omegaconf==2.3.0 -opencv-python==4.11.0.86 -overrides==7.7.0 -packaging==26.2 -peft==0.13.2 -pillow==12.2.0 -platformdirs==4.9.6 -pluggy==1.6.0 -propcache==0.5.2 -protobuf==7.35.0 -psutil==7.2.2 -pyaml==26.2.1 -pydantic==2.13.4 -pydantic-core==2.46.4 -pygments==2.20.0 -pyparsing==3.3.2 -pytest==9.0.3 -python-coreml-stable-diffusion @ git+https://github.com/apple/ml-stable-diffusion.git@e5d960c41a6a4ab200b8db379194127607b1c590 -python-dateutil==2.9.0.post0 -pywavelets==1.8.0 -pyyaml==6.0.3 -regex==2026.5.9 -requests==2.34.2 -safetensors==0.7.0 -scikit-learn==1.7.2 -scipy==1.15.3 -sentencepiece==0.2.1 -sentry-sdk==2.60.0 -setuptools==81.0.0 -six==1.17.0 -smmap==5.0.3 -sympy==1.14.0 -tabulate==0.10.0 -threadpoolctl==3.6.0 -tokenizers==0.19.1 -torch==2.0.1 -torchsde==0.2.6 -torchvision==0.15.2 -tqdm==4.67.3 -trampoline==0.1.2 -transformers==4.44.2 -typing-extensions==4.15.0 -typing-inspection==0.4.2 -urllib3==2.7.0 -wadler-lindig==0.1.7 -wandb==0.27.0 -yarl==1.24.2 -zipp==4.1.0 diff --git a/bench/env/capture.sh b/bench/env/capture.sh deleted file mode 100755 index aec3d9b..0000000 --- a/bench/env/capture.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -# Phase 1 baseline environment capture. -# -# Default mode: query the project's .venv via `uv pip` (matches how uv builds -# the venv — without a bundled pip executable). For a non-uv setup, point -# PYTHON_BIN at the right interpreter; pip freeze then falls back to -# `python -m pip` if available, otherwise `uv pip`. -# -# Usage: -# bash bench/env/capture.sh # uses .venv (uv-managed) -# PYTHON_BIN=apple_env/bin/python bash bench/env/capture.sh -# -# Writes bench/env/baseline-.txt with: this repo's git sha, -# ComfyUI's git sha (override path with COMFY_DIR=...), python + macOS -# versions, the resolved git commit of python-coreml-stable-diffusion as -# installed, and the full freeze of the active venv (for fresh-venv -# reproducibility). -set -euo pipefail - -REPO_ROOT="$(git rev-parse --show-toplevel)" -cd "$REPO_ROOT" - -SHA_SHORT="$(git rev-parse --short HEAD)" -OUT_DIR="bench/env" -OUT="${OUT_DIR}/baseline-${SHA_SHORT}.txt" -mkdir -p "$OUT_DIR" - -COMFY_DIR="${COMFY_DIR:-$(cd ../.. && pwd)}" -PYTHON_BIN="${PYTHON_BIN:-$REPO_ROOT/.venv/bin/python}" - -if [ ! -x "$PYTHON_BIN" ]; then - echo "PYTHON_BIN not executable: $PYTHON_BIN" >&2 - exit 1 -fi - -# Prefer `python -m pip` (works in classic venvs). Fall back to `uv pip`, -# which queries any interpreter without needing pip installed in it. -run_freeze() { - if "$PYTHON_BIN" -m pip --version >/dev/null 2>&1; then - "$PYTHON_BIN" -m pip freeze - elif command -v uv >/dev/null 2>&1; then - uv pip freeze --python "$PYTHON_BIN" - else - echo "neither pip nor uv available for freeze" - fi -} - -run_show_msd() { - if "$PYTHON_BIN" -m pip --version >/dev/null 2>&1; then - "$PYTHON_BIN" -m pip show python-coreml-stable-diffusion 2>/dev/null || echo "not-installed" - elif command -v uv >/dev/null 2>&1; then - uv pip show --python "$PYTHON_BIN" python-coreml-stable-diffusion 2>/dev/null || echo "not-installed" - else - echo "neither pip nor uv available for show" - fi -} - -{ - echo "# Phase 1 baseline environment capture" - echo "timestamp_utc: $(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "repo_sha: $(git rev-parse HEAD)" - echo "repo_branch: $(git rev-parse --abbrev-ref HEAD)" - echo "comfyui_dir: ${COMFY_DIR}" - echo "comfyui_sha: $(git -C "$COMFY_DIR" rev-parse HEAD 2>/dev/null || echo unknown)" - echo "python_bin: ${PYTHON_BIN}" - echo "python_version: $("$PYTHON_BIN" --version 2>&1)" - echo "platform_machine: $(uname -m)" - echo "platform_uname: $(uname -a)" - if command -v sw_vers >/dev/null 2>&1; then - echo "macos_product: $(sw_vers -productName)" - echo "macos_version: $(sw_vers -productVersion)" - echo "macos_build: $(sw_vers -buildVersion)" - fi - echo - echo "## python-coreml-stable-diffusion (resolved)" - run_show_msd - echo - echo "## python-coreml-stable-diffusion installed git metadata" - DIST_INFO="$("$PYTHON_BIN" -c "import importlib.metadata as m; d=m.distribution('python-coreml-stable-diffusion'); print(d._path)" 2>/dev/null || true)" - if [ -n "${DIST_INFO}" ] && [ -f "${DIST_INFO}/direct_url.json" ]; then - cat "${DIST_INFO}/direct_url.json" - echo - else - echo "direct_url.json not found" - fi - echo - echo "## coremltools version" - "$PYTHON_BIN" -c "import coremltools; print('coremltools', coremltools.__version__)" 2>&1 || true - echo - echo "## torch version" - "$PYTHON_BIN" -c "import torch; print('torch', torch.__version__)" 2>&1 || true - echo - echo "## numpy version" - "$PYTHON_BIN" -c "import numpy; print('numpy', numpy.__version__)" 2>&1 || true - echo - echo "## freeze" - run_freeze -} > "$OUT" - -echo "wrote $OUT" diff --git a/bench/env/pytest-unit-1e5791d.txt b/bench/env/pytest-unit-1e5791d.txt deleted file mode 100644 index f0e81e6..0000000 --- a/bench/env/pytest-unit-1e5791d.txt +++ /dev/null @@ -1,98 +0,0 @@ -============================= test session starts ============================== -platform darwin -- Python 3.12.11, pytest-9.0.3, pluggy-1.6.0 -- /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/bin/python -cachedir: .pytest_cache -rootdir: /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite -configfile: pyproject.toml -plugins: jaxtyping-0.3.9 -collecting ... collected 88 items - -tests/unit/test_characterization_controlnet.py::test_expand_inputs_doubles_singleton_numpy PASSED [ 1%] -tests/unit/test_characterization_controlnet.py::test_expand_inputs_doubles_singleton_torch PASSED [ 2%] -tests/unit/test_characterization_controlnet.py::test_expand_inputs_doubles_singleton_list PASSED [ 3%] -tests/unit/test_characterization_controlnet.py::test_expand_inputs_skips_already_batched PASSED [ 4%] -tests/unit/test_characterization_controlnet.py::test_expand_inputs_preserves_unknown_value_types PASSED [ 5%] -tests/unit/test_characterization_controlnet.py::test_no_control_returns_zero_fp16_for_residuals PASSED [ 6%] -tests/unit/test_characterization_controlnet.py::test_no_control_returns_empty_when_no_residuals PASSED [ 7%] -tests/unit/test_characterization_controlnet.py::test_extract_residual_kwargs_empty_when_model_has_no_residual_inputs PASSED [ 9%] -tests/unit/test_characterization_controlnet.py::test_extract_residual_kwargs_none_control_returns_no_control_shapes PASSED [ 10%] -tests/unit/test_characterization_controlnet.py::test_extract_residual_kwargs_flattens_output_then_middle_and_casts_fp16 PASSED [ 11%] -tests/unit/test_characterization_controlnet.py::test_chunk_control_none_returns_list_of_nones_with_length_target PASSED [ 12%] -tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[1-2-1] PASSED [ 13%] -tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[2-2-1] PASSED [ 14%] -tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[3-2-2] PASSED [ 15%] -tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[4-2-2] PASSED [ 17%] -tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[5-3-2] PASSED [ 18%] -tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[9-4-3] PASSED [ 19%] -tests/unit/test_characterization_controlnet.py::test_chunk_control_preserves_keys_order PASSED [ 20%] -tests/unit/test_characterization_controlnet.py::test_chunk_control_zero_pads_remainder PASSED [ 21%] -tests/unit/test_characterization_inputs.py::test_coreml_kwargs_sd15_shapes_and_fp16 PASSED [ 22%] -tests/unit/test_characterization_inputs.py::test_coreml_kwargs_sd15_with_controlnet_emits_residuals PASSED [ 23%] -tests/unit/test_characterization_inputs.py::test_coreml_kwargs_sd15_without_controlnet_zero_fills_residuals PASSED [ 25%] -tests/unit/test_characterization_inputs.py::test_coreml_kwargs_lcm_adds_timestep_cond PASSED [ 26%] -tests/unit/test_characterization_inputs.py::test_coreml_kwargs_lcm_skips_timestep_cond_when_not_provided PASSED [ 27%] -tests/unit/test_characterization_inputs.py::test_coreml_kwargs_sdxl_base_emits_time_ids_and_text_embeds PASSED [ 28%] -tests/unit/test_characterization_inputs.py::test_coreml_kwargs_sdxl_refiner_uses_len5_time_ids PASSED [ 29%] -tests/unit/test_characterization_inputs.py::test_chunks_sd15_pad_to_batch2_returns_one_chunk PASSED [ 30%] -tests/unit/test_characterization_inputs.py::test_chunks_sd15_with_controlnet_chunks_residuals_too PASSED [ 31%] -tests/unit/test_characterization_inputs.py::test_chunks_lcm_carries_timestep_cond_per_chunk PASSED [ 32%] -tests/unit/test_characterization_inputs.py::test_chunks_sdxl_base_propagates_time_ids_and_text_embeds PASSED [ 34%] -tests/unit/test_characterization_inputs.py::test_chunks_sdxl_refiner_uses_len5_time_ids PASSED [ 35%] -tests/unit/test_characterization_inputs.py::test_chunks_sdxl_synthesizes_zero_time_ids_when_caller_omits PASSED [ 36%] -tests/unit/test_characterization_inputs.py::test_chunks_splits_batch_into_multiple_target2_chunks PASSED [ 37%] -tests/unit/test_characterization_inputs.py::test_chunks_timestep_is_broadcast_from_first_value PASSED [ 38%] -tests/unit/test_characterization_latents.py::test_chunk_batch_passthrough_when_shape_matches PASSED [ 39%] -tests/unit/test_characterization_latents.py::test_chunk_batch_pads_single_chunk_when_input_smaller PASSED [ 40%] -tests/unit/test_characterization_latents.py::test_chunk_batch_splits_exact_multiple PASSED [ 42%] -tests/unit/test_characterization_latents.py::test_chunk_batch_pads_remainder_chunk PASSED [ 43%] -tests/unit/test_characterization_latents.py::test_chunk_batch_pad_region_is_zero[1-4-1] PASSED [ 44%] -tests/unit/test_characterization_latents.py::test_chunk_batch_pad_region_is_zero[3-2-2] PASSED [ 45%] -tests/unit/test_characterization_latents.py::test_chunk_batch_pad_region_is_zero[5-3-2] PASSED [ 46%] -tests/unit/test_characterization_latents.py::test_chunk_batch_pad_region_is_zero[9-4-3] PASSED [ 47%] -tests/unit/test_characterization_latents.py::test_merge_chunks_exact_concat PASSED [ 48%] -tests/unit/test_characterization_latents.py::test_merge_chunks_truncates_padding PASSED [ 50%] -tests/unit/test_characterization_latents.py::test_merge_chunks_singleton_returns_equal_copy_when_shape_matches PASSED [ 51%] -tests/unit/test_characterization_out_name.py::test_attention_suffix[SPLIT_EINSUM-se] PASSED [ 52%] -tests/unit/test_characterization_out_name.py::test_attention_suffix[SPLIT_EINSUM_V2-se2] PASSED [ 53%] -tests/unit/test_characterization_out_name.py::test_attention_suffix[ORIGINAL-orig] PASSED [ 54%] -tests/unit/test_characterization_out_name.py::test_includes_batch_and_size PASSED [ 55%] -tests/unit/test_characterization_out_name.py::test_appends_cn_suffix_when_controlnet_support_true PASSED [ 56%] -tests/unit/test_characterization_out_name.py::test_drops_extension_at_first_period PASSED [ 57%] -tests/unit/test_characterization_out_name.py::test_replaces_spaces_with_underscores PASSED [ 59%] -tests/unit/test_characterization_out_name.py::test_single_lora PASSED [ 60%] -tests/unit/test_characterization_out_name.py::test_multiple_loras_sorted PASSED [ 61%] -tests/unit/test_characterization_out_name.py::test_lora_plus_controlnet PASSED [ 62%] -tests/unit/test_characterization_out_name.py::test_sdxl_1024_original_gpu PASSED [ 63%] -tests/unit/test_characterization_out_name.py::test_lora_names_from_params_sorts_by_name PASSED [ 64%] -tests/unit/test_characterization_out_name.py::test_lora_names_from_params_empty_list PASSED [ 65%] -tests/unit/test_characterization_sdxl_options.py::test_build_time_ids_base_defaults PASSED [ 67%] -tests/unit/test_characterization_sdxl_options.py::test_build_time_ids_base_respects_overrides PASSED [ 68%] -tests/unit/test_characterization_sdxl_options.py::test_build_time_ids_refiner_defaults PASSED [ 69%] -tests/unit/test_characterization_sdxl_options.py::test_build_time_ids_refiner_respects_aesthetic_score PASSED [ 70%] -tests/unit/test_characterization_sdxl_options.py::test_build_time_ids_neither_base_nor_refiner_returns_len4 PASSED [ 71%] -tests/unit/test_characterization_sdxl_options.py::test_text_embeds_concat_pos_then_neg PASSED [ 72%] -tests/unit/test_characterization_sdxl_options.py::test_wrapper_captures_time_ids_text_embeds_refiner_via_closure PASSED [ 73%] -tests/unit/test_characterization_sdxl_options.py::test_wrapper_returns_zero_when_context_missing PASSED [ 75%] -tests/unit/test_characterization_sdxl_options.py::test_wrapper_refiner_truncates_context_to_g_clip PASSED [ 76%] -tests/unit/test_chunks.py::test_batch_chunking[1] PASSED [ 77%] -tests/unit/test_chunks.py::test_batch_chunking[2] PASSED [ 78%] -tests/unit/test_chunks.py::test_batch_chunking[4] PASSED [ 79%] -tests/unit/test_chunks.py::test_batch_chunking[5] PASSED [ 80%] -tests/unit/test_chunks.py::test_batch_chunking[9] PASSED [ 81%] -tests/unit/test_chunks.py::test_merge_chunks[1] PASSED [ 82%] -tests/unit/test_chunks.py::test_merge_chunks[2] PASSED [ 84%] -tests/unit/test_chunks.py::test_merge_chunks[4] PASSED [ 85%] -tests/unit/test_chunks.py::test_merge_chunks[5] PASSED [ 86%] -tests/unit/test_chunks.py::test_merge_chunks[9] PASSED [ 87%] -tests/unit/test_chunks.py::test_chunking_controlnet[1-2-1] PASSED [ 88%] -tests/unit/test_chunks.py::test_chunking_controlnet[1-1-1] PASSED [ 89%] -tests/unit/test_chunks.py::test_chunking_controlnet[2-2-1] PASSED [ 90%] -tests/unit/test_chunks.py::test_chunking_controlnet[3-2-2] PASSED [ 92%] -tests/unit/test_chunks.py::test_chunking_controlnet[4-2-2] PASSED [ 93%] -tests/unit/test_chunks.py::test_chunking_controlnet[5-3-2] PASSED [ 94%] -tests/unit/test_chunks.py::test_chunking_controlnet[9-4-3] PASSED [ 95%] -tests/unit/test_chunks.py::test_chunking_no_control PASSED [ 96%] -tests/unit/test_chunks.py::test_chunking_inputs PASSED [ 97%] -tests/unit/test_controlnet.py::test_no_control PASSED [ 98%] -tests/unit/test_tier0_purity.py::test_no_framework_modules_loaded_by_unit_tier PASSED [100%] - -============================== 88 passed in 1.83s ============================== diff --git a/bench/env/pytest-unit-ef2a18c.txt b/bench/env/pytest-unit-ef2a18c.txt deleted file mode 100644 index 984defe..0000000 --- a/bench/env/pytest-unit-ef2a18c.txt +++ /dev/null @@ -1,39 +0,0 @@ -============================= test session starts ============================== -platform darwin -- Python 3.11.14, pytest-9.0.3, pluggy-1.6.0 -- /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/bin/python -cachedir: .pytest_cache -rootdir: /Users/aszczepanski/dev/ComfyUI -configfile: pytest.ini -plugins: jaxtyping-0.3.9 -collecting ... collected 20 items - -tests/unit/test_chunks.py::test_batch_chunking[1] PASSED -tests/unit/test_chunks.py::test_batch_chunking[2] PASSED -tests/unit/test_chunks.py::test_batch_chunking[4] PASSED -tests/unit/test_chunks.py::test_batch_chunking[5] PASSED -tests/unit/test_chunks.py::test_batch_chunking[9] PASSED -tests/unit/test_chunks.py::test_merge_chunks[1] PASSED -tests/unit/test_chunks.py::test_merge_chunks[2] PASSED -tests/unit/test_chunks.py::test_merge_chunks[4] PASSED -tests/unit/test_chunks.py::test_merge_chunks[5] PASSED -tests/unit/test_chunks.py::test_merge_chunks[9] PASSED -tests/unit/test_chunks.py::test_chunking_controlnet[1-2-1] PASSED -tests/unit/test_chunks.py::test_chunking_controlnet[1-1-1] PASSED -tests/unit/test_chunks.py::test_chunking_controlnet[2-2-1] PASSED -tests/unit/test_chunks.py::test_chunking_controlnet[3-2-2] PASSED -tests/unit/test_chunks.py::test_chunking_controlnet[4-2-2] PASSED -tests/unit/test_chunks.py::test_chunking_controlnet[5-3-2] PASSED -tests/unit/test_chunks.py::test_chunking_controlnet[9-4-3] PASSED -tests/unit/test_chunks.py::test_chunking_no_control PASSED -tests/unit/test_chunks.py::test_chunking_inputs PASSED -tests/unit/test_controlnet.py::test_no_control PASSED - -=============================== warnings summary =============================== -:241 - :241: DeprecationWarning: builtin type SwigPyPacked has no __module__ attribute - -:241 - :241: DeprecationWarning: builtin type SwigPyObject has no __module__ attribute - --- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html -======================== 20 passed, 2 warnings in 2.81s ======================== -sys:1: DeprecationWarning: builtin type swigvarlink has no __module__ attribute diff --git a/bench/prompts.json b/bench/prompts.json deleted file mode 100644 index 6a4017e..0000000 --- a/bench/prompts.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "_comment": "Fixed prompt set for Phase 1 baseline reproducibility. Do not edit without bumping the bench version: changing prompts invalidates cross-baseline PSNR comparisons.", - "seed": 0, - "negative_prompt": "blurry, low quality, text, watermark", - "prompts": [ - { - "id": "p01_portrait", - "text": "a portrait of an astronaut in a sunflower field, soft natural light, 35mm film" - }, - { - "id": "p02_cityscape", - "text": "a futuristic city skyline at sunset, glass towers, volumetric lighting, cinematic" - }, - { - "id": "p03_still_life", - "text": "a still life of fruit on a wooden table, oil painting, dramatic chiaroscuro" - } - ] -} diff --git a/bench/results/1e5791d.json b/bench/results/1e5791d.json deleted file mode 100644 index a3e1fcd..0000000 --- a/bench/results/1e5791d.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "git_sha": "1e5791d", - "timestamp": "2026-05-24T14:37:16+00:00", - "host": { - "machine": "arm64", - "system": "Darwin", - "release": "25.1.0", - "python": "3.12.11" - }, - "tool_versions": { - "coremltools": "9.0", - "numpy": "1.26.4" - }, - "settings": { - "repeats": 30, - "assumed_steps": 20, - "input_seed": 0, - "compute_units": [ - "CPU_AND_NE", - "CPU_AND_GPU" - ] - }, - "results": [ - { - "label": "v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_NE", - "model_path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc", - "compute_unit": "CPU_AND_NE", - "error": null, - "model_size_bytes": 1720796055, - "load_time_s": 0.2086, - "kind": "sd15", - "expected_inputs": { - "sample": [ - 1, - 4, - 64, - 64 - ], - "timestep": [ - 1 - ], - "encoder_hidden_states": [ - 1, - 768, - 1, - 77 - ] - }, - "warmup_ms": 219.471, - "forward_ms": { - "mean": 199.435, - "std": 5.333, - "min": 196.825, - "median": 197.493, - "n": 30 - }, - "est_e2e_ms": 3949.9, - "peak_rss_mb": 399.0 - }, - { - "label": "v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_GPU", - "model_path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc", - "compute_unit": "CPU_AND_GPU", - "error": null, - "model_size_bytes": 1720796055, - "load_time_s": 1.2334, - "kind": "sd15", - "expected_inputs": { - "sample": [ - 1, - 4, - 64, - 64 - ], - "timestep": [ - 1 - ], - "encoder_hidden_states": [ - 1, - 768, - 1, - 77 - ] - }, - "warmup_ms": 1392.959, - "forward_ms": { - "mean": 281.433, - "std": 11.822, - "min": 268.273, - "median": 278.577, - "n": 30 - }, - "est_e2e_ms": 5571.5, - "peak_rss_mb": 2113.5 - } - ], - "notes": [ - "forward_ms is the latency of a single Core ML UNet forward pass with synthetic inputs; with classifier-free guidance one sampler step typically maps to one batched forward (cond+uncond) \u2014 interpret est_e2e_ms accordingly.", - "peak_rss_mb is process RSS only and excludes ANE/wired GPU memory; treat as a floor.", - "quality_psnr is null unless --reference-images and --candidate-images are provided.", - "Latency varies run-to-run; 'min' is the most reproducible figure for comparisons." - ] -} \ No newline at end of file diff --git a/bench/results/1e5791d.md b/bench/results/1e5791d.md deleted file mode 100644 index 1b854df..0000000 --- a/bench/results/1e5791d.md +++ /dev/null @@ -1,19 +0,0 @@ -# CoreMLSuite baseline — `1e5791d` - -- timestamp: 2026-05-24T14:37:16+00:00 -- host: arm64 / Darwin 25.1.0 -- python: 3.12.11 -- coremltools: 9.0, numpy: 1.26.4 -- settings: repeats=30, input_seed=0, assumed_steps=20 - -| label | kind | compute | size (MB) | load (s) | warmup (ms) | fwd median (ms) | fwd min (ms) | est e2e (ms) | peak RSS (MB) | error | -|---|---|---|---|---|---|---|---|---|---|---| -| v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_NE | sd15 | CPU_AND_NE | 1641.1 | 0.2086 | 219.471 | 197.493 | 196.825 | 3949.9 | 399.0 | | -| v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_GPU | sd15 | CPU_AND_GPU | 1641.1 | 1.2334 | 1392.959 | 278.577 | 268.273 | 5571.5 | 2113.5 | | - -## Notes - -- forward_ms is the latency of a single Core ML UNet forward pass with synthetic inputs; with classifier-free guidance one sampler step typically maps to one batched forward (cond+uncond) — interpret est_e2e_ms accordingly. -- peak_rss_mb is process RSS only and excludes ANE/wired GPU memory; treat as a floor. -- quality_psnr is null unless --reference-images and --candidate-images are provided. -- Latency varies run-to-run; 'min' is the most reproducible figure for comparisons. diff --git a/bench/results/ef2a18c.json b/bench/results/ef2a18c.json deleted file mode 100644 index b5f52ae..0000000 --- a/bench/results/ef2a18c.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "git_sha": "ef2a18c", - "timestamp": "2026-05-22T13:07:25+00:00", - "host": { - "machine": "arm64", - "system": "Darwin", - "release": "25.1.0", - "python": "3.11.14" - }, - "tool_versions": { - "coremltools": "8.2", - "numpy": "1.23.5" - }, - "settings": { - "repeats": 30, - "assumed_steps": 20, - "input_seed": 0, - "compute_units": [ - "CPU_AND_NE", - "CPU_AND_GPU" - ] - }, - "results": [ - { - "label": "v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_NE", - "model_path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc", - "compute_unit": "CPU_AND_NE", - "error": null, - "model_size_bytes": 1720795925, - "load_time_s": 0.2841, - "kind": "sd15", - "expected_inputs": { - "sample": [ - 1, - 4, - 64, - 64 - ], - "timestep": [ - 1 - ], - "encoder_hidden_states": [ - 1, - 768, - 1, - 77 - ] - }, - "warmup_ms": 225.201, - "forward_ms": { - "mean": 197.126, - "std": 0.476, - "min": 196.812, - "median": 196.969, - "n": 30 - }, - "est_e2e_ms": 3939.4, - "peak_rss_mb": 349.7 - }, - { - "label": "v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_GPU", - "model_path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc", - "compute_unit": "CPU_AND_GPU", - "error": null, - "model_size_bytes": 1720795925, - "load_time_s": 19.959, - "kind": "sd15", - "expected_inputs": { - "sample": [ - 1, - 4, - 64, - 64 - ], - "timestep": [ - 1 - ], - "encoder_hidden_states": [ - 1, - 768, - 1, - 77 - ] - }, - "warmup_ms": 558.865, - "forward_ms": { - "mean": 270.275, - "std": 1.312, - "min": 267.398, - "median": 270.259, - "n": 30 - }, - "est_e2e_ms": 5405.2, - "peak_rss_mb": 2132.9 - } - ], - "notes": [ - "forward_ms is the latency of a single Core ML UNet forward pass with synthetic inputs; with classifier-free guidance one sampler step typically maps to one batched forward (cond+uncond) \u2014 interpret est_e2e_ms accordingly.", - "peak_rss_mb is process RSS only and excludes ANE/wired GPU memory; treat as a floor.", - "quality_psnr is null unless --reference-images and --candidate-images are provided.", - "Latency varies run-to-run; 'min' is the most reproducible figure for comparisons." - ] -} \ No newline at end of file diff --git a/bench/results/ef2a18c.md b/bench/results/ef2a18c.md deleted file mode 100644 index 9db2035..0000000 --- a/bench/results/ef2a18c.md +++ /dev/null @@ -1,19 +0,0 @@ -# CoreMLSuite baseline — `ef2a18c` - -- timestamp: 2026-05-22T13:07:25+00:00 -- host: arm64 / Darwin 25.1.0 -- python: 3.11.14 -- coremltools: 8.2, numpy: 1.23.5 -- settings: repeats=30, input_seed=0, assumed_steps=20 - -| label | kind | compute | size (MB) | load (s) | warmup (ms) | fwd median (ms) | fwd min (ms) | est e2e (ms) | peak RSS (MB) | error | -|---|---|---|---|---|---|---|---|---|---|---| -| v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_NE | sd15 | CPU_AND_NE | 1641.1 | 0.2841 | 225.201 | 196.969 | 196.812 | 3939.4 | 349.7 | | -| v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_GPU | sd15 | CPU_AND_GPU | 1641.1 | 19.959 | 558.865 | 270.259 | 267.398 | 5405.2 | 2132.9 | | - -## Notes - -- forward_ms is the latency of a single Core ML UNet forward pass with synthetic inputs; with classifier-free guidance one sampler step typically maps to one batched forward (cond+uncond) — interpret est_e2e_ms accordingly. -- peak_rss_mb is process RSS only and excludes ANE/wired GPU memory; treat as a floor. -- quality_psnr is null unless --reference-images and --candidate-images are provided. -- Latency varies run-to-run; 'min' is the most reproducible figure for comparisons. diff --git a/bench/results/quant_matrix_0bbd8d8.json b/bench/results/quant_matrix_0bbd8d8.json deleted file mode 100644 index 8055500..0000000 --- a/bench/results/quant_matrix_0bbd8d8.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "git_sha": "0bbd8d8", - "compute_unit": "CPU_AND_NE", - "repeats": 20, - "input_seed": 0, - "ckpt_stem": "v1-5-pruned-emaonly", - "rows": [ - { - "nbits": "none", - "path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc", - "size_bytes": 1720796055, - "load_time_s": 0.1997, - "fwd_ms_median": 196.651, - "fwd_ms_min": 196.588, - "psnr_db_vs_none": null, - "size_ratio_vs_none": 1.0 - }, - { - "nbits": "8", - "path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_q8_unet.mlmodelc", - "size_bytes": 861946375, - "load_time_s": 0.1475, - "fwd_ms_median": 185.223, - "fwd_ms_min": 183.373, - "psnr_db_vs_none": 53.54, - "size_ratio_vs_none": 0.501 - }, - { - "nbits": "6", - "path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_q6_unet.mlmodelc", - "size_bytes": 647068433, - "load_time_s": 0.1457, - "fwd_ms_median": 181.091, - "fwd_ms_min": 180.977, - "psnr_db_vs_none": 40.2, - "size_ratio_vs_none": 0.376 - }, - { - "nbits": "4", - "path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_q4_unet.mlmodelc", - "size_bytes": 432280915, - "load_time_s": 0.1427, - "fwd_ms_median": 178.953, - "fwd_ms_min": 178.753, - "psnr_db_vs_none": 27.47, - "size_ratio_vs_none": 0.251 - } - ] -} \ No newline at end of file diff --git a/bench/results/quant_matrix_0bbd8d8.md b/bench/results/quant_matrix_0bbd8d8.md deleted file mode 100644 index 0e14a57..0000000 --- a/bench/results/quant_matrix_0bbd8d8.md +++ /dev/null @@ -1,12 +0,0 @@ -# Quantization tradeoff matrix — `0bbd8d8` - -- compute unit: CPU_AND_NE -- repeats per variant: 20 -- ckpt: v1-5-pruned-emaonly, 1x512x512, attn=se - -| nbits | size (MB) | size vs none | load (s) | fwd median (ms) | fwd min (ms) | PSNR vs none (dB) | error | -|---|---|---|---|---|---|---|---| -| none | 1641.1 | 1.0 | 0.1997 | 196.651 | 196.588 | - | | -| 8 | 822.0 | 0.501 | 0.1475 | 185.223 | 183.373 | 53.54 | | -| 6 | 617.1 | 0.376 | 0.1457 | 181.091 | 180.977 | 40.2 | | -| 4 | 412.3 | 0.251 | 0.1427 | 178.953 | 178.753 | 27.47 | | diff --git a/bench/results/smoke/ef2a18c/E2E-1.5-CoreML_00002_.png b/bench/results/smoke/ef2a18c/E2E-1.5-CoreML_00002_.png deleted file mode 100644 index a561e16..0000000 Binary files a/bench/results/smoke/ef2a18c/E2E-1.5-CoreML_00002_.png and /dev/null differ diff --git a/bench/results/smoke/ef2a18c/report.json b/bench/results/smoke/ef2a18c/report.json deleted file mode 100644 index e86e3be..0000000 --- a/bench/results/smoke/ef2a18c/report.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "git_sha": "ef2a18c", - "seed": 42, - "ckpt_name": "v1-5-pruned-emaonly.safetensors", - "workflow": "tests/integration/workflows/e2e-1.5-basic-conversion.json", - "coreml_image": "bench/results/smoke/ef2a18c/E2E-1.5-CoreML_00002_.png", - "mps_image": null, - "psnr_db": null, - "wall_seconds": 21.1 -} \ No newline at end of file diff --git a/bench/run.py b/bench/run.py deleted file mode 100644 index 37223c1..0000000 --- a/bench/run.py +++ /dev/null @@ -1,417 +0,0 @@ -#!/usr/bin/env python3 -""" -bench/run.py — Phase 1 baseline benchmark harness for ComfyUI-CoreMLSuite. - -WHAT THIS MEASURES (the deterministic, easy-to-trust numbers) -------------------------------------------------------------- -For each converted Core ML UNet model in the matrix, in-process: - * model load time (includes mlpackage compile; mlmodelc is precompiled) - * model file size on disk - * warmup (first forward) time - * steady-state UNet forward latency (mean/std/min/median over N repeats) - * an end-to-end estimate (median forward * assumed sampler steps) - * peak process RSS during the run - -It drives the Core ML model DIRECTLY with synthetic inputs shaped from the -model's own `expected_inputs`. This isolates the ANE/Core ML UNet latency — -the exact quantity that quantization (Phase 6) and the coremltools upgrade -(Phase 5) are expected to move — WITHOUT needing a live ComfyUI server, CLIP, -VAE, or a checkpoint. It is therefore the cleanest, most reproducible signal. - -WHAT THIS DOES NOT MEASURE --------------------------- - * Image quality. PSNR is a separate, decoupled concern: pass --reference-images - and --candidate-images (e.g. the E2E-1.5-MPS / E2E-1.5-CoreML PNGs your - existing integration workflow already produces) and PSNR is computed per - matching filename. Without them, quality_psnr is null (and that's honest). - * ANE / wired GPU memory. peak_rss_mb is *process* RSS only; treat as a coarse - floor, not the true device footprint. - -REQUIREMENTS ------------- - * Run on Apple Silicon (macOS) with the suite's deps installed - (coremltools + apple/ml-stable-diffusion providing `python_coreml_stable_diffusion`). - * numpy is required. psutil and Pillow are optional (graceful fallback). - -USAGE ------ - # Auto-discover every .mlmodelc/.mlpackage under a dir, sweep compute units: - python bench/run.py --models-dir /path/to/ComfyUI/models/unet - - # Explicit matrix (recommended for a stable, committed baseline): - python bench/run.py --matrix bench/matrix.json --models-dir /path/to/models/unet - - # With quality from already-produced images: - python bench/run.py --models-dir ... \ - --reference-images /path/to/mps_pngs --candidate-images /path/to/coreml_pngs - -matrix.json format (a list of configs): - [ - {"label": "sd15-se", "model": "dreamshaper_8_1x512x512_se.mlmodelc", "compute_unit": "CPU_AND_NE"}, - {"label": "sdxl-orig","model": "sdxl_base_1x1024x1024_orig.mlmodelc", "compute_unit": "CPU_AND_GPU"} - ] -Each "model" is resolved against --models-dir unless it is an absolute path. - -Output: bench/results/.json and bench/results/.md -""" - -import argparse -import dataclasses -import datetime as _dt -import glob -import json -import os -import platform -import statistics -import subprocess -import sys -import time -from pathlib import Path -from typing import Any, Optional - -import numpy as np - -# ----- optional deps (graceful) --------------------------------------------- -try: - import psutil # type: ignore - - _PROC = psutil.Process(os.getpid()) -except Exception: # pragma: no cover - optional - psutil = None - _PROC = None - -DEFAULT_COMPUTE_UNITS = ["CPU_AND_NE", "CPU_AND_GPU"] -COREML_EXTS = (".mlmodelc", ".mlpackage") - - -# ----- small helpers -------------------------------------------------------- -def git_sha() -> str: - try: - out = subprocess.check_output( - ["git", "rev-parse", "--short", "HEAD"], stderr=subprocess.DEVNULL - ) - return out.decode().strip() or "nogit" - except Exception: - return "nogit" - - -def coremltools_version() -> str: - try: - import coremltools as ct # noqa: WPS433 (local import is intentional) - - return getattr(ct, "__version__", "unknown") - except Exception as exc: # pragma: no cover - return f"import-failed: {exc}" - - -def dir_size_bytes(path: Path) -> int: - """Core ML models are directories (.mlmodelc / .mlpackage).""" - if path.is_file(): - return path.stat().st_size - total = 0 - for root, _dirs, files in os.walk(path): - for f in files: - try: - total += (Path(root) / f).stat().st_size - except OSError: - pass - return total - - -def sample_rss_mb() -> Optional[float]: - if _PROC is None: - return None - try: - return _PROC.memory_info().rss / (1024 * 1024) - except Exception: - return None - - -def detect_kind(expected_inputs: dict) -> str: - """Mirror the suite's detection WITHOUT importing comfy (keeps bench light).""" - if "time_ids" in expected_inputs and "text_embeds" in expected_inputs: - try: - n = expected_inputs["time_ids"]["shape"][1] - except Exception: - n = -1 - return "sdxl_base" if n == 6 else "sdxl_refiner" if n == 5 else "sdxl_unknown" - if "timestep_cond" in expected_inputs: - return "lcm" - return "sd15" - - -def make_inputs(expected_inputs: dict, rng: np.random.Generator) -> dict[str, np.ndarray]: - """Build synthetic fp16 inputs matching the model's expected shapes. - - Values are random; they do not affect latency in any meaningful way, and - we feed fp16 to match how CoreMLModelWrapper feeds the model at runtime. - """ - inputs: dict[str, np.ndarray] = {} - for name, spec in expected_inputs.items(): - shape = tuple(int(d) for d in spec["shape"]) - inputs[name] = rng.standard_normal(shape).astype(np.float16) - return inputs - - -def psnr(img_a: "np.ndarray", img_b: "np.ndarray") -> float: - mse = float(np.mean((img_a.astype(np.float64) - img_b.astype(np.float64)) ** 2)) - if mse == 0: - return 100.0 - return 20.0 * float(np.log10(255.0 / np.sqrt(mse))) - - -# ----- core measurement ----------------------------------------------------- -@dataclasses.dataclass -class Config: - label: str - model_path: str - compute_unit: str - - -def resolve_matrix(args) -> list[Config]: - configs: list[Config] = [] - - if args.matrix: - entries = json.loads(Path(args.matrix).read_text()) - for e in entries: - model = e["model"] - if not os.path.isabs(model): - if not args.models_dir: - raise SystemExit("matrix uses relative model names but --models-dir not given") - model = os.path.join(args.models_dir, model) - configs.append(Config(e.get("label", Path(model).stem), model, e.get("compute_unit", "CPU_AND_NE"))) - return configs - - if args.model: - for m in args.model: - for cu in args.compute_units: - configs.append(Config(f"{Path(m).stem}-{cu}", m, cu)) - return configs - - if args.models_dir: - found: list[str] = [] - for ext in COREML_EXTS: - found += glob.glob(os.path.join(args.models_dir, f"*{ext}")) - found = sorted(set(found)) - if not found: - raise SystemExit(f"No {COREML_EXTS} models found under {args.models_dir}") - for m in found: - for cu in args.compute_units: - configs.append(Config(f"{Path(m).stem}-{cu}", m, cu)) - return configs - - raise SystemExit("Provide one of: --matrix, --model, or --models-dir") - - -def measure(cfg: Config, args, rng: np.random.Generator) -> dict[str, Any]: - from python_coreml_stable_diffusion.coreml_model import CoreMLModel # local import: M2 only - - result: dict[str, Any] = { - "label": cfg.label, - "model_path": cfg.model_path, - "compute_unit": cfg.compute_unit, - "error": None, - } - path = Path(cfg.model_path) - if not path.exists(): - result["error"] = "model path does not exist" - return result - - sources = "compiled" if cfg.model_path.endswith(".mlmodelc") else "packages" - peak_rss = sample_rss_mb() - - try: - result["model_size_bytes"] = dir_size_bytes(path) - - t0 = time.perf_counter() - model = CoreMLModel(cfg.model_path, cfg.compute_unit, sources) - result["load_time_s"] = round(time.perf_counter() - t0, 4) - peak_rss = max(filter(None, [peak_rss, sample_rss_mb()]), default=None) - - expected = dict(model.expected_inputs) - result["kind"] = detect_kind(expected) - result["expected_inputs"] = {k: list(v["shape"]) for k, v in expected.items()} - - # Pre-generate all input sets so timing excludes array allocation. - warm_in = make_inputs(expected, rng) - step_inputs = [make_inputs(expected, rng) for _ in range(args.repeats)] - - # Warmup (first forward — ANE prepares its graph here). - t0 = time.perf_counter() - out = model(**warm_in) - result["warmup_ms"] = round((time.perf_counter() - t0) * 1000.0, 3) - if not (isinstance(out, dict) and "noise_pred" in out): - result["error"] = "unexpected output (no 'noise_pred')" - peak_rss = max(filter(None, [peak_rss, sample_rss_mb()]), default=None) - - # Steady-state forward latency. - times_ms: list[float] = [] - for inp in step_inputs: - t0 = time.perf_counter() - model(**inp) - times_ms.append((time.perf_counter() - t0) * 1000.0) - rss = sample_rss_mb() - if rss is not None: - peak_rss = rss if peak_rss is None else max(peak_rss, rss) - - result["forward_ms"] = { - "mean": round(statistics.fmean(times_ms), 3), - "std": round(statistics.pstdev(times_ms), 3) if len(times_ms) > 1 else 0.0, - "min": round(min(times_ms), 3), - "median": round(statistics.median(times_ms), 3), - "n": len(times_ms), - } - result["est_e2e_ms"] = round(result["forward_ms"]["median"] * args.assumed_steps, 1) - result["peak_rss_mb"] = round(peak_rss, 1) if peak_rss is not None else None - - except Exception as exc: # keep one bad model from killing the whole run - result["error"] = f"{type(exc).__name__}: {exc}" - - return result - - -def compute_quality(reference_dir: str, candidate_dir: str) -> dict[str, Any]: - try: - from PIL import Image # type: ignore - except Exception as exc: # pragma: no cover - return {"error": f"Pillow not available: {exc}"} - - ref = {p.name: p for p in Path(reference_dir).glob("*.png")} - cand = {p.name: p for p in Path(candidate_dir).glob("*.png")} - common = sorted(set(ref) & set(cand)) - pairs: dict[str, Any] = {} - for name in common: - try: - a = np.array(Image.open(ref[name]).convert("RGB")) - b = np.array(Image.open(cand[name]).convert("RGB")) - if a.shape != b.shape: - pairs[name] = {"error": f"shape mismatch {a.shape} vs {b.shape}"} - continue - pairs[name] = {"psnr_db": round(psnr(a, b), 2)} - except Exception as exc: - pairs[name] = {"error": str(exc)} - return { - "reference_dir": reference_dir, - "candidate_dir": candidate_dir, - "matched": len(common), - "pairs": pairs, - } - - -# ----- reporting ------------------------------------------------------------ -def to_markdown(report: dict[str, Any]) -> str: - lines = [ - f"# CoreMLSuite baseline — `{report['git_sha']}`", - "", - f"- timestamp: {report['timestamp']}", - f"- host: {report['host']['machine']} / {report['host']['system']} {report['host']['release']}", - f"- python: {report['host']['python']}", - f"- coremltools: {report['tool_versions']['coremltools']}, numpy: {report['tool_versions']['numpy']}", - f"- settings: repeats={report['settings']['repeats']}, " - f"input_seed={report['settings']['input_seed']}, " - f"assumed_steps={report['settings']['assumed_steps']}", - "", - "| label | kind | compute | size (MB) | load (s) | warmup (ms) | fwd median (ms) | fwd min (ms) | est e2e (ms) | peak RSS (MB) | error |", - "|---|---|---|---|---|---|---|---|---|---|---|", - ] - for r in report["results"]: - if r.get("error") and "forward_ms" not in r: - lines.append( - f"| {r['label']} | - | {r['compute_unit']} | - | - | - | - | - | - | - | {r['error']} |" - ) - continue - size_mb = round(r.get("model_size_bytes", 0) / (1024 * 1024), 1) - fwd = r.get("forward_ms", {}) - lines.append( - f"| {r['label']} | {r.get('kind','?')} | {r['compute_unit']} | {size_mb} | " - f"{r.get('load_time_s','-')} | {r.get('warmup_ms','-')} | {fwd.get('median','-')} | " - f"{fwd.get('min','-')} | {r.get('est_e2e_ms','-')} | {r.get('peak_rss_mb','-')} | " - f"{r.get('error') or ''} |" - ) - - q = report.get("quality") - if q and q.get("pairs"): - lines += ["", "## Quality (PSNR vs reference)", "", "| image | PSNR (dB) |", "|---|---|"] - for name, val in q["pairs"].items(): - lines.append(f"| {name} | {val.get('psnr_db', val.get('error','?'))} |") - - if report.get("notes"): - lines += ["", "## Notes", ""] + [f"- {n}" for n in report["notes"]] - return "\n".join(lines) + "\n" - - -def main(argv: Optional[list[str]] = None) -> int: - ap = argparse.ArgumentParser(description="CoreMLSuite Phase-1 baseline benchmark") - ap.add_argument("--matrix", help="path to matrix.json (list of {label, model, compute_unit})") - ap.add_argument("--models-dir", help="dir holding .mlmodelc/.mlpackage models") - ap.add_argument("--model", action="append", help="explicit model path (repeatable)") - ap.add_argument("--compute-units", nargs="+", default=DEFAULT_COMPUTE_UNITS, - help="compute units to sweep in auto/--model mode") - ap.add_argument("--repeats", type=int, default=30, help="steady-state forward passes") - ap.add_argument("--assumed-steps", type=int, default=20, - help="sampler steps used ONLY for the est_e2e_ms estimate") - ap.add_argument("--input-seed", type=int, default=0, help="seed for synthetic input generation") - ap.add_argument("--reference-images", help="dir of reference PNGs (e.g. MPS) for optional PSNR") - ap.add_argument("--candidate-images", help="dir of candidate PNGs (e.g. CoreML) for optional PSNR") - ap.add_argument("--out-dir", default="bench/results", help="where to write .json/.md") - args = ap.parse_args(argv) - - sha = git_sha() - rng = np.random.default_rng(args.input_seed) - configs = resolve_matrix(args) - - notes = [ - "forward_ms is the latency of a single Core ML UNet forward pass with synthetic " - "inputs; with classifier-free guidance one sampler step typically maps to one " - "batched forward (cond+uncond) — interpret est_e2e_ms accordingly.", - "peak_rss_mb is process RSS only and excludes ANE/wired GPU memory; treat as a floor.", - "quality_psnr is null unless --reference-images and --candidate-images are provided.", - "Latency varies run-to-run; 'min' is the most reproducible figure for comparisons.", - ] - if psutil is None: - notes.append("psutil not installed -> peak_rss_mb is null. `pip install psutil` to capture it.") - - print(f"[bench] git={sha} coremltools={coremltools_version()} configs={len(configs)}", file=sys.stderr) - - results = [] - for cfg in configs: - print(f"[bench] measuring {cfg.label} ({cfg.compute_unit}) ...", file=sys.stderr) - results.append(measure(cfg, args, rng)) - - report: dict[str, Any] = { - "git_sha": sha, - "timestamp": _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds"), - "host": { - "machine": platform.machine(), - "system": platform.system(), - "release": platform.release(), - "python": platform.python_version(), - }, - "tool_versions": {"coremltools": coremltools_version(), "numpy": np.__version__}, - "settings": { - "repeats": args.repeats, - "assumed_steps": args.assumed_steps, - "input_seed": args.input_seed, - "compute_units": args.compute_units, - }, - "results": results, - "notes": notes, - } - - if args.reference_images and args.candidate_images: - report["quality"] = compute_quality(args.reference_images, args.candidate_images) - - out_dir = Path(args.out_dir) - out_dir.mkdir(parents=True, exist_ok=True) - json_path = out_dir / f"{sha}.json" - md_path = out_dir / f"{sha}.md" - json_path.write_text(json.dumps(report, indent=2)) - md_path.write_text(to_markdown(report)) - - print(f"[bench] wrote {json_path}", file=sys.stderr) - print(f"[bench] wrote {md_path}", file=sys.stderr) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/bench/scripts/convert_sd15.py b/bench/scripts/convert_sd15.py deleted file mode 100644 index 5e9388e..0000000 --- a/bench/scripts/convert_sd15.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -"""bench/scripts/convert_sd15.py — Phase 1 baseline UNet conversion. - -Converts a SD1.5 checkpoint to a Core ML UNet (.mlmodelc) for the bench harness. -Bypasses the ComfyUI node graph and calls coreml_suite.converter directly so -the conversion is reproducible from a single command. - -Run from the repo root with the project venv: - .venv/bin/python bench/scripts/convert_sd15.py - -Environment overrides: - COMFY_DIR=... (default: ../..) - CKPT_NAME=v1-5-pruned-emaonly.safetensors - ATTN=SPLIT_EINSUM (SPLIT_EINSUM | SPLIT_EINSUM_V2 | ORIGINAL) - HEIGHT=512 WIDTH=512 BATCH_SIZE=1 - CONTROLNET=0 -""" -import logging -import os -import sys -from pathlib import Path - -logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") -log = logging.getLogger("convert_sd15") - -REPO_ROOT = Path(__file__).resolve().parents[2] -COMFY_DIR = Path(os.environ.get("COMFY_DIR", REPO_ROOT.parents[1])).resolve() - -if str(COMFY_DIR) not in sys.path: - sys.path.insert(0, str(COMFY_DIR)) -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - -import folder_paths # noqa: E402 (ComfyUI module — needs sys.path above) - -# Ensure ComfyUI's folder_paths is pointed at the real install for checkpoints. -folder_paths.base_path = str(COMFY_DIR) -folder_paths.add_model_folder_path("checkpoints", str(COMFY_DIR / "models" / "checkpoints")) -folder_paths.add_model_folder_path("unet", str(COMFY_DIR / "models" / "unet")) - -from coreml_suite import converter # noqa: E402 -from coreml_suite.config import ModelVersion # noqa: E402 - - -def main() -> int: - ckpt_name = os.environ.get("CKPT_NAME", "v1-5-pruned-emaonly.safetensors") - attn = os.environ.get("ATTN", "SPLIT_EINSUM") - height = int(os.environ.get("HEIGHT", "512")) - width = int(os.environ.get("WIDTH", "512")) - batch_size = int(os.environ.get("BATCH_SIZE", "1")) - controlnet = os.environ.get("CONTROLNET", "0") not in ("0", "false", "False", "") - quantize_nbits = os.environ.get("QUANT_NBITS", "none") - - ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name) - if not ckpt_path: - log.error("checkpoint not found: %s under %s", ckpt_name, COMFY_DIR / "models" / "checkpoints") - return 2 - - from coreml_suite.core.naming import compose_out_name - - out_name = compose_out_name( - ckpt_name=ckpt_name, - batch_size=batch_size, - width=width, - height=height, - controlnet_support=controlnet, - attention_implementation=attn, - quantize_nbits=quantize_nbits, - ) - unet_out_path = converter.get_out_path("unet", out_name) - - log.info("repo_root=%s comfy_dir=%s", REPO_ROOT, COMFY_DIR) - log.info("ckpt=%s out_name=%s", ckpt_path, out_name) - log.info( - "attn=%s size=%dx%d batch=%d controlnet=%s quant=%s", - attn, width, height, batch_size, controlnet, quantize_nbits, - ) - - converter.convert( - ckpt_path=ckpt_path, - model_version=ModelVersion.SD15, - unet_out_path=unet_out_path, - batch_size=batch_size, - sample_size=(height // 8, width // 8), - controlnet_support=controlnet, - lora_weights=[], - attn_impl=attn, - config_path=None, - quantize_nbits=quantize_nbits, - ) - - target_path = converter.compile_model(out_path=unet_out_path, out_name=out_name, submodule_name="unet") - log.info("compiled: %s", target_path) - print(target_path) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/bench/scripts/quant_matrix.py b/bench/scripts/quant_matrix.py deleted file mode 100644 index d35456a..0000000 --- a/bench/scripts/quant_matrix.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python3 -"""bench/scripts/quant_matrix.py — Phase 6 quantization tradeoff matrix. - -For each variant in {none, 8, 6, 4}, load the corresponding .mlmodelc, -run a synthetic forward pass with a fixed seed, and record: - - on-disk size - - load time - - forward-pass median latency - - PSNR of noise_pred vs the `none` baseline (proxy for output drift) - -Writes a JSON + Markdown summary into bench/results/quant_matrix_.{json,md}. - -Assumes converted .mlmodelc files exist under $COMFY_DIR/models/unet/ -following the convention from coreml_suite.core.naming.compose_out_name -(e.g. v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc for the baseline, -v1-5-pruned-emaonly_1x512x512_se_q4_unet.mlmodelc for 4-bit, etc.). -""" -import json -import os -import statistics -import subprocess -import sys -import time -from pathlib import Path -from typing import Any - -import numpy as np - -REPO_ROOT = Path(__file__).resolve().parents[2] -COMFY_DIR = Path(os.environ.get("COMFY_DIR", REPO_ROOT.parents[1])).resolve() -MODELS_DIR = COMFY_DIR / "models" / "unet" - -VARIANTS = ["none", "8", "6", "4"] -CKPT_STEM = os.environ.get("CKPT_STEM", "v1-5-pruned-emaonly") -BATCH = int(os.environ.get("BATCH", "1")) -WIDTH = int(os.environ.get("WIDTH", "512")) -HEIGHT = int(os.environ.get("HEIGHT", "512")) -ATTN_SUFFIX = os.environ.get("ATTN_SUFFIX", "se") -COMPUTE_UNIT = os.environ.get("COMPUTE_UNIT", "CPU_AND_NE") -REPEATS = int(os.environ.get("REPEATS", "20")) -INPUT_SEED = int(os.environ.get("INPUT_SEED", "0")) - - -def variant_path(nbits: str) -> Path: - quant_suffix = f"_q{nbits}" if nbits != "none" else "" - name = f"{CKPT_STEM}_{BATCH}x{WIDTH}x{HEIGHT}_{ATTN_SUFFIX}{quant_suffix}_unet.mlmodelc" - return MODELS_DIR / name - - -def dir_size_bytes(path: Path) -> int: - total = 0 - for root, _dirs, files in os.walk(path): - for f in files: - try: - total += (Path(root) / f).stat().st_size - except OSError: - pass - return total - - -def psnr(a: np.ndarray, b: np.ndarray) -> float: - """Float PSNR for noise_pred tensors normalized to a [-1, 1]-ish range.""" - a = a.astype(np.float64) - b = b.astype(np.float64) - peak = max(float(np.max(np.abs(a))), float(np.max(np.abs(b))), 1.0) - mse = float(np.mean((a - b) ** 2)) - if mse == 0: - return 100.0 - return 20.0 * float(np.log10(peak / np.sqrt(mse))) - - -def measure_variant( - nbits: str, ref_output: "np.ndarray | None" -) -> "tuple[dict[str, Any], np.ndarray | None]": - from python_coreml_stable_diffusion.coreml_model import CoreMLModel - - path = variant_path(nbits) - result: dict[str, Any] = {"nbits": nbits, "path": str(path)} - if not path.exists(): - result["error"] = f"model not found at {path}" - return result, None - result["size_bytes"] = dir_size_bytes(path) - - rng = np.random.default_rng(INPUT_SEED) - t0 = time.perf_counter() - model = CoreMLModel(str(path), COMPUTE_UNIT, "compiled") - result["load_time_s"] = round(time.perf_counter() - t0, 4) - - expected = dict(model.expected_inputs) - fixed_inputs = { - name: rng.standard_normal(tuple(int(d) for d in spec["shape"])).astype(np.float16) - for name, spec in expected.items() - } - - # Warmup - out = model(**fixed_inputs) - times = [] - for _ in range(REPEATS): - # Fresh randomness per repeat for steady-state timing, but keep - # output capture deterministic (we run fixed_inputs at end). - live = { - name: rng.standard_normal(tuple(int(d) for d in spec["shape"])).astype(np.float16) - for name, spec in expected.items() - } - t0 = time.perf_counter() - model(**live) - times.append((time.perf_counter() - t0) * 1000.0) - result["fwd_ms_median"] = round(statistics.median(times), 3) - result["fwd_ms_min"] = round(min(times), 3) - - # Deterministic forward for PSNR comparison across variants. - deterministic_out = model(**fixed_inputs)["noise_pred"] - if ref_output is None: - result["psnr_db_vs_none"] = None - else: - result["psnr_db_vs_none"] = round(psnr(ref_output, deterministic_out), 2) - return result, deterministic_out - - -def main() -> int: - try: - sha = subprocess.check_output( - ["git", "-C", str(REPO_ROOT), "rev-parse", "--short", "HEAD"], - stderr=subprocess.DEVNULL, - ).decode().strip() - except Exception: - sha = "nogit" - - out_dir = REPO_ROOT / "bench" / "results" - out_dir.mkdir(parents=True, exist_ok=True) - - print(f"[quant-matrix] git={sha} compute_unit={COMPUTE_UNIT} repeats={REPEATS}", file=sys.stderr) - - rows: list[dict[str, Any]] = [] - ref_output: np.ndarray | None = None - for v in VARIANTS: - print(f"[quant-matrix] measuring nbits={v} ...", file=sys.stderr) - try: - row, out_tensor = measure_variant(v, ref_output) - if v == "none": - ref_output = out_tensor - rows.append(row) - except Exception as exc: - rows.append({"nbits": v, "error": f"{type(exc).__name__}: {exc}"}) - - baseline_size = next((r["size_bytes"] for r in rows if r.get("nbits") == "none" and "size_bytes" in r), None) - for r in rows: - if "size_bytes" in r and baseline_size: - r["size_ratio_vs_none"] = round(r["size_bytes"] / baseline_size, 3) - - report = { - "git_sha": sha, - "compute_unit": COMPUTE_UNIT, - "repeats": REPEATS, - "input_seed": INPUT_SEED, - "ckpt_stem": CKPT_STEM, - "rows": rows, - } - (out_dir / f"quant_matrix_{sha}.json").write_text(json.dumps(report, indent=2)) - - lines = [ - f"# Quantization tradeoff matrix — `{sha}`", - "", - f"- compute unit: {COMPUTE_UNIT}", - f"- repeats per variant: {REPEATS}", - f"- ckpt: {CKPT_STEM}, {BATCH}x{WIDTH}x{HEIGHT}, attn={ATTN_SUFFIX}", - "", - "| nbits | size (MB) | size vs none | load (s) | fwd median (ms) | fwd min (ms) | PSNR vs none (dB) | error |", - "|---|---|---|---|---|---|---|---|", - ] - for r in rows: - if "error" in r and "size_bytes" not in r: - lines.append(f"| {r.get('nbits','?')} | - | - | - | - | - | - | {r['error']} |") - continue - size_mb = round(r.get("size_bytes", 0) / (1024 * 1024), 1) - lines.append( - f"| {r['nbits']} | {size_mb} | {r.get('size_ratio_vs_none','-')} | " - f"{r.get('load_time_s','-')} | {r.get('fwd_ms_median','-')} | " - f"{r.get('fwd_ms_min','-')} | {r.get('psnr_db_vs_none') if r.get('psnr_db_vs_none') is not None else '-'} | " - f"{r.get('error','')} |" - ) - (out_dir / f"quant_matrix_{sha}.md").write_text("\n".join(lines) + "\n") - print(f"[quant-matrix] wrote {out_dir / f'quant_matrix_{sha}.md'}", file=sys.stderr) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/bench/scripts/smoke_image.py b/bench/scripts/smoke_image.py deleted file mode 100644 index 74fa1fa..0000000 --- a/bench/scripts/smoke_image.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python3 -"""bench/scripts/smoke_image.py — Phase 1 known-good-workflow image smoke test. - -Posts a Stable Diffusion 1.5 workflow (CLIP → Core ML UNet → VAE → PNG) to a -locally running ComfyUI server, waits for the queue to drain, then copies the -generated CoreML + MPS reference images into bench/results/smoke// -and writes a small report.json with the PSNR between them. This is the Phase 1 -"known-good workflow still produces an image [M2-ANE]" artifact. - -Prereqs: - - ComfyUI server running locally on $COMFY_HOST:$COMFY_PORT (default - http://localhost:8188), started against the project .venv so it picks up - the pinned ml-stable-diffusion + Core ML Suite nodes. - - The Core ML UNet has already been converted (see convert_sd15.py); the - workflow's Core ML Converter node will skip if the .mlmodelc exists. - -Run from the repo root: - .venv/bin/python bench/scripts/smoke_image.py -""" -import json -import os -import shutil -import subprocess -import sys -import time -import urllib.error -import urllib.request -from pathlib import Path - -import numpy as np -from PIL import Image - -REPO_ROOT = Path(__file__).resolve().parents[2] -COMFY_DIR = Path(os.environ.get("COMFY_DIR", REPO_ROOT.parents[1])).resolve() -COMFY_HOST = os.environ.get("COMFY_HOST", "localhost") -COMFY_PORT = int(os.environ.get("COMFY_PORT", "8188")) -COMFY_URL = f"http://{COMFY_HOST}:{COMFY_PORT}" - -CKPT_NAME = os.environ.get("CKPT_NAME", "v1-5-pruned-emaonly.safetensors") -WORKFLOW_PATH = Path(os.environ.get( - "WORKFLOW_PATH", - REPO_ROOT / "tests" / "integration" / "workflows" / "e2e-1.5-basic-conversion.json", -)) -TIMEOUT_S = int(os.environ.get("TIMEOUT_S", "1800")) # 30min cap - - -def git_sha() -> str: - try: - return subprocess.check_output( - ["git", "-C", str(REPO_ROOT), "rev-parse", "--short", "HEAD"], - stderr=subprocess.DEVNULL, - ).decode().strip() - except Exception: - return "nogit" - - -def http_get_json(path: str) -> dict: - with urllib.request.urlopen(f"{COMFY_URL}{path}", timeout=60) as r: - return json.loads(r.read().decode()) - - -def http_post_json(path: str, payload: dict) -> dict: - data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request( - f"{COMFY_URL}{path}", data=data, - headers={"Content-Type": "application/json"}, method="POST", - ) - # Long timeout: first POST blocks while the server warms model loaders + - # the Core ML compile-on-load pass (~60-90s on a cold cache). - with urllib.request.urlopen(req, timeout=300) as r: - return json.loads(r.read().decode()) - - -def psnr(a: np.ndarray, b: np.ndarray) -> float: - mse = float(np.mean((a.astype(np.float64) - b.astype(np.float64)) ** 2)) - if mse == 0: - return 100.0 - return 20.0 * float(np.log10(255.0 / np.sqrt(mse))) - - -def find_image(out_dir: Path, prefix: str) -> Path | None: - matches = sorted(out_dir.glob(f"{prefix}_*.png"), reverse=True) - return matches[0] if matches else None - - -def main() -> int: - sha = git_sha() - out_root = REPO_ROOT / "bench" / "results" / "smoke" / sha - out_root.mkdir(parents=True, exist_ok=True) - - workflow = json.loads(WORKFLOW_PATH.read_text()) - # Point both checkpoint nodes at the maintainer's available SD1.5 ckpt. - for nid in ("4", "10"): - if nid in workflow: - workflow[nid]["inputs"]["ckpt_name"] = CKPT_NAME - # Fixed seed for reproducibility (Phase 1 baseline). - seed = int(os.environ.get("SEED", "42")) - for nid in ("3", "11"): - if nid in workflow and "seed" in workflow[nid].get("inputs", {}): - workflow[nid]["inputs"]["seed"] = seed - # Phase 1 only needs a Core ML image. The reference MPS branch (nodes - # 3/8/9) trips a known MPS f16/f32 mismatch on torch 2.0.1 + macOS 26+, - # which would crash the whole server. Strip those nodes so the queue - # only runs the Core ML path; SKIP_MPS=0 to opt back in. - if os.environ.get("SKIP_MPS", "1") not in ("0", "false", "False"): - for nid in ("3", "8", "9"): - workflow.pop(nid, None) - - print(f"[smoke] git={sha} server={COMFY_URL} ckpt={CKPT_NAME} seed={seed}", file=sys.stderr) - - resp = http_post_json("/prompt", {"prompt": workflow}) - print(f"[smoke] queued: {resp}", file=sys.stderr) - - t0 = time.time() - while True: - if time.time() - t0 > TIMEOUT_S: - print(f"[smoke] TIMEOUT after {TIMEOUT_S}s waiting for queue drain", file=sys.stderr) - return 2 - try: - q = http_get_json("/prompt") - remaining = q.get("exec_info", {}).get("queue_remaining", -1) - if remaining == 0: - break - except (urllib.error.URLError, json.JSONDecodeError) as exc: - print(f"[smoke] poll error: {exc}", file=sys.stderr) - time.sleep(2) - - comfy_out = COMFY_DIR / "output" - coreml_png = find_image(comfy_out, "E2E-1.5-CoreML") - if coreml_png is None: - print(f"[smoke] missing Core ML image under {comfy_out}", file=sys.stderr) - return 3 - coreml_dst = out_root / coreml_png.name - shutil.copy2(coreml_png, coreml_dst) - - mps_png = find_image(comfy_out, "E2E-1.5-MPS") - mps_dst = None - psnr_db = None - if mps_png is not None: - mps_dst = out_root / mps_png.name - shutil.copy2(mps_png, mps_dst) - a = np.array(Image.open(coreml_dst).convert("RGB")) - b = np.array(Image.open(mps_dst).convert("RGB")) - if a.shape == b.shape: - psnr_db = round(psnr(a, b), 2) - - report = { - "git_sha": sha, - "seed": seed, - "ckpt_name": CKPT_NAME, - "workflow": str(WORKFLOW_PATH.relative_to(REPO_ROOT)), - "coreml_image": str(coreml_dst.relative_to(REPO_ROOT)), - "mps_image": str(mps_dst.relative_to(REPO_ROOT)) if mps_dst else None, - "psnr_db": psnr_db, - "wall_seconds": round(time.time() - t0, 1), - } - (out_root / "report.json").write_text(json.dumps(report, indent=2)) - print(json.dumps(report, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/constraints/comfy-ceiling.txt b/constraints/comfy-ceiling.txt index c582dec..013d2b6 100644 --- a/constraints/comfy-ceiling.txt +++ b/constraints/comfy-ceiling.txt @@ -3,14 +3,14 @@ # # ComfyUI's requirements.txt requests bare `torch`/`torchvision`/`torchaudio` # and `numpy>=1.25.0`, which would float past the versions coremltools 9 / -# apple-ml-stable-diffusion have been validated against (see docs/deps.md). +# apple-ml-stable-diffusion have been validated against. # These constraints cap the resolution so the canary keeps testing the same # toolchain the suite actually ships. # # If upstream ComfyUI ever hard-requires something beyond these bounds, the # install FAILS — and that failure is the signal we want: it means the host # outgrew the pinned toolchain and coremltools / ml-stable-diffusion need a -# deliberate bump (a Phase 5-style upgrade), not a silent float. +# deliberate bump, not a silent float. torch>=2.7,<2.8 torchvision>=0.22,<0.23 torchaudio>=2.7,<2.8 diff --git a/coreml_suite/controlnet.py b/coreml_suite/controlnet.py index 141b471..dbac4c4 100644 --- a/coreml_suite/controlnet.py +++ b/coreml_suite/controlnet.py @@ -1,4 +1,4 @@ -"""Phase 3 compatibility shim — re-exports from coreml_suite.core.controlnet.""" +"""Compatibility shim — re-exports from coreml_suite.core.controlnet.""" from coreml_suite.core.controlnet import ( chunk_control, expand_inputs, diff --git a/coreml_suite/converter.py b/coreml_suite/converter.py index c2a5b55..1af74cd 100644 --- a/coreml_suite/converter.py +++ b/coreml_suite/converter.py @@ -307,8 +307,8 @@ def convert_unet( gc.collect() if quantize_nbits != "none": - # Phase 6: opt-in k-means weight palettization. Default path - # (quantize_nbits="none") is byte-for-byte unchanged from Phase 5. + # Opt-in k-means weight palettization. The default path + # (quantize_nbits="none") leaves the traced UNet untouched. from coremltools.optimize.coreml import ( OpPalettizerConfig, OptimizationConfig, diff --git a/coreml_suite/core/controlnet.py b/coreml_suite/core/controlnet.py index 0f063d3..72a1ba5 100644 --- a/coreml_suite/core/controlnet.py +++ b/coreml_suite/core/controlnet.py @@ -1,7 +1,7 @@ """Pure helpers around the ControlNet residual inputs of the Core ML UNet. -Moved from coreml_suite.controlnet in Phase 3. Behavior unchanged — Phase 2 -characterization tests cover shapes, dtype (fp16), and zero-fill fallback. +Re-exported by coreml_suite.controlnet. Characterization tests cover +shapes, dtype (fp16), and zero-fill fallback. """ from itertools import chain from math import ceil diff --git a/coreml_suite/core/inputs.py b/coreml_suite/core/inputs.py index 9263a47..b875c5a 100644 --- a/coreml_suite/core/inputs.py +++ b/coreml_suite/core/inputs.py @@ -1,7 +1,6 @@ """Pure transform from torch sampler inputs to Core ML UNet kwargs. -Moved from coreml_suite.models in Phase 3. Behavior unchanged — Phase 2 -characterization tests cover SD1.5 / SDXL base / SDXL refiner / LCM +Characterization tests cover SD1.5 / SDXL base / SDXL refiner / LCM variants and the chunked-batch fan-out. """ import numpy as np diff --git a/coreml_suite/core/latents.py b/coreml_suite/core/latents.py index 58f72ce..d6ad528 100644 --- a/coreml_suite/core/latents.py +++ b/coreml_suite/core/latents.py @@ -1,8 +1,8 @@ """Pure batch-chunking helpers for Core ML's fixed-shape UNet inputs. -Moved from coreml_suite.latents in Phase 3. Behavior unchanged — Phase 2 -characterization tests cover the contract (padding-zero regions, truncation -in merge_chunks, identity-passthrough when shape already matches). +Re-exported by coreml_suite.latents. Characterization tests cover the +contract (padding-zero regions, truncation in merge_chunks, +identity-passthrough when shape already matches). """ import torch diff --git a/coreml_suite/core/naming.py b/coreml_suite/core/naming.py index 16f2024..71d186d 100644 --- a/coreml_suite/core/naming.py +++ b/coreml_suite/core/naming.py @@ -1,6 +1,6 @@ """Pure out_name composition for the Core ML UNet artifact. -Extracted from CoreMLConverter.convert in Phase 3 so the filename contract +Extracted from CoreMLConverter.convert so the filename contract can be tested + reused without instantiating the node. The string is the cache key: every workflow that references a converted .mlpackage depends on it staying byte-for-byte identical. @@ -13,8 +13,8 @@ ATTN_SUFFIX = { "ORIGINAL": "orig", } -# Phase 6: palettization bits. "none" = no quantization (default; keeps the -# pre-Phase-6 filename intact so existing workflows still resolve their +# Palettization bits. "none" = no quantization (default; keeps the +# unquantized filename intact so existing workflows still resolve their # cached .mlpackage). Numeric values append a `_q` suffix. QUANT_NBITS_VALUES = ("none", "8", "6", "4") @@ -32,7 +32,7 @@ def compose_out_name( ) -> str: """Build the .mlpackage stem from convert() parameters. - Locked behaviour (Phase 2 characterization tests): + Locked behaviour (characterization tests): - first '.' in ckpt_name wins (`a.b.c.safetensors` -> `a`) - spaces collapse to underscores - LoRA names are taken stem-only, sorted, joined with '_' and @@ -41,7 +41,7 @@ def compose_out_name( - controlnet adds `_cn` - attn suffix is `_se` | `_se2` | `_orig` - Phase 6 addition: + Quantization: - quantize_nbits "none" (default) appends nothing — existing unquantized .mlpackages keep the old filename - "4" / "6" / "8" appends `_q` after the attn suffix diff --git a/coreml_suite/core/sdxl.py b/coreml_suite/core/sdxl.py index 7ea16eb..fd1e8d2 100644 --- a/coreml_suite/core/sdxl.py +++ b/coreml_suite/core/sdxl.py @@ -1,9 +1,9 @@ """Pure SDXL detection + time_ids/text_embeds assembly. -Moved from coreml_suite.models in Phase 3. The framework-coupled adapter -`add_sdxl_model_options` lives in models.py and now delegates the math -here. Phase 2 characterization tests cover base (len 6) vs refiner -(len 5) and the closure free-vars produced by `sdxl_model_function_wrapper`. +The framework-coupled adapter `add_sdxl_model_options` lives in models.py +and delegates the math here. Characterization tests cover base (len 6) vs +refiner (len 5) and the closure free-vars produced by +`sdxl_model_function_wrapper`. """ import torch diff --git a/coreml_suite/latents.py b/coreml_suite/latents.py index d2896df..b3b55d6 100644 --- a/coreml_suite/latents.py +++ b/coreml_suite/latents.py @@ -1,4 +1,4 @@ -"""Phase 3 compatibility shim — re-exports from coreml_suite.core.latents.""" +"""Compatibility shim — re-exports from coreml_suite.core.latents.""" from coreml_suite.core.latents import chunk_batch, merge_chunks __all__ = ["chunk_batch", "merge_chunks"] diff --git a/coreml_suite/models.py b/coreml_suite/models.py index f4bc360..358d7b1 100644 --- a/coreml_suite/models.py +++ b/coreml_suite/models.py @@ -1,7 +1,7 @@ """Framework-coupled glue between Core ML UNets and ComfyUI's sampler stack. Pure math (CoreMLInputs, SDXL detection, time_ids/text_embeds assembly, -sdxl_model_function_wrapper) lives in coreml_suite.core.* after Phase 3. +sdxl_model_function_wrapper) lives in coreml_suite.core.*. This module is what touches comfy.*: model_base, ModelPatcher, the diffusion_model wrapper, and the maintainer-facing add_sdxl_model_options adapter. diff --git a/coreml_suite/nodes.py b/coreml_suite/nodes.py index 96f1d78..00025fc 100644 --- a/coreml_suite/nodes.py +++ b/coreml_suite/nodes.py @@ -249,12 +249,11 @@ class CoreMLConverter(COREML_NODE): "controlnet_support": ("BOOLEAN", {"default": False}), }, "optional": { - # Phase 6: k-means weight palettization. Kept optional so - # workflows saved before Phase 6 still validate — ComfyUI - # rejects a prompt that omits any `required` input. When - # omitted it defaults to "none", identical to pre-Phase-6 - # behavior and filename, so existing cached .mlpackages - # still resolve. + # k-means weight palettization. Kept optional so workflows + # that omit it still validate — ComfyUI rejects a prompt that + # omits any `required` input. When omitted it defaults to + # "none", identical to unquantized behavior and filename, so + # existing cached .mlpackages still resolve. "quantize_nbits": (list(QUANT_NBITS_VALUES), {"default": "none"}), "lora_params": ("LORA_PARAMS",), }, diff --git a/docs/ci-m2.md b/docs/ci-m2.md deleted file mode 100644 index dee42cc..0000000 --- a/docs/ci-m2.md +++ /dev/null @@ -1,137 +0,0 @@ -# Self-hosted M2 runner — setup - -Tier 2 (ANE + integration + bench) runs on a self-hosted GitHub Actions -runner registered against the maintainer's M-series Mac. Hosted macOS -runners on GitHub do not expose the Apple Neural Engine, so the ANE -half of the matrix has to live on real hardware. - -## One-time runner setup - -1. **Install dependencies on the Mac.** Python 3.12.x (matching the - `requires-python = ">=3.12,<3.13"` pin), `uv`, and `git`. The workflow - clones and manages the ComfyUI checkout itself (see step 3), so you do - not pre-install ComfyUI. - - ```bash - brew install python@3.12 uv git - ``` - -2. **Register the runner.** From the repo Settings → Actions → Runners - → New self-hosted runner, follow the macOS-ARM instructions. Add the - labels exactly: `self-hosted`, `macOS`, `ARM64`, `coreml` (the - workflow `runs-on` clause requires all four). - - ```bash - mkdir ~/actions-runner && cd ~/actions-runner - curl -O -L https://github.com/actions/runner/releases/download/v2.317.0/actions-runner-osx-arm64-2.317.0.tar.gz - tar xzf actions-runner-osx-arm64-2.317.0.tar.gz - ./config.sh --url https://github.com// \ - --token \ - --labels self-hosted,macOS,ARM64,coreml \ - --name "$(hostname)-m2" - ./svc.sh install && ./svc.sh start # run as a launchd service - ``` - -3. **Persist `COMFY_DIR` for the runner.** Point it at a **dedicated, - runner-owned** ComfyUI directory — **not** your personal dev checkout. - The workflow does `git reset --hard` on it and rewrites its - `custom_nodes/ComfyUI-CoreMLSuite` symlink, so it must be disposable. - Add it to the runner's `.env`: - - ```bash - echo 'COMFY_DIR=/Users//actions-runner/comfyui' >> ~/actions-runner/.env - ``` - - You don't have to clone ComfyUI yourself: the `Set up ComfyUI checkout` - step clones it on first run, checks out the right ref (latest or the - pinned SHA — see *ComfyUI version under test*), installs ComfyUI's deps, - and symlinks `$COMFY_DIR/custom_nodes/ComfyUI-CoreMLSuite` → - `$GITHUB_WORKSPACE` so the server always loads the checked-out PR. If - `COMFY_DIR` is a real node directory rather than a symlink, the step - fails loudly instead of deleting it. - -4. **Provide the SD1.5 checkpoint.** Drop - `v1-5-pruned-emaonly.safetensors` (~4 GB) into - `$COMFY_DIR/models/checkpoints/`. This is the one heavy artifact that - stays as a runner-local cache; everything derived from it (the - `.mlmodelc` UNet variants) is converted automatically by the - `Convert UNet variants if missing` step and cached across runs. Override - the filename with `CKPT_NAME` in the runner `.env` if needed. - - Order is free: do this before or after the first run. The setup step - initialises the ComfyUI repo **in place** (`git init`, not `git clone`, - which would refuse a non-empty directory) and uses `checkout -f`, which - never removes untracked files — so a checkpoint you dropped in first is - preserved. - -## Triggers - -The Tier 2 workflow (`.github/workflows/tier2.yml`) runs: - -- **On PR label `run-m2`** — maintainers add the label to opt a PR - into the ANE lane (the runner is not free; default off). -- **Nightly at 04:00 UTC** via `schedule:`. -- **Manually** via the workflow_dispatch button. - -## ComfyUI version under test - -Tier 2 runs a **hybrid** strategy, keyed on the trigger, so the suite tracks a -moving host without making PRs flaky to overnight upstream drift: - -| Trigger | ComfyUI ref | ComfyUI deps | -|---|---|---| -| **schedule** (nightly) | latest `origin/master` | its own `requirements.txt`, capped by `constraints/comfy-ceiling.txt` | -| **PR label `run-m2`** / **dispatch** | pinned `requires-comfyui` SHA | the frozen `comfy` uv group | - -- **Nightly = canary.** It pulls the latest ComfyUI and installs *ComfyUI's - own* dependency set. The frozen `comfy` uv group cannot track a moving host - by hand (a latest checkout needs e.g. `comfyui-frontend-package==1.44.19`, - `comfy_aimdo`, `alembic`, `blake3` that an older pin never listed), so latest - mode defers to upstream's `requirements.txt`. Upstream API or dependency - breakage surfaces here, in CI, instead of in a user's install. -- **PR / dispatch = reproducible gate.** It checks out the pinned - `requires-comfyui` SHA and uses the frozen `comfy` group — the known-good - combination — so a PR fails for its own reasons, not because ComfyUI moved. - -The resolved ComfyUI SHA and mode are written to the job step summary (and -`COMFY_SHA` in the env), so any failure names the exact commit it hit. - -**The toolchain ceiling is deliberate.** `constraints/comfy-ceiling.txt` caps -`torch<2.8` / `numpy<2` / `coremltools 9` while installing latest ComfyUI's -requirements. If upstream ever hard-requires something past those bounds the -nightly install *fails on purpose* — that is the signal that coremltools / -`ml-stable-diffusion` need a deliberate Phase-5-style bump, not a silent float -that would break the ANE path (see `docs/deps.md`). - -`pyproject.toml`'s `requires-comfyui` is both the PR-gate ref and the -published-compatibility declaration for the Comfy registry. Bump it once a -newer ComfyUI is validated (the nightly canary is what tells you it's safe). - -## Artifacts - -- Bench JSON/MD are uploaded as `bench-results`. -- M2 golden image regressions surface as test failures in - `tests/m2/test_golden_image.py`; diff PNG is written next to the - golden under `tests/m2/_latest_generated.png` (gitignored). - -## When the runner is down - -If the maintainer's Mac is offline, the workflow queues until the -runner comes back. Cancel a stuck run from the Actions UI; the gate is -not blocking by default (Tier 0 + Tier 1 carry PR status). Tier 2 is -"good to merge once it goes green," not "blocked until then." - -## Replacing the integration e2e - -The legacy `tests/integration/test_basic_conversion_1_5.py` checked -CoreML output against an MPS reference image at PSNR > 25 dB. That -reference path is broken on macOS 26.x with torch 2.0.1 (see Phase 1 -Gate report). Phase 4 moves the same coverage to -`tests/m2/test_golden_image.py`, which: - -- runs the Core ML pipeline only (no MPS reference), -- asserts SHA256 against `tests/m2/goldens/sd15_seed42.sha256`, -- falls back to PSNR ≥ 40 dB if the hash drifts. - -This removes the human-eyeball dependency: a regression is now a -numerical fail, not a "looks different to me." diff --git a/docs/deps.md b/docs/deps.md deleted file mode 100644 index 3606707..0000000 --- a/docs/deps.md +++ /dev/null @@ -1,163 +0,0 @@ -# Dependency decisions - -## Phase 5: ml-stable-diffusion compatibility under the bumped toolchain - -### Scope - -Bumped: Python 3.11 → 3.12, torch 2.0.1 → 2.7.1, coremltools 8.2 → 9.0. -**Not bumped:** numpy stays in the 1.24..1.x range. Phase 5 spec asked -for numpy 2 as a goal; in practice the bump is decoupled — see -"Why not numpy 2" below. - -### Problem - -`python_coreml_stable_diffusion` pins three transitive dependencies in -its `setup.py` that block any modern install: - -```python -install_requires=[ - "coremltools>=8.0", - "diffusers[torch]==0.30.2", - "transformers==4.44.2", - "huggingface-hub==0.24.6", - "numpy<1.24", - ... -] -``` - -The exact-pin lines on diffusers/transformers/huggingface-hub block the -Python-3.12 / torch-2.7 / coremltools-9 combo. The numpy ceiling at -1.24 also blocks newer numpy. - -The upstream `main` branch as of 2026-05-24 (commit `e12202c1f`) has -the same pins as our Phase 1 `e5d960c4` SHA — no working alternative -exists upstream. No maintained fork on PyPI either. - -### Decision - -**Keep the SHA pinned at `e5d960c4`** (Phase 1 baseline) and apply -`[tool.uv] override-dependencies` to relax the four blocking pins: - -```toml -[tool.uv] -override-dependencies = [ - "numpy>=1.24,<2", - "diffusers>=0.30", - "transformers>=4.44", - "huggingface-hub>=0.24", -] -``` - -### Why not bump the SHA, vendor the code, or fork? - -- **Bump SHA:** the only newer SHA on `main` (`e12202c1f`) has identical - pins. No upstream fix. -- **Vendor the imports:** our code only uses `unet.UNet2DConditionModel`, - `unet.UNet2DConditionModelXL`, `unet.AttentionImplementations`, - `unet.calculate_conv2d_output_shape`, and `coreml_model.CoreMLModel`. - Copying these into `coreml_suite/_vendor/` would work but adds - hundreds of lines of code, owns a maintenance burden we don't want - yet, and detaches us from upstream bug fixes. -- **Public fork:** would also need maintenance + CI to keep current with - Apple's main. - -The override is the smallest workable patch: it trusts that the API -surface we touch (`unet.*` + `coreml_model.CoreMLModel`) is stable -across the bumped versions — verified empirically by Tier 1 (synthetic -UNet round-trip through `ct.convert`) and Tier 2 (real SD1.5 conversion -+ golden image PSNR ≥ 25 dB). - -### Why not numpy 2 - -The Phase 5 spec asked for numpy 2 alongside the Python / coremltools / -torch bumps. Tier 0 + Tier 1 are happy on numpy 2 — none of our own -modules touch numpy in a way that breaks. The SD UNet conversion path -is not: - -1. `coremltools 9.0 (and 8.3) `_cast(int)` in - `converters/mil/frontend/torch/ops.py` does - `dtype(x.val)` where `x.val` is a numpy ndarray with a single - element. Under numpy 2 that raises `TypeError: only 0-dimensional - arrays can be converted to Python scalars`. Fixable via a - `np.ndarray.item()` shim. -2. After patching (1), `view` then trips - `mb.cast(x=shape, dtype="int32")` where `shape` is a Python list of - non-scalar `Var`s — a code path the upstream guard - `all([isinstance(dim, Var) and len(dim.shape) == 0 for dim in shape])` - doesn't cover. Different bug class; needs a separate, larger shim - on the `view` op (and probably more elsewhere — each patch reveals - the next). - -Neither bug is a Phase-5 deliverable. None of our code paths require -numpy 2. **Decision: hold numpy at >=1.24,<2 and unblock the rest of -the bump.** numpy 2 stays as an explicit follow-up once coremltools -ships an upstream fix (or we sign up for the shim work). - -### Tier 2 PSNR threshold - -Phase 2 anchored the golden image as a SHA256 + 40 dB PSNR fallback. -The toolchain bump changes the bit-exact output: the SD1.5 baseline at -seed=42 lands at ~29 dB against the Phase 2 PNG. The image is -visually identical (same composition, same colors, sub-pixel drift), -just numerically different — typical for a coremltools/torch upgrade. - -`tests/m2/test_golden_image.py` was bumped to a 25 dB threshold and the -golden anchor was re-captured against the bumped toolchain. Refactor PRs -(Phase 3-style "should not change math") should raise the threshold via -`GOLDEN_PSNR_MIN_DB=40` (or higher); future toolchain bumps can repeat -the Phase 5 dance and recapture. - -### Bench diff vs Phase 1 - -| metric | Phase 1 (ct8.2/np1.23/py3.11) | Phase 5 (ct9.0/np1.26/py3.12/torch2.7) | delta | -|---|---:|---:|---:| -| SD1.5 NE fwd median (ms) | 196.97 | 197.32 | +0.2% | -| SD1.5 GPU fwd median (ms) | 270.26 | 272.01 | +0.6% | -| Model size (MB) | 1641 | 1641 | 0 | - -Within run-to-run noise (Phase 1 reproducibility check landed at -+0.0% / -0.3% across two consecutive runs). The bump is performance- -neutral on the SD1.5 baseline. - -### Rollback - -Reverting Phase 5 = restoring the Phase 1 pin set: - -```toml -requires-python = ">=3.11,<3.12" -dependencies = [ - "python-coreml-stable-diffusion @ git+https://github.com/apple/ml-stable-diffusion.git@e5d960c41a6a4ab200b8db379194127607b1c590", - "torch==2.0.1", - "coremltools==8.2", - "numpy<1.25", - "overrides", - "diffusers>=0.22", - "peft>=0.6.2", - "omegaconf>=2.3", -] -# Drop the [tool.uv] override-dependencies block entirely. -# Restore tests/m2/test_golden_image.py GOLDEN_PSNR_MIN_DB to 40. -# Restore tests/m2/goldens/sd15_seed42.{png,sha256} from the -# `modernize/phase4-tiered-ci` branch tip (the Phase 2 anchor). -``` - -Concretely: `git revert ` followed by -`rm -rf .venv && uv venv --python 3.11 && uv sync` will restore the -Phase 1 environment. The Phase 1 `bench/results/ef2a18c.json` is the -performance reference the rollback gets you back to. - -## Symbols we depend on from ml-stable-diffusion - -If we ever do need to vendor (option above), this is the surface to -copy: - -| Import path | Used in | -|---|---| -| `python_coreml_stable_diffusion.unet.UNet2DConditionModel` | `coreml_suite/converter.py` | -| `python_coreml_stable_diffusion.unet.UNet2DConditionModelXL` | `coreml_suite/converter.py` | -| `python_coreml_stable_diffusion.unet.AttentionImplementations` | `coreml_suite/converter.py`, `coreml_suite/nodes.py` | -| `python_coreml_stable_diffusion.unet.calculate_conv2d_output_shape` | `coreml_suite/converter.py` | -| `python_coreml_stable_diffusion.coreml_model.CoreMLModel` | `coreml_suite/nodes.py` | - -Plus the LCM-specific conversion helpers in -`coreml_suite/lcm/converter.py` (same `unet.*` symbols). diff --git a/docs/spikes/01-multifunction-models.md b/docs/spikes/01-multifunction-models.md deleted file mode 100644 index a1b4b07..0000000 --- a/docs/spikes/01-multifunction-models.md +++ /dev/null @@ -1,122 +0,0 @@ -# Spike 1 — MultiFunction models - -**Question (from spec):** Can one `.mlpackage` hold multiple batch sizes / -controlnet-on-off variants sharing weights, replacing the -"reconvert-per-config / encode-in-filename" approach? - -**Verdict:** Prototype for **controlnet-on/off only**. Use enumerated shapes -(spike 2), not multifunction, for batch size. Hard-gated behind **macOS 15 / -iOS 18**. Medium effort, medium payoff. - -## What it is - -MultiFunction packs several "functions" into one `mlprogram` `.mlpackage` and -**deduplicates identical weights across them**, so disk and in-memory footprint -shrink rather than just bundling N models. - -- Introduced: **coremltools 8.0** (2024-09-16). Unchanged through 9.0. -- Runtime minimum: **macOS 15 / iOS 18** (enforced — `save_multifunction` forces - `spec_version = max(spec_version, iOS18)`). -- `mlprogram` only (all of this suite's `.mlpackage`s already are). - -Build a multifunction package: - -```python -from coremltools.utils import MultiFunctionDescriptor, save_multifunction - -desc = MultiFunctionDescriptor() -desc.add_function(model_path="unet.mlpackage", - source_function_name="main", target_function_name="plain") -desc.add_function(model_path="unet_cn.mlpackage", - source_function_name="main", target_function_name="controlnet") -desc.default_function_name = "plain" # mandatory -save_multifunction(desc, "unet_multi.mlpackage") -``` - -Load a specific function (the save-time `CPU_ONLY` default does **not** bind -runtime compute units — reload with the unit you want): - -```python -import coremltools as ct -m = ct.models.MLModel("unet_multi.mlpackage", - function_name="controlnet", - compute_units=ct.ComputeUnit.CPU_AND_NE) -``` - -Swift runtime selection: `MLModelConfiguration.functionName`. - -## Does it actually share weights? - -Yes, genuinely — not just a bundle. `save_multifunction` runs the MIL pass -`const_deduplication` across functions. Dedup is **value-based (weight-hash)**, -not reference-based: two *independently converted* packages dedup their -bit-identical tensors at merge time. - -**Critical caveat for this suite:** dedup only fires if the shared tensors are -**byte-identical**. Different palettization/quantization (this repo's -`quantize_nbits` k-means path, `converter.py:309-325`) between two functions -defeats the hash match — you silently get a bundle, not a shared-weight model, -with no error. To benefit, convert the shared UNet trunk with identical -precision for both functions. - -Apple's documented benefit is **smaller size + shared in-memory weights** (load -base once, switch functions cheaply). No Apple source claims an ANE *compute* -speedup beyond that memory/load win. - -## How the two targets map - -| Target | Fit | Why | -|--------|-----|-----| -| **controlnet-on / controlnet-off** | **Good** | Structurally the adapter case Apple pushes (WWDC24 10159/10161): shared UNet trunk + extra control inputs in one function. Different I/O signatures per function are fully supported. This is the prototype candidate. | -| **different batch sizes** | **Awkward — wrong tool** | Batch is a *shape* difference, not a graph/weight difference. The idiomatic answer is `EnumeratedShapes` on a single function (spike 2): same weights, multiple shapes, no merge step, works on older OS targets. | - -Current per-config grounding: `converter.py:288-289` adds controlnet inputs via -`add_cnet_support`; `naming.py:22-63` encodes `_cn` into the filename cache key. -A 2-function package would replace the two separate `*_cn` / non-`cn` artifacts -with one — but only for macOS 15+ users. - -## Pipeline support - -`python_coreml_stable_diffusion` / `apple/ml-stable-diffusion` has **no -multifunction support** — it still emits separate `.mlpackage`s per component -and per config (`--unet-batch-one`, separate `*_control-unet.mlpackage`). You -would convert each variant as today, then add your **own** post-merge step -calling `MultiFunctionDescriptor`/`save_multifunction`. - -## Effort / risk - -- **Hard OS gate (macOS 15 / iOS 18).** Single biggest adoption risk — many Mac - SD users lag on OS. Likely need to keep the per-config path as fallback, which - erodes the simplicity win. The suite currently targets macOS 13. -- **Dedup defeated by mismatched quant** between functions (see above). -- `default_function_name` is mandatory; `source_function_name` is almost always - `"main"` for `ct.convert` output — verify per package. -- Spec version is forced up to the max of inputs (≥ iOS18); cannot keep a lower - target after merge. - -## Recommendation - -Low priority. If pursued: prototype **controlnet-on/off as a two-function -package**, converting the shared trunk with identical precision so dedup fires; -gate behind a macOS 15+ check; keep per-config reconvert as fallback. Do **not** -use multifunction for batch size — that is spike 2's job. Measure ANE engagement -and first-load compile time on real hardware before committing — those are the -make-or-break unknowns. - -## Unverified (needs [M2-ANE]) - -- Real-world ANE behavior of a merged SD UNet — no public report either way. -- Whether Core ML compiles a multifunction package per-function or whole-package - on first load (SD UNet first-load compile is minutes — could worsen startup). -- Exact `@available` version string for Swift `MLModelConfiguration.functionName` - (property confirmed; version inferred iOS 18 from the feature gate). - -## Sources - -- coremltools MultiFunction guide — https://apple.github.io/coremltools/docs-guides/source/multifunction-models.html -- `coremltools/models/utils.py` source — https://github.com/apple/coremltools/blob/main/coremltools/models/utils.py -- coremltools 8.0 release notes — https://github.com/apple/coremltools/releases/tag/8.0 -- WWDC24 10159 (Bring your ML/AI models to Apple silicon) — https://developer.apple.com/videos/play/wwdc2024/10159/ -- WWDC24 10161 (Deploy ML/AI models on-device with Core ML) — https://developer.apple.com/videos/play/wwdc2024/10161/ -- `MLModelConfiguration.functionName` — https://developer.apple.com/documentation/coreml/mlmodelconfiguration/functionname -- apple/ml-stable-diffusion `torch2coreml.py` — https://github.com/apple/ml-stable-diffusion/blob/main/python_coreml_stable_diffusion/torch2coreml.py diff --git a/docs/spikes/02-flexible-shapes.md b/docs/spikes/02-flexible-shapes.md deleted file mode 100644 index 6f2b72f..0000000 --- a/docs/spikes/02-flexible-shapes.md +++ /dev/null @@ -1,135 +0,0 @@ -# Spike 2 — Flexible / enumerated shapes - -**Question (from spec):** Can enumerated/dynamic shapes + -`materialize_dynamic_shape_mlmodel` remove the fixed 512×512 / per-size -reconversion and simplify the `chunk_batch` padding? - -**Verdict:** Use `ct.EnumeratedShapes` over a small set of **spatial** latent -sizes to kill per-size reconversion while keeping the ANE. Keep `chunk_batch` -for the **batch** dimension. Medium effort, medium–high payoff. The batch logic -is only partially eliminable and not worth the OS-gate cost. - -## The three mechanisms - -```python -import coremltools as ct - -# (a) RangeDim — continuous bounded range per dim -shape = ct.Shape(shape=(1, 4, - ct.RangeDim(lower_bound=64, upper_bound=128, default=64), - ct.RangeDim(lower_bound=64, upper_bound=128, default=64))) - -# (b) EnumeratedShapes — fixed finite set (≤128 shapes) -shape = ct.EnumeratedShapes( - shapes=[[1,4,64,64],[1,4,96,96],[1,4,128,128]], default=[1,4,64,64]) - -# (c) fixed — what converter.py:80 uses today -ct.TensorType(name="sample", shape=(1,4,64,64)) -``` - -## The ANE crux (confirmed, not inferred) - -coremltools maintainers confirmed in issue -[#2370](https://github.com/apple/coremltools/issues/2370) (tested coremltools -8.0 / macOS 15 / M3 Pro): - -- **`EnumeratedShapes` → runs every listed shape on the ANE.** This is the only - flexible option Apple endorses for an ANE model, and the only one their own - SD repo points to. -- **`RangeDim` → only the *default* shape runs on ANE.** Non-default shapes fall - to GPU/CPU (measured ~10× slower). The `ReshapeFrequency.Infrequent` - optimization hint (iOS 17.4+) lets RangeDim use the ANE for arbitrary shapes, - but **each new shape triggers an ANE recompile** (latency spike). - -coremltools FAQ, verbatim: *"Use `EnumeratedShapes` for best performance ... -The converted model will run on the NE, unless the conversion introduces dynamic -layers not supported on the NE, such as converting a static reshape to a fully -dynamic reshape."* - -## `materialize_dynamic_shape_mlmodel` - -Alternative path: convert once with symbolic dims, then bake out N fixed-shape -**functions** into one weight-shared package. - -```python -from coremltools.utils import materialize_dynamic_shape_mlmodel -materialize_dynamic_shape_mlmodel( - dynamic_shape_mlmodel, - {"size_64": {"sample": (1,4,64,64)}, "size_96": {"sample": (1,4,96,96)}}, - "unet_materialized.mlpackage") -``` - -- Introduced coremltools **8.0**. Tensor I/O only (SD UNet is `MLMultiArray` — OK). -- Deployment target: materializing **>1 function** (or non-`main` source) emits a - multifunction model → forced to **iOS 18 / macOS 15**. Materializing exactly - **one** function from `"main"` stays unifunction at the **original** target - (macOS 13 stays viable for the single-shape case). -- Materialized functions are fully static → run on ANE like normal fixed-shape - models, with shared weights across functions. - -## Does this kill `chunk_batch` or just per-size reconversion? - -Current grounding: -- `converter.py:266-271` converts for a fixed `(batch, C, H, W)` `sample_shape`. -- `latents.py:10-35` `chunk_batch` pads/splits the **batch** dim to the fixed - converted batch size. - -| Approach | ANE | Kills per-size reconv? | Kills `chunk_batch`? | Min target | Effort | -|----------|-----|------------------------|----------------------|------------|--------| -| `EnumeratedShapes` (spatial only) | Yes (all shapes) | **Yes** | No | macOS 13 | Med — `group_norm` hurdle | -| `EnumeratedShapes` (spatial × batch) | Yes | Yes | Partially | **macOS 15** (multi-input) | Med–High | -| `RangeDim` | Default only / `Infrequent`=recompile | Yes but slow | Partially | iOS 17.4 (hint) | Low code, bad ANE | -| `materialize_dynamic_shape_mlmodel` | Yes (static funcs) | Yes (1 package) | No | macOS 15 if >1 func | Med | - -- **Per-size (spatial H/W) reconversion → realistically eliminable** via - `EnumeratedShapes` over a curated size set, keeping the ANE. -- **`chunk_batch` batch padding → only partially eliminable, higher risk.** - Reasons: (1) batch must co-vary across multiple UNet inputs (`sample`, - `encoder_hidden_states`, `timestep`, SDXL/LCM/cnet extras), which hits the - **pre-iOS 18 single-`EnumeratedShapes`-input limit** — a hard wall unless you - target macOS 15+; (2) enumeration only covers the exact batch sizes you list, - so you still need `chunk_batch`-style fallback for unlisted counts; - (3) batch × spatial multiplies the enumerated-shape count, inflating compile - time and package size. - -## What breaks / risk - -- **`group_norm` conversion failure** is documented for flexible-shape SD UNet - (ml-stable-diffusion [#70](https://github.com/apple/ml-stable-diffusion/issues/70)): - flexible shapes failed at `group_norm`, and `SPLIT_EINSUM` with non-standard - dims caused kernel panics. Flexible *spatial* dims on the SD UNet are known to - be fiddly, not turnkey. (This suite defaults to `SPLIT_EINSUM` — - `converter.py:339`.) -- One **unreproduced** report in #2370 of `EnumeratedShapes` on `mlprogram` - running only the default shape on ANE; the maintainer could not reproduce. - Treat the documented "all shapes on ANE" as expected but **benchmark on target - hardware** — this is the single biggest verification gap. - -## Recommendation - -Highest-value direction of the three packaging spikes. If pursued: -`EnumeratedShapes` over a curated set of latent **spatial** sizes (e.g. 64²/96²/ -128²), keep batch fixed at the CFG-doubled value, keep `chunk_batch` for batch. -Stays on macOS 13. Budget time for the `group_norm` conversion hurdle and -**prove ANE residency on real hardware first** before removing any per-size -reconversion. `materialize_dynamic_shape_mlmodel` is a clean fallback for -"ship N fixed sizes in one weight-shared package" but forces macOS 15 once you -bake >1 function. - -## Unverified (needs [M2-ANE]) - -- `EnumeratedShapes`-on-`mlprogram` ANE residency for a *real* SD UNet - (the #2370 regression — unreproduced by maintainer). -- That materialized functions run on ANE (strongly implied; no benchmark found). -- `.mlmodelc` size / compile-time growth per enumerated shape (qualitative only). -- Whether batch-as-flexible-dim is ANE-safe for the SD UNet (no API prohibition, - no positive confirmation). - -## Sources - -- coremltools flexible inputs guide — https://apple.github.io/coremltools/docs-guides/source/flexible-inputs.html -- coremltools #2370 (Flexible shapes on Neural Engine) — https://github.com/apple/coremltools/issues/2370 -- coremltools #2386 (docs fix) — https://github.com/apple/coremltools/pull/2386 -- `materialize_dynamic_shape_mlmodel` source — https://github.com/apple/coremltools/blob/main/coremltools/models/utils.py -- coremltools 8.0 release notes — https://github.com/apple/coremltools/releases/tag/8.0 -- ml-stable-diffusion #70 (flexible-shape group_norm failure) — https://github.com/apple/ml-stable-diffusion/issues/70 diff --git a/docs/spikes/03-mlx-interop.md b/docs/spikes/03-mlx-interop.md deleted file mode 100644 index bda9e04..0000000 --- a/docs/spikes/03-mlx-interop.md +++ /dev/null @@ -1,106 +0,0 @@ -# Spike 3 — MLX interop decision - -**Question (from spec):** Should the suite stay pure Core ML/ANE, or add an -interop path with DiffusionKit / ComfyUI-MLX for SD3/Flux (GPU/MLX)? Recommend, -don't implement. - -**Verdict:** **Stay pure Core ML/ANE.** Document an off-ramp (point users to -existing MLX nodes for Flux); do **not** absorb MLX into the suite. The niche -MLX cannot reach — low-power, background, GPU-free — is exactly this suite's -moat. The optional on-brand build is **SD3 Medium on ANE** (upstream-supported), -plus a power benchmark to make the moat quantitative. - -## The two are complements, not substitutes - -MLX runs on **GPU + CPU only — it cannot target the ANE** (confirmed: MLX -devices are `mx.cpu` / `mx.gpu`; the M5 "Neural Accelerators" are inside the -GPU via Metal 4, *not* the ANE). So MLX structurally cannot serve this suite's -niche. - -| Axis | Core ML / ANE (this suite) | MLX (mflux / DiffusionKit) | -|------|----------------------------|----------------------------| -| Compute target | **ANE** (or GPU) | GPU + CPU only | -| Power draw | **Very low** (~5 W GEMM-level on M4) | High (~24 W; saturates GPU) | -| Thermal / fanless / background | **Wins** — cool, doesn't block GPU | Pins the GPU | -| Concurrent with GPU work | **Yes** (game/render/train running) | No — competes for GPU | -| Raw speed | Competitive on SD1.5/SDXL | **Faster** for Flux/SDXL when GPU free | -| Model coverage | SD1.5 / SDXL / **SD3** | **Flux**, FLUX.2, Qwen-Image, Z-Image; no SD3 | -| Conversion friction | High (convert+chunk+palettize ahead) | Low (load HF weights, quant on the fly) | - -The power figures are **GEMM-level, not full-pipeline diffusion** — directional, -not citable per-image. See "Unverified". - -## Ecosystem reality (2024–2026) - -- **DiffusionKit** (argmaxinc) — MIT. Python side runs SD3 + Flux on **MLX**; - Swift side does SD3 on Core ML (upstreamed to apple/ml-stable-diffusion). - **Repo archived (read-only) March 2026** — effectively end-of-life. Building - interop on it means depending on a dead repo. -- **mflux** (filipstrand) — MIT, **active** (v0.17.x, Apr 2026, ~2k stars). The - healthy MLX-image engine: FLUX.1/FLUX.2, Qwen-Image, Z-Image, 4/8-bit quant. - No SD3. **GPU only.** -- **Mflux-ComfyUI** (raysers) — MIT, wraps mflux; the most-starred Comfy MLX-Flux - node. The Flux-on-Mac-ComfyUI niche is **already occupied** here. -- The original `thoddnn/ComfyUI-MLX` that blogs cite ("70% faster") **no longer - exists** (404, deliberately removed). Treat those claims as referencing a dead - repo. - -## Can SD3 / Flux run on ANE via Core ML? - -- **apple/ml-stable-diffusion** — MIT, 17.8k stars. Not abandoned but coasting: - **last tagged release 1.1.1 (May 2024)**, main branch commits through - **July 2025**. Maintained at a trickle. -- **SD3 on Core ML/ANE: YES.** The README has a `--sd3-version` conversion path - (MMDiT, 16-ch VAE, T5), upstreamed from DiffusionKit. SD3 Medium ~2B params — - within ANE feasibility with palettization/chunking, same playbook as SDXL. -- **Flux on Core ML/ANE: effectively NO.** Flux is ~12B params (~24 GB fp16 - transformer + ~9 GB T5). ANE wants aggressive chunking (this suite already - bisects the 1.72 GB SD1.5 UNet into sub-1 GB chunks via `bisect_model`). - Flux would need heavy low-bit palettization **and** a dozen-plus chunks just to - fit, with precision risk on MMDiT attention. No maintained community Core ML - Flux exists. Flux on Mac in practice = MLX/GPU (mflux). - -## Maintenance burden if we went interop (rejected) - -- Two heavy new deps: **MLX** (large native, Apple-silicon-only, moves fast) + - **mflux** (DiffusionKit is archived, so mflux is the only live option). Both - violate the spec's "no new runtime dependencies without flagging" rule - (`MODERNIZATION_SPEC.md` §0.7) and would coexist with the already-fragile - `python_coreml_stable_diffusion` git dependency — doubling the moving surface. -- The interop would be a thin re-wrap of `Mflux-ComfyUI`, which already exists - and is MIT. Zero differentiation for real maintenance cost. - -## Recommendation - -| Option | Effort | Payoff | Verdict | -|--------|--------|--------|---------| -| **A. Stay pure Core ML/ANE; optionally extend to SD3 Medium** | Low–Med | Med | **Do** | -| **B. Document an off-ramp** in README (Flux/SD3-on-GPU → mflux / Mflux-ComfyUI; this suite = low-power / background / GPU-free / SD1.5/SDXL/SD3 on ANE) | Trivial | High | **Do** | -| C. Build MLX interop inside the suite (mflux dep) | High | Low | **Reject** | -| D. Build Flux-on-ANE Core ML conversion | Very high (research) | Low/uncertain | **Reject** | - -Net: stay pure, lean into the moat, point users elsewhere for Flux. The market -already built the MLX-Flux bridge; this suite's value is the orthogonal axis MLX -provably cannot reach. The one new build worth considering is **SD3 Medium on -ANE** (feasible, upstream-supported, on-brand) plus a **power benchmark** to -make the moat quantitative. - -## Unverified - -- A clean full-pipeline **diffusion** joules-per-image ANE-vs-MLX-GPU benchmark - (only GEMM-level ~5 W vs ~24 W on M4 exists). `bench/run.py` already measures - `CPU_AND_NE` vs `CPU_AND_GPU` — generating this would be the highest-value - artifact to prove the moat. -- Any working community Core ML Flux conversion (none found). -- Whether Argmax has a successor to DiffusionKit (archived Mar 2026, no announced - replacement; the live MLX-Flux torch is carried by mflux). - -## Sources - -- argmaxinc/DiffusionKit (archived) — https://github.com/argmaxinc/DiffusionKit -- apple/ml-stable-diffusion (+ releases) — https://github.com/apple/ml-stable-diffusion / https://github.com/apple/ml-stable-diffusion/releases -- filipstrand/mflux — https://github.com/filipstrand/mflux -- raysers/Mflux-ComfyUI — https://github.com/raysers/Mflux-ComfyUI -- MLX (unified memory / devices) — https://ml-explore.github.io/mlx/ -- Apple ML research, LLMs with MLX on M5 — https://machinelearning.apple.com/research/exploring-llms-mlx-m5 -- NPU vs GPU power (GEMM-level, directional) — arXiv 2511.13450 diff --git a/docs/spikes/README.md b/docs/spikes/README.md deleted file mode 100644 index f464ce9..0000000 --- a/docs/spikes/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Phase 7 — Strategic spikes (research only) - -Research-only investigations per `MODERNIZATION_SPEC.md` Phase 7. None of these -are merged behavior changes. Each spike delivers a written recommendation with -effort/payoff so the maintainer can choose which (if any) become real phases. - -| # | Spike | Verdict | Effort | Payoff | -|---|-------|---------|--------|--------| -| 1 | [MultiFunction models](01-multifunction-models.md) | Prototype for **controlnet-on/off only**; use enumerated shapes for batch | Medium | Medium (gated by macOS 15) | -| 2 | [Flexible / enumerated shapes](02-flexible-shapes.md) | `EnumeratedShapes` for **spatial size**; keep `chunk_batch` for batch | Medium | Medium–High | -| 3 | [MLX interop decision](03-mlx-interop.md) | **Stay pure Core ML/ANE.** Document an off-ramp; do not absorb MLX | Trivial (docs) | High (positioning) | - -## Cross-cutting finding - -Spikes 1 and 2 both run into the same wall: the high-value packaging features -(`MultiFunctionDescriptor`, `materialize_dynamic_shape_mlmodel`, multi-input -`EnumeratedShapes`) require **macOS 15 / iOS 18** at runtime, while the suite -today targets **macOS 13** (`coreml_suite/converter.py:110`). Adopting either -means either raising the floor (drops older-OS users — a real cost for a -community ComfyUI pack) or keeping the current per-config path as a fallback -(erodes the "one artifact" simplicity win). This OS gate, not the API, is the -dominant decision factor. - -## Gate 7 report - -``` -## Gate 7 report -- Spike 1 (MultiFunction): recommendation written — prototype controlnet-on/off - as 2-function package; NOT for batch size. Gated macOS 15. No code merged. -- Spike 2 (flexible shapes): recommendation written — EnumeratedShapes for - spatial size, retains ANE per coremltools maintainer confirmation; keep - chunk_batch for batch. No code merged. -- Spike 3 (MLX interop): recommendation written — stay pure Core ML/ANE, - document off-ramp to mflux/Mflux-ComfyUI for Flux. Reject in-suite interop. - No code merged. -- Open verification gaps requiring on-hardware [M2-ANE] measurement are listed - per spike under "Unverified". -``` - -## Verification debt (must measure on [M2-ANE] before any promotion) - -- EnumeratedShapes ANE residency for a *real* SD UNet (one unreproduced - coremltools regression report exists — spike 2). -- Multifunction SD UNet ANE behavior + first-load compile time (no public - report either way — spike 1). -- Full-pipeline diffusion power draw ANE vs MLX-GPU (only GEMM-level figures - exist — spike 3). `bench/run.py` already measures `CPU_AND_NE` vs - `CPU_AND_GPU`, so this is the highest-value artifact to make the moat - quantitative. diff --git a/pyproject.toml b/pyproject.toml index 8eb088b..79aeea4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,11 +6,11 @@ license = "MIT" requires-python = ">=3.12,<3.13" packages = [{ include = "coreml_suite" }] dependencies = [ - # Phase 5 toolchain bump: Python 3.12, coremltools 9, torch 2.7. + # Python 3.12, coremltools 9, torch 2.7. # numpy stays in the 1.24..1.x range — none of our modules need # numpy 2, and coremltools + ml-stable-diffusion's SD UNet trace # hit hard bugs under numpy 2 (`_cast` int(ndarray) strictness and - # `view` mixed-Var shape lists). See docs/deps.md. + # `view` mixed-Var shape lists). # torch 2.7 is the latest version coremltools 9's PyTorch frontend # has been tested against. "python-coreml-stable-diffusion @ git+https://github.com/apple/ml-stable-diffusion.git@e5d960c41a6a4ab200b8db379194127607b1c590", @@ -31,8 +31,7 @@ Repository = "https://github.com/aszc-dev/ComfyUI-CoreMLSuite" PublisherId = "aszc-dev" DisplayName = "ComfyUI-CoreMLSuite" Icon = "" -# Pinned to the ComfyUI commit this Phase 1 baseline was validated against. -# Bump together with the toolchain upgrade in Phase 5. +# Pinned to the ComfyUI commit this toolchain was validated against. requires-comfyui = "==ab5413351eee61f3d7f10c74e75286df0058bb18" [dependency-groups] @@ -41,7 +40,7 @@ dev = [ "psutil>=7.2.2", ] # ComfyUI runtime deps that aren't part of our package's runtime contract -# but are needed to spin up the ComfyUI server for Tier 2 / smoke_image. +# but are needed to spin up the ComfyUI server for Tier 2 integration tests. # Kept in a uv group so `uv sync --group comfy` brings them in without # polluting the published metadata (and without re-bumping our torch pin # via `uv pip install -r ComfyUI/requirements.txt`, which would float to @@ -63,11 +62,11 @@ comfy = [ ] [tool.uv] -# Phase 5 toolchain bump: ml-stable-diffusion's setup.py hard-pins -# numpy<1.24, diffusers==0.30.2 and transformers==4.44.2, which blocks -# the modern torch / coremltools combo on Python 3.12. Override the -# four blocking pins; the .unet / .coreml_model symbols we actually -# import (see docs/deps.md) are stable across the bumped versions. +# ml-stable-diffusion's setup.py hard-pins numpy<1.24, diffusers==0.30.2 +# and transformers==4.44.2, which blocks the modern torch / coremltools +# combo on Python 3.12. Override the four blocking pins; the .unet / +# .coreml_model symbols we actually import are stable across the bumped +# versions. override-dependencies = [ "numpy>=1.24,<2", "diffusers>=0.30", @@ -76,14 +75,13 @@ override-dependencies = [ ] [tool.pytest.ini_options] -# Phase 2: tier markers gate which environment a test needs. -# - unit: framework-free pure-logic tests (Tier 0; future Phase 3 will run -# these without ComfyUI on Linux). For now they may still transitively -# import comfy. +# Tier markers gate which environment a test needs. +# - unit: framework-free pure-logic tests (Tier 0; run without ComfyUI on +# Linux). # - m2: needs an Apple Silicon Mac with the Neural Engine (Tier 2), -# typically the maintainer's self-hosted runner or local M-series box. +# typically a self-hosted runner or local M-series box. # - smoke: lightweight checks that need Apple Silicon + coremltools but no -# ANE/real model (reserved for Phase 4 Tier 1 smoke harness). +# ANE/real model (Tier 1). markers = [ "unit: framework-free unit test (Tier 0)", "m2: requires Apple Silicon + Neural Engine (Tier 2)", diff --git a/tests/conftest.py b/tests/conftest.py index 1587338..5fd4a7c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,14 +1,12 @@ """Pytest bootstrap for ComfyUI-CoreMLSuite tests. -- Adds the ComfyUI checkout to sys.path so production modules that - transitively import `comfy.*` resolve when pytest is invoked from this - package's root. Phase 3 will split pure logic into a comfy-free core and - this hack can go. +- Adds the ComfyUI checkout to sys.path so the framework-coupled modules + that transitively import `comfy.*` resolve when pytest is invoked from + this package's root. - Auto-applies tier markers based on the directory a test lives in, so individual files don't have to repeat @pytest.mark.unit / .m2. -- Skips the maintainer's in-progress test scaffolds so they don't break - collection (they reference modules / venv layouts that aren't part of - Phase 2 scope). +- Skips in-progress test scaffolds so they don't break collection (they + reference modules / venv layouts that aren't part of this suite). """ import sys from pathlib import Path @@ -23,9 +21,8 @@ for p in (str(COMFY_DIR), str(REPO_ROOT)): sys.path.insert(0, p) -# Skip WIP test scaffolds left in the tree by the maintainer; they import -# modules (coreml_suite.experiments, convert_apple) that are not part of -# Phase 2 scope. +# Skip WIP test scaffolds; they import modules (coreml_suite.experiments, +# convert_apple) that are not part of this suite. collect_ignore_glob = [ "unit/test_experiments.py", "unit/test_unet_conversion.py", diff --git a/tests/m2/test_golden_image.py b/tests/m2/test_golden_image.py index 7bb4dcc..a43495e 100644 --- a/tests/m2/test_golden_image.py +++ b/tests/m2/test_golden_image.py @@ -1,4 +1,4 @@ -"""Phase 2 [M2-ANE] golden-image anchor. +"""[M2-ANE] golden-image anchor. Runs the e2e SD1.5 + CoreML workflow against a local ComfyUI server, fetches the generated PNG, and asserts both: @@ -14,8 +14,8 @@ regression. The 25 dB default reflects empirical drift between Core ML toolchains (8.2/np1.23/py3.11/torch2.0 → 9.0/np1.26/py3.12/torch2.7 measured at ~29 dB on the SD1.5 baseline; the image is visually identical at that -level). Bump the threshold up for refactor PRs (Phase 3-style "should -not change math"), down for toolchain PRs. +level). Bump the threshold up for refactor PRs ("should not change +math"), down for toolchain PRs. Skips entirely on non-Apple-Silicon hosts or when the server / converted model is missing, so the unit lane on Linux still passes. @@ -111,8 +111,7 @@ def _post_workflow_and_collect_png() -> bytes: for nid in ("3", "11"): if nid in workflow and "seed" in workflow[nid].get("inputs", {}): workflow[nid]["inputs"]["seed"] = SEED - # Drop the MPS reference branch — torch 2.0.1 MPS path is broken on macOS 26 - # (see Phase 1 gate report). Only the Core ML pipeline is needed here. + # Drop the MPS reference branch — only the Core ML pipeline is needed here. for nid in ("3", "8", "9"): workflow.pop(nid, None) diff --git a/tests/unit/test_characterization_controlnet.py b/tests/unit/test_characterization_controlnet.py index bb46258..dfdcecb 100644 --- a/tests/unit/test_characterization_controlnet.py +++ b/tests/unit/test_characterization_controlnet.py @@ -1,4 +1,4 @@ -"""Phase 2 characterization tests for coreml_suite.controlnet. +"""Characterization tests for coreml_suite.controlnet. Locks shapes + dtypes + zero-fill behavior of expand_inputs / no_control / extract_residual_kwargs / chunk_control. These pure helpers feed the Core ML diff --git a/tests/unit/test_characterization_inputs.py b/tests/unit/test_characterization_inputs.py index 8209b1d..2f47b5f 100644 --- a/tests/unit/test_characterization_inputs.py +++ b/tests/unit/test_characterization_inputs.py @@ -1,10 +1,10 @@ -"""Phase 2 characterization tests for coreml_suite.models.CoreMLInputs. +"""Characterization tests for coreml_suite.models.CoreMLInputs. Locks the shape transforms applied by chunks() and coreml_kwargs() for the four model variants the suite supports: SD1.5, LCM (SD1.5 + timestep_cond), SDXL base (time_ids len 6), and SDXL refiner (time_ids len 5). -These contracts feed the Core ML UNet at runtime; if Phase 3 silently +These contracts feed the Core ML UNet at runtime; if a refactor silently re-shapes them, generation breaks. """ import numpy as np diff --git a/tests/unit/test_characterization_latents.py b/tests/unit/test_characterization_latents.py index 24e3df7..0ad40a0 100644 --- a/tests/unit/test_characterization_latents.py +++ b/tests/unit/test_characterization_latents.py @@ -1,7 +1,7 @@ -"""Phase 2 characterization tests for coreml_suite.latents. +"""Characterization tests for coreml_suite.latents. Locks the *current* behavior of chunk_batch / merge_chunks — including the -zero-pad regions and the truncation in merge — so the Phase 3 refactor +zero-pad regions and the truncation in merge — so a refactor cannot silently shift either contract. """ import pytest @@ -110,7 +110,7 @@ def test_merge_chunks_truncates_padding(): def test_merge_chunks_singleton_returns_equal_copy_when_shape_matches(): """A singleton chunk list still goes through torch.cat, so we get a new - tensor equal to the input — locked here because Phase 3 might be tempted + tensor equal to the input — locked here because a refactor might be tempted to short-circuit and accidentally return the same object.""" x = _const_tensor(2) out = merge_chunks([x], x.shape) diff --git a/tests/unit/test_characterization_out_name.py b/tests/unit/test_characterization_out_name.py index dce672b..a246a50 100644 --- a/tests/unit/test_characterization_out_name.py +++ b/tests/unit/test_characterization_out_name.py @@ -1,9 +1,9 @@ -"""Phase 2 characterization tests, Phase 3 re-pointed. +"""Characterization tests for the .mlpackage filename composition. -After Phase 3 the .mlpackage filename composition is the pure +The filename composition is the pure coreml_suite.core.naming.compose_out_name function. CoreMLConverter.convert -calls it; the previous Phase 2 test had to monkey-patch heavy converter -internals just to capture the string, which made the test framework-coupled. +calls it; testing the pure function avoids monkey-patching heavy converter +internals just to capture the string. """ import pytest @@ -145,11 +145,11 @@ def test_lora_names_from_params_empty_list(): assert lora_names_from_params([]) == [] -# ---------- Phase 6: quantize_nbits suffix --------------------------------- +# ---------- quantize_nbits suffix ------------------------------------------ def test_quantize_nbits_none_appends_nothing(): - """'none' is the default and must keep the pre-Phase-6 filename so + """'none' is the default and must keep the unquantized filename so existing cached .mlpackages still resolve.""" out = compose_out_name( ckpt_name="dreamshaper_8.safetensors", diff --git a/tests/unit/test_characterization_sdxl_options.py b/tests/unit/test_characterization_sdxl_options.py index c51c374..85cbd1b 100644 --- a/tests/unit/test_characterization_sdxl_options.py +++ b/tests/unit/test_characterization_sdxl_options.py @@ -1,6 +1,6 @@ -"""Phase 2 characterization tests, Phase 3 re-pointed. +"""Characterization tests for the SDXL options math. -After Phase 3 the SDXL time_ids / text_embeds math lives in +The SDXL time_ids / text_embeds math lives in coreml_suite.core.sdxl as pure builders. The framework adapter add_sdxl_model_options (in models.py) is exercised separately by the m2 golden image test; here we just lock the pure math. @@ -95,7 +95,7 @@ def test_wrapper_captures_time_ids_text_embeds_refiner_via_closure(): def test_wrapper_returns_zero_when_context_missing(): """When c_crossattn is None the wrapper short-circuits to zeros_like(x). - Locked here because Phase 3 mustn't change this default.""" + Locked here because the refactor mustn't change this default.""" wrapper = sdxl_model_function_wrapper(torch.zeros(2, 6), torch.zeros(2, 1280)) x = torch.randn(2, 4, 16, 16) out = wrapper( diff --git a/tests/unit/test_chunks.py b/tests/unit/test_chunks.py index 8ead7ed..f1f1dd7 100644 --- a/tests/unit/test_chunks.py +++ b/tests/unit/test_chunks.py @@ -1,8 +1,7 @@ -"""Original smoke tests, re-pointed at coreml_suite.core.* in Phase 3. +"""Smoke tests for the pure batch-chunking helpers in coreml_suite.core. -Drops the `from comfy.model_management import get_torch_device` import -(replaced with torch.device('cpu') so Tier 0 runs without ComfyUI) and the -dead `model_config` fixture (defined in Phase 1 but never consumed). +Uses torch.device('cpu') instead of comfy.model_management.get_torch_device +so Tier 0 runs without ComfyUI. """ import pytest import torch diff --git a/tests/unit/test_tier0_purity.py b/tests/unit/test_tier0_purity.py index ffa0690..cff52f5 100644 --- a/tests/unit/test_tier0_purity.py +++ b/tests/unit/test_tier0_purity.py @@ -1,4 +1,4 @@ -"""Phase 3 gate: prove the Tier-0 lane is framework-free. +"""Gate: prove the Tier-0 lane is framework-free. In a pure `pytest -m unit` run, none of the banned runtime modules (comfy, coremltools, python_coreml_stable_diffusion, folder_paths,