diff --git a/bench/env/baseline-ef2a18c.txt b/bench/env/baseline-ef2a18c.txt new file mode 100644 index 0000000..f8bc2b9 --- /dev/null +++ b/bench/env/baseline-ef2a18c.txt @@ -0,0 +1,129 @@ +# Phase 1 baseline environment capture +timestamp_utc: 2026-05-22T13:06:37Z +repo_sha: ef2a18cff3d8b563ae053d27c1c5c233bd05d47f +repo_branch: modernize/phase1-freeze-baseline +comfyui_dir: /Users/aszczepanski/dev/ComfyUI +comfyui_sha: ab5413351eee61f3d7f10c74e75286df0058bb18 +python_bin: /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/bin/python +python_version: Python 3.11.14 +platform_machine: arm64 +platform_uname: Darwin mac-2.home 25.1.0 Darwin Kernel Version 25.1.0: Mon Oct 20 19:33:00 PDT 2025; root:xnu-12377.41.6~2/RELEASE_ARM64_T6020 arm64 +macos_product: macOS +macos_version: 26.1 +macos_build: 25B78 + +## python-coreml-stable-diffusion (resolved) +Name: python-coreml-stable-diffusion +Version: 1.1.0 +Location: /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/lib/python3.11/site-packages +Requires: coremltools, diffusers, diffusionkit, huggingface-hub, invisible-watermark, matplotlib, numpy, pytest, safetensors, scikit-learn, scipy, torch, transformers +Required-by: + +## python-coreml-stable-diffusion installed git metadata +{"url":"https://github.com/apple/ml-stable-diffusion.git","vcs_info":{"vcs":"git","commit_id":"e5d960c41a6a4ab200b8db379194127607b1c590","requested_revision":"e5d960c41a6a4ab200b8db379194127607b1c590"}} + +## coremltools version +scikit-learn version 1.7.2 is not supported. Minimum required version: 0.17. Maximum required version: 1.5.1. Disabling scikit-learn conversion API. +Minimum required torch version for importing coremltools.optimize.torch is 2.1.0. Got torch version 2.0.1. +coremltools 8.2 + +## torch version +torch 2.0.1 + +## numpy version +numpy 1.23.5 + +## freeze +accelerate==1.13.0 +aiohappyeyeballs==2.6.2 +aiohttp==3.13.5 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +argmaxtools==0.1.23 +attrs==26.1.0 +beartype==0.22.9 +cattrs==26.1.0 +certifi==2026.5.20 +charset-normalizer==3.4.7 +click==8.4.1 +comfyui-frontend-package==1.14.6 +contourpy==1.3.2 +coremltools==8.2 +cycler==0.12.1 +diffusers==0.30.2 +diffusionkit==0.4.0 +einops==0.8.2 +filelock==3.29.0 +fonttools==4.63.0 +frozenlist==1.8.0 +fsspec==2026.4.0 +gitdb==4.0.12 +gitpython==3.1.50 +huggingface-hub==0.24.6 +idna==3.16 +importlib-metadata==9.0.0 +iniconfig==2.3.0 +invisible-watermark==0.2.0 +jaxtyping==0.3.9 +jinja2==3.1.6 +joblib==1.5.3 +kiwisolver==1.5.0 +kornia==0.8.3 +kornia-rs==0.1.14 +markupsafe==3.0.3 +matplotlib==3.10.9 +mlx==0.31.2 +mlx-metal==0.31.2 +mpmath==1.3.0 +multidict==6.7.1 +networkx==3.6.1 +numpy==1.23.5 +omegaconf==2.3.0 +opencv-python==4.11.0.86 +overrides==7.7.0 +packaging==26.2 +peft==0.13.2 +pillow==12.2.0 +platformdirs==4.9.6 +pluggy==1.6.0 +propcache==0.5.2 +protobuf==7.35.0 +psutil==7.2.2 +pyaml==26.2.1 +pydantic==2.13.4 +pydantic-core==2.46.4 +pygments==2.20.0 +pyparsing==3.3.2 +pytest==9.0.3 +python-coreml-stable-diffusion @ git+https://github.com/apple/ml-stable-diffusion.git@e5d960c41a6a4ab200b8db379194127607b1c590 +python-dateutil==2.9.0.post0 +pywavelets==1.8.0 +pyyaml==6.0.3 +regex==2026.5.9 +requests==2.34.2 +safetensors==0.7.0 +scikit-learn==1.7.2 +scipy==1.15.3 +sentencepiece==0.2.1 +sentry-sdk==2.60.0 +setuptools==81.0.0 +six==1.17.0 +smmap==5.0.3 +sympy==1.14.0 +tabulate==0.10.0 +threadpoolctl==3.6.0 +tokenizers==0.19.1 +torch==2.0.1 +torchsde==0.2.6 +torchvision==0.15.2 +tqdm==4.67.3 +trampoline==0.1.2 +transformers==4.44.2 +typing-extensions==4.15.0 +typing-inspection==0.4.2 +urllib3==2.7.0 +wadler-lindig==0.1.7 +wandb==0.27.0 +yarl==1.24.2 +zipp==4.1.0 diff --git a/bench/env/pytest-unit-ef2a18c.txt b/bench/env/pytest-unit-ef2a18c.txt new file mode 100644 index 0000000..984defe --- /dev/null +++ b/bench/env/pytest-unit-ef2a18c.txt @@ -0,0 +1,39 @@ +============================= test session starts ============================== +platform darwin -- Python 3.11.14, pytest-9.0.3, pluggy-1.6.0 -- /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/bin/python +cachedir: .pytest_cache +rootdir: /Users/aszczepanski/dev/ComfyUI +configfile: pytest.ini +plugins: jaxtyping-0.3.9 +collecting ... collected 20 items + +tests/unit/test_chunks.py::test_batch_chunking[1] PASSED +tests/unit/test_chunks.py::test_batch_chunking[2] PASSED +tests/unit/test_chunks.py::test_batch_chunking[4] PASSED +tests/unit/test_chunks.py::test_batch_chunking[5] PASSED +tests/unit/test_chunks.py::test_batch_chunking[9] PASSED +tests/unit/test_chunks.py::test_merge_chunks[1] PASSED +tests/unit/test_chunks.py::test_merge_chunks[2] PASSED +tests/unit/test_chunks.py::test_merge_chunks[4] PASSED +tests/unit/test_chunks.py::test_merge_chunks[5] PASSED +tests/unit/test_chunks.py::test_merge_chunks[9] PASSED +tests/unit/test_chunks.py::test_chunking_controlnet[1-2-1] PASSED +tests/unit/test_chunks.py::test_chunking_controlnet[1-1-1] PASSED +tests/unit/test_chunks.py::test_chunking_controlnet[2-2-1] PASSED +tests/unit/test_chunks.py::test_chunking_controlnet[3-2-2] PASSED +tests/unit/test_chunks.py::test_chunking_controlnet[4-2-2] PASSED +tests/unit/test_chunks.py::test_chunking_controlnet[5-3-2] PASSED +tests/unit/test_chunks.py::test_chunking_controlnet[9-4-3] PASSED +tests/unit/test_chunks.py::test_chunking_no_control PASSED +tests/unit/test_chunks.py::test_chunking_inputs PASSED +tests/unit/test_controlnet.py::test_no_control PASSED + +=============================== warnings summary =============================== +:241 + :241: DeprecationWarning: builtin type SwigPyPacked has no __module__ attribute + +:241 + :241: DeprecationWarning: builtin type SwigPyObject has no __module__ attribute + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +======================== 20 passed, 2 warnings in 2.81s ======================== +sys:1: DeprecationWarning: builtin type swigvarlink has no __module__ attribute diff --git a/bench/results/ef2a18c.json b/bench/results/ef2a18c.json new file mode 100644 index 0000000..b5f52ae --- /dev/null +++ b/bench/results/ef2a18c.json @@ -0,0 +1,103 @@ +{ + "git_sha": "ef2a18c", + "timestamp": "2026-05-22T13:07:25+00:00", + "host": { + "machine": "arm64", + "system": "Darwin", + "release": "25.1.0", + "python": "3.11.14" + }, + "tool_versions": { + "coremltools": "8.2", + "numpy": "1.23.5" + }, + "settings": { + "repeats": 30, + "assumed_steps": 20, + "input_seed": 0, + "compute_units": [ + "CPU_AND_NE", + "CPU_AND_GPU" + ] + }, + "results": [ + { + "label": "v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_NE", + "model_path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc", + "compute_unit": "CPU_AND_NE", + "error": null, + "model_size_bytes": 1720795925, + "load_time_s": 0.2841, + "kind": "sd15", + "expected_inputs": { + "sample": [ + 1, + 4, + 64, + 64 + ], + "timestep": [ + 1 + ], + "encoder_hidden_states": [ + 1, + 768, + 1, + 77 + ] + }, + "warmup_ms": 225.201, + "forward_ms": { + "mean": 197.126, + "std": 0.476, + "min": 196.812, + "median": 196.969, + "n": 30 + }, + "est_e2e_ms": 3939.4, + "peak_rss_mb": 349.7 + }, + { + "label": "v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_GPU", + "model_path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc", + "compute_unit": "CPU_AND_GPU", + "error": null, + "model_size_bytes": 1720795925, + "load_time_s": 19.959, + "kind": "sd15", + "expected_inputs": { + "sample": [ + 1, + 4, + 64, + 64 + ], + "timestep": [ + 1 + ], + "encoder_hidden_states": [ + 1, + 768, + 1, + 77 + ] + }, + "warmup_ms": 558.865, + "forward_ms": { + "mean": 270.275, + "std": 1.312, + "min": 267.398, + "median": 270.259, + "n": 30 + }, + "est_e2e_ms": 5405.2, + "peak_rss_mb": 2132.9 + } + ], + "notes": [ + "forward_ms is the latency of a single Core ML UNet forward pass with synthetic inputs; with classifier-free guidance one sampler step typically maps to one batched forward (cond+uncond) \u2014 interpret est_e2e_ms accordingly.", + "peak_rss_mb is process RSS only and excludes ANE/wired GPU memory; treat as a floor.", + "quality_psnr is null unless --reference-images and --candidate-images are provided.", + "Latency varies run-to-run; 'min' is the most reproducible figure for comparisons." + ] +} \ No newline at end of file diff --git a/bench/results/ef2a18c.md b/bench/results/ef2a18c.md new file mode 100644 index 0000000..9db2035 --- /dev/null +++ b/bench/results/ef2a18c.md @@ -0,0 +1,19 @@ +# CoreMLSuite baseline — `ef2a18c` + +- timestamp: 2026-05-22T13:07:25+00:00 +- host: arm64 / Darwin 25.1.0 +- python: 3.11.14 +- coremltools: 8.2, numpy: 1.23.5 +- settings: repeats=30, input_seed=0, assumed_steps=20 + +| label | kind | compute | size (MB) | load (s) | warmup (ms) | fwd median (ms) | fwd min (ms) | est e2e (ms) | peak RSS (MB) | error | +|---|---|---|---|---|---|---|---|---|---|---| +| v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_NE | sd15 | CPU_AND_NE | 1641.1 | 0.2841 | 225.201 | 196.969 | 196.812 | 3939.4 | 349.7 | | +| v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_GPU | sd15 | CPU_AND_GPU | 1641.1 | 19.959 | 558.865 | 270.259 | 267.398 | 5405.2 | 2132.9 | | + +## Notes + +- forward_ms is the latency of a single Core ML UNet forward pass with synthetic inputs; with classifier-free guidance one sampler step typically maps to one batched forward (cond+uncond) — interpret est_e2e_ms accordingly. +- peak_rss_mb is process RSS only and excludes ANE/wired GPU memory; treat as a floor. +- quality_psnr is null unless --reference-images and --candidate-images are provided. +- Latency varies run-to-run; 'min' is the most reproducible figure for comparisons. diff --git a/bench/results/smoke/ef2a18c/E2E-1.5-CoreML_00002_.png b/bench/results/smoke/ef2a18c/E2E-1.5-CoreML_00002_.png new file mode 100644 index 0000000..a561e16 Binary files /dev/null and b/bench/results/smoke/ef2a18c/E2E-1.5-CoreML_00002_.png differ diff --git a/bench/results/smoke/ef2a18c/report.json b/bench/results/smoke/ef2a18c/report.json new file mode 100644 index 0000000..e86e3be --- /dev/null +++ b/bench/results/smoke/ef2a18c/report.json @@ -0,0 +1,10 @@ +{ + "git_sha": "ef2a18c", + "seed": 42, + "ckpt_name": "v1-5-pruned-emaonly.safetensors", + "workflow": "tests/integration/workflows/e2e-1.5-basic-conversion.json", + "coreml_image": "bench/results/smoke/ef2a18c/E2E-1.5-CoreML_00002_.png", + "mps_image": null, + "psnr_db": null, + "wall_seconds": 21.1 +} \ No newline at end of file