diff --git a/bench/env/baseline-1e5791d.txt b/bench/env/baseline-1e5791d.txt new file mode 100644 index 0000000..ad2819d --- /dev/null +++ b/bench/env/baseline-1e5791d.txt @@ -0,0 +1,134 @@ +# Phase 1 baseline environment capture +timestamp_utc: 2026-05-24T14:36:51Z +repo_sha: 1e5791d1087b4dfb095e7761f05ac4e82c92cf7a +repo_branch: modernize/phase5-toolchain-bump +comfyui_dir: /Users/aszczepanski/dev/ComfyUI +comfyui_sha: ab5413351eee61f3d7f10c74e75286df0058bb18 +python_bin: /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/bin/python +python_version: Python 3.12.11 +platform_machine: arm64 +platform_uname: Darwin mac-2.home 25.1.0 Darwin Kernel Version 25.1.0: Mon Oct 20 19:33:00 PDT 2025; root:xnu-12377.41.6~2/RELEASE_ARM64_T6020 arm64 +macos_product: macOS +macos_version: 26.1 +macos_build: 25B78 + +## python-coreml-stable-diffusion (resolved) +Name: python-coreml-stable-diffusion +Version: 1.1.0 +Location: /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/lib/python3.12/site-packages +Requires: coremltools, diffusers, diffusionkit, huggingface-hub, invisible-watermark, matplotlib, numpy, pytest, safetensors, scikit-learn, scipy, torch, transformers +Required-by: + +## python-coreml-stable-diffusion installed git metadata +{"url":"https://github.com/apple/ml-stable-diffusion.git","vcs_info":{"vcs":"git","commit_id":"e5d960c41a6a4ab200b8db379194127607b1c590","requested_revision":"e5d960c41a6a4ab200b8db379194127607b1c590"}} + +## coremltools version +scikit-learn version 1.7.2 is not supported. Minimum required version: 0.17. Maximum required version: 1.5.1. Disabling scikit-learn conversion API. +Torch version 2.7.1 has not been tested with coremltools. You may run into unexpected errors. Torch 2.7.0 is the most recent version that has been tested. +coremltools 9.0 + +## torch version +torch 2.7.1 + +## numpy version +numpy 1.26.4 + +## freeze +accelerate==1.13.0 +aiohappyeyeballs==2.6.2 +aiohttp==3.13.5 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +argmaxtools==0.1.23 +attrs==26.1.0 +beartype==0.22.9 +cattrs==26.1.0 +certifi==2026.5.20 +cffi==2.0.0 +charset-normalizer==3.4.7 +click==8.4.1 +comfyui-frontend-package==1.14.6 +contourpy==1.3.2 +coremltools==9.0 +cycler==0.12.1 +diffusers==0.30.2 +diffusionkit==0.4.0 +einops==0.8.2 +filelock==3.29.0 +fonttools==4.63.0 +frozenlist==1.8.0 +fsspec==2026.4.0 +gitdb==4.0.12 +gitpython==3.1.50 +huggingface-hub==0.24.6 +idna==3.16 +importlib-metadata==9.0.0 +iniconfig==2.3.0 +invisible-watermark==0.2.0 +jaxtyping==0.3.9 +jinja2==3.1.6 +joblib==1.5.3 +kiwisolver==1.5.0 +kornia==0.8.3 +kornia-rs==0.1.14 +markupsafe==3.0.3 +matplotlib==3.10.9 +mlx==0.31.2 +mlx-metal==0.31.2 +mpmath==1.3.0 +multidict==6.7.1 +networkx==3.6.1 +numpy==1.26.4 +omegaconf==2.3.0 +opencv-python==4.11.0.86 +overrides==7.7.0 +packaging==26.2 +peft==0.13.2 +pillow==12.2.0 +platformdirs==4.9.6 +pluggy==1.6.0 +propcache==0.5.2 +protobuf==7.35.0 +psutil==7.2.2 +pyaml==26.2.1 +pycparser==3.0 +pydantic==2.13.4 +pydantic-core==2.46.4 +pygments==2.20.0 +pyparsing==3.3.2 +pytest==9.0.3 +python-coreml-stable-diffusion @ git+https://github.com/apple/ml-stable-diffusion.git@e5d960c41a6a4ab200b8db379194127607b1c590 +python-dateutil==2.9.0.post0 +pywavelets==1.8.0 +pyyaml==6.0.3 +regex==2026.5.9 +requests==2.34.2 +safetensors==0.7.0 +scikit-learn==1.7.2 +scipy==1.15.3 +sentencepiece==0.2.1 +sentry-sdk==2.60.0 +setuptools==81.0.0 +six==1.17.0 +smmap==5.0.3 +soundfile==0.13.1 +spandrel==0.4.2 +sympy==1.14.0 +tabulate==0.10.0 +threadpoolctl==3.6.0 +tokenizers==0.19.1 +torch==2.7.1 +torchaudio==2.11.0 +torchsde==0.2.6 +torchvision==0.22.1 +tqdm==4.67.3 +trampoline==0.1.2 +transformers==4.44.2 +typing-extensions==4.15.0 +typing-inspection==0.4.2 +urllib3==2.7.0 +wadler-lindig==0.1.7 +wandb==0.27.0 +yarl==1.24.2 +zipp==4.1.0 diff --git a/bench/env/pytest-unit-1e5791d.txt b/bench/env/pytest-unit-1e5791d.txt new file mode 100644 index 0000000..f0e81e6 --- /dev/null +++ b/bench/env/pytest-unit-1e5791d.txt @@ -0,0 +1,98 @@ +============================= test session starts ============================== +platform darwin -- Python 3.12.11, pytest-9.0.3, pluggy-1.6.0 -- /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite/.venv/bin/python +cachedir: .pytest_cache +rootdir: /Users/aszczepanski/dev/ComfyUI/custom_nodes/ComfyUI-CoreMLSuite +configfile: pyproject.toml +plugins: jaxtyping-0.3.9 +collecting ... collected 88 items + +tests/unit/test_characterization_controlnet.py::test_expand_inputs_doubles_singleton_numpy PASSED [ 1%] +tests/unit/test_characterization_controlnet.py::test_expand_inputs_doubles_singleton_torch PASSED [ 2%] +tests/unit/test_characterization_controlnet.py::test_expand_inputs_doubles_singleton_list PASSED [ 3%] +tests/unit/test_characterization_controlnet.py::test_expand_inputs_skips_already_batched PASSED [ 4%] +tests/unit/test_characterization_controlnet.py::test_expand_inputs_preserves_unknown_value_types PASSED [ 5%] +tests/unit/test_characterization_controlnet.py::test_no_control_returns_zero_fp16_for_residuals PASSED [ 6%] +tests/unit/test_characterization_controlnet.py::test_no_control_returns_empty_when_no_residuals PASSED [ 7%] +tests/unit/test_characterization_controlnet.py::test_extract_residual_kwargs_empty_when_model_has_no_residual_inputs PASSED [ 9%] +tests/unit/test_characterization_controlnet.py::test_extract_residual_kwargs_none_control_returns_no_control_shapes PASSED [ 10%] +tests/unit/test_characterization_controlnet.py::test_extract_residual_kwargs_flattens_output_then_middle_and_casts_fp16 PASSED [ 11%] +tests/unit/test_characterization_controlnet.py::test_chunk_control_none_returns_list_of_nones_with_length_target PASSED [ 12%] +tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[1-2-1] PASSED [ 13%] +tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[2-2-1] PASSED [ 14%] +tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[3-2-2] PASSED [ 15%] +tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[4-2-2] PASSED [ 17%] +tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[5-3-2] PASSED [ 18%] +tests/unit/test_characterization_controlnet.py::test_chunk_control_shapes_after_chunking[9-4-3] PASSED [ 19%] +tests/unit/test_characterization_controlnet.py::test_chunk_control_preserves_keys_order PASSED [ 20%] +tests/unit/test_characterization_controlnet.py::test_chunk_control_zero_pads_remainder PASSED [ 21%] +tests/unit/test_characterization_inputs.py::test_coreml_kwargs_sd15_shapes_and_fp16 PASSED [ 22%] +tests/unit/test_characterization_inputs.py::test_coreml_kwargs_sd15_with_controlnet_emits_residuals PASSED [ 23%] +tests/unit/test_characterization_inputs.py::test_coreml_kwargs_sd15_without_controlnet_zero_fills_residuals PASSED [ 25%] +tests/unit/test_characterization_inputs.py::test_coreml_kwargs_lcm_adds_timestep_cond PASSED [ 26%] +tests/unit/test_characterization_inputs.py::test_coreml_kwargs_lcm_skips_timestep_cond_when_not_provided PASSED [ 27%] +tests/unit/test_characterization_inputs.py::test_coreml_kwargs_sdxl_base_emits_time_ids_and_text_embeds PASSED [ 28%] +tests/unit/test_characterization_inputs.py::test_coreml_kwargs_sdxl_refiner_uses_len5_time_ids PASSED [ 29%] +tests/unit/test_characterization_inputs.py::test_chunks_sd15_pad_to_batch2_returns_one_chunk PASSED [ 30%] +tests/unit/test_characterization_inputs.py::test_chunks_sd15_with_controlnet_chunks_residuals_too PASSED [ 31%] +tests/unit/test_characterization_inputs.py::test_chunks_lcm_carries_timestep_cond_per_chunk PASSED [ 32%] +tests/unit/test_characterization_inputs.py::test_chunks_sdxl_base_propagates_time_ids_and_text_embeds PASSED [ 34%] +tests/unit/test_characterization_inputs.py::test_chunks_sdxl_refiner_uses_len5_time_ids PASSED [ 35%] +tests/unit/test_characterization_inputs.py::test_chunks_sdxl_synthesizes_zero_time_ids_when_caller_omits PASSED [ 36%] +tests/unit/test_characterization_inputs.py::test_chunks_splits_batch_into_multiple_target2_chunks PASSED [ 37%] +tests/unit/test_characterization_inputs.py::test_chunks_timestep_is_broadcast_from_first_value PASSED [ 38%] +tests/unit/test_characterization_latents.py::test_chunk_batch_passthrough_when_shape_matches PASSED [ 39%] +tests/unit/test_characterization_latents.py::test_chunk_batch_pads_single_chunk_when_input_smaller PASSED [ 40%] +tests/unit/test_characterization_latents.py::test_chunk_batch_splits_exact_multiple PASSED [ 42%] +tests/unit/test_characterization_latents.py::test_chunk_batch_pads_remainder_chunk PASSED [ 43%] +tests/unit/test_characterization_latents.py::test_chunk_batch_pad_region_is_zero[1-4-1] PASSED [ 44%] +tests/unit/test_characterization_latents.py::test_chunk_batch_pad_region_is_zero[3-2-2] PASSED [ 45%] +tests/unit/test_characterization_latents.py::test_chunk_batch_pad_region_is_zero[5-3-2] PASSED [ 46%] +tests/unit/test_characterization_latents.py::test_chunk_batch_pad_region_is_zero[9-4-3] PASSED [ 47%] +tests/unit/test_characterization_latents.py::test_merge_chunks_exact_concat PASSED [ 48%] +tests/unit/test_characterization_latents.py::test_merge_chunks_truncates_padding PASSED [ 50%] +tests/unit/test_characterization_latents.py::test_merge_chunks_singleton_returns_equal_copy_when_shape_matches PASSED [ 51%] +tests/unit/test_characterization_out_name.py::test_attention_suffix[SPLIT_EINSUM-se] PASSED [ 52%] +tests/unit/test_characterization_out_name.py::test_attention_suffix[SPLIT_EINSUM_V2-se2] PASSED [ 53%] +tests/unit/test_characterization_out_name.py::test_attention_suffix[ORIGINAL-orig] PASSED [ 54%] +tests/unit/test_characterization_out_name.py::test_includes_batch_and_size PASSED [ 55%] +tests/unit/test_characterization_out_name.py::test_appends_cn_suffix_when_controlnet_support_true PASSED [ 56%] +tests/unit/test_characterization_out_name.py::test_drops_extension_at_first_period PASSED [ 57%] +tests/unit/test_characterization_out_name.py::test_replaces_spaces_with_underscores PASSED [ 59%] +tests/unit/test_characterization_out_name.py::test_single_lora PASSED [ 60%] +tests/unit/test_characterization_out_name.py::test_multiple_loras_sorted PASSED [ 61%] +tests/unit/test_characterization_out_name.py::test_lora_plus_controlnet PASSED [ 62%] +tests/unit/test_characterization_out_name.py::test_sdxl_1024_original_gpu PASSED [ 63%] +tests/unit/test_characterization_out_name.py::test_lora_names_from_params_sorts_by_name PASSED [ 64%] +tests/unit/test_characterization_out_name.py::test_lora_names_from_params_empty_list PASSED [ 65%] +tests/unit/test_characterization_sdxl_options.py::test_build_time_ids_base_defaults PASSED [ 67%] +tests/unit/test_characterization_sdxl_options.py::test_build_time_ids_base_respects_overrides PASSED [ 68%] +tests/unit/test_characterization_sdxl_options.py::test_build_time_ids_refiner_defaults PASSED [ 69%] +tests/unit/test_characterization_sdxl_options.py::test_build_time_ids_refiner_respects_aesthetic_score PASSED [ 70%] +tests/unit/test_characterization_sdxl_options.py::test_build_time_ids_neither_base_nor_refiner_returns_len4 PASSED [ 71%] +tests/unit/test_characterization_sdxl_options.py::test_text_embeds_concat_pos_then_neg PASSED [ 72%] +tests/unit/test_characterization_sdxl_options.py::test_wrapper_captures_time_ids_text_embeds_refiner_via_closure PASSED [ 73%] +tests/unit/test_characterization_sdxl_options.py::test_wrapper_returns_zero_when_context_missing PASSED [ 75%] +tests/unit/test_characterization_sdxl_options.py::test_wrapper_refiner_truncates_context_to_g_clip PASSED [ 76%] +tests/unit/test_chunks.py::test_batch_chunking[1] PASSED [ 77%] +tests/unit/test_chunks.py::test_batch_chunking[2] PASSED [ 78%] +tests/unit/test_chunks.py::test_batch_chunking[4] PASSED [ 79%] +tests/unit/test_chunks.py::test_batch_chunking[5] PASSED [ 80%] +tests/unit/test_chunks.py::test_batch_chunking[9] PASSED [ 81%] +tests/unit/test_chunks.py::test_merge_chunks[1] PASSED [ 82%] +tests/unit/test_chunks.py::test_merge_chunks[2] PASSED [ 84%] +tests/unit/test_chunks.py::test_merge_chunks[4] PASSED [ 85%] +tests/unit/test_chunks.py::test_merge_chunks[5] PASSED [ 86%] +tests/unit/test_chunks.py::test_merge_chunks[9] PASSED [ 87%] +tests/unit/test_chunks.py::test_chunking_controlnet[1-2-1] PASSED [ 88%] +tests/unit/test_chunks.py::test_chunking_controlnet[1-1-1] PASSED [ 89%] +tests/unit/test_chunks.py::test_chunking_controlnet[2-2-1] PASSED [ 90%] +tests/unit/test_chunks.py::test_chunking_controlnet[3-2-2] PASSED [ 92%] +tests/unit/test_chunks.py::test_chunking_controlnet[4-2-2] PASSED [ 93%] +tests/unit/test_chunks.py::test_chunking_controlnet[5-3-2] PASSED [ 94%] +tests/unit/test_chunks.py::test_chunking_controlnet[9-4-3] PASSED [ 95%] +tests/unit/test_chunks.py::test_chunking_no_control PASSED [ 96%] +tests/unit/test_chunks.py::test_chunking_inputs PASSED [ 97%] +tests/unit/test_controlnet.py::test_no_control PASSED [ 98%] +tests/unit/test_tier0_purity.py::test_no_framework_modules_loaded_by_unit_tier PASSED [100%] + +============================== 88 passed in 1.83s ============================== diff --git a/bench/results/1e5791d.json b/bench/results/1e5791d.json new file mode 100644 index 0000000..a3e1fcd --- /dev/null +++ b/bench/results/1e5791d.json @@ -0,0 +1,103 @@ +{ + "git_sha": "1e5791d", + "timestamp": "2026-05-24T14:37:16+00:00", + "host": { + "machine": "arm64", + "system": "Darwin", + "release": "25.1.0", + "python": "3.12.11" + }, + "tool_versions": { + "coremltools": "9.0", + "numpy": "1.26.4" + }, + "settings": { + "repeats": 30, + "assumed_steps": 20, + "input_seed": 0, + "compute_units": [ + "CPU_AND_NE", + "CPU_AND_GPU" + ] + }, + "results": [ + { + "label": "v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_NE", + "model_path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc", + "compute_unit": "CPU_AND_NE", + "error": null, + "model_size_bytes": 1720796055, + "load_time_s": 0.2086, + "kind": "sd15", + "expected_inputs": { + "sample": [ + 1, + 4, + 64, + 64 + ], + "timestep": [ + 1 + ], + "encoder_hidden_states": [ + 1, + 768, + 1, + 77 + ] + }, + "warmup_ms": 219.471, + "forward_ms": { + "mean": 199.435, + "std": 5.333, + "min": 196.825, + "median": 197.493, + "n": 30 + }, + "est_e2e_ms": 3949.9, + "peak_rss_mb": 399.0 + }, + { + "label": "v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_GPU", + "model_path": "/Users/aszczepanski/dev/ComfyUI/models/unet/v1-5-pruned-emaonly_1x512x512_se_unet.mlmodelc", + "compute_unit": "CPU_AND_GPU", + "error": null, + "model_size_bytes": 1720796055, + "load_time_s": 1.2334, + "kind": "sd15", + "expected_inputs": { + "sample": [ + 1, + 4, + 64, + 64 + ], + "timestep": [ + 1 + ], + "encoder_hidden_states": [ + 1, + 768, + 1, + 77 + ] + }, + "warmup_ms": 1392.959, + "forward_ms": { + "mean": 281.433, + "std": 11.822, + "min": 268.273, + "median": 278.577, + "n": 30 + }, + "est_e2e_ms": 5571.5, + "peak_rss_mb": 2113.5 + } + ], + "notes": [ + "forward_ms is the latency of a single Core ML UNet forward pass with synthetic inputs; with classifier-free guidance one sampler step typically maps to one batched forward (cond+uncond) \u2014 interpret est_e2e_ms accordingly.", + "peak_rss_mb is process RSS only and excludes ANE/wired GPU memory; treat as a floor.", + "quality_psnr is null unless --reference-images and --candidate-images are provided.", + "Latency varies run-to-run; 'min' is the most reproducible figure for comparisons." + ] +} \ No newline at end of file diff --git a/bench/results/1e5791d.md b/bench/results/1e5791d.md new file mode 100644 index 0000000..1b854df --- /dev/null +++ b/bench/results/1e5791d.md @@ -0,0 +1,19 @@ +# CoreMLSuite baseline — `1e5791d` + +- timestamp: 2026-05-24T14:37:16+00:00 +- host: arm64 / Darwin 25.1.0 +- python: 3.12.11 +- coremltools: 9.0, numpy: 1.26.4 +- settings: repeats=30, input_seed=0, assumed_steps=20 + +| label | kind | compute | size (MB) | load (s) | warmup (ms) | fwd median (ms) | fwd min (ms) | est e2e (ms) | peak RSS (MB) | error | +|---|---|---|---|---|---|---|---|---|---|---| +| v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_NE | sd15 | CPU_AND_NE | 1641.1 | 0.2086 | 219.471 | 197.493 | 196.825 | 3949.9 | 399.0 | | +| v1-5-pruned-emaonly_1x512x512_se_unet-CPU_AND_GPU | sd15 | CPU_AND_GPU | 1641.1 | 1.2334 | 1392.959 | 278.577 | 268.273 | 5571.5 | 2113.5 | | + +## Notes + +- forward_ms is the latency of a single Core ML UNet forward pass with synthetic inputs; with classifier-free guidance one sampler step typically maps to one batched forward (cond+uncond) — interpret est_e2e_ms accordingly. +- peak_rss_mb is process RSS only and excludes ANE/wired GPU memory; treat as a floor. +- quality_psnr is null unless --reference-images and --candidate-images are provided. +- Latency varies run-to-run; 'min' is the most reproducible figure for comparisons.