diff --git a/.github/workflows/comfyui_compat.yml b/.github/workflows/comfyui_compat.yml index 5e59f61..a26f55c 100644 --- a/.github/workflows/comfyui_compat.yml +++ b/.github/workflows/comfyui_compat.yml @@ -97,6 +97,8 @@ jobs: python -m pip install -r ComfyUI/requirements.txt git -C ComfyUI log -1 --format='ComfyUI %h %cd %s' --date=short python -m pip list 2>/dev/null | grep -i -E "^(comfyui|torch) " || true + # Which CPU ran it: float results can differ by a last bit between CPUs. + lscpu | grep -E "^Model name" || true - name: Everything the pack takes from ComfyUI is there working-directory: ComfyUI-AusBoss run: AUSBOSS_COMFY_ROOT="$GITHUB_WORKSPACE/ComfyUI" python tests/test_comfy_core_contract.py diff --git a/tests/fixtures/stitch_released_images.md b/tests/fixtures/stitch_released_images.md index a3ed568..bab6e84 100644 --- a/tests/fixtures/stitch_released_images.md +++ b/tests/fixtures/stitch_released_images.md @@ -27,10 +27,29 @@ difference was 5.0068e-6; all masks were exact, and no rounded 8-bit channels changed. Thus images allow `atol=1e-5, rtol=0` (0.00255 of an 8-bit step). This is a CPU regression test, not a promise of bitwise GPU reproducibility. Shape, dtype and non-finite output errors still fail. A mutation check confirms -a single channel changed by 1/255 fails. Mask hashes remain byte-exact. +a single channel changed by 1/255 fails. The masks were byte-exact in all of +these runs, which shared one CPU; see Blend masks below. See [PyTorch numerical accuracy](https://docs.pytorch.org/docs/2.11/notes/numerical_accuracy.html) for why floating-point kernels need not agree bit for bit. +## Blend masks + +`stitch_released_masks.npz` holds the 11 float32 blend masks of the same runs: +`classic__mask` (one per case; the mask is the same at every strength) +and `blend_in__mask`. Each is the mask whose hash the test pins, and +`test_the_mask_references_are_the_pinned_masks` holds the file to those +hashes, so it carries the released code's masks byte for byte. + +A run is compared with these masks at `atol=1e-5, rtol=0`, like the images, +and no longer by hash. On a GitHub-hosted runner two masks came out with +another hash on unchanged code: the straight transform and Crop For Inpaint, +the two whose feather is a blur. The blur's bell curve is `torch.exp`, which +is accurate to under one float32 step but is not rounded the same way on every +CPU. Moving each value of that curve by one float32 step on the reference +machine changes the hash of exactly those two masks, moves them by at most +2.4e-7, and leaves the other four untouched. A blur made 2% wider still fails +both of those cases at every strength. + ## Reproducing the references Run from the repository root with the environment above. Never regenerate from @@ -84,3 +103,28 @@ for mode, ref in ( arrays.update({key: data[key] for key in data.files}) np.savez_compressed("stitch_released_images.candidate.npz", **arrays) ``` + +The masks file needs no old checkout: the pins already say what the releases +gave. Run the test's own builders and keep a mask only when its hash is the +pinned one. On a CPU that rounds the blur differently the assert fails and +nothing is written. + +```python +import runpy +import numpy as np + +m = runpy.run_path("tests/test_stitch_classic_pins.py") +arrays = {} +for index, (name, build) in enumerate(m["CASES"].items()): + for strength in m["STRENGTHS"]: + stitcher, patch = build() + _image, mask = m["stitch"](stitcher, patch, strength) + assert m["digest"](mask) == m["PINS"][(name, strength)][1], name + arrays[f"classic_{index}_mask"] = mask.numpy().copy() + if name in m["BLEND_IN_CASES"]: + stitcher, patch = build() + _image, mask = m["stitch"](stitcher, patch, 0.0, seam="blend in") + assert m["digest"](mask) == m["BLEND_IN_PINS"][name][1], name + arrays[f"blend_in_{index}_mask"] = mask.numpy().copy() +np.savez_compressed("stitch_released_masks.candidate.npz", **arrays) +``` diff --git a/tests/fixtures/stitch_released_masks.npz b/tests/fixtures/stitch_released_masks.npz new file mode 100644 index 0000000..1de3d72 Binary files /dev/null and b/tests/fixtures/stitch_released_masks.npz differ diff --git a/tests/test_memory_helpers.py b/tests/test_memory_helpers.py index 06771f4..8851eda 100644 --- a/tests/test_memory_helpers.py +++ b/tests/test_memory_helpers.py @@ -13,6 +13,14 @@ if "nodes" in sys.modules and not hasattr(sys.modules["nodes"], "__path__"): from nodes._memory_helpers import WILDCARD, free_gpu_memory +try: + # Loaded before any test swaps sys.modules: patch.dict forgets whatever + # was first imported inside it, and torch cannot be imported twice in + # one process (the second import can end the process without a word). + import torch # noqa: F401 +except Exception: # these tests also run where torch is not installed + pass + class WildcardTypeTests(unittest.TestCase): def test_it_is_the_star_string(self): diff --git a/tests/test_stitch_classic_pins.py b/tests/test_stitch_classic_pins.py index 782b625..9c110e1 100644 --- a/tests/test_stitch_classic_pins.py +++ b/tests/test_stitch_classic_pins.py @@ -13,9 +13,12 @@ the published Qwen Image 2.1 Rotate + Outpaint workflow stitches that way. Tone-matched float32 output varies slightly with CPU kernels and thread counts, including on unchanged releases. Images are compared with frozen release tensors at an absolute tolerance of 1e-5 (0.00255 of an 8-bit step), -with no relative tolerance. Masks still use exact hashes. The fixtures come -from released code, never the implementation under review; see -fixtures/stitch_released_images.md for provenance and regeneration. +with no relative tolerance. A feathered blend mask moves the same way, by a +float32 step or two: its blur takes an exp, and CPUs round that differently. +Masks are compared with frozen release masks at the same tolerance, and each +of those is held to its pinned hash. The fixtures come from released code, +never the implementation under review; see fixtures/stitch_released_images.md +for provenance and regeneration. """ from __future__ import annotations @@ -45,9 +48,11 @@ STRENGTHS = (0.0, 0.5, 1.0) IMAGE_ATOL = 1e-5 REFERENCE_IMAGES = ROOT / "tests" / "fixtures" / "stitch_released_images.npz" +REFERENCE_MASKS = ROOT / "tests" / "fixtures" / "stitch_released_masks.npz" -# Historical image hashes are retained for --print diagnostics. Only masks -# are byte-exact across CPU backends; images use the release fixtures below. +# Historical image hashes are retained for --print diagnostics; images use +# the release fixtures below. A mask hash says which mask the release gave: +# the mask fixture must have that hash, and a run is compared with the fixture. # (case, color_match) -> (image, blend_mask), first 20 hex digits of SHA-256. PINS = { ("load image + pad", 0.0): ("bceaded74f00fa7af3df", "da3307917bed14a6f6ac"), @@ -175,6 +180,12 @@ def reference_image(name, strength, seam="classic"): return torch.from_numpy(references[key]) +def reference_mask(name, seam="classic"): + key = f"{seam}_{tuple(CASES).index(name)}_mask" + with np.load(REFERENCE_MASKS, allow_pickle=False) as references: + return torch.from_numpy(references[key]) + + def assert_released_image(image, expected): # No relative tolerance: bright pixels get no extra allowance. Shapes, # dtypes and non-finite values must also match (equal_nan defaults false). @@ -190,7 +201,14 @@ class ClassicSeamPinTests(unittest.TestCase): stitcher, patch_image = build() image, blend_mask = stitch(stitcher, patch_image, strength) assert_released_image(image, reference_image(name, strength)) - self.assertEqual(digest(blend_mask), PINS[(name, strength)][1]) + assert_released_image(blend_mask, reference_mask(name)) + + def test_the_mask_references_are_the_pinned_masks(self): + # Reads the files only, so every machine gives the same answer. + for (name, _strength), (_image, mask) in PINS.items(): + self.assertEqual(digest(reference_mask(name)), mask) + for name, (_image, mask) in BLEND_IN_PINS.items(): + self.assertEqual(digest(reference_mask(name, "blend_in")), mask) def test_release_pixels_with_one_thread_and_native_convolution(self): # Exercise the reduction/kernel choices that break raw image hashes. @@ -265,7 +283,7 @@ class BlendInToneMatchOffPinTests(unittest.TestCase): stitcher, patch_image = CASES[name]() image, blend_mask = stitch(stitcher, patch_image, 0.0, seam="blend in") assert_released_image(image, reference_image(name, 0.0, "blend_in")) - self.assertEqual(digest(blend_mask), BLEND_IN_PINS[name][1]) + assert_released_image(blend_mask, reference_mask(name, "blend_in")) def test_release_pixels_without_mkldnn(self): with torch.backends.mkldnn.flags(enabled=False):