From 9ea0be07131056460a515f321b3e33641f4e83ba Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Tue, 4 Aug 2026 22:41:45 +0300 Subject: [PATCH] feat: a non-finite latent stops the run where it happened MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A chunk that samples to completion and returns NaN/Inf currently sails through everything: the scene blend spreads it into the already-finished scenes, the VAE decodes it, and the first thing that objects is ffmpeg's AAC encoder — "Input contains (near) NaN/+-Inf", naming nothing that produced it, after the whole montage has been paid for. The video half never objects at all: NaN through astype(np.uint8) is undefined but silent, so it degrades with a warning. _sample_chunk now checks the returned latent and raises with the run-phase label ("scene 2/3 · pass 1 of 2"), which stream, how many values and what dtype, plus the causes worth testing in order — mismatched LoRA, a mis-inferred checkpoint layout, a dtype the model cannot hold. It raises rather than warns because a non-finite latent is unrecoverable and blending it forward destroys the scenes that were still good. One isfinite() per chunk against a multi-second sample is not a cost worth weighing; a latent the helper cannot read is ignored, never fatal. Co-Authored-By: Claude Opus 5 --- samplers.py | 55 +++++++++++++++++++++++++++ tests/test_finite_sample_guard.py | 62 +++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 tests/test_finite_sample_guard.py diff --git a/samplers.py b/samplers.py index 8cb6db4d..baeb8ab5 100644 --- a/samplers.py +++ b/samplers.py @@ -3025,6 +3025,60 @@ class FunPackLTXAVSceneChainSampler: except Exception: # noqa: BLE001 pass + def _current_phase_label(self): + """The progress label this chunk is running under ("scene 2/3 · pass 1 of 2"), for + error messages. Best-effort like _set_phase — never let a readout break a render.""" + try: + try: + from . import run_phase as _rp + except ImportError: + import run_phase as _rp + return str((_rp.current() or {}).get("label") or "") + except Exception: # noqa: BLE001 + return "" + + def _assert_finite_sample(self, sampled): + """Stop the run the moment a chunk comes back with NaN/Inf in it. + + Nothing downstream notices: the blend spreads it into every previously finished + scene, the VAE decodes it, and the first thing that objects is ffmpeg's AAC encoder + with a message naming nothing that produced it ("Input contains (near) NaN/+-Inf") — + after the whole montage has been paid for. The video half is worse: NaN through + `astype(np.uint8)` is undefined but silent, so it degrades without complaining. + + A non-finite latent is unrecoverable, so this raises rather than warns. One + isfinite() per chunk against a multi-second sample is not a cost worth weighing. + """ + try: + tensors = self._latent_tensors({"samples": sampled}) + except Exception: # noqa: BLE001 — never fail the run over the check itself + return + names = ("video", "audio") + bad = [] + for i, t in enumerate(tensors): + if not isinstance(t, torch.Tensor): + continue + finite = torch.isfinite(t) + if bool(finite.all()): + continue + n_bad = int((~finite).sum().item()) + label = names[i] if i < len(names) else f"stream {i}" + bad.append(f"{label} ({n_bad}/{t.numel()} values, dtype={t.dtype})") + if not bad: + return + where = self._current_phase_label() or "this chunk" + raise RuntimeError( + f"[FunPackSceneChain] {where}: the sampler returned a non-finite latent — " + f"{'; '.join(bad)}. Sampling itself ran to completion, so this is corrupt " + f"MATH, not a wiring or memory problem. Usual causes, in the order worth " + f"testing: a LoRA that does not match this checkpoint (ComfyUI logs the keys " + f"it could not apply at load — check the log), a base checkpoint whose layout " + f"the loader mis-inferred (third-party repacks of quantised weights are the " + f"common case), or a VAE/model dtype the model cannot hold. Stopping here " + f"rather than blending this into the finished scenes and failing later in " + f"ffmpeg with nothing to point at." + ) + def _sample_chunk(self, model, sampler, sigmas, seed, cfg, positive, negative, latent, pbar=None, step_offset=0, alg_guide_tail_frames=0, alg_guide_blur_strength=2.0, alg_guide_blur_sigma_threshold=0.975, @@ -3065,6 +3119,7 @@ class FunPackLTXAVSceneChainSampler: ) finally: self._remove_bounded_attention(_ba_handles) + self._assert_finite_sample(sampled) latent["samples"] = sampled latent.pop("noise_mask", None) return latent diff --git a/tests/test_finite_sample_guard.py b/tests/test_finite_sample_guard.py new file mode 100644 index 00000000..5e240fd5 --- /dev/null +++ b/tests/test_finite_sample_guard.py @@ -0,0 +1,62 @@ +"""A chunk that comes back non-finite stops the run where it happened. + +Nothing downstream notices a NaN latent on its own: the scene blend spreads it into every +already-finished scene, the VAE decodes it, and the first thing to object is ffmpeg's AAC +encoder — naming nothing that produced it, after the whole montage has been paid for. The +video half never objects at all (NaN through astype(np.uint8) is undefined but silent). +""" +import sys +import types +from pathlib import Path + +import pytest +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.modules.setdefault("folder_paths", types.SimpleNamespace(models_dir="")) + +from samplers import FunPackLTXAVSceneChainSampler # noqa: E402 + + +@pytest.fixture +def node(): + return FunPackLTXAVSceneChainSampler() + + +def test_a_clean_latent_passes_silently(node): + node._assert_finite_sample(torch.zeros(1, 4, 2, 8, 8)) + + +def test_nan_in_the_video_stream_raises(node): + x = torch.zeros(1, 4, 2, 8, 8) + x[0, 0, 0, 0, 0] = float("nan") + with pytest.raises(RuntimeError) as e: + node._assert_finite_sample(x) + assert "non-finite latent" in str(e.value) + assert "video" in str(e.value) + + +def test_inf_counts_too(node): + """+-Inf is what the AAC encoder actually names, and it is just as unrecoverable.""" + x = torch.zeros(1, 4, 2, 8, 8) + x[0, 0, 0, 0, 0] = float("inf") + with pytest.raises(RuntimeError): + node._assert_finite_sample(x) + + +def test_the_message_says_how_many_and_which_stream(node): + x = torch.zeros(2, 4, 2, 4, 4) + x[0, 0, 0, 0, 0] = float("nan") + x[0, 0, 0, 0, 1] = float("nan") + with pytest.raises(RuntimeError) as e: + node._assert_finite_sample(x) + msg = str(e.value) + assert "2/" in msg # count of bad values + assert "LoRA" in msg # the first thing worth testing + assert str(x.dtype) in msg + + +def test_the_check_never_fails_the_run_on_its_own(node): + """A latent shape the helper cannot read is not a reason to kill a good render.""" + node._assert_finite_sample("not a tensor") + node._assert_finite_sample(None)