feat: a non-finite latent stops the run where it happened

A chunk that samples to completion and returns NaN/Inf currently sails through
everything: the scene blend spreads it into the already-finished scenes, the VAE
decodes it, and the first thing that objects is ffmpeg's AAC encoder — "Input
contains (near) NaN/+-Inf", naming nothing that produced it, after the whole
montage has been paid for. The video half never objects at all: NaN through
astype(np.uint8) is undefined but silent, so it degrades with a warning.

_sample_chunk now checks the returned latent and raises with the run-phase label
("scene 2/3 · pass 1 of 2"), which stream, how many values and what dtype, plus
the causes worth testing in order — mismatched LoRA, a mis-inferred checkpoint
layout, a dtype the model cannot hold. It raises rather than warns because a
non-finite latent is unrecoverable and blending it forward destroys the scenes
that were still good. One isfinite() per chunk against a multi-second sample is
not a cost worth weighing; a latent the helper cannot read is ignored, never
fatal.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
digital-garbage
2026-08-04 22:41:45 +03:00
co-authored by Claude Opus 5
parent 8e8a4a5471
commit 9ea0be0713
2 changed files with 117 additions and 0 deletions
+55
View File
@@ -3025,6 +3025,60 @@ class FunPackLTXAVSceneChainSampler:
except Exception: # noqa: BLE001
pass
def _current_phase_label(self):
"""The progress label this chunk is running under ("scene 2/3 · pass 1 of 2"), for
error messages. Best-effort like _set_phase — never let a readout break a render."""
try:
try:
from . import run_phase as _rp
except ImportError:
import run_phase as _rp
return str((_rp.current() or {}).get("label") or "")
except Exception: # noqa: BLE001
return ""
def _assert_finite_sample(self, sampled):
"""Stop the run the moment a chunk comes back with NaN/Inf in it.
Nothing downstream notices: the blend spreads it into every previously finished
scene, the VAE decodes it, and the first thing that objects is ffmpeg's AAC encoder
with a message naming nothing that produced it ("Input contains (near) NaN/+-Inf") —
after the whole montage has been paid for. The video half is worse: NaN through
`astype(np.uint8)` is undefined but silent, so it degrades without complaining.
A non-finite latent is unrecoverable, so this raises rather than warns. One
isfinite() per chunk against a multi-second sample is not a cost worth weighing.
"""
try:
tensors = self._latent_tensors({"samples": sampled})
except Exception: # noqa: BLE001 — never fail the run over the check itself
return
names = ("video", "audio")
bad = []
for i, t in enumerate(tensors):
if not isinstance(t, torch.Tensor):
continue
finite = torch.isfinite(t)
if bool(finite.all()):
continue
n_bad = int((~finite).sum().item())
label = names[i] if i < len(names) else f"stream {i}"
bad.append(f"{label} ({n_bad}/{t.numel()} values, dtype={t.dtype})")
if not bad:
return
where = self._current_phase_label() or "this chunk"
raise RuntimeError(
f"[FunPackSceneChain] {where}: the sampler returned a non-finite latent — "
f"{'; '.join(bad)}. Sampling itself ran to completion, so this is corrupt "
f"MATH, not a wiring or memory problem. Usual causes, in the order worth "
f"testing: a LoRA that does not match this checkpoint (ComfyUI logs the keys "
f"it could not apply at load — check the log), a base checkpoint whose layout "
f"the loader mis-inferred (third-party repacks of quantised weights are the "
f"common case), or a VAE/model dtype the model cannot hold. Stopping here "
f"rather than blending this into the finished scenes and failing later in "
f"ffmpeg with nothing to point at."
)
def _sample_chunk(self, model, sampler, sigmas, seed, cfg, positive, negative, latent,
pbar=None, step_offset=0, alg_guide_tail_frames=0,
alg_guide_blur_strength=2.0, alg_guide_blur_sigma_threshold=0.975,
@@ -3065,6 +3119,7 @@ class FunPackLTXAVSceneChainSampler:
)
finally:
self._remove_bounded_attention(_ba_handles)
self._assert_finite_sample(sampled)
latent["samples"] = sampled
latent.pop("noise_mask", None)
return latent
+62
View File
@@ -0,0 +1,62 @@
"""A chunk that comes back non-finite stops the run where it happened.
Nothing downstream notices a NaN latent on its own: the scene blend spreads it into every
already-finished scene, the VAE decodes it, and the first thing to object is ffmpeg's AAC
encoder — naming nothing that produced it, after the whole montage has been paid for. The
video half never objects at all (NaN through astype(np.uint8) is undefined but silent).
"""
import sys
import types
from pathlib import Path
import pytest
import torch
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.modules.setdefault("folder_paths", types.SimpleNamespace(models_dir=""))
from samplers import FunPackLTXAVSceneChainSampler # noqa: E402
@pytest.fixture
def node():
return FunPackLTXAVSceneChainSampler()
def test_a_clean_latent_passes_silently(node):
node._assert_finite_sample(torch.zeros(1, 4, 2, 8, 8))
def test_nan_in_the_video_stream_raises(node):
x = torch.zeros(1, 4, 2, 8, 8)
x[0, 0, 0, 0, 0] = float("nan")
with pytest.raises(RuntimeError) as e:
node._assert_finite_sample(x)
assert "non-finite latent" in str(e.value)
assert "video" in str(e.value)
def test_inf_counts_too(node):
"""+-Inf is what the AAC encoder actually names, and it is just as unrecoverable."""
x = torch.zeros(1, 4, 2, 8, 8)
x[0, 0, 0, 0, 0] = float("inf")
with pytest.raises(RuntimeError):
node._assert_finite_sample(x)
def test_the_message_says_how_many_and_which_stream(node):
x = torch.zeros(2, 4, 2, 4, 4)
x[0, 0, 0, 0, 0] = float("nan")
x[0, 0, 0, 0, 1] = float("nan")
with pytest.raises(RuntimeError) as e:
node._assert_finite_sample(x)
msg = str(e.value)
assert "2/" in msg # count of bad values
assert "LoRA" in msg # the first thing worth testing
assert str(x.dtype) in msg
def test_the_check_never_fails_the_run_on_its_own(node):
"""A latent shape the helper cannot read is not a reason to kill a good render."""
node._assert_finite_sample("not a tensor")
node._assert_finite_sample(None)