feat: a non-finite latent stops the run where it happened
A chunk that samples to completion and returns NaN/Inf currently sails through
everything: the scene blend spreads it into the already-finished scenes, the VAE
decodes it, and the first thing that objects is ffmpeg's AAC encoder — "Input
contains (near) NaN/+-Inf", naming nothing that produced it, after the whole
montage has been paid for. The video half never objects at all: NaN through
astype(np.uint8) is undefined but silent, so it degrades with a warning.
_sample_chunk now checks the returned latent and raises with the run-phase label
("scene 2/3 · pass 1 of 2"), which stream, how many values and what dtype, plus
the causes worth testing in order — mismatched LoRA, a mis-inferred checkpoint
layout, a dtype the model cannot hold. It raises rather than warns because a
non-finite latent is unrecoverable and blending it forward destroys the scenes
that were still good. One isfinite() per chunk against a multi-second sample is
not a cost worth weighing; a latent the helper cannot read is ignored, never
fatal.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
8e8a4a5471
commit
9ea0be0713
+55
@@ -3025,6 +3025,60 @@ class FunPackLTXAVSceneChainSampler:
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
def _current_phase_label(self):
|
||||
"""The progress label this chunk is running under ("scene 2/3 · pass 1 of 2"), for
|
||||
error messages. Best-effort like _set_phase — never let a readout break a render."""
|
||||
try:
|
||||
try:
|
||||
from . import run_phase as _rp
|
||||
except ImportError:
|
||||
import run_phase as _rp
|
||||
return str((_rp.current() or {}).get("label") or "")
|
||||
except Exception: # noqa: BLE001
|
||||
return ""
|
||||
|
||||
def _assert_finite_sample(self, sampled):
|
||||
"""Stop the run the moment a chunk comes back with NaN/Inf in it.
|
||||
|
||||
Nothing downstream notices: the blend spreads it into every previously finished
|
||||
scene, the VAE decodes it, and the first thing that objects is ffmpeg's AAC encoder
|
||||
with a message naming nothing that produced it ("Input contains (near) NaN/+-Inf") —
|
||||
after the whole montage has been paid for. The video half is worse: NaN through
|
||||
`astype(np.uint8)` is undefined but silent, so it degrades without complaining.
|
||||
|
||||
A non-finite latent is unrecoverable, so this raises rather than warns. One
|
||||
isfinite() per chunk against a multi-second sample is not a cost worth weighing.
|
||||
"""
|
||||
try:
|
||||
tensors = self._latent_tensors({"samples": sampled})
|
||||
except Exception: # noqa: BLE001 — never fail the run over the check itself
|
||||
return
|
||||
names = ("video", "audio")
|
||||
bad = []
|
||||
for i, t in enumerate(tensors):
|
||||
if not isinstance(t, torch.Tensor):
|
||||
continue
|
||||
finite = torch.isfinite(t)
|
||||
if bool(finite.all()):
|
||||
continue
|
||||
n_bad = int((~finite).sum().item())
|
||||
label = names[i] if i < len(names) else f"stream {i}"
|
||||
bad.append(f"{label} ({n_bad}/{t.numel()} values, dtype={t.dtype})")
|
||||
if not bad:
|
||||
return
|
||||
where = self._current_phase_label() or "this chunk"
|
||||
raise RuntimeError(
|
||||
f"[FunPackSceneChain] {where}: the sampler returned a non-finite latent — "
|
||||
f"{'; '.join(bad)}. Sampling itself ran to completion, so this is corrupt "
|
||||
f"MATH, not a wiring or memory problem. Usual causes, in the order worth "
|
||||
f"testing: a LoRA that does not match this checkpoint (ComfyUI logs the keys "
|
||||
f"it could not apply at load — check the log), a base checkpoint whose layout "
|
||||
f"the loader mis-inferred (third-party repacks of quantised weights are the "
|
||||
f"common case), or a VAE/model dtype the model cannot hold. Stopping here "
|
||||
f"rather than blending this into the finished scenes and failing later in "
|
||||
f"ffmpeg with nothing to point at."
|
||||
)
|
||||
|
||||
def _sample_chunk(self, model, sampler, sigmas, seed, cfg, positive, negative, latent,
|
||||
pbar=None, step_offset=0, alg_guide_tail_frames=0,
|
||||
alg_guide_blur_strength=2.0, alg_guide_blur_sigma_threshold=0.975,
|
||||
@@ -3065,6 +3119,7 @@ class FunPackLTXAVSceneChainSampler:
|
||||
)
|
||||
finally:
|
||||
self._remove_bounded_attention(_ba_handles)
|
||||
self._assert_finite_sample(sampled)
|
||||
latent["samples"] = sampled
|
||||
latent.pop("noise_mask", None)
|
||||
return latent
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""A chunk that comes back non-finite stops the run where it happened.
|
||||
|
||||
Nothing downstream notices a NaN latent on its own: the scene blend spreads it into every
|
||||
already-finished scene, the VAE decodes it, and the first thing to object is ffmpeg's AAC
|
||||
encoder — naming nothing that produced it, after the whole montage has been paid for. The
|
||||
video half never objects at all (NaN through astype(np.uint8) is undefined but silent).
|
||||
"""
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
sys.modules.setdefault("folder_paths", types.SimpleNamespace(models_dir=""))
|
||||
|
||||
from samplers import FunPackLTXAVSceneChainSampler # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def node():
|
||||
return FunPackLTXAVSceneChainSampler()
|
||||
|
||||
|
||||
def test_a_clean_latent_passes_silently(node):
|
||||
node._assert_finite_sample(torch.zeros(1, 4, 2, 8, 8))
|
||||
|
||||
|
||||
def test_nan_in_the_video_stream_raises(node):
|
||||
x = torch.zeros(1, 4, 2, 8, 8)
|
||||
x[0, 0, 0, 0, 0] = float("nan")
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
node._assert_finite_sample(x)
|
||||
assert "non-finite latent" in str(e.value)
|
||||
assert "video" in str(e.value)
|
||||
|
||||
|
||||
def test_inf_counts_too(node):
|
||||
"""+-Inf is what the AAC encoder actually names, and it is just as unrecoverable."""
|
||||
x = torch.zeros(1, 4, 2, 8, 8)
|
||||
x[0, 0, 0, 0, 0] = float("inf")
|
||||
with pytest.raises(RuntimeError):
|
||||
node._assert_finite_sample(x)
|
||||
|
||||
|
||||
def test_the_message_says_how_many_and_which_stream(node):
|
||||
x = torch.zeros(2, 4, 2, 4, 4)
|
||||
x[0, 0, 0, 0, 0] = float("nan")
|
||||
x[0, 0, 0, 0, 1] = float("nan")
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
node._assert_finite_sample(x)
|
||||
msg = str(e.value)
|
||||
assert "2/" in msg # count of bad values
|
||||
assert "LoRA" in msg # the first thing worth testing
|
||||
assert str(x.dtype) in msg
|
||||
|
||||
|
||||
def test_the_check_never_fails_the_run_on_its_own(node):
|
||||
"""A latent shape the helper cannot read is not a reason to kill a good render."""
|
||||
node._assert_finite_sample("not a tensor")
|
||||
node._assert_finite_sample(None)
|
||||
Reference in New Issue
Block a user