diff --git a/movie_editor/frontend/pipeline_caps.js b/movie_editor/frontend/pipeline_caps.js index 518f43fc..6cd6255b 100644 --- a/movie_editor/frontend/pipeline_caps.js +++ b/movie_editor/frontend/pipeline_caps.js @@ -106,7 +106,10 @@ // Same idea for non-boolean knobs whose neutral value means "off". function h3DeadValueIssues(si) { const out = []; - if (Math.abs(Number(si.v2a_grad_scale ?? 1) - 1) > 1e-6) { + // Only when joyai_audio_memory is on: v2a_grad_scale is that feature's coupling knob and + // does nothing without it on ANY model, so flagging a left-over value as an H3 limitation + // blames the wrong thing — it would be equally inert on LTX. + if (si.joyai_audio_memory && Math.abs(Number(si.v2a_grad_scale ?? 1) - 1) > 1e-6) { out.push(["v2a_grad_scale", "v2a_grad_scale scales LTXAV's video→audio cross-attention module; H3 has no separate " + "video→audio attention, only joint rows."]); diff --git a/samplers.py b/samplers.py index baeb8ab5..baea6fb9 100644 --- a/samplers.py +++ b/samplers.py @@ -3037,6 +3037,71 @@ class FunPackLTXAVSceneChainSampler: except Exception: # noqa: BLE001 return "" + def _nonfinite_report(self, samples): + """['video (12/1000 values, dtype=...)'] for each stream carrying NaN/Inf. Empty when + clean, and empty when the latent is not readable — the check must never be the thing + that kills a render.""" + try: + tensors = self._latent_tensors({"samples": samples}) + except Exception: # noqa: BLE001 + return [] + names = ("video", "audio") + bad = [] + for i, t in enumerate(tensors): + if not isinstance(t, torch.Tensor): + continue + finite = torch.isfinite(t) + if bool(finite.all()): + continue + n_bad = int((~finite).sum().item()) + label = names[i] if i < len(names) else f"stream {i}" + bad.append(f"{label} ({n_bad}/{t.numel()} values, dtype={t.dtype})") + return bad + + def _assert_finite_inputs(self, samples, sigmas): + """Check what the chunk is ABOUT TO be sampled from, so a corrupt run says whether the + damage arrived or was produced here. + + Without this the output check alone cannot tell "the model computed garbage" from "the + model was handed garbage and faithfully propagated it" — and those have disjoint + suspect lists. The schedule is checked too because it is hand-typed: our own solvers + divide by sigma (er = sigma_next / sigma), so an INTERIOR zero or a repeat is an + instant Inf that then poisons every value in the tensor. A trailing zero is fine — it + is only ever the target of the last step, never a divisor. + """ + where = self._current_phase_label() or "this chunk" + bad = self._nonfinite_report(samples) + if bad: + raise RuntimeError( + f"[FunPackSceneChain] {where}: the latent handed to the sampler is ALREADY " + f"non-finite before any sampling — {'; '.join(bad)}. The model has not run " + f"yet, so this is not the checkpoint, the LoRA or the sampler. Look at what " + f"produced this latent: the empty-latent node, the i2v anchor encode, or (on " + f"a later scene) the previous scene's output." + ) + if not isinstance(sigmas, torch.Tensor) or sigmas.numel() < 2: + return + s = sigmas.detach().float().reshape(-1).cpu() + if not bool(torch.isfinite(s).all()): + raise RuntimeError( + f"[FunPackSceneChain] {where}: the sigma schedule contains NaN/Inf " + f"({[float(v) for v in s]}). Fix the schedule — nothing downstream can." + ) + interior = s[:-1] + if bool((interior <= 0).any()): + raise RuntimeError( + f"[FunPackSceneChain] {where}: the sigma schedule has a zero or negative " + f"value before its last entry ({[float(v) for v in s]}). Every value except " + f"the final one is divided by, so this produces Inf on that step and NaN " + f"everywhere after it. Only the LAST sigma may be 0." + ) + if bool((s[1:] >= s[:-1]).any()): + raise RuntimeError( + f"[FunPackSceneChain] {where}: the sigma schedule is not strictly " + f"descending ({[float(v) for v in s]}). A repeated value makes a zero-length " + f"step the solvers divide by; an ascending one integrates backwards." + ) + def _assert_finite_sample(self, sampled): """Stop the run the moment a chunk comes back with NaN/Inf in it. @@ -3049,28 +3114,14 @@ class FunPackLTXAVSceneChainSampler: A non-finite latent is unrecoverable, so this raises rather than warns. One isfinite() per chunk against a multi-second sample is not a cost worth weighing. """ - try: - tensors = self._latent_tensors({"samples": sampled}) - except Exception: # noqa: BLE001 — never fail the run over the check itself - return - names = ("video", "audio") - bad = [] - for i, t in enumerate(tensors): - if not isinstance(t, torch.Tensor): - continue - finite = torch.isfinite(t) - if bool(finite.all()): - continue - n_bad = int((~finite).sum().item()) - label = names[i] if i < len(names) else f"stream {i}" - bad.append(f"{label} ({n_bad}/{t.numel()} values, dtype={t.dtype})") + bad = self._nonfinite_report(sampled) if not bad: return where = self._current_phase_label() or "this chunk" raise RuntimeError( f"[FunPackSceneChain] {where}: the sampler returned a non-finite latent — " - f"{'; '.join(bad)}. Sampling itself ran to completion, so this is corrupt " - f"MATH, not a wiring or memory problem. Usual causes, in the order worth " + f"{'; '.join(bad)}. Its inputs were checked and were clean, so this was produced " + f"HERE, by the model or the sampler math. Usual causes, in the order worth " f"testing: a LoRA that does not match this checkpoint (ComfyUI logs the keys " f"it could not apply at load — check the log), a base checkpoint whose layout " f"the loader mis-inferred (third-party repacks of quantised weights are the " @@ -3089,6 +3140,7 @@ class FunPackLTXAVSceneChainSampler: raise ValueError("sigmas input must be a SIGMAS tensor.") latent = self._clone_latent(latent) samples = latent["samples"] + self._assert_finite_inputs(samples, sigmas) noise = comfy.sample.prepare_noise(samples, int(seed)) def _progress_cb(step, _denoised, _x, _total_steps): @@ -5488,7 +5540,12 @@ class FunPackLTXAVSceneChainSampler: "against LTX's 4096-wide cross-attention context, and the overlap " "tokens need LTX's patchifier; H3's native ref2va reference blocks " "are the equivalent and are not wired yet)") - if v2a_grad_scale is not None and abs(float(v2a_grad_scale) - 1.0) > 1e-6: + # Gated on joyai_audio_memory, not on the value alone: v2a_grad_scale is that + # feature's coupling knob and is never installed without it (see the call site), + # so reporting a left-over value as an H3 limitation blames the model for a knob + # that would be equally inert on LTX. + if (joyai_audio_memory and v2a_grad_scale is not None + and abs(float(v2a_grad_scale) - 1.0) > 1e-6): _dead.append("v2a_grad_scale (hooks LTXAV's video_to_audio_attn submodule; H3 " "has no separate video->audio cross-attention to scale)") if segmented_detailing: diff --git a/tests/test_finite_sample_guard.py b/tests/test_finite_sample_guard.py index 5e240fd5..aa158f3f 100644 --- a/tests/test_finite_sample_guard.py +++ b/tests/test_finite_sample_guard.py @@ -60,3 +60,58 @@ def test_the_check_never_fails_the_run_on_its_own(node): """A latent shape the helper cannot read is not a reason to kill a good render.""" node._assert_finite_sample("not a tensor") node._assert_finite_sample(None) + + +# ── inputs ──────────────────────────────────────────────────────────────────── +# The output check alone cannot tell "the model computed garbage" from "the model was handed +# garbage and propagated it faithfully", and those have disjoint suspect lists. + +def _sigmas(*vals): + return torch.tensor(list(vals), dtype=torch.float32) + + +def test_clean_inputs_pass(node): + node._assert_finite_inputs(torch.zeros(1, 4, 2, 8, 8), _sigmas(1.0, 0.5, 0.0)) + + +def test_a_latent_that_arrives_nan_blames_its_producer_not_the_model(node): + x = torch.zeros(1, 4, 2, 8, 8) + x[0, 0, 0, 0, 0] = float("nan") + with pytest.raises(RuntimeError) as e: + node._assert_finite_inputs(x, _sigmas(1.0, 0.0)) + msg = str(e.value) + assert "ALREADY" in msg + assert "has not run yet" in msg + # It names the model-side suspects only to RULE THEM OUT, and points at the producer. + assert "not the checkpoint, the LoRA or the sampler" in msg + assert "empty-latent node" in msg + + +def test_nan_in_the_schedule_is_named_as_the_schedule(node): + with pytest.raises(RuntimeError) as e: + node._assert_finite_inputs(torch.zeros(1, 4), _sigmas(1.0, float("nan"), 0.0)) + assert "sigma schedule contains NaN" in str(e.value) + + +def test_an_interior_zero_sigma_is_caught_before_it_divides(node): + """Our solvers divide by sigma, so a 0 anywhere but the end is an instant Inf.""" + with pytest.raises(RuntimeError) as e: + node._assert_finite_inputs(torch.zeros(1, 4), _sigmas(1.0, 0.0, 0.5, 0.0)) + assert "before its last entry" in str(e.value) + + +def test_a_trailing_zero_is_fine(node): + """The last sigma is only ever a target, never a divisor — the normal schedule shape.""" + node._assert_finite_inputs(torch.zeros(1, 4), _sigmas(1.0, 0.6, 0.3, 0.0)) + + +def test_a_repeated_sigma_is_caught(node): + with pytest.raises(RuntimeError) as e: + node._assert_finite_inputs(torch.zeros(1, 4), _sigmas(1.0, 0.5, 0.5, 0.0)) + assert "strictly descending" in str(e.value) + + +def test_an_ascending_schedule_is_caught(node): + with pytest.raises(RuntimeError) as e: + node._assert_finite_inputs(torch.zeros(1, 4), _sigmas(0.3, 0.9, 0.0)) + assert "strictly descending" in str(e.value)