From e7b624d684e95276fcd3e2b0d4fad37b9ad03b65 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:39:47 +0300 Subject: [PATCH] fix: drop guide keyframes on a resized second pass; unify the two ALG switches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two unrelated fixes. **Guide keyframes vs a resolution-changing second_pass_op.** Guide keyframes are recorded as TOKEN indices into pass 1's latent grid, so upscale_2x — which doubles each spatial dim and thus quadruples the tokens per latent frame — leaves them addressing the wrong tokens. Since the LTX-2.5 commit core rejects that outright ("keyframe_idxs holds N tokens, which is not a whole number of M-token latent frames"), which is how this surfaced; older cores silently mis-placed the guides instead, which is worse. Pass 2 now drops them and says so in the scene report, exactly as segmented detailing already does for its crop — so detailing's stripper becomes public rather than being duplicated. Only pass 2 is affected; pass 1 ran at the grid the guides were recorded against and keeps every one. The pin loss and the guide loss now share one predicate (_latent_spatial_changed) so they cannot drift apart, and an unreadable shape answers "no change" so a probe failure never costs a guide. **One ALG control instead of two.** The Distilled Flow panel had its own alg_enabled while the chain sampler's alg_anchor did the same guidance on any sampler and already drove the Distilled Flow one when set — two switches for one behaviour, with precedence depending on which you had touched. The Editor now shows only alg_anchor. The node input stays for hand-built graphs; what moved is the Editor's control. A project still carrying the old switch is migrated once, strength and threshold included, and told so, rather than left running ALG with nothing on screen to turn it off. The migration is deferred out of the render pass and guarded by a module-level set of project ids: every store write re-renders the pane synchronously, so migrating inline recursed until the stack blew (caught in the browser, not by the suite). Co-Authored-By: Claude Opus 5 --- detailing.py | 20 ++++- movie_editor/frontend/engine_settings.js | 49 +++++++++++- movie_editor/frontend/sampler_panel.js | 18 ++--- samplers.py | 39 +++++++++- tests/test_detailing.py | 4 +- tests/test_second_pass_layout_conds.py | 95 ++++++++++++++++++++++++ 6 files changed, 204 insertions(+), 21 deletions(-) create mode 100644 tests/test_second_pass_layout_conds.py diff --git a/detailing.py b/detailing.py index 647dde8c..62cab65a 100644 --- a/detailing.py +++ b/detailing.py @@ -334,12 +334,28 @@ def find_tube(heat, latent_h, latent_w, threshold=DEFAULT_THRESHOLD, # Refine # --------------------------------------------------------------------------- -def _strip_layout_conds(conds): +def has_layout_conds(conds): + """True when any entry carries a full-frame layout key — i.e. when + strip_layout_conds() would actually drop something. Callers use this to + report the loss instead of silently discarding an anchor or a guide. + """ + for entry in conds or []: + if isinstance(entry, (list, tuple)) and len(entry) == 2 and isinstance(entry[1], dict): + if any(k in entry[1] for k in _LAYOUT_COND_KEYS): + return True + return False + + +def strip_layout_conds(conds): """Copy conditioning minus full-frame layout keys (anchors/guides). The crop refine is its own small denoise: text still applies, but keyframe indices and guide latents describe positions in the full-frame latent and would fight (or crash) the crop-sized pass. + + Public because the Scene Chain sampler needs the same treatment for a + resolution-changing second_pass_op — any pass run on a differently-sized + latent than the one these keys were recorded against has the same problem. """ out = [] for entry in conds or []: @@ -508,7 +524,7 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, tail_sigmas = torch.tensor(stage2_sigmas(renoise_sigma), dtype=torch.float32) refined = chain._sample_chunk( model, sampler, tail_sigmas, int(seed) + 7777, cfg, - _strip_layout_conds(positive), _strip_layout_conds(negative), crop_latent) + strip_layout_conds(positive), strip_layout_conds(negative), crop_latent) refined_crop = chain._latent_tensors(refined)[0] cost_note = f"repair: renoise={renoise_sigma:.2f}" diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 1ed628bb..b54bcf3a 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -7,6 +7,12 @@ const S = window.Store; const API = window.MovieEditorAPI; + // Project ids whose legacy Distilled-Flow ALG switch has been folded into the + // sampler-wide alg_anchor this session. Module scope, not render scope: every store + // write re-renders this pane synchronously, so the guard has to outlive a render or + // the migration re-enters itself and recurses until the stack blows. + const _ALG_MIGRATED = new Set(); + // Easy Gen has no rating UI at all (by design — see easy_gen/frontend/), so every // setting that is a no-op without a trained refinement key / rated history is hidden // there, not just made harder to find. window.FunPackAppName is the same discriminator @@ -648,6 +654,36 @@ else S.unsetSamplerInput("second_pass"); } + // ALG used to have two switches: the Distilled Flow panel's own alg_enabled, and the + // chain sampler's alg_anchor, which is the same guidance on ANY sampler and already + // drove the Distilled Flow one whenever it was on. Two controls for one behaviour, and + // which of them won depended on which you had set — so the Editor now shows only + // alg_anchor. The node input still exists for hand-built graphs; what moves is the + // Editor's control. A project carrying the old switch is migrated once, with its + // strength and threshold, rather than left running ALG with nothing on screen to + // turn it off. + function migrateDistilledAlg(cfg) { + const pid = String(S.get()?.project?.id || ""); + const dc = cfg?.high?.distilled; + if (!dc || !dc.alg_enabled) return _ALG_MIGRATED.has(pid); + // Mark and clear BEFORE writing anything: each setter notifies the store, which + // re-renders this pane synchronously and comes straight back through here. + _ALG_MIGRATED.add(pid); + const strength = dc.alg_strength; + const threshold = dc.alg_sigma_threshold; + dc.alg_enabled = false; + // Out of the render pass entirely — persisting mid-render is what caused the + // recursion above, and a deferred write costs one extra repaint instead. + setTimeout(() => { + S.setSamplerInputNow("alg_anchor", true); + if (strength != null) S.setSamplerInputNow("alg_anchor_strength", Number(strength)); + if (threshold != null) S.setSamplerInputNow("alg_anchor_sigma_threshold", Number(threshold)); + persistSamplers(cfg, false); + }, 0); + return true; + } + const algMigrated = migrateDistilledAlg(samplers); + try { window.SamplerPanel.render(box, samplers, (s) => persistSamplers(s, true), @@ -664,8 +700,17 @@ // it is never hidden behind a particular choice up there. const algG = group(pane, "Anchor blur (ALG)"); renderKnobList(algG, st, ["alg_anchor", "alg_anchor_strength", "alg_anchor_sigma_threshold"]); - algG.append(hintEl("The same blur for guide and JoyAI-memory frames is under Experimental " - + "(“Blur i2v guides and JoyAI memory”) — it has its own strength and window.")); + if (algMigrated) { + const moved = hintEl("Moved here from the Distilled Flow panel's own ALG switch — it was " + + "the same blur, and this one works on every sampler. Your strength and threshold " + + "came with it; nothing changed about how the scene samples."); + moved.style.color = "var(--amber)"; + algG.append(moved); + } + algG.append(hintEl("This is the only ALG anchor control — on Distilled Flow it runs inside " + + "the sampler loop, on every other sampler through a denoiser proxy. The same blur for " + + "guide and JoyAI-memory frames is under Experimental (“Blur i2v guides and JoyAI " + + "memory”) — it has its own strength and window.")); // Second pass lives here rather than under Experimental: at its defaults the split is // behaviour-neutral (pass 2 resumes from exactly the state pass 1 handed over), so it // is a sampler setting, not a gamble. The only control is the schedule field in the diff --git a/movie_editor/frontend/sampler_panel.js b/movie_editor/frontend/sampler_panel.js index 52f457c9..1279080c 100644 --- a/movie_editor/frontend/sampler_panel.js +++ b/movie_editor/frontend/sampler_panel.js @@ -332,18 +332,12 @@ (v) => { dc.s_noise = v; save(); })); renderVelocityBlock(container, dk + "-d", dc, save, saveNow); - sectionTag(container, "ALG (experimental)"); - row(container, "enabled", - checkCtrl(!!dc.alg_enabled, dk + "-dalge", (v) => { dc.alg_enabled = v; saveNow(); })); - hint(container, "Adaptive Low-Pass Guidance (arXiv:2506.08456). Blurs the i2v anchor frame during the earliest steps so the model can't shortcut to a near-static video that just matches it. No effect without an i2v anchor. Same guidance as 'Anchor blur (ALG)' below, which works on every sampler — turning that on drives this one, so you only need one of the two."); - if (dc.alg_enabled) { - row(container, "blur strength", - numCtrl(dc.alg_strength, 1.0, 4, 0.1, dk + "-dalgs", - (v) => { dc.alg_strength = v; save(); })); - row(container, "sigma threshold", - numCtrl(dc.alg_sigma_threshold, 0.5, 0.999, 0.005, dk + "-dalgt", - (v) => { dc.alg_sigma_threshold = v; save(); })); - } + // No ALG block here on purpose. This sampler has its own alg_enabled input, but the + // chain sampler's "Anchor blur (ALG)" is the same guidance, works on every sampler, + // and already drives this one when it is on — so showing both put two switches on + // screen for one behaviour, with precedence that depended on which you had set. + // The single control lives in the "Anchor blur (ALG)" group just below this panel; + // engine_settings.js migrates a project that still carries the old switch. sectionTag(container, "Momentum Guidance (experimental)"); row(container, "enabled", diff --git a/samplers.py b/samplers.py index ddaebd2a..84c7513b 100644 --- a/samplers.py +++ b/samplers.py @@ -3713,6 +3713,17 @@ class FunPackLTXAVSceneChainSampler: result["samples"] = tensors[0] return result, note + def _latent_spatial_changed(self, state, chunk): + """True when `state`'s video stream no longer has `chunk`'s H/W — i.e. a + second_pass_op resized it. Everything recorded against the old grid (the i2v + pin, guide keyframe token indices) is invalid past this point. Unreadable + shapes answer False so a probe failure never drops a guide on its own.""" + try: + return (self._latent_tensors(state)[0].shape[-2:] + != self._latent_tensors(chunk)[0].shape[-2:]) + except Exception: + return False + def _restore_pinned_prefix(self, state, chunk): """Put the i2v pin back on a latent that is about to be handed to a second pass. @@ -3729,7 +3740,7 @@ class FunPackLTXAVSceneChainSampler: # A resolution-changing second_pass_op (upscale_2x) leaves the chunk's anchor and # mask the wrong size. Rescaling them here would be inventing an anchor; the # caller reports the lost pin instead. - if self._latent_tensors(state)[0].shape[-2:] != self._latent_tensors(chunk)[0].shape[-2:]: + if self._latent_spatial_changed(state, chunk): return state except Exception: return state @@ -6791,6 +6802,28 @@ class FunPackLTXAVSceneChainSampler: "second_pass NOTE: the i2v anchor could not stay pinned for pass 2 " "because second_pass_op changed the latent resolution — pass 2 runs " "unpinned and may drift from the reference image.") + # Same resolution change, second casualty: guide keyframes are recorded as + # TOKEN indices into the pass-1 grid, and 2x spatial means 4x the tokens per + # latent frame, so those indices now address the wrong tokens entirely. + # Since the LTX-2.5 commit core rejects it outright ("keyframe_idxs holds N + # tokens, which is not a whole number of M-token latent frames") and tells + # you to crop the guides before upscaling; older cores silently mis-placed + # them, which is worse. Drop them for pass 2, exactly as segmented detailing + # does for its crop — and only for pass 2, so pass 1 keeps every guide. + _sp_positive, _sp_negative = scene_positive, scene_negative + if self._latent_spatial_changed(_sp_state, chunk): + try: + from .detailing import has_layout_conds, strip_layout_conds + except ImportError: + from detailing import has_layout_conds, strip_layout_conds + if has_layout_conds(scene_positive) or has_layout_conds(scene_negative): + _sp_positive = strip_layout_conds(scene_positive) + _sp_negative = strip_layout_conds(scene_negative) + run_mechanisms.append( + "second_pass NOTE: mid-scene guides / guide keyframes were " + "dropped for pass 2 because second_pass_op changed the latent " + "resolution — their token positions describe the pass-1 grid. " + "Pass 1 used them in full.") _sp_kw = dict(_sample_kwargs) _sp_kw["step_offset"] = _sample_kwargs["step_offset"] + (int(sigmas.numel()) - 1) # Announced BEFORE it runs: the after-the-fact run_mechanisms line says @@ -6800,8 +6833,8 @@ class FunPackLTXAVSceneChainSampler: f"{' on ' + _scene_label if _scene_label else ''}: " f"{int(_sp_b.numel()) - 1} steps from sigma {float(_sp_b[0].item()):g}") sampled = self._sample_chunk( - model, sampler, _sp_b, scene_seed + 4242, cfg, scene_positive, - scene_negative, _sp_state, **_sp_kw) + model, sampler, _sp_b, scene_seed + 4242, cfg, _sp_positive, + _sp_negative, _sp_state, **_sp_kw) run_mechanisms.append( f"second_pass({int(sigmas.numel()) - 1} steps + " f"{int(_sp_b.numel()) - 1} steps = " diff --git a/tests/test_detailing.py b/tests/test_detailing.py index 84cabc21..89e61153 100644 --- a/tests/test_detailing.py +++ b/tests/test_detailing.py @@ -220,13 +220,13 @@ def test_resolve_download_failure_raises_actionable_error(monkeypatch, tmp_path) # --------------------------------------------------------------------------- -# _strip_layout_conds / _downscale_to +# strip_layout_conds / _downscale_to # --------------------------------------------------------------------------- def test_strip_layout_conds_removes_guide_keys_only(): cond = [[torch.zeros(1, 4), {"keyframe_idxs": "kf", "guiding_latent": "g", "pooled_output": "keep"}]] - out = detailing._strip_layout_conds(cond) + out = detailing.strip_layout_conds(cond) assert out[0][1] == {"pooled_output": "keep"} # Original untouched (the scene's own conds must not lose their guides). assert "keyframe_idxs" in cond[0][1] diff --git a/tests/test_second_pass_layout_conds.py b/tests/test_second_pass_layout_conds.py new file mode 100644 index 00000000..49d39843 --- /dev/null +++ b/tests/test_second_pass_layout_conds.py @@ -0,0 +1,95 @@ +"""Guide keyframes must not survive a resolution-changing second_pass_op. + +Guide keyframes are recorded as TOKEN indices into pass 1's latent grid. `upscale_2x` +doubles each spatial dim, so a latent frame is suddenly 4x as many tokens and those indices +address the wrong ones. Since the LTX-2.5 commit core raises on it outright ("keyframe_idxs +holds N tokens, which is not a whole number of M-token latent frames"); before that it +silently mis-placed the guides, which is worse. Pass 2 therefore drops them — and only +pass 2, because pass 1 ran at the grid they were recorded against. +""" +import sys +import types +from pathlib import Path + +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import detailing # noqa: E402 +import samplers # noqa: E402 + + +def _cond(**meta): + return [[torch.zeros(1, 4, 128), dict(meta)]] + + +def _latent(h, w, frames=3): + return {"samples": torch.zeros(1, 128, frames, h, w)} + + +def _chain(): + return samplers.FunPackLTXAVSceneChainSampler() + + +# --- the shared predicate ------------------------------------------------------------------ + +def test_spatial_change_is_detected(): + chain = _chain() + assert chain._latent_spatial_changed(_latent(32, 48), _latent(16, 24)) is True + + +def test_same_size_is_not_a_change(): + chain = _chain() + assert chain._latent_spatial_changed(_latent(16, 24), _latent(16, 24)) is False + + +def test_more_frames_alone_is_not_a_spatial_change(): + """Only H/W matter: token-per-frame count is spatial. A differing frame count must not + be read as a resize, or 'sharpen' runs would drop guides for no reason.""" + chain = _chain() + assert chain._latent_spatial_changed(_latent(16, 24, frames=5), _latent(16, 24, frames=3)) is False + + +def test_unreadable_shapes_report_no_change(): + """A probe failure must not cost the user their guides.""" + chain = _chain() + assert chain._latent_spatial_changed({"samples": "not a tensor"}, _latent(16, 24)) is False + + +# --- the stripper ------------------------------------------------------------------------- + +def test_layout_keys_are_detected_and_stripped(): + conds = _cond(keyframe_idxs=torch.zeros(1, 3, 512), pooled_output=None) + assert detailing.has_layout_conds(conds) is True + stripped = detailing.strip_layout_conds(conds) + assert "keyframe_idxs" not in stripped[0][1] + assert "pooled_output" in stripped[0][1], "only layout keys go; text conditioning stays" + + +def test_every_layout_key_is_covered(): + conds = _cond(**{k: 1 for k in detailing._LAYOUT_COND_KEYS}) + assert detailing.strip_layout_conds(conds)[0][1] == {} + + +def test_plain_text_conditioning_has_no_layout_keys(): + """The common case: no guides, so pass 2 must be told nothing was dropped.""" + assert detailing.has_layout_conds(_cond(pooled_output=None)) is False + + +def test_stripping_does_not_mutate_the_input(): + """Pass 1's conditioning is the same object; mutating it would retroactively strip the + pass that legitimately used the guides.""" + conds = _cond(keyframe_idxs=torch.zeros(1, 3, 512)) + detailing.strip_layout_conds(conds) + assert "keyframe_idxs" in conds[0][1] + + +def test_tensor_identity_is_preserved(): + """Only the meta dict is rebuilt — the conditioning tensor must not be copied.""" + conds = _cond(keyframe_idxs=1) + assert detailing.strip_layout_conds(conds)[0][0] is conds[0][0] + + +def test_non_entry_items_pass_through_untouched(): + assert detailing.strip_layout_conds([None, "junk"]) == [None, "junk"] + assert detailing.has_layout_conds([None, "junk"]) is False