diff --git a/detailing.py b/detailing.py index 647dde8c..62cab65a 100644 --- a/detailing.py +++ b/detailing.py @@ -334,12 +334,28 @@ def find_tube(heat, latent_h, latent_w, threshold=DEFAULT_THRESHOLD, # Refine # --------------------------------------------------------------------------- -def _strip_layout_conds(conds): +def has_layout_conds(conds): + """True when any entry carries a full-frame layout key — i.e. when + strip_layout_conds() would actually drop something. Callers use this to + report the loss instead of silently discarding an anchor or a guide. + """ + for entry in conds or []: + if isinstance(entry, (list, tuple)) and len(entry) == 2 and isinstance(entry[1], dict): + if any(k in entry[1] for k in _LAYOUT_COND_KEYS): + return True + return False + + +def strip_layout_conds(conds): """Copy conditioning minus full-frame layout keys (anchors/guides). The crop refine is its own small denoise: text still applies, but keyframe indices and guide latents describe positions in the full-frame latent and would fight (or crash) the crop-sized pass. + + Public because the Scene Chain sampler needs the same treatment for a + resolution-changing second_pass_op — any pass run on a differently-sized + latent than the one these keys were recorded against has the same problem. """ out = [] for entry in conds or []: @@ -508,7 +524,7 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, tail_sigmas = torch.tensor(stage2_sigmas(renoise_sigma), dtype=torch.float32) refined = chain._sample_chunk( model, sampler, tail_sigmas, int(seed) + 7777, cfg, - _strip_layout_conds(positive), _strip_layout_conds(negative), crop_latent) + strip_layout_conds(positive), strip_layout_conds(negative), crop_latent) refined_crop = chain._latent_tensors(refined)[0] cost_note = f"repair: renoise={renoise_sigma:.2f}" diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 1ed628bb..b54bcf3a 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -7,6 +7,12 @@ const S = window.Store; const API = window.MovieEditorAPI; + // Project ids whose legacy Distilled-Flow ALG switch has been folded into the + // sampler-wide alg_anchor this session. Module scope, not render scope: every store + // write re-renders this pane synchronously, so the guard has to outlive a render or + // the migration re-enters itself and recurses until the stack blows. + const _ALG_MIGRATED = new Set(); + // Easy Gen has no rating UI at all (by design — see easy_gen/frontend/), so every // setting that is a no-op without a trained refinement key / rated history is hidden // there, not just made harder to find. window.FunPackAppName is the same discriminator @@ -648,6 +654,36 @@ else S.unsetSamplerInput("second_pass"); } + // ALG used to have two switches: the Distilled Flow panel's own alg_enabled, and the + // chain sampler's alg_anchor, which is the same guidance on ANY sampler and already + // drove the Distilled Flow one whenever it was on. Two controls for one behaviour, and + // which of them won depended on which you had set — so the Editor now shows only + // alg_anchor. The node input still exists for hand-built graphs; what moves is the + // Editor's control. A project carrying the old switch is migrated once, with its + // strength and threshold, rather than left running ALG with nothing on screen to + // turn it off. + function migrateDistilledAlg(cfg) { + const pid = String(S.get()?.project?.id || ""); + const dc = cfg?.high?.distilled; + if (!dc || !dc.alg_enabled) return _ALG_MIGRATED.has(pid); + // Mark and clear BEFORE writing anything: each setter notifies the store, which + // re-renders this pane synchronously and comes straight back through here. + _ALG_MIGRATED.add(pid); + const strength = dc.alg_strength; + const threshold = dc.alg_sigma_threshold; + dc.alg_enabled = false; + // Out of the render pass entirely — persisting mid-render is what caused the + // recursion above, and a deferred write costs one extra repaint instead. + setTimeout(() => { + S.setSamplerInputNow("alg_anchor", true); + if (strength != null) S.setSamplerInputNow("alg_anchor_strength", Number(strength)); + if (threshold != null) S.setSamplerInputNow("alg_anchor_sigma_threshold", Number(threshold)); + persistSamplers(cfg, false); + }, 0); + return true; + } + const algMigrated = migrateDistilledAlg(samplers); + try { window.SamplerPanel.render(box, samplers, (s) => persistSamplers(s, true), @@ -664,8 +700,17 @@ // it is never hidden behind a particular choice up there. const algG = group(pane, "Anchor blur (ALG)"); renderKnobList(algG, st, ["alg_anchor", "alg_anchor_strength", "alg_anchor_sigma_threshold"]); - algG.append(hintEl("The same blur for guide and JoyAI-memory frames is under Experimental " - + "(“Blur i2v guides and JoyAI memory”) — it has its own strength and window.")); + if (algMigrated) { + const moved = hintEl("Moved here from the Distilled Flow panel's own ALG switch — it was " + + "the same blur, and this one works on every sampler. Your strength and threshold " + + "came with it; nothing changed about how the scene samples."); + moved.style.color = "var(--amber)"; + algG.append(moved); + } + algG.append(hintEl("This is the only ALG anchor control — on Distilled Flow it runs inside " + + "the sampler loop, on every other sampler through a denoiser proxy. The same blur for " + + "guide and JoyAI-memory frames is under Experimental (“Blur i2v guides and JoyAI " + + "memory”) — it has its own strength and window.")); // Second pass lives here rather than under Experimental: at its defaults the split is // behaviour-neutral (pass 2 resumes from exactly the state pass 1 handed over), so it // is a sampler setting, not a gamble. The only control is the schedule field in the diff --git a/movie_editor/frontend/sampler_panel.js b/movie_editor/frontend/sampler_panel.js index 52f457c9..1279080c 100644 --- a/movie_editor/frontend/sampler_panel.js +++ b/movie_editor/frontend/sampler_panel.js @@ -332,18 +332,12 @@ (v) => { dc.s_noise = v; save(); })); renderVelocityBlock(container, dk + "-d", dc, save, saveNow); - sectionTag(container, "ALG (experimental)"); - row(container, "enabled", - checkCtrl(!!dc.alg_enabled, dk + "-dalge", (v) => { dc.alg_enabled = v; saveNow(); })); - hint(container, "Adaptive Low-Pass Guidance (arXiv:2506.08456). Blurs the i2v anchor frame during the earliest steps so the model can't shortcut to a near-static video that just matches it. No effect without an i2v anchor. Same guidance as 'Anchor blur (ALG)' below, which works on every sampler — turning that on drives this one, so you only need one of the two."); - if (dc.alg_enabled) { - row(container, "blur strength", - numCtrl(dc.alg_strength, 1.0, 4, 0.1, dk + "-dalgs", - (v) => { dc.alg_strength = v; save(); })); - row(container, "sigma threshold", - numCtrl(dc.alg_sigma_threshold, 0.5, 0.999, 0.005, dk + "-dalgt", - (v) => { dc.alg_sigma_threshold = v; save(); })); - } + // No ALG block here on purpose. This sampler has its own alg_enabled input, but the + // chain sampler's "Anchor blur (ALG)" is the same guidance, works on every sampler, + // and already drives this one when it is on — so showing both put two switches on + // screen for one behaviour, with precedence that depended on which you had set. + // The single control lives in the "Anchor blur (ALG)" group just below this panel; + // engine_settings.js migrates a project that still carries the old switch. sectionTag(container, "Momentum Guidance (experimental)"); row(container, "enabled", diff --git a/samplers.py b/samplers.py index ddaebd2a..84c7513b 100644 --- a/samplers.py +++ b/samplers.py @@ -3713,6 +3713,17 @@ class FunPackLTXAVSceneChainSampler: result["samples"] = tensors[0] return result, note + def _latent_spatial_changed(self, state, chunk): + """True when `state`'s video stream no longer has `chunk`'s H/W — i.e. a + second_pass_op resized it. Everything recorded against the old grid (the i2v + pin, guide keyframe token indices) is invalid past this point. Unreadable + shapes answer False so a probe failure never drops a guide on its own.""" + try: + return (self._latent_tensors(state)[0].shape[-2:] + != self._latent_tensors(chunk)[0].shape[-2:]) + except Exception: + return False + def _restore_pinned_prefix(self, state, chunk): """Put the i2v pin back on a latent that is about to be handed to a second pass. @@ -3729,7 +3740,7 @@ class FunPackLTXAVSceneChainSampler: # A resolution-changing second_pass_op (upscale_2x) leaves the chunk's anchor and # mask the wrong size. Rescaling them here would be inventing an anchor; the # caller reports the lost pin instead. - if self._latent_tensors(state)[0].shape[-2:] != self._latent_tensors(chunk)[0].shape[-2:]: + if self._latent_spatial_changed(state, chunk): return state except Exception: return state @@ -6791,6 +6802,28 @@ class FunPackLTXAVSceneChainSampler: "second_pass NOTE: the i2v anchor could not stay pinned for pass 2 " "because second_pass_op changed the latent resolution — pass 2 runs " "unpinned and may drift from the reference image.") + # Same resolution change, second casualty: guide keyframes are recorded as + # TOKEN indices into the pass-1 grid, and 2x spatial means 4x the tokens per + # latent frame, so those indices now address the wrong tokens entirely. + # Since the LTX-2.5 commit core rejects it outright ("keyframe_idxs holds N + # tokens, which is not a whole number of M-token latent frames") and tells + # you to crop the guides before upscaling; older cores silently mis-placed + # them, which is worse. Drop them for pass 2, exactly as segmented detailing + # does for its crop — and only for pass 2, so pass 1 keeps every guide. + _sp_positive, _sp_negative = scene_positive, scene_negative + if self._latent_spatial_changed(_sp_state, chunk): + try: + from .detailing import has_layout_conds, strip_layout_conds + except ImportError: + from detailing import has_layout_conds, strip_layout_conds + if has_layout_conds(scene_positive) or has_layout_conds(scene_negative): + _sp_positive = strip_layout_conds(scene_positive) + _sp_negative = strip_layout_conds(scene_negative) + run_mechanisms.append( + "second_pass NOTE: mid-scene guides / guide keyframes were " + "dropped for pass 2 because second_pass_op changed the latent " + "resolution — their token positions describe the pass-1 grid. " + "Pass 1 used them in full.") _sp_kw = dict(_sample_kwargs) _sp_kw["step_offset"] = _sample_kwargs["step_offset"] + (int(sigmas.numel()) - 1) # Announced BEFORE it runs: the after-the-fact run_mechanisms line says @@ -6800,8 +6833,8 @@ class FunPackLTXAVSceneChainSampler: f"{' on ' + _scene_label if _scene_label else ''}: " f"{int(_sp_b.numel()) - 1} steps from sigma {float(_sp_b[0].item()):g}") sampled = self._sample_chunk( - model, sampler, _sp_b, scene_seed + 4242, cfg, scene_positive, - scene_negative, _sp_state, **_sp_kw) + model, sampler, _sp_b, scene_seed + 4242, cfg, _sp_positive, + _sp_negative, _sp_state, **_sp_kw) run_mechanisms.append( f"second_pass({int(sigmas.numel()) - 1} steps + " f"{int(_sp_b.numel()) - 1} steps = " diff --git a/tests/test_detailing.py b/tests/test_detailing.py index 84cabc21..89e61153 100644 --- a/tests/test_detailing.py +++ b/tests/test_detailing.py @@ -220,13 +220,13 @@ def test_resolve_download_failure_raises_actionable_error(monkeypatch, tmp_path) # --------------------------------------------------------------------------- -# _strip_layout_conds / _downscale_to +# strip_layout_conds / _downscale_to # --------------------------------------------------------------------------- def test_strip_layout_conds_removes_guide_keys_only(): cond = [[torch.zeros(1, 4), {"keyframe_idxs": "kf", "guiding_latent": "g", "pooled_output": "keep"}]] - out = detailing._strip_layout_conds(cond) + out = detailing.strip_layout_conds(cond) assert out[0][1] == {"pooled_output": "keep"} # Original untouched (the scene's own conds must not lose their guides). assert "keyframe_idxs" in cond[0][1] diff --git a/tests/test_second_pass_layout_conds.py b/tests/test_second_pass_layout_conds.py new file mode 100644 index 00000000..49d39843 --- /dev/null +++ b/tests/test_second_pass_layout_conds.py @@ -0,0 +1,95 @@ +"""Guide keyframes must not survive a resolution-changing second_pass_op. + +Guide keyframes are recorded as TOKEN indices into pass 1's latent grid. `upscale_2x` +doubles each spatial dim, so a latent frame is suddenly 4x as many tokens and those indices +address the wrong ones. Since the LTX-2.5 commit core raises on it outright ("keyframe_idxs +holds N tokens, which is not a whole number of M-token latent frames"); before that it +silently mis-placed the guides, which is worse. Pass 2 therefore drops them — and only +pass 2, because pass 1 ran at the grid they were recorded against. +""" +import sys +import types +from pathlib import Path + +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import detailing # noqa: E402 +import samplers # noqa: E402 + + +def _cond(**meta): + return [[torch.zeros(1, 4, 128), dict(meta)]] + + +def _latent(h, w, frames=3): + return {"samples": torch.zeros(1, 128, frames, h, w)} + + +def _chain(): + return samplers.FunPackLTXAVSceneChainSampler() + + +# --- the shared predicate ------------------------------------------------------------------ + +def test_spatial_change_is_detected(): + chain = _chain() + assert chain._latent_spatial_changed(_latent(32, 48), _latent(16, 24)) is True + + +def test_same_size_is_not_a_change(): + chain = _chain() + assert chain._latent_spatial_changed(_latent(16, 24), _latent(16, 24)) is False + + +def test_more_frames_alone_is_not_a_spatial_change(): + """Only H/W matter: token-per-frame count is spatial. A differing frame count must not + be read as a resize, or 'sharpen' runs would drop guides for no reason.""" + chain = _chain() + assert chain._latent_spatial_changed(_latent(16, 24, frames=5), _latent(16, 24, frames=3)) is False + + +def test_unreadable_shapes_report_no_change(): + """A probe failure must not cost the user their guides.""" + chain = _chain() + assert chain._latent_spatial_changed({"samples": "not a tensor"}, _latent(16, 24)) is False + + +# --- the stripper ------------------------------------------------------------------------- + +def test_layout_keys_are_detected_and_stripped(): + conds = _cond(keyframe_idxs=torch.zeros(1, 3, 512), pooled_output=None) + assert detailing.has_layout_conds(conds) is True + stripped = detailing.strip_layout_conds(conds) + assert "keyframe_idxs" not in stripped[0][1] + assert "pooled_output" in stripped[0][1], "only layout keys go; text conditioning stays" + + +def test_every_layout_key_is_covered(): + conds = _cond(**{k: 1 for k in detailing._LAYOUT_COND_KEYS}) + assert detailing.strip_layout_conds(conds)[0][1] == {} + + +def test_plain_text_conditioning_has_no_layout_keys(): + """The common case: no guides, so pass 2 must be told nothing was dropped.""" + assert detailing.has_layout_conds(_cond(pooled_output=None)) is False + + +def test_stripping_does_not_mutate_the_input(): + """Pass 1's conditioning is the same object; mutating it would retroactively strip the + pass that legitimately used the guides.""" + conds = _cond(keyframe_idxs=torch.zeros(1, 3, 512)) + detailing.strip_layout_conds(conds) + assert "keyframe_idxs" in conds[0][1] + + +def test_tensor_identity_is_preserved(): + """Only the meta dict is rebuilt — the conditioning tensor must not be copied.""" + conds = _cond(keyframe_idxs=1) + assert detailing.strip_layout_conds(conds)[0][0] is conds[0][0] + + +def test_non_entry_items_pass_through_untouched(): + assert detailing.strip_layout_conds([None, "junk"]) == [None, "junk"] + assert detailing.has_layout_conds([None, "junk"]) is False