diff --git a/detailing.py b/detailing.py index 4d24ff4a..634ad719 100644 --- a/detailing.py +++ b/detailing.py @@ -25,11 +25,36 @@ import torch import torch.nn.functional as F # Official LTX-2.3 two-stage distilled workflow, stage-2 ManualSigmas: re-noise the -# upsampled latent to 0.85 and run the tail of the 8-step distilled schedule. Using -# Lightricks' published re-entry point instead of inventing one: too much noise and -# the region redraws (drifts from the surrounding frame), too little and no detail -# is gained. -STAGE2_SIGMAS = (0.85, 0.725, 0.421875, 0.0) +# upsampled latent to 0.85 and run the tail of the 8-step distilled schedule. +# Lightricks designed this re-entry point to SHARPEN already-correct structure at +# higher resolution, not to REPAIR a malformed region — our use case (a mangled +# hand) needs more genuine freedom to redraw than their use case (a clean shot +# that just needs finer detail). 0.85 is the built-in default (matches the +# official recipe / prior behavior exactly), but it is a real knob +# (detail_denoise on the sampler) precisely because the right value depends on +# how wrong the source region already is: too high and the region drifts from +# its surroundings; too low and the tail just smooths the upsampled +# interpolation back down without adding anything (upscaled-looking, not +# detailed - the failure mode this constant existed to describe from day one). +DEFAULT_RENOISE_SIGMA = 0.85 +# Shape of the tail preserved as ratios of the entry sigma, not fixed values, so +# raising/lowering the entry point scales the whole tail schedule consistently +# instead of only moving its first step. +_STAGE2_RATIOS = (1.0, 0.725 / 0.85, 0.421875 / 0.85, 0.0) + + +def stage2_sigmas(renoise_sigma=DEFAULT_RENOISE_SIGMA): + """The re-denoise tail schedule for a given re-entry sigma. + + stage2_sigmas(0.85) reproduces the exact official STAGE2_SIGMAS values. + """ + r = max(0.0, min(1.0, float(renoise_sigma))) + return tuple(r * ratio for ratio in _STAGE2_RATIOS) + + +# Kept for anyone importing the old name directly (tests, prior callers) - identical +# to stage2_sigmas() at the default re-entry point. +STAGE2_SIGMAS = stage2_sigmas(DEFAULT_RENOISE_SIGMA) # CLIPSeg heat above this (post-sigmoid) counts as "the named thing is here". DEFAULT_THRESHOLD = 0.35 @@ -359,7 +384,7 @@ def _downscale_to(video, h, w): def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, detail_targets, upsampler, seed, cfg, threshold=DEFAULT_THRESHOLD, strength=1.0, area_cap=MAX_TUBE_AREA, - debug=False): + renoise_sigma=DEFAULT_RENOISE_SIGMA, debug=False): """Run the segmented-detail pass on one finished scene latent. Returns (latent, note): the refined latent dict plus a run_mechanisms note. @@ -415,11 +440,14 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, crop = video[:, :, :, y0:y1, x0:x1] crop_up = _run_upsampler(upsampler, crop, vae, debug=debug) - # 3) Re-denoise the crop over the official stage-2 tail. The clean crop goes - # in as the latent; CONST noise_scaling (sigma*noise + (1-sigma)*x) re-noises - # it to sigma=0.85 exactly like the official two-stage workflow's stage 2. - # The scene's audio stream rides along so joint attention sees a legal AV - # latent, but its refined output is discarded below - audio is protected by + # 3) Re-denoise the crop over a stage-2 tail shaped like the official two-stage + # workflow's, but re-entering at `renoise_sigma` (default 0.85, their exact + # value) rather than a hardcoded one: how much genuine freedom the model needs + # to fix this crop depends on how wrong it already is, which only the person + # looking at the result can judge. CONST noise_scaling (sigma*noise + + # (1-sigma)*x) re-noises the clean crop to that sigma automatically. The + # scene's audio stream rides along so joint attention sees a legal AV latent, + # but its refined output is discarded below - audio is protected by # construction, not by masking. import comfy.nested_tensor @@ -427,7 +455,7 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, if len(tensors) > 1: crop_latent["samples"] = comfy.nested_tensor.NestedTensor( [crop_up] + [t.detach().clone() for t in tensors[1:]]) - tail_sigmas = torch.tensor(STAGE2_SIGMAS, dtype=torch.float32) + tail_sigmas = torch.tensor(stage2_sigmas(renoise_sigma), dtype=torch.float32) refined = chain._sample_chunk( model, sampler, tail_sigmas, int(seed) + 7777, cfg, _strip_layout_conds(positive), _strip_layout_conds(negative), crop_latent) diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 701a071f..3cb81293 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -206,6 +206,8 @@ hint: "CLIPSeg match confidence required before a region counts as found. Its raw score for a real region is often well under 0.5 — if the scene report shows 'no match: max CLIPSeg score X < threshold', lower this toward X rather than assuming nothing is there." }, { name: "detail_max_area", label: "Detail max area", kind: "float", default: 0.35, min: 0.05, max: 1.0, step: 0.05, dependsOn: "segmented_detailing", hint: "Ceiling on how much of the frame the region may cover before it's refused, as a fraction of the frame. Cost-only guardrail (a bigger region costs more, roughly 4× its area × 3 steps) — never a judgment call about whether it's worth detailing. If the scene report shows a region refused at some %, raise this above that % to detail it anyway. 1.0 = no cap." }, + { name: "detail_denoise", label: "Detail re-noise strength", kind: "float", default: 0.85, min: 0.3, max: 0.99, step: 0.05, dependsOn: "segmented_detailing", + hint: "How much noise the crop gets re-noised to before its 3-step refine (0.85 is the official LTX 2.3 recipe's own value). Higher = more freedom to genuinely reconstruct the region (fix bad anatomy), risking drift from the surrounding frame; lower = closer to a plain upscale — looks 'detailed' as interpolation but doesn't actually repair it. If the result looks upscaled but not corrected, raise this." }, ]; const SAMPLER_KNOB_MAP = Object.fromEntries(SAMPLER_KNOBS.map((k) => [k.name, k])); @@ -318,7 +320,7 @@ chain_timing: ["frame_overlap", "transition_duration", "use_same_seed"], chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "taste_nearest_prompt", "output_guidance", "output_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold"], chain_decode: ["decode_noise_scale", "decode_timestep", "decode_tile_size"], - chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "detail_max_area", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], + chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "detail_max_area", "detail_denoise", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], }; function countChainView(p, id) { diff --git a/samplers.py b/samplers.py index 4bbf4e5f..b978aeee 100644 --- a/samplers.py +++ b/samplers.py @@ -2474,6 +2474,10 @@ class FunPackLTXAVSceneChainSampler: "default": 0.35, "min": 0.05, "max": 1.0, "step": 0.05, "tooltip": "Ceiling on how much of the frame the detected region may cover before the pass refuses it, as a fraction of frame area. This is a COST guardrail only (cost ~= 4x area x 3 steps, so a large region starts to rival a second full render), never a judgment about whether the region is worth detailing — if the scene report shows a region refused at some %, raise this above that % to detail it anyway (up to 1.0 = no cap, full-frame allowed).", }), + "detail_denoise": ("FLOAT", { + "default": 0.85, "min": 0.3, "max": 0.99, "step": 0.05, + "tooltip": "How much noise the crop is re-noised to before the 3-step refine tail (the official LTX 2.3 two-stage recipe's own value, 0.85, is the default). Higher = more freedom for the model to genuinely reconstruct the region (fix bad anatomy) at the cost of possibly drifting from the surrounding frame; lower = closer to a plain upscale (looks 'detailed' as interpolation, but doesn't actually repair the region — if that's what you're seeing, raise this).", + }), }, "hidden": { "unique_id": "UNIQUE_ID", @@ -4674,7 +4678,7 @@ class FunPackLTXAVSceneChainSampler: taste_nearest_prompt=False, segmented_detailing=False, detail_targets="hands", detail_upsampler="None", detail_strength=1.0, detail_threshold=0.35, - detail_max_area=0.35, + detail_max_area=0.35, detail_denoise=0.85, unique_id=None, prompt=None): if not isinstance(positive, list) or not positive: raise ValueError("positive conditioning must contain at least one scene entry.") @@ -5143,7 +5147,8 @@ class FunPackLTXAVSceneChainSampler: self, model, vae, sampler, scene_positive, scene_negative, sampled, detail_targets, _detail_upsampler_model, scene_seed, cfg, threshold=detail_threshold, strength=detail_strength, - area_cap=detail_max_area, debug=debug_log) + area_cap=detail_max_area, renoise_sigma=detail_denoise, + debug=debug_log) # detail_refine_scene always returns a diagnostic note when it actually # ran detection (miss or hit) — None only means it never attempted # detection at all (which can't happen on this branch). diff --git a/tests/test_detailing.py b/tests/test_detailing.py index 5fdcc83c..ce210c73 100644 --- a/tests/test_detailing.py +++ b/tests/test_detailing.py @@ -58,6 +58,29 @@ def test_parse_targets_splits_and_strips(): assert detailing.parse_targets(None) == [] +# --------------------------------------------------------------------------- +# stage2_sigmas +# --------------------------------------------------------------------------- + +def test_stage2_sigmas_default_matches_official_recipe(): + # Must reproduce the exact official two-stage workflow values unless the + # user raises/lowers detail_denoise — no behavior change for existing runs. + assert torch.allclose(torch.tensor(detailing.stage2_sigmas(0.85)), + torch.tensor(detailing.STAGE2_SIGMAS)) + assert detailing.stage2_sigmas(0.85)[-1] == 0.0 + +def test_stage2_sigmas_scales_with_renoise_sigma(): + # Raising the re-entry point should raise every step in the tail proportionally + # (more genuine freedom to reconstruct, not just a bigger first jump). + low = detailing.stage2_sigmas(0.5) + high = detailing.stage2_sigmas(0.95) + assert high[0] > low[0] > 0 + assert high[1] > low[1] + assert low[-1] == high[-1] == 0.0 + # Shape (ratios between steps) preserved regardless of entry point. + assert abs(low[1] / low[0] - high[1] / high[0]) < 1e-9 + + # --------------------------------------------------------------------------- # find_tube # --------------------------------------------------------------------------- @@ -334,6 +357,17 @@ def _patch_detection_and_upsampler(monkeypatch): crop, scale_factor=(1, 2, 2), mode="nearest")) return heat +def test_refine_passes_renoise_sigma_to_the_tail_schedule(monkeypatch): + chain = _FakeChain() + latent, _ = _scene_latent() + _patch_detection_and_upsampler(monkeypatch) + detailing.detail_refine_scene( + chain, "model", "vae", "sampler", [], [], latent, "hands", object(), + seed=0, cfg=1.0, renoise_sigma=0.95) + sigmas = chain.chunk_calls[0]["sigmas"] + assert abs(float(sigmas[0]) - 0.95) < 1e-6 + assert torch.allclose(sigmas, torch.tensor(detailing.stage2_sigmas(0.95))) + def test_refine_round_trip_touches_only_the_tube(monkeypatch): chain = _FakeChain() latent, video = _scene_latent()