From c864f62480f36c99e227ddbf14acebd2da72e8c7 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sat, 18 Jul 2026 18:29:28 +0300 Subject: [PATCH] Segmented detailing: auto-resolve/download upsampler + loud skips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first live attempt silently no-oped: detail_upsampler defaulted to 'None' (nothing installed on the rental) and the guard skipped without a word. Two fixes: - detail_upsampler default is now 'auto': prefer the official installed file, else the highest-versioned spatial upscaler (version-aware compare — plain sort ranked ltx2_3_… over ltx-2.3-…-1.1), else hf_hub_download of Lightricks/LTX-2.3 ltx-2.3-spatial-upscaler-x2-1.1.safetensors (~1 GB, once) with an actionable error if the repo is gated. Legacy 'None' = auto. - With the toggle on, every outcome is visible: upsampler choice and skips print to console AND land in the scene report (segmented_detail(SKIPPED: …) / (no region matched)); resolve/load failure disables the pass for the run instead of retrying the download per scene. A failed pass still never costs the scene. Co-Authored-By: Claude Fable 5 --- detailing.py | 57 ++++++++++++++++++++++++ movie_editor/frontend/engine_settings.js | 11 ++--- samplers.py | 38 +++++++++++----- tests/test_detailing.py | 49 ++++++++++++++++++++ 4 files changed, 139 insertions(+), 16 deletions(-) diff --git a/detailing.py b/detailing.py index 832226f5..96adea14 100644 --- a/detailing.py +++ b/detailing.py @@ -66,6 +66,63 @@ def _log(debug, msg): # Models # --------------------------------------------------------------------------- +# Official Lightricks stage-2 spatial upscaler — the exact file the two-stage +# distilled workflows use. ~1 GB, fetched once into models/latent_upscale_models +# when nothing suitable is installed. +DEFAULT_UPSAMPLER_REPO = "Lightricks/LTX-2.3" +DEFAULT_UPSAMPLER_FILE = "ltx-2.3-spatial-upscaler-x2-1.1.safetensors" + + +def resolve_upsampler_name(model_name=None): + """Turn the node's detail_upsampler widget value into a loadable filename. + + Explicit filename -> returned as-is. "auto" (and the legacy "None" default) + -> prefer an installed spatial upscaler (highest version wins), else any + installed latent upscale model, else download the official file from HF. + Raises with a user-actionable message when nothing can be obtained — callers + surface it, they must NOT silently skip. + """ + import os + + import folder_paths + + name = str(model_name or "").strip() + if name and name not in ("None", "auto"): + return name + files = folder_paths.get_filename_list("latent_upscale_models") + if DEFAULT_UPSAMPLER_FILE in files: + return DEFAULT_UPSAMPLER_FILE + spatial = [f for f in files if "spatial" in f.lower() and "upscal" in f.lower()] + if spatial: + # Highest embedded version wins (…-1.1 over …-1.0); plain lexicographic would + # rank "ltx2_3_…" above "ltx-2.3-…-1.1" ('2' sorts after '-'). + import re + return max(spatial, key=lambda f: ([int(n) for n in re.findall(r"\d+", f)], f)) + if files: + return files[0] + try: + from huggingface_hub import hf_hub_download + except ImportError as exc: + raise RuntimeError( + "no model in models/latent_upscale_models and huggingface_hub is not " + f"installed — pip install huggingface_hub, or download {DEFAULT_UPSAMPLER_FILE} " + f"from {DEFAULT_UPSAMPLER_REPO} into that folder manually") from exc + dest_dir = folder_paths.get_folder_paths("latent_upscale_models")[0] + os.makedirs(dest_dir, exist_ok=True) + print(f"[FunPackDetail] no latent upscale model installed — downloading " + f"{DEFAULT_UPSAMPLER_FILE} (~1 GB) from {DEFAULT_UPSAMPLER_REPO} into {dest_dir} …") + try: + hf_hub_download(repo_id=DEFAULT_UPSAMPLER_REPO, filename=DEFAULT_UPSAMPLER_FILE, + local_dir=dest_dir) + except Exception as exc: + raise RuntimeError( + f"auto-download of {DEFAULT_UPSAMPLER_FILE} failed ({exc}) — if the repo is " + "gated, accept the license on huggingface.co and set HF_TOKEN, or place the " + f"file in {dest_dir} manually") from exc + print(f"[FunPackDetail] download complete: {DEFAULT_UPSAMPLER_FILE}") + return DEFAULT_UPSAMPLER_FILE + + def load_latent_upsampler(model_name): """Load Lightricks' LatentUpsampler from models/latent_upscale_models. diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 29b39564..8d166504 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -197,7 +197,7 @@ { name: "plateau_cache_threshold", label: "Plateau threshold (sigma)", kind: "float", default: 0.975, min: 0.5, max: 0.999, step: 0.005, dependsOn: "plateau_cache", hint: "Steps with sigma at or above this count as the reusable plateau. Higher = fewer steps cached (safer); lower = more cached (faster, more approximation). 0.975 catches the documented noise plateau while leaving structure formation fully computed." }, { name: "segmented_detailing", label: "Segmented detailing (region refine)", kind: "bool", default: false, - hint: "EXPERIMENTAL ADetailer-for-video: after each scene renders, CLIPSeg finds the regions named below (hands, feet, …), cuts them out of the latent as a tube, refines them at 2× working resolution through Lightricks' latent upsampler (3 extra steps on the crop only), and pastes them back through a feathered silhouette. Final resolution never changes. Cost ≈ 4 × region area × 3 steps (hands ~+15%); regions over 35% of the frame are refused. Needs the detail upsampler model below. UNVALIDATED LIVE." }, + hint: "EXPERIMENTAL ADetailer-for-video: after each scene renders, CLIPSeg finds the regions named below (hands, feet, …), cuts them out of the latent as a tube, refines them at 2× working resolution through Lightricks' latent upsampler (3 extra steps on the crop only), and pastes them back through a feathered silhouette. Final resolution never changes. Cost ≈ 4 × region area × 3 steps (hands ~+15%); regions over 35% of the frame are refused. The upsampler model below is found — or downloaded (~1 GB, once) — automatically; skips are reported in the scene report. UNVALIDATED LIVE." }, { name: "detail_targets", label: "Detail targets", kind: "text", default: "hands", dependsOn: "segmented_detailing", placeholder: "hands, feet", hint: "Comma-separated regions to detail, in plain words. Each becomes a CLIPSeg text query — malformed anatomy still matches its name. Also editable from Composer ▸ Compose while detailing is on." }, { name: "detail_strength", label: "Detail strength", kind: "float", default: 1.0, min: 0, max: 1, step: 0.05, dependsOn: "segmented_detailing", @@ -617,16 +617,17 @@ return; } const sel = el("select"); sel.dataset.k = "si-detail_upsampler"; + const cur = si.detail_upsampler && si.detail_upsampler !== "None" ? si.detail_upsampler : "auto"; _detailUpsamplerChoices.forEach((c) => { const o = el("option", null, c); o.value = c; - if (c === (si.detail_upsampler || "None")) o.selected = true; + if (c === cur) o.selected = true; sel.append(o); }); sel.onchange = () => S.setSamplerInputNow("detail_upsampler", sel.value); g.append(field("Latent upsampler", sel, - "The LTX 2.3 spatial upsampler from models/latent_upscale_models (the official two-stage workflows use the same file). Detailing no-ops while this is None.")); - if ((si.detail_upsampler || "None") === "None" && _detailUpsamplerChoices.length <= 1) { - g.append(hintEl("No models found in models/latent_upscale_models on the server — download the LTX 2.3 latent upsampler there, then reopen this pane.")); + "The LTX 2.3 spatial upsampler from models/latent_upscale_models (the official two-stage workflows use the same file). 'auto' picks the newest installed spatial upscaler — or downloads the official one (~1 GB, once) when the folder is empty.")); + if (_detailUpsamplerChoices.length <= 1) { + g.append(hintEl("Nothing installed in models/latent_upscale_models yet — the first detailed run downloads the official upsampler automatically (watch the ComfyUI console).")); } } diff --git a/samplers.py b/samplers.py index ce6242b7..58a78d93 100644 --- a/samplers.py +++ b/samplers.py @@ -2452,15 +2452,15 @@ class FunPackLTXAVSceneChainSampler: }), "segmented_detailing": ("BOOLEAN", { "default": False, - "tooltip": "EXPERIMENTAL ADetailer-for-video: after each scene finishes denoising, CLIPSeg (text-prompted segmentation) locates the regions named in detail_targets on a few decoded keyframes; the matched region is cut out of the video latent as a spatiotemporal tube, pushed through Lightricks' trained latent upsampler (2x spatial — the official two-stage pipeline's stage-2 model), re-noised to the official stage-2 sigma (0.85) and re-denoised for the 3-step tail schedule, then downscaled back to its ORIGINAL latent size and pasted through the feathered CLIPSeg silhouette. Final resolution never changes — the upsample only lets the model resolve structure (fingers, small faces) at a higher working resolution. Cost ~= 4 x tube area fraction x 3 steps (hands ~+15%); tubes over 35% of the frame are refused (that's a re-render, not a detail pass). Audio untouched by construction. Requires detail_upsampler; silent no-op when nothing matches. UNVALIDATED LIVE.", + "tooltip": "EXPERIMENTAL ADetailer-for-video: after each scene finishes denoising, CLIPSeg (text-prompted segmentation) locates the regions named in detail_targets on a few decoded keyframes; the matched region is cut out of the video latent as a spatiotemporal tube, pushed through Lightricks' trained latent upsampler (2x spatial — the official two-stage pipeline's stage-2 model), re-noised to the official stage-2 sigma (0.85) and re-denoised for the 3-step tail schedule, then downscaled back to its ORIGINAL latent size and pasted through the feathered CLIPSeg silhouette. Final resolution never changes — the upsample only lets the model resolve structure (fingers, small faces) at a higher working resolution. Cost ~= 4 x tube area fraction x 3 steps (hands ~+15%); tubes over 35% of the frame are refused (that's a re-render, not a detail pass). Audio untouched by construction. detail_upsampler 'auto' finds or downloads the official Lightricks upsampler (~1 GB, once); skips are reported loudly in console + scene report. UNVALIDATED LIVE.", }), "detail_targets": ("STRING", { "default": "hands", "tooltip": "Comma-separated regions to detail, in plain words ('hands', 'hands, feet', 'face'). Each becomes a CLIPSeg text query; matched regions merge into one tube per scene. CLIPSeg matches broad CLIP semantics, so malformed anatomy still lights up for its name.", }), "detail_upsampler": (cls._detail_upsampler_choices(), { - "default": "None", - "tooltip": "Latent upsampler checkpoint from models/latent_upscale_models (the LTX 2.3 spatial upsampler used by the official two-stage workflows). Required — segmented_detailing no-ops while this is None.", + "default": "auto", + "tooltip": "Latent upsampler checkpoint from models/latent_upscale_models (the LTX 2.3 spatial upsampler used by the official two-stage workflows). 'auto' picks the newest installed spatial upscaler, or downloads the official file (~1 GB, once) when the folder is empty. Pick a file explicitly to pin it.", }), "detail_strength": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05, @@ -2492,11 +2492,14 @@ class FunPackLTXAVSceneChainSampler: @classmethod def _detail_upsampler_choices(cls): + # "auto": prefer an installed spatial upscaler, else download the official + # Lightricks file on first use. ("None" from older saved workflows/projects is + # treated as auto too — the enable toggle is the only off switch.) try: import folder_paths - return ["None"] + folder_paths.get_filename_list("latent_upscale_models") + return ["auto"] + folder_paths.get_filename_list("latent_upscale_models") except Exception: - return ["None"] + return ["auto"] def _is_nested(self, samples): return bool(getattr(samples, "is_nested", False)) @@ -4851,7 +4854,8 @@ class FunPackLTXAVSceneChainSampler: scene_runs: list = [] joyai_bank = _JoyAIMemoryBank(joyai_memory_size, joyai_fix_frames) if joyai_memory else None _identity_overlap_state: dict = {} - _detail_upsampler_model = None # lazy: loaded at the first detailed scene + _detail_upsampler_model = None # lazy: resolved+loaded at the first detailed scene + _detail_disabled_reason = None # set on resolve/load failure: don't retry per scene for scene_index, scene_cond in enumerate(scene_conditionings): scene_positive = [scene_cond] @@ -5113,15 +5117,18 @@ class FunPackLTXAVSceneChainSampler: # Segmented detailing runs on the clean, fully-cropped scene (guide/audio # tails gone, overlap still present — the tube is spatial, so carried head # frames detail together with the rest) and BEFORE the JoyAI harvest, so - # cross-shot memory banks the improved frames. - if segmented_detailing and detail_strength > 0 and detail_upsampler != "None": + # cross-shot memory banks the improved frames. Every skip is LOUD (console + + # scene report): with the toggle on, silence must mean "ran and found nothing". + if segmented_detailing and detail_strength > 0 and not _detail_disabled_reason: try: try: from . import detailing as _detailing except ImportError: import detailing as _detailing if _detail_upsampler_model is None: - _detail_upsampler_model = _detailing.load_latent_upsampler(detail_upsampler) + _resolved = _detailing.resolve_upsampler_name(detail_upsampler) + _detail_upsampler_model = _detailing.load_latent_upsampler(_resolved) + print(f"[FunPackSceneChain] segmented detailing upsampler: {_resolved}") _t_detail0 = _time.perf_counter() sampled, _detail_note = _detailing.detail_refine_scene( self, model, vae, sampler, scene_positive, scene_negative, sampled, @@ -5130,9 +5137,18 @@ class FunPackLTXAVSceneChainSampler: if _detail_note: run_mechanisms.append(_detail_note) _phase_sampling += _time.perf_counter() - _t_detail0 + else: + run_mechanisms.append("segmented_detail(no region matched)") except Exception as _detail_exc: - # A failed detail pass must never cost the scene itself. - print(f"[FunPackSceneChain] segmented_detailing failed, scene kept as-is: {_detail_exc}") + # A failed detail pass must never cost the scene itself — but it must + # never fail silently either. Model resolution/load failures disable + # the pass for the rest of the run (no per-scene download retries). + if _detail_upsampler_model is None: + _detail_disabled_reason = str(_detail_exc) + print(f"[FunPackSceneChain] SEGMENTED DETAILING SKIPPED (scene kept as-is): {_detail_exc}") + run_mechanisms.append(f"segmented_detail(SKIPPED: {_detail_exc})") + elif segmented_detailing and _detail_disabled_reason: + run_mechanisms.append("segmented_detail(SKIPPED: upsampler unavailable, see first scene)") if joyai_bank is not None: # Harvest from the clean, fully-cropped scene so injected memory tails never re-enter # the bank. Scene 0 seeds the pinned anchor (num_fix); later scenes roll in. The audio diff --git a/tests/test_detailing.py b/tests/test_detailing.py index d168f04f..1430600f 100644 --- a/tests/test_detailing.py +++ b/tests/test_detailing.py @@ -99,6 +99,55 @@ def test_find_tube_enforces_minimum_edge(): assert (x1 - x0) >= detailing.MIN_TUBE_EDGE +# --------------------------------------------------------------------------- +# resolve_upsampler_name +# --------------------------------------------------------------------------- + +def _patch_folder_paths(monkeypatch, files, folder="/models/latent_upscale_models"): + fp = sys.modules["folder_paths"] + monkeypatch.setattr(fp, "get_filename_list", lambda kind: list(files), raising=False) + monkeypatch.setattr(fp, "get_folder_paths", lambda kind: [folder], raising=False) + +def test_resolve_explicit_name_passes_through(monkeypatch): + _patch_folder_paths(monkeypatch, []) + assert detailing.resolve_upsampler_name("my_upscaler.safetensors") == "my_upscaler.safetensors" + +def test_resolve_auto_prefers_newest_spatial_upscaler(monkeypatch): + _patch_folder_paths(monkeypatch, [ + "other_model.safetensors", + "ltx-2.3-spatial-upscaler-x2-1.0.safetensors", + "ltx-2.3-spatial-upscaler-x2-1.1.safetensors", + ]) + for alias in ("auto", "None", "", None): + assert detailing.resolve_upsampler_name(alias) == "ltx-2.3-spatial-upscaler-x2-1.1.safetensors" + +def test_resolve_auto_falls_back_to_any_installed(monkeypatch): + _patch_folder_paths(monkeypatch, ["something_else.safetensors"]) + assert detailing.resolve_upsampler_name("auto") == "something_else.safetensors" + +def test_resolve_auto_downloads_when_folder_empty(monkeypatch, tmp_path): + _patch_folder_paths(monkeypatch, [], folder=str(tmp_path / "lup")) + calls = {} + hub = types.ModuleType("huggingface_hub") + hub.hf_hub_download = lambda repo_id, filename, local_dir: calls.update( + repo=repo_id, file=filename, dest=local_dir) + monkeypatch.setitem(sys.modules, "huggingface_hub", hub) + name = detailing.resolve_upsampler_name("auto") + assert name == detailing.DEFAULT_UPSAMPLER_FILE + assert calls["repo"] == detailing.DEFAULT_UPSAMPLER_REPO + assert calls["file"] == detailing.DEFAULT_UPSAMPLER_FILE + +def test_resolve_download_failure_raises_actionable_error(monkeypatch, tmp_path): + _patch_folder_paths(monkeypatch, [], folder=str(tmp_path / "lup")) + hub = types.ModuleType("huggingface_hub") + def _boom(**kw): + raise OSError("401 gated") + hub.hf_hub_download = _boom + monkeypatch.setitem(sys.modules, "huggingface_hub", hub) + with pytest.raises(RuntimeError, match="manually"): + detailing.resolve_upsampler_name("auto") + + # --------------------------------------------------------------------------- # _strip_layout_conds / _downscale_to # ---------------------------------------------------------------------------