Segmented detailing: auto-resolve/download upsampler + loud skips
The first live attempt silently no-oped: detail_upsampler defaulted to 'None' (nothing installed on the rental) and the guard skipped without a word. Two fixes: - detail_upsampler default is now 'auto': prefer the official installed file, else the highest-versioned spatial upscaler (version-aware compare — plain sort ranked ltx2_3_… over ltx-2.3-…-1.1), else hf_hub_download of Lightricks/LTX-2.3 ltx-2.3-spatial-upscaler-x2-1.1.safetensors (~1 GB, once) with an actionable error if the repo is gated. Legacy 'None' = auto. - With the toggle on, every outcome is visible: upsampler choice and skips print to console AND land in the scene report (segmented_detail(SKIPPED: …) / (no region matched)); resolve/load failure disables the pass for the run instead of retrying the download per scene. A failed pass still never costs the scene. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
digital-garbage
co-authored by
Claude Fable 5
Claude Fable 5
parent
db121973f1
commit
c864f62480
@@ -66,6 +66,63 @@ def _log(debug, msg):
|
||||
# Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Official Lightricks stage-2 spatial upscaler — the exact file the two-stage
|
||||
# distilled workflows use. ~1 GB, fetched once into models/latent_upscale_models
|
||||
# when nothing suitable is installed.
|
||||
DEFAULT_UPSAMPLER_REPO = "Lightricks/LTX-2.3"
|
||||
DEFAULT_UPSAMPLER_FILE = "ltx-2.3-spatial-upscaler-x2-1.1.safetensors"
|
||||
|
||||
|
||||
def resolve_upsampler_name(model_name=None):
|
||||
"""Turn the node's detail_upsampler widget value into a loadable filename.
|
||||
|
||||
Explicit filename -> returned as-is. "auto" (and the legacy "None" default)
|
||||
-> prefer an installed spatial upscaler (highest version wins), else any
|
||||
installed latent upscale model, else download the official file from HF.
|
||||
Raises with a user-actionable message when nothing can be obtained — callers
|
||||
surface it, they must NOT silently skip.
|
||||
"""
|
||||
import os
|
||||
|
||||
import folder_paths
|
||||
|
||||
name = str(model_name or "").strip()
|
||||
if name and name not in ("None", "auto"):
|
||||
return name
|
||||
files = folder_paths.get_filename_list("latent_upscale_models")
|
||||
if DEFAULT_UPSAMPLER_FILE in files:
|
||||
return DEFAULT_UPSAMPLER_FILE
|
||||
spatial = [f for f in files if "spatial" in f.lower() and "upscal" in f.lower()]
|
||||
if spatial:
|
||||
# Highest embedded version wins (…-1.1 over …-1.0); plain lexicographic would
|
||||
# rank "ltx2_3_…" above "ltx-2.3-…-1.1" ('2' sorts after '-').
|
||||
import re
|
||||
return max(spatial, key=lambda f: ([int(n) for n in re.findall(r"\d+", f)], f))
|
||||
if files:
|
||||
return files[0]
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"no model in models/latent_upscale_models and huggingface_hub is not "
|
||||
f"installed — pip install huggingface_hub, or download {DEFAULT_UPSAMPLER_FILE} "
|
||||
f"from {DEFAULT_UPSAMPLER_REPO} into that folder manually") from exc
|
||||
dest_dir = folder_paths.get_folder_paths("latent_upscale_models")[0]
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
print(f"[FunPackDetail] no latent upscale model installed — downloading "
|
||||
f"{DEFAULT_UPSAMPLER_FILE} (~1 GB) from {DEFAULT_UPSAMPLER_REPO} into {dest_dir} …")
|
||||
try:
|
||||
hf_hub_download(repo_id=DEFAULT_UPSAMPLER_REPO, filename=DEFAULT_UPSAMPLER_FILE,
|
||||
local_dir=dest_dir)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"auto-download of {DEFAULT_UPSAMPLER_FILE} failed ({exc}) — if the repo is "
|
||||
"gated, accept the license on huggingface.co and set HF_TOKEN, or place the "
|
||||
f"file in {dest_dir} manually") from exc
|
||||
print(f"[FunPackDetail] download complete: {DEFAULT_UPSAMPLER_FILE}")
|
||||
return DEFAULT_UPSAMPLER_FILE
|
||||
|
||||
|
||||
def load_latent_upsampler(model_name):
|
||||
"""Load Lightricks' LatentUpsampler from models/latent_upscale_models.
|
||||
|
||||
|
||||
@@ -197,7 +197,7 @@
|
||||
{ name: "plateau_cache_threshold", label: "Plateau threshold (sigma)", kind: "float", default: 0.975, min: 0.5, max: 0.999, step: 0.005, dependsOn: "plateau_cache",
|
||||
hint: "Steps with sigma at or above this count as the reusable plateau. Higher = fewer steps cached (safer); lower = more cached (faster, more approximation). 0.975 catches the documented noise plateau while leaving structure formation fully computed." },
|
||||
{ name: "segmented_detailing", label: "Segmented detailing (region refine)", kind: "bool", default: false,
|
||||
hint: "EXPERIMENTAL ADetailer-for-video: after each scene renders, CLIPSeg finds the regions named below (hands, feet, …), cuts them out of the latent as a tube, refines them at 2× working resolution through Lightricks' latent upsampler (3 extra steps on the crop only), and pastes them back through a feathered silhouette. Final resolution never changes. Cost ≈ 4 × region area × 3 steps (hands ~+15%); regions over 35% of the frame are refused. Needs the detail upsampler model below. UNVALIDATED LIVE." },
|
||||
hint: "EXPERIMENTAL ADetailer-for-video: after each scene renders, CLIPSeg finds the regions named below (hands, feet, …), cuts them out of the latent as a tube, refines them at 2× working resolution through Lightricks' latent upsampler (3 extra steps on the crop only), and pastes them back through a feathered silhouette. Final resolution never changes. Cost ≈ 4 × region area × 3 steps (hands ~+15%); regions over 35% of the frame are refused. The upsampler model below is found — or downloaded (~1 GB, once) — automatically; skips are reported in the scene report. UNVALIDATED LIVE." },
|
||||
{ name: "detail_targets", label: "Detail targets", kind: "text", default: "hands", dependsOn: "segmented_detailing", placeholder: "hands, feet",
|
||||
hint: "Comma-separated regions to detail, in plain words. Each becomes a CLIPSeg text query — malformed anatomy still matches its name. Also editable from Composer ▸ Compose while detailing is on." },
|
||||
{ name: "detail_strength", label: "Detail strength", kind: "float", default: 1.0, min: 0, max: 1, step: 0.05, dependsOn: "segmented_detailing",
|
||||
@@ -617,16 +617,17 @@
|
||||
return;
|
||||
}
|
||||
const sel = el("select"); sel.dataset.k = "si-detail_upsampler";
|
||||
const cur = si.detail_upsampler && si.detail_upsampler !== "None" ? si.detail_upsampler : "auto";
|
||||
_detailUpsamplerChoices.forEach((c) => {
|
||||
const o = el("option", null, c); o.value = c;
|
||||
if (c === (si.detail_upsampler || "None")) o.selected = true;
|
||||
if (c === cur) o.selected = true;
|
||||
sel.append(o);
|
||||
});
|
||||
sel.onchange = () => S.setSamplerInputNow("detail_upsampler", sel.value);
|
||||
g.append(field("Latent upsampler", sel,
|
||||
"The LTX 2.3 spatial upsampler from models/latent_upscale_models (the official two-stage workflows use the same file). Detailing no-ops while this is None."));
|
||||
if ((si.detail_upsampler || "None") === "None" && _detailUpsamplerChoices.length <= 1) {
|
||||
g.append(hintEl("No models found in models/latent_upscale_models on the server — download the LTX 2.3 latent upsampler there, then reopen this pane."));
|
||||
"The LTX 2.3 spatial upsampler from models/latent_upscale_models (the official two-stage workflows use the same file). 'auto' picks the newest installed spatial upscaler — or downloads the official one (~1 GB, once) when the folder is empty."));
|
||||
if (_detailUpsamplerChoices.length <= 1) {
|
||||
g.append(hintEl("Nothing installed in models/latent_upscale_models yet — the first detailed run downloads the official upsampler automatically (watch the ComfyUI console)."));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+27
-11
@@ -2452,15 +2452,15 @@ class FunPackLTXAVSceneChainSampler:
|
||||
}),
|
||||
"segmented_detailing": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "EXPERIMENTAL ADetailer-for-video: after each scene finishes denoising, CLIPSeg (text-prompted segmentation) locates the regions named in detail_targets on a few decoded keyframes; the matched region is cut out of the video latent as a spatiotemporal tube, pushed through Lightricks' trained latent upsampler (2x spatial — the official two-stage pipeline's stage-2 model), re-noised to the official stage-2 sigma (0.85) and re-denoised for the 3-step tail schedule, then downscaled back to its ORIGINAL latent size and pasted through the feathered CLIPSeg silhouette. Final resolution never changes — the upsample only lets the model resolve structure (fingers, small faces) at a higher working resolution. Cost ~= 4 x tube area fraction x 3 steps (hands ~+15%); tubes over 35% of the frame are refused (that's a re-render, not a detail pass). Audio untouched by construction. Requires detail_upsampler; silent no-op when nothing matches. UNVALIDATED LIVE.",
|
||||
"tooltip": "EXPERIMENTAL ADetailer-for-video: after each scene finishes denoising, CLIPSeg (text-prompted segmentation) locates the regions named in detail_targets on a few decoded keyframes; the matched region is cut out of the video latent as a spatiotemporal tube, pushed through Lightricks' trained latent upsampler (2x spatial — the official two-stage pipeline's stage-2 model), re-noised to the official stage-2 sigma (0.85) and re-denoised for the 3-step tail schedule, then downscaled back to its ORIGINAL latent size and pasted through the feathered CLIPSeg silhouette. Final resolution never changes — the upsample only lets the model resolve structure (fingers, small faces) at a higher working resolution. Cost ~= 4 x tube area fraction x 3 steps (hands ~+15%); tubes over 35% of the frame are refused (that's a re-render, not a detail pass). Audio untouched by construction. detail_upsampler 'auto' finds or downloads the official Lightricks upsampler (~1 GB, once); skips are reported loudly in console + scene report. UNVALIDATED LIVE.",
|
||||
}),
|
||||
"detail_targets": ("STRING", {
|
||||
"default": "hands",
|
||||
"tooltip": "Comma-separated regions to detail, in plain words ('hands', 'hands, feet', 'face'). Each becomes a CLIPSeg text query; matched regions merge into one tube per scene. CLIPSeg matches broad CLIP semantics, so malformed anatomy still lights up for its name.",
|
||||
}),
|
||||
"detail_upsampler": (cls._detail_upsampler_choices(), {
|
||||
"default": "None",
|
||||
"tooltip": "Latent upsampler checkpoint from models/latent_upscale_models (the LTX 2.3 spatial upsampler used by the official two-stage workflows). Required — segmented_detailing no-ops while this is None.",
|
||||
"default": "auto",
|
||||
"tooltip": "Latent upsampler checkpoint from models/latent_upscale_models (the LTX 2.3 spatial upsampler used by the official two-stage workflows). 'auto' picks the newest installed spatial upscaler, or downloads the official file (~1 GB, once) when the folder is empty. Pick a file explicitly to pin it.",
|
||||
}),
|
||||
"detail_strength": ("FLOAT", {
|
||||
"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05,
|
||||
@@ -2492,11 +2492,14 @@ class FunPackLTXAVSceneChainSampler:
|
||||
|
||||
@classmethod
|
||||
def _detail_upsampler_choices(cls):
|
||||
# "auto": prefer an installed spatial upscaler, else download the official
|
||||
# Lightricks file on first use. ("None" from older saved workflows/projects is
|
||||
# treated as auto too — the enable toggle is the only off switch.)
|
||||
try:
|
||||
import folder_paths
|
||||
return ["None"] + folder_paths.get_filename_list("latent_upscale_models")
|
||||
return ["auto"] + folder_paths.get_filename_list("latent_upscale_models")
|
||||
except Exception:
|
||||
return ["None"]
|
||||
return ["auto"]
|
||||
|
||||
def _is_nested(self, samples):
|
||||
return bool(getattr(samples, "is_nested", False))
|
||||
@@ -4851,7 +4854,8 @@ class FunPackLTXAVSceneChainSampler:
|
||||
scene_runs: list = []
|
||||
joyai_bank = _JoyAIMemoryBank(joyai_memory_size, joyai_fix_frames) if joyai_memory else None
|
||||
_identity_overlap_state: dict = {}
|
||||
_detail_upsampler_model = None # lazy: loaded at the first detailed scene
|
||||
_detail_upsampler_model = None # lazy: resolved+loaded at the first detailed scene
|
||||
_detail_disabled_reason = None # set on resolve/load failure: don't retry per scene
|
||||
|
||||
for scene_index, scene_cond in enumerate(scene_conditionings):
|
||||
scene_positive = [scene_cond]
|
||||
@@ -5113,15 +5117,18 @@ class FunPackLTXAVSceneChainSampler:
|
||||
# Segmented detailing runs on the clean, fully-cropped scene (guide/audio
|
||||
# tails gone, overlap still present — the tube is spatial, so carried head
|
||||
# frames detail together with the rest) and BEFORE the JoyAI harvest, so
|
||||
# cross-shot memory banks the improved frames.
|
||||
if segmented_detailing and detail_strength > 0 and detail_upsampler != "None":
|
||||
# cross-shot memory banks the improved frames. Every skip is LOUD (console +
|
||||
# scene report): with the toggle on, silence must mean "ran and found nothing".
|
||||
if segmented_detailing and detail_strength > 0 and not _detail_disabled_reason:
|
||||
try:
|
||||
try:
|
||||
from . import detailing as _detailing
|
||||
except ImportError:
|
||||
import detailing as _detailing
|
||||
if _detail_upsampler_model is None:
|
||||
_detail_upsampler_model = _detailing.load_latent_upsampler(detail_upsampler)
|
||||
_resolved = _detailing.resolve_upsampler_name(detail_upsampler)
|
||||
_detail_upsampler_model = _detailing.load_latent_upsampler(_resolved)
|
||||
print(f"[FunPackSceneChain] segmented detailing upsampler: {_resolved}")
|
||||
_t_detail0 = _time.perf_counter()
|
||||
sampled, _detail_note = _detailing.detail_refine_scene(
|
||||
self, model, vae, sampler, scene_positive, scene_negative, sampled,
|
||||
@@ -5130,9 +5137,18 @@ class FunPackLTXAVSceneChainSampler:
|
||||
if _detail_note:
|
||||
run_mechanisms.append(_detail_note)
|
||||
_phase_sampling += _time.perf_counter() - _t_detail0
|
||||
else:
|
||||
run_mechanisms.append("segmented_detail(no region matched)")
|
||||
except Exception as _detail_exc:
|
||||
# A failed detail pass must never cost the scene itself.
|
||||
print(f"[FunPackSceneChain] segmented_detailing failed, scene kept as-is: {_detail_exc}")
|
||||
# A failed detail pass must never cost the scene itself — but it must
|
||||
# never fail silently either. Model resolution/load failures disable
|
||||
# the pass for the rest of the run (no per-scene download retries).
|
||||
if _detail_upsampler_model is None:
|
||||
_detail_disabled_reason = str(_detail_exc)
|
||||
print(f"[FunPackSceneChain] SEGMENTED DETAILING SKIPPED (scene kept as-is): {_detail_exc}")
|
||||
run_mechanisms.append(f"segmented_detail(SKIPPED: {_detail_exc})")
|
||||
elif segmented_detailing and _detail_disabled_reason:
|
||||
run_mechanisms.append("segmented_detail(SKIPPED: upsampler unavailable, see first scene)")
|
||||
if joyai_bank is not None:
|
||||
# Harvest from the clean, fully-cropped scene so injected memory tails never re-enter
|
||||
# the bank. Scene 0 seeds the pinned anchor (num_fix); later scenes roll in. The audio
|
||||
|
||||
@@ -99,6 +99,55 @@ def test_find_tube_enforces_minimum_edge():
|
||||
assert (x1 - x0) >= detailing.MIN_TUBE_EDGE
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_upsampler_name
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _patch_folder_paths(monkeypatch, files, folder="/models/latent_upscale_models"):
|
||||
fp = sys.modules["folder_paths"]
|
||||
monkeypatch.setattr(fp, "get_filename_list", lambda kind: list(files), raising=False)
|
||||
monkeypatch.setattr(fp, "get_folder_paths", lambda kind: [folder], raising=False)
|
||||
|
||||
def test_resolve_explicit_name_passes_through(monkeypatch):
|
||||
_patch_folder_paths(monkeypatch, [])
|
||||
assert detailing.resolve_upsampler_name("my_upscaler.safetensors") == "my_upscaler.safetensors"
|
||||
|
||||
def test_resolve_auto_prefers_newest_spatial_upscaler(monkeypatch):
|
||||
_patch_folder_paths(monkeypatch, [
|
||||
"other_model.safetensors",
|
||||
"ltx-2.3-spatial-upscaler-x2-1.0.safetensors",
|
||||
"ltx-2.3-spatial-upscaler-x2-1.1.safetensors",
|
||||
])
|
||||
for alias in ("auto", "None", "", None):
|
||||
assert detailing.resolve_upsampler_name(alias) == "ltx-2.3-spatial-upscaler-x2-1.1.safetensors"
|
||||
|
||||
def test_resolve_auto_falls_back_to_any_installed(monkeypatch):
|
||||
_patch_folder_paths(monkeypatch, ["something_else.safetensors"])
|
||||
assert detailing.resolve_upsampler_name("auto") == "something_else.safetensors"
|
||||
|
||||
def test_resolve_auto_downloads_when_folder_empty(monkeypatch, tmp_path):
|
||||
_patch_folder_paths(monkeypatch, [], folder=str(tmp_path / "lup"))
|
||||
calls = {}
|
||||
hub = types.ModuleType("huggingface_hub")
|
||||
hub.hf_hub_download = lambda repo_id, filename, local_dir: calls.update(
|
||||
repo=repo_id, file=filename, dest=local_dir)
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", hub)
|
||||
name = detailing.resolve_upsampler_name("auto")
|
||||
assert name == detailing.DEFAULT_UPSAMPLER_FILE
|
||||
assert calls["repo"] == detailing.DEFAULT_UPSAMPLER_REPO
|
||||
assert calls["file"] == detailing.DEFAULT_UPSAMPLER_FILE
|
||||
|
||||
def test_resolve_download_failure_raises_actionable_error(monkeypatch, tmp_path):
|
||||
_patch_folder_paths(monkeypatch, [], folder=str(tmp_path / "lup"))
|
||||
hub = types.ModuleType("huggingface_hub")
|
||||
def _boom(**kw):
|
||||
raise OSError("401 gated")
|
||||
hub.hf_hub_download = _boom
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", hub)
|
||||
with pytest.raises(RuntimeError, match="manually"):
|
||||
detailing.resolve_upsampler_name("auto")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _strip_layout_conds / _downscale_to
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user