From 80329c9f74454da02bb841adb806ea6e3c68c3e5 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:55:22 +0300 Subject: [PATCH 01/24] =?UTF-8?q?Loop=20roll=20v2:=20content-only=20roll?= =?UTF-8?q?=20=E2=80=94=20guides=20and=20audio=20memory=20stay=20pinned?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Guide-carrying scenes no longer disable the loop temporal style. Appended guide frames always live at the tail of the video stream's T range, and keyframe_idxs is per-token — so the wrapper derives the tail frame count (tokens / (H*W), same math as comfy's get_keyframe_idxs) and rolls only the content region [0, T - tail); the tail, keyframe_idxs, and attention entries that reference it stay canonical. The audio stream's appended memory (JoyAI, rides a trailing mask=0 run) is detected the same way from audio_denoise_mask. Guides keep informing every rolled position — their influence smears across the cycle, which is what identity/style references should do in a loop — and are never dragged into the rendered content. Unparseable guide contexts (malformed keyframe_idxs, entries without idxs) still run canonical: never risk a corrupting roll. Content-region size gates the minimum-frames check now, so a short clip whose frames are mostly guide tail stays canonical too. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 10 +++ ltx_enhancements.py | 147 +++++++++++++++++++++++++++++---------- samplers.py | 10 +-- tests/test_loop_style.py | 74 +++++++++++++++++++- 4 files changed, 197 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76c8d39a..097072c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [Unreleased] + +### Changed +- **Loop temporal style now works alongside guides.** Scenes carrying appended guide frames + (carry-i2v-guides, mid-scene guide, JoyAI memory, custom guide stacks) no longer disable + the loop roll: only the content region rolls while the pinned guide tail (and the audio + memory tail) stays canonical, so guides keep informing every position of the cycle without + ever being rendered into it. The i2v anchor was always supported - it rides the denoise + mask, which rolls in step with the latent. + ## [3.3.0] - 2026-07-15 ### Added diff --git a/ltx_enhancements.py b/ltx_enhancements.py index 87d763c6..79857cfa 100644 --- a/ltx_enhancements.py +++ b/ltx_enhancements.py @@ -198,11 +198,15 @@ def make_pulse_temporal_wrapper(old_wrapper, segment_count=None, peak_mult=None, # # Roll starts below the near-noise plateau: content is ~pure noise up there (nothing to # smooth) and the plateau step-cache reuses base forwards, which a per-step roll would -# invalidate. Guides pin absolute frame positions (keyframe_idxs / attention entries -# reference appended frames inside the same T range), so calls carrying them are left -# canonical rather than rolled. +# invalidate. Appended guide frames (carry_i2v_guides / mid-scene guide / JoyAI memory / +# custom stacks) live at the TAIL of the same T range, pinned to absolute positions by +# keyframe_idxs — so only the content region [0, T - tail) rolls and the tail (plus +# keyframe_idxs and attention entries, which reference tail tokens) stays canonical. +# Guides keep informing every rolled position (their influence smears across the cycle), +# which is exactly what identity/style references should do in a loop; they are never +# rendered as frames themselves. LOOP_ROLL_MAX_SIGMA = 0.95 -LOOP_ROLL_MIN_FRAMES = 4 # below this many latent frames there is nothing to loop +LOOP_ROLL_MIN_FRAMES = 4 # below this many CONTENT latent frames there is nothing to loop def _van_der_corput(n): @@ -242,37 +246,94 @@ def _loop_stream_shapes(args): return shapes -def _loop_roll_packed(x, shapes, frac, direction): - """Cyclically roll every stream's temporal dim (dim 2) of a packed [B,1,N] tensor by - round(frac * T_stream) frames (video and audio each by their own frame count, so the - two stay time-aligned). direction=+1 rolls, -1 unrolls. Returns a new tensor.""" +def _loop_content_shift(frac, t, tail): + """Frame shift for a stream with `t` total frames of which the last `tail` are pinned + guide frames. 0 when the content region is too small to roll.""" + tc = int(t) - max(0, int(tail)) + if tc < 2: + return 0, tc + return int(round(frac * tc)) % tc, tc + + +def _loop_roll_stream(stream, frac, direction, tail): + """Roll only the content region [0, T-tail) of one stream's temporal dim (dim 2); + the pinned guide tail stays canonical. Returns the input unchanged when there is + nothing to roll.""" + shift, tc = _loop_content_shift(frac, stream.shape[2], tail) + if not shift: + return stream + out = stream.clone() + out[:, :, :tc] = torch.roll(stream[:, :, :tc], shifts=direction * shift, dims=2) + return out + + +def _loop_roll_packed(x, shapes, frac, direction, tails=None): + """Cyclically roll every stream's content region of a packed [B,1,N] tensor by + round(frac * T_content) frames (video and audio each by their own content length, so + the two stay time-aligned). `tails` gives per-stream pinned tail-frame counts aligned + with `shapes` (appended guides / audio memory — left canonical). direction=+1 rolls, + -1 unrolls. Returns a new tensor.""" import math as _math if x.ndim == 5: # unpacked single stream - t = x.shape[2] - shift = int(round(frac * t)) % t - return torch.roll(x, shifts=direction * shift, dims=2) if shift else x + return _loop_roll_stream(x, frac, direction, (tails or [0])[0]) out = x.clone() off = 0 - for dims in shapes: + for i, dims in enumerate(shapes): sz = _math.prod(dims[1:]) if len(dims) >= 3 and dims[2] >= 2: - t = dims[2] - shift = int(round(frac * t)) % t - if shift: - stream = x[..., off:off + sz].reshape([x.shape[0]] + list(dims[1:])) - out[..., off:off + sz] = torch.roll( - stream, shifts=direction * shift, dims=2).reshape(x.shape[0], 1, sz) + stream = x[..., off:off + sz].reshape([x.shape[0]] + list(dims[1:])) + rolled = _loop_roll_stream(stream, frac, direction, (tails or [])[i] if tails else 0) + if rolled is not stream: + out[..., off:off + sz] = rolled.reshape(x.shape[0], 1, sz) off += sz return out -def _loop_roll_mask(mask, frac, direction): - """Roll a denoise mask ([B,1,T,H,W] video / [B,C,T,F] audio) in step with its stream.""" +def _loop_roll_mask(mask, frac, direction, tail=0): + """Roll a denoise mask ([B,1,T,H,W] video / [B,C,T,F] audio) in step with its stream, + leaving the pinned tail region canonical like the stream itself.""" if not isinstance(mask, torch.Tensor) or mask.ndim < 3 or mask.shape[2] < 2: return mask - t = mask.shape[2] - shift = int(round(frac * t)) % t - return torch.roll(mask, shifts=direction * shift, dims=2) if shift else mask + return _loop_roll_stream(mask, frac, direction, tail) + + +def _loop_video_tail_frames(c, video_shape): + """Number of appended guide latent frames pinned at the video stream's tail. 0 when + the call carries no guides; None when a guide context exists but can't be parsed + (caller then leaves the call canonical rather than risking a corrupting roll). + keyframe_idxs is per-TOKEN — frames = tokens / (H*W), same math as comfy's + get_keyframe_idxs.""" + kf = c.get("keyframe_idxs") + if kf is None: + # Attention entries without keyframe_idxs shouldn't happen; treat as unparseable. + return None if c.get("guide_attention_entries") else 0 + try: + if len(video_shape) != 5 or not hasattr(kf, "shape") or len(kf.shape) < 3: + return None + tokens_per_frame = int(video_shape[-2]) * int(video_shape[-1]) + n = int(kf.shape[2]) // max(1, tokens_per_frame) + return n if 0 < n < int(video_shape[2]) else None + except Exception: + return None + + +def _loop_audio_tail_frames(mask): + """Appended audio-memory frames at the tail of the audio stream = the trailing run of + all-zero frames in its denoise mask (JoyAI paired audio memory rides mask=0; genuine + audio content is never fully masked). 0 when there is no mask or no zero tail.""" + if not isinstance(mask, torch.Tensor) or mask.ndim < 3 or mask.shape[2] < 2: + return 0 + try: + other_dims = [d for d in range(mask.ndim) if d != 2] + per_frame = mask.float().abs().amax(dim=other_dims) # [T] + tail = 0 + for v in reversed(per_frame.tolist()): + if v > 1e-4: + break + tail += 1 + return tail if tail < mask.shape[2] else 0 + except Exception: + return 0 def make_loop_temporal_wrapper(old_wrapper): @@ -299,36 +360,46 @@ def make_loop_temporal_wrapper(old_wrapper): state["last_sigma"] = sigma c = args.get("c") or {} - if ( - sigma > LOOP_ROLL_MAX_SIGMA - or c.get("keyframe_idxs") is not None - or c.get("guide_attention_entries") is not None - ): + if sigma > LOOP_ROLL_MAX_SIGMA: return _call(args) shapes = _loop_stream_shapes(args) if not shapes: return _call(args) - # Gate on the VIDEO stream's frame count (the 5-dim shape); audio rides along. - t_video = max((s[2] for s in shapes if len(s) == 5), - default=max((s[2] for s in shapes if len(s) >= 3), default=0)) - if t_video < LOOP_ROLL_MIN_FRAMES: + # Per-stream pinned tails: appended guide frames (video, counted from the + # per-token keyframe_idxs) and appended audio memory (trailing mask=0 run). + # Only the content region in front of each tail rolls; the tail — and the + # keyframe_idxs / attention entries that reference it — stays canonical. + video_shape = next((s for s in shapes if len(s) == 5), shapes[0]) + v_tail = _loop_video_tail_frames(c, video_shape) + if v_tail is None: + return _call(args) # guide context we can't parse: never risk a bad roll + tails = [ + (v_tail if len(s) == 5 else _loop_audio_tail_frames(c.get("audio_denoise_mask"))) + for s in shapes + ] + t_content = int(video_shape[2]) - v_tail if len(video_shape) >= 3 else 0 + if t_content < LOOP_ROLL_MIN_FRAMES: return _call(args) state["count"] += 1 frac = _van_der_corput(state["count"]) if not state["logged"]: state["logged"] = True - print(f"[FunPack] loop temporal style: Mobius latent roll active (T={t_video})") + tail_note = f", {v_tail} guide frame(s) pinned" if v_tail else "" + print(f"[FunPack] loop temporal style: Mobius latent roll active (T={t_content}{tail_note})") try: new_c = dict(c) - for key in ("denoise_mask", "audio_denoise_mask"): - if isinstance(new_c.get(key), torch.Tensor): - new_c[key] = _loop_roll_mask(new_c[key], frac, 1) + if isinstance(new_c.get("denoise_mask"), torch.Tensor): + new_c["denoise_mask"] = _loop_roll_mask(new_c["denoise_mask"], frac, 1, tail=v_tail) + if isinstance(new_c.get("audio_denoise_mask"), torch.Tensor): + new_c["audio_denoise_mask"] = _loop_roll_mask( + new_c["audio_denoise_mask"], frac, 1, + tail=_loop_audio_tail_frames(new_c["audio_denoise_mask"])) rolled = dict(args) - rolled["input"] = _loop_roll_packed(args["input"], shapes, frac, 1) + rolled["input"] = _loop_roll_packed(args["input"], shapes, frac, 1, tails=tails) rolled["c"] = new_c out = _call(rolled) - return _loop_roll_packed(out, shapes, frac, -1) + return _loop_roll_packed(out, shapes, frac, -1, tails=tails) except Exception as e: if state.get("roll_error") is None: state["roll_error"] = True diff --git a/samplers.py b/samplers.py index f93e1412..4c78c5e7 100644 --- a/samplers.py +++ b/samplers.py @@ -4953,10 +4953,10 @@ class FunPackLTXAVSceneChainSampler: # Loop temporal style (auto director's funpack_temporal_loop): Mobius latent # roll. Installed BELOW the guidance wrappers (embed guidance / slider / # dynashift / output guidance) so they see canonical-orientation inputs and - # predictions — the roll exists only around the base forward. Calls that - # carry guides (keyframe_idxs pins appended frames to absolute positions) - # are left canonical by the wrapper itself, so on guide-carrying scenes the - # roll is inert rather than corrupting. + # predictions — the roll exists only around the base forward. Appended guide + # frames stay pinned: the wrapper rolls only the content region in front of + # the guide tail (counted from keyframe_idxs / the audio mask), so guides + # keep informing the whole cycle without ever being dragged into it. if self._scene_temporal_loop(scene_cond): try: from .ltx_enhancements import make_loop_temporal_wrapper @@ -4967,7 +4967,7 @@ class FunPackLTXAVSceneChainSampler: make_loop_temporal_wrapper(_loop_base), _loop_base) _loop_pinned = guide_tail > 0 or carried > 0 or carried_guide_frames > 0 run_mechanisms.append( - "temporal_loop_roll(inert: guides pin frames)" if _loop_pinned + "temporal_loop_roll(content-only: guide tail pinned)" if _loop_pinned else "temporal_loop_roll") # taste_nearest_prompt: swap the single global liked direction for the one # learned on this scene's closest rated prompts (resolved from the ORIGINAL diff --git a/tests/test_loop_style.py b/tests/test_loop_style.py index 6a8b6266..bff6eddc 100644 --- a/tests/test_loop_style.py +++ b/tests/test_loop_style.py @@ -94,7 +94,7 @@ def test_wrapper_rolls_in_unrolls_out(): assert torch.equal(seen["input"][..., : video.numel()].reshape(VSHAPE), torch.roll(video, 4, dims=2)) -def test_wrapper_gates_plateau_guides_and_tiny_clips(): +def test_wrapper_gates_plateau_unparseable_guides_and_tiny_clips(): video, _, packed = _packed() seen = {} @@ -105,8 +105,10 @@ def test_wrapper_gates_plateau_guides_and_tiny_clips(): w = make_loop_temporal_wrapper(None) w(fake_apply, _args(0.99, packed)) # near-noise plateau assert torch.equal(seen["input"], packed) + # malformed keyframe_idxs (too few dims to count tokens): canonical, never a blind roll w(fake_apply, _args(0.725, packed, extra_c={"keyframe_idxs": torch.zeros(1, 1)})) assert torch.equal(seen["input"], packed) + # attention entries without keyframe_idxs: guide context we can't size -> canonical w(fake_apply, _args(0.725, packed, extra_c={"guide_attention_entries": [object()]})) assert torch.equal(seen["input"], packed) @@ -118,6 +120,76 @@ def test_wrapper_gates_plateau_guides_and_tiny_clips(): assert torch.equal(seen["input"], tiny) +def _keyframe_idxs_for(n_frames, h=4, w=6): + # per-token layout: [B, coords, n_tokens, start/end] — the wrapper only reads shape[2] + return torch.zeros(1, 1, n_frames * h * w, 2) + + +def test_wrapper_content_only_roll_with_guide_tail(): + """Guide frames appended at the video tail stay pinned; only content rolls.""" + v_tail = 2 + video, audio, packed = _packed() # video T=16 -> content 14; audio T=32, no tail + mask = torch.rand(1, 1, 16, 4, 6) + seen = {} + + def fake_apply(x, ts, **c): + seen["input"], seen["mask"] = x, c.get("denoise_mask") + return x * 1.0 + + w = make_loop_temporal_wrapper(None) + out = w(fake_apply, _args(0.909, packed, mask, + extra_c={"keyframe_idxs": _keyframe_idxs_for(v_tail)})) + v_in = seen["input"][..., : video.numel()].reshape(VSHAPE) + # first eligible step: frac 0.5 of the 14 content frames -> shift 7 + assert torch.equal(v_in[:, :, :14], torch.roll(video[:, :, :14], 7, dims=2)) + assert torch.equal(v_in[:, :, 14:], video[:, :, 14:]) # guide tail untouched + assert torch.equal(seen["mask"][:, :, :14], torch.roll(mask[:, :, :14], 7, dims=2)) + assert torch.equal(seen["mask"][:, :, 14:], mask[:, :, 14:]) # tail mask untouched + # audio has no tail: rolls fully by its own content length (0.5 * 32 = 16) + a_in = seen["input"][..., video.numel():].reshape(ASHAPE) + assert torch.equal(a_in, torch.roll(audio, 16, dims=2)) + assert torch.equal(out, packed) # unrolled on the way out + + +def test_wrapper_audio_memory_tail_stays_pinned(): + """A trailing mask=0 run on the audio stream (JoyAI memory) is treated as pinned.""" + a_tail = 4 + video, audio, packed = _packed() + amask = torch.ones(1, 1, 32, 16) + amask[:, :, -a_tail:] = 0.0 + seen = {} + + def fake_apply(x, ts, **c): + seen["input"], seen["amask"] = x, c.get("audio_denoise_mask") + return x * 1.0 + + w = make_loop_temporal_wrapper(None) + out = w(fake_apply, _args(0.909, packed, extra_c={"audio_denoise_mask": amask})) + a_in = seen["input"][..., video.numel():].reshape(ASHAPE) + # audio content 28 frames -> frac 0.5 -> shift 14; tail canonical + assert torch.equal(a_in[:, :, :28], torch.roll(audio[:, :, :28], 14, dims=2)) + assert torch.equal(a_in[:, :, 28:], audio[:, :, 28:]) + assert torch.equal(seen["amask"], amask) # all-ones content rolls invisibly; zeros pinned + assert torch.equal(out, packed) + + +def test_wrapper_guide_tail_reduces_content_gate(): + """A clip whose content region (T - tail) is under the minimum stays canonical.""" + shape = (1, 128, 5, 4, 6) + x = torch.rand(1, 1, math.prod(shape[1:])) + seen = {} + + def fake_apply(x_, ts, **c): + seen["input"] = x_ + return x_ * 1.0 + + w = make_loop_temporal_wrapper(None) + w(fake_apply, {"input": x, "timestep": torch.tensor([0.7]), + "c": {"latent_shapes": [torch.Size(shape)], + "keyframe_idxs": _keyframe_idxs_for(2)}}) # content 3 < 4 + assert torch.equal(seen["input"], x) + + def test_wrapper_resets_shift_sequence_on_new_scene(): video, _, packed = _packed() seen = {} From 302028a491ed3ad447a1c9fba2b55a04e3d27558 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Wed, 15 Jul 2026 22:38:58 +0300 Subject: [PATCH 02/24] Add rapid_start/rapid_end/rapid_start_end temporal styles Punchy frame_rate mult held over the first/last slice of a scene's denoise progress, easing to natural in the middle - same progress- from-sigma mechanism as pulse, shaped as a single edge ramp instead of repeated segments. Wired through build_enhancements (single-scene), the Scene Chain Sampler's per-scene wrapper dispatch, and both Studio and Editor temporal-style dropdowns. Co-Authored-By: Claude Sonnet 5 --- conditioning.py | 23 ++++++++- ltx_enhancements.py | 60 +++++++++++++++++++++++- movie_editor/frontend/engine_settings.js | 2 +- samplers.py | 13 ++++- web/funpack_studio.js | 2 +- 5 files changed, 92 insertions(+), 8 deletions(-) diff --git a/conditioning.py b/conditioning.py index 9d2baf29..1a77e40a 100644 --- a/conditioning.py +++ b/conditioning.py @@ -6484,10 +6484,10 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): "default": "", "tooltip": "Optional user feedback describing what was specifically wrong with the previous output (e.g. 'he was supposed to hold her hand, not her head'). Has highest priority in the advisor system prompt.", }), - "temporal_style": (["natural", "auto", "accelerate", "decelerate", "loop", "freeze", "pulse"], { + "temporal_style": (["natural", "auto", "accelerate", "decelerate", "loop", "freeze", "pulse", "rapid_start", "rapid_end", "rapid_start_end"], { "default": "natural", "label": "Temporal Style", - "tooltip": "Controls how the model perceives motion timing via frame_rate RoPE manipulation. natural=no change, auto=per-scene director picks motion energy from each scene's prompt (needs the Scene Chain Sampler), accelerate=faster motion, decelerate=heavier motion, loop=circular temporal coords, freeze=highly compressed time, pulse=repeated ease-down motion holds per scene (needs the Scene Chain Sampler).", + "tooltip": "Controls how the model perceives motion timing via frame_rate RoPE manipulation. natural=no change, auto=per-scene director picks motion energy from each scene's prompt (needs the Scene Chain Sampler), accelerate=faster motion, decelerate=heavier motion, loop=circular temporal coords, freeze=highly compressed time, pulse=repeated ease-down motion holds per scene (needs the Scene Chain Sampler), rapid_start=speeds up just the beginning of each scene then settles to natural, rapid_end=natural then speeds up just the ending, rapid_start_end=speeds up both ends with natural motion in the middle.", }), "split_by_transitions": ("BOOLEAN", { "default": False, @@ -13355,6 +13355,24 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): out.append([cond, meta] if isinstance(entry, list) else (cond, meta)) return out + def _v2_apply_rapid_temporal(self, conditioning_list, temporal_style): + """temporal_style in (rapid_start, rapid_end, rapid_start_end): tag each scene + entry so the Scene Chain Sampler installs a rapid-edge ease temporal wrapper + per scene (mirrors _v2_apply_pulse_temporal).""" + style = str(temporal_style or "natural").strip().lower() + if style not in ("rapid_start", "rapid_end", "rapid_start_end"): + return conditioning_list + out = [] + for entry in conditioning_list or []: + if not (isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict)): + out.append(entry) + continue + cond, meta = entry[0], dict(entry[1]) + meta["funpack_temporal_mode"] = style + meta["funpack_temporal_label"] = style + out.append([cond, meta] if isinstance(entry, list) else (cond, meta)) + return out + def _v2_apply_auto_temporal(self, conditioning_list, temporal_style, fallback_text=""): """temporal_style="auto": bake a per-scene frame_rate multiplier into each conditioning entry's metadata so the Scene Chain Sampler can apply per-scene motion @@ -13452,6 +13470,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): _v2_apply_scene_refinement_keys); scenes without bound keys keep the project key.""" mode = str(steer_mode or "relative").lower() out = self._v2_apply_pulse_temporal(conditioning_list, temporal_style) + out = self._v2_apply_rapid_temporal(out, temporal_style) out = self._v2_apply_auto_temporal(out, temporal_style, fallback_text=temporal_fallback_text) out = self._v2_apply_bounded_attention(out, clip, fallback_text=temporal_fallback_text, encode_cache=encode_cache) if mode in ("relative", "both"): diff --git a/ltx_enhancements.py b/ltx_enhancements.py index 79857cfa..831aa0bf 100644 --- a/ltx_enhancements.py +++ b/ltx_enhancements.py @@ -6,7 +6,8 @@ Technique 2: Per-block attention temperature Technique 3: Temporal RoPE style Manipulate frame_rate in the model's positional embedding to change motion character. - Styles: natural / accelerate / decelerate / loop / freeze / pulse + Styles: natural / accelerate / decelerate / loop / freeze / pulse / + rapid_start / rapid_end / rapid_start_end Technique 4: Denoise creativity mask Spatial noise weighting derived from latent variance: high-variance regions get more freedom. @@ -25,7 +26,10 @@ from hashlib import md5 import torch -TEMPORAL_STYLES = ["natural", "auto", "accelerate", "decelerate", "loop", "freeze", "pulse"] +TEMPORAL_STYLES = [ + "natural", "auto", "accelerate", "decelerate", "loop", "freeze", "pulse", + "rapid_start", "rapid_end", "rapid_start_end", +] # frame_rate multiplier fed to the LTX RoPE per style. >1 => model assumes more # frames cover the same wall-clock time => smaller inter-frame deltas => smoother / @@ -45,6 +49,12 @@ PULSE_SEGMENT_COUNT = 3 PULSE_PEAK_MULT = 0.88 PULSE_FLOOR_MULT = 1.65 +# Rapid start/end: punchy (low) frame_rate mult held over the first/last slice of the +# scene's denoise progress, easing to natural (1.0) elsewhere. Same progress-from-sigma +# mechanism as pulse, just shaped as a single ramp instead of repeated segments. +RAPID_MULT = 0.65 +RAPID_FRACTION = 0.35 # portion of denoise progress the rapid ramp covers at each edge + # Keyword -> motion-energy intent for the "auto" director. The classifier scans a # scene's prompt and emits a continuous frame_rate multiplier (plus a loop flag). @@ -118,6 +128,22 @@ def pulse_mult_for_progress(progress, segment_count=None, peak_mult=None, floor_ return peak + (floor - peak) * _ease_down(local_t) +def rapid_mult_for_progress(progress, mode, fraction=None, mult=None): + """Map denoise progress [0,1] to a frame_rate multiplier for the rapid_start / + rapid_end / rapid_start_end styles: `mult` (punchy) held over the first and/or + last `fraction` of progress, easing to natural (1.0) toward the middle.""" + frac = RAPID_FRACTION if fraction is None else max(1e-3, min(0.5, float(fraction))) + rapid = RAPID_MULT if mult is None else float(mult) + progress = max(0.0, min(1.0, float(progress))) + start = mode in ("rapid_start", "rapid_start_end") + end = mode in ("rapid_end", "rapid_start_end") + if start and progress < frac: + return rapid + (1.0 - rapid) * _ease_down(progress / frac) + if end and progress > 1.0 - frac: + return rapid + (1.0 - rapid) * _ease_down((1.0 - progress) / frac) + return 1.0 + + def _scale_frame_rate_in_args(args, mult): """Return a shallow-copied args dict with frame_rate scaled by mult (no-op on failure).""" c = args.get("c") @@ -188,6 +214,33 @@ def make_pulse_temporal_wrapper(old_wrapper, segment_count=None, peak_mult=None, return _pulse_wrapper + +def make_rapid_temporal_wrapper(old_wrapper, mode, fraction=None, mult=None): + """Build a model_function_wrapper for rapid_start / rapid_end / rapid_start_end: + punchy frame_rate mult over the first/last slice of denoise progress (derived from + sigma, same as pulse), natural (1.0, i.e. no-op) elsewhere. A fresh wrapper should + be installed per scene so sigma bounds reset between scenes.""" + frac = RAPID_FRACTION if fraction is None else fraction + rapid = RAPID_MULT if mult is None else mult + sigma_start = [None] + + def _rapid_wrapper(apply_fn, args, _old=old_wrapper, _mode=mode, _frac=frac, _rapid=rapid): + ts = args.get("timestep") + try: + sigma = float(ts.max().item()) if ts is not None else 1.0 + except Exception: + sigma = 1.0 + if sigma_start[0] is None: + sigma_start[0] = max(sigma, 1e-6) + progress = max(0.0, min(1.0, 1.0 - sigma / sigma_start[0])) + mult_here = rapid_mult_for_progress(progress, _mode, _frac, _rapid) + args = _scale_frame_rate_in_args(args, mult_here) + if _old is not None: + return _old(apply_fn, args) + return apply_fn(args["input"], args["timestep"], **args.get("c", {})) + + return _rapid_wrapper + # --- Loop temporal style: Mobius-style latent roll (arXiv:2502.20307) --------------- # Seamless looping without training or extra forward passes: on eligible denoise steps # the latent is cyclically rolled along its temporal dim before the forward and the @@ -1437,6 +1490,9 @@ def build_enhancements(model, rating_profile, temporal_style, refinement_key, re elif temporal_style == "loop": model.model_options["model_function_wrapper"] = make_loop_temporal_wrapper( model.model_options.get("model_function_wrapper")) + elif temporal_style in ("rapid_start", "rapid_end", "rapid_start_end"): + model.model_options["model_function_wrapper"] = make_rapid_temporal_wrapper( + model.model_options.get("model_function_wrapper"), temporal_style) elif temporal_style not in ("natural", "auto"): mult = TEMPORAL_STYLE_MULT.get(temporal_style, 1.0) wrapper = make_temporal_wrapper(model.model_options.get("model_function_wrapper"), mult) diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 763cf600..a2c81538 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -58,7 +58,7 @@ { name: "absolute_strength", label: "Absolute strength", kind: "float", default: 0.6, min: 0, max: 1, step: 0.05, dependsOn: "steer_mode", dependsVals: ["absolute", "both"] }, { name: "temporal_style", label: "Temporal style", kind: "combo", - choices: ["natural", "auto", "accelerate", "decelerate", "loop", "freeze", "pulse"], default: "natural" }, + choices: ["natural", "auto", "accelerate", "decelerate", "loop", "freeze", "pulse", "rapid_start", "rapid_end", "rapid_start_end"], default: "natural" }, { name: "split_transition_placement", label: "Transition placement", kind: "combo", choices: ["start", "end", "silent"], default: "start" }, ]; diff --git a/samplers.py b/samplers.py index 4c78c5e7..49ddad3d 100644 --- a/samplers.py +++ b/samplers.py @@ -5001,8 +5001,9 @@ class FunPackLTXAVSceneChainSampler: # corrects whatever prediction those already produced, not the raw base one. run_mechanisms.append(f"output_guidance({output_guidance_strength})") self._build_output_guidance_wrapper(model, _output_value_fn, output_guidance_strength) - # Per-scene temporal style (auto / pulse): layer a frame_rate wrapper on top - # of whatever is installed (e.g. embed guidance). + # Per-scene temporal style (auto / pulse / rapid_start / rapid_end / + # rapid_start_end): layer a frame_rate wrapper on top of whatever is + # installed (e.g. embed guidance). _scene_mode = self._scene_temporal_mode(scene_cond) _scene_mult = self._scene_temporal_mult(scene_cond) _cur_wrapper = model.model_options.get("model_function_wrapper") @@ -5014,6 +5015,14 @@ class FunPackLTXAVSceneChainSampler: _tw = make_pulse_temporal_wrapper(_cur_wrapper) if _tw is not None: model.model_options["model_function_wrapper"] = _tag_scene_wrapper(_tw, _cur_wrapper) + elif _scene_mode in ("rapid_start", "rapid_end", "rapid_start_end"): + try: + from .ltx_enhancements import make_rapid_temporal_wrapper + except ImportError: + from ltx_enhancements import make_rapid_temporal_wrapper + _tw = make_rapid_temporal_wrapper(_cur_wrapper, _scene_mode) + if _tw is not None: + model.model_options["model_function_wrapper"] = _tag_scene_wrapper(_tw, _cur_wrapper) elif _scene_mult is not None and abs(_scene_mult - 1.0) >= 1e-3: try: from .ltx_enhancements import make_temporal_wrapper diff --git a/web/funpack_studio.js b/web/funpack_studio.js index ee2ed5f2..0acc05ac 100644 --- a/web/funpack_studio.js +++ b/web/funpack_studio.js @@ -8,7 +8,7 @@ const LORA_TYPES = ["general", "action", "style", "quality", "character"]; const ADVISOR_DTYPES = ["bfloat16", "float16", "float32"]; const REFINER_MODES = ["Refine", "Prompt only", "Learning"]; const ADVISOR_MODES = ["Off", "Only diagnostics", "Only prompt", "Full"]; -const TEMPORAL_STYLES = ["natural", "auto", "accelerate", "decelerate", "loop", "freeze", "pulse"]; +const TEMPORAL_STYLES = ["natural", "auto", "accelerate", "decelerate", "loop", "freeze", "pulse", "rapid_start", "rapid_end", "rapid_start_end"]; const CATEGORY_ORDER = ["action", "camera", "subject", "appearance", "environment", "style", "quality", "details"]; const TABS = ["Session", "Shortcuts", "Split markers", "Refiner", "Advisor", "LoRA", "Sampler", "Adjustments", "Timeline"]; const SAMPLER_TYPES = ["Hybrid Euler 2S", "Distilled Flow", "KSampler"]; From a6e4a934f132cbfac472347d2fa82c2fb2c4b50e Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sat, 18 Jul 2026 18:10:10 +0300 Subject: [PATCH 03/24] Segmented detailing: CLIPSeg-targeted latent crop-refine on Scene Chain ADetailer-for-video probe core. After a scene denoises, CLIPSeg locates user-named regions (hands, feet, ...) on decoded keyframes; the region is cropped as a spatiotemporal tube, 2x-upsampled through Lightricks' trained LatentUpsampler (comfy core, no port needed), re-denoised over the official stage-2 tail (0.85/0.725/0.4219/0), downscaled back and pasted through the feathered CLIPSeg silhouette. Final resolution unchanged; audio protected by construction (refined audio discarded); tubes >35% frame refused; empty targets/no-detection = bit-identical no-op. UNVALIDATED LIVE. Co-Authored-By: Claude Fable 5 --- detailing.py | 349 ++++++++++++++++++++++++++++++++++++++++ samplers.py | 50 ++++++ tests/test_detailing.py | 248 ++++++++++++++++++++++++++++ 3 files changed, 647 insertions(+) create mode 100644 detailing.py create mode 100644 tests/test_detailing.py diff --git a/detailing.py b/detailing.py new file mode 100644 index 00000000..832226f5 --- /dev/null +++ b/detailing.py @@ -0,0 +1,349 @@ +"""Segmented detailing: ADetailer-style region refine for the Scene Chain sampler. + +After a scene finishes denoising, CLIPSeg (text-prompted segmentation, the same +model family face-swap stacks use for occlusion masking) locates user-named +regions ("hands", "feet", ...) on a few decoded keyframes. The matched region is +cut out of the video latent as a spatiotemporal tube, pushed through Lightricks' +own trained LatentUpsampler (2x spatial, latent space - the official two-stage +pipeline's stage-2 model), re-noised to the official stage-2 re-entry sigma and +re-denoised for the tail schedule, then downscaled back to its original latent +size and pasted through the feathered CLIPSeg soft mask. Final resolution never +changes: the upsample exists only so the model can resolve structure (fingers, +edges) at a higher working resolution before the crop returns to the grid. + +Boundary law: this is a visual-behavior op, so it lives on the Chain Sampler. +Audio is never modified - the refine pass carries the scene's audio stream so +joint attention sees a legal AV latent, but its audio output is discarded. + +No-op guarantee: empty targets, no upsampler, or no detection above threshold +returns the input latent object untouched (bit-identical run). +""" + +import math + +import torch +import torch.nn.functional as F + +# Official LTX-2.3 two-stage distilled workflow, stage-2 ManualSigmas: re-noise the +# upsampled latent to 0.85 and run the tail of the 8-step distilled schedule. Using +# Lightricks' published re-entry point instead of inventing one: too much noise and +# the region redraws (drifts from the surrounding frame), too little and no detail +# is gained. +STAGE2_SIGMAS = (0.85, 0.725, 0.421875, 0.0) + +# CLIPSeg heat above this (post-sigmoid) counts as "the named thing is here". +DEFAULT_THRESHOLD = 0.35 + +# Tubes larger than this fraction of the frame are refused: at that size the pass +# stops being region detailing and becomes a second full render (cost ~= 4x area +# fraction x tail steps), which belongs to a different feature. +MAX_TUBE_AREA = 0.35 + +# Latent cells of context padding around the thresholded region, and the minimum +# tube edge. Padding gives the refine pass surrounding context to blend against; +# the minimum keeps the crop a legal latent for conv stacks and attention. +TUBE_PAD = 2 +MIN_TUBE_EDGE = 4 + +# Keyframes decoded per scene for detection. Single latent frames decode cheaply +# through the causal VAE (treated as stills) - enough for a temporal-union tube. +DETECTION_FRAMES = 3 + +# Cond keys that tie a scene's conditioning to the full-frame latent layout +# (i2v anchors / guide keyframes). The crop is its own small latent with no +# appended guide frames, so these must not travel into the refine pass. +_LAYOUT_COND_KEYS = ("keyframe_idxs", "guiding_latent", "concat_latent_image", "concat_mask") + +_CLIPSEG = None # (processor, model), lazy - loaded on first enabled run only + + +def _log(debug, msg): + if debug: + print(f"[FunPackDetail] {msg}") + + +# --------------------------------------------------------------------------- +# Models +# --------------------------------------------------------------------------- + +def load_latent_upsampler(model_name): + """Load Lightricks' LatentUpsampler from models/latent_upscale_models. + + Mirrors the LTX branch of comfy's LatentUpscaleModelLoader (safe_load + + config from safetensors metadata); the architecture itself ships in comfy + core (comfy.ldm.lightricks.latent_upsampler), so there is nothing to port. + """ + import json + + import comfy.model_management + import comfy.utils + import folder_paths + from comfy.ldm.lightricks.latent_upsampler import LatentUpsampler + + model_path = folder_paths.get_full_path_or_raise("latent_upscale_models", model_name) + sd, metadata = comfy.utils.load_torch_file(model_path, safe_load=True, return_metadata=True) + if "post_upsample_res_blocks.0.conv2.bias" not in sd: + raise ValueError( + f"{model_name} is not an LTX latent upsampler (expected post_upsample_res_blocks keys).") + config = json.loads(metadata["config"]) + model = LatentUpsampler.from_config(config).to( + dtype=comfy.model_management.vae_dtype(allowed_dtypes=[torch.bfloat16, torch.float32])) + model.load_state_dict(sd) + model.eval() + return model + + +def _get_clipseg(): + """CLIPSeg processor+model, loaded once and kept on CPU. + + CPU on purpose: ~150M params, a handful of 352px frames per scene - the cost + is milliseconds against a video denoise, and it never competes with the + diffusion model for VRAM. Loaded straight from transformers (which comfy + already ships); no GPL face-swap code involved. + """ + global _CLIPSEG + if _CLIPSEG is None: + from transformers import CLIPSegForImageSegmentation, CLIPSegProcessor + + name = "CIDAS/clipseg-rd64-refined" + processor = CLIPSegProcessor.from_pretrained(name) + model = CLIPSegForImageSegmentation.from_pretrained(name) + model.eval() + _CLIPSEG = (processor, model) + return _CLIPSEG + + +# --------------------------------------------------------------------------- +# Detection +# --------------------------------------------------------------------------- + +def parse_targets(detail_targets): + """Comma-separated user text -> clean prompt list ('' -> []).""" + if not detail_targets: + return [] + return [t.strip() for t in str(detail_targets).split(",") if t.strip()] + + +def _decode_detection_frames(vae, video, count, debug=False): + """Decode `count` evenly spaced single latent frames to [H, W, C] float images. + + Each latent frame is decoded alone as [B, C, 1, h, w]: the causal VAE treats + a length-1 clip as a still, which is exactly good enough for segmentation and + avoids decoding the whole scene just to find a region. + """ + frames = [] + f = int(video.shape[2]) + idxs = sorted({min(f - 1, round(i * (f - 1) / max(1, count - 1))) for i in range(max(1, count))}) + for i in idxs: + try: + decoded = vae.decode(video[:, :, i:i + 1]) + except Exception as exc: + _log(debug, f"keyframe {i} decode failed ({exc}); skipping") + continue + img = decoded + if img.dim() == 5: # [B, T, H, W, C] + img = img[0, 0] + elif img.dim() == 4: # [B, H, W, C] + img = img[0] + frames.append((i, img.detach().float().clamp(0.0, 1.0).cpu())) + return frames + + +def _clipseg_heat(frames, prompts, debug=False): + """Max-over-frames, max-over-prompts CLIPSeg heat map. + + Returns [H, W] in [0, 1] at the first frame's pixel size, or None when + nothing was decodable. Max (not mean) across frames on purpose: the tube + must cover the region wherever it appears during the scene. + """ + import numpy as np + from PIL import Image + + if not frames or not prompts: + return None + processor, model = _get_clipseg() + h, w = frames[0][1].shape[0], frames[0][1].shape[1] + heat = None + with torch.no_grad(): + for _, img in frames: + pil = Image.fromarray((img.numpy() * 255.0).astype(np.uint8)) + inputs = processor( + text=list(prompts), images=[pil] * len(prompts), return_tensors="pt") + logits = model(**inputs).logits # [P, 352, 352] (or [352, 352] for one prompt) + if logits.dim() == 2: + logits = logits[None] + frame_heat = torch.sigmoid(logits).amax(dim=0)[None, None] # union over prompts + frame_heat = F.interpolate(frame_heat, size=(h, w), mode="bilinear", align_corners=False)[0, 0] + heat = frame_heat if heat is None else torch.maximum(heat, frame_heat) + return heat + + +def find_tube(heat, latent_h, latent_w, threshold=DEFAULT_THRESHOLD, + area_cap=MAX_TUBE_AREA, debug=False): + """Threshold pixel-space heat into a latent-space tube box + soft paste mask. + + Returns (y0, y1, x0, x1, mask) in latent coords - mask is [1, 1, 1, th, tw] + matching the tube, feathered from the CLIPSeg heat itself so the paste + follows the actual silhouette instead of the rectangle. None when nothing + crossed the threshold or the tube breached the area cap. + """ + if heat is None: + return None + # Heat -> latent grid first; box math happens where the crop happens. + lat_heat = F.interpolate(heat[None, None], size=(latent_h, latent_w), + mode="bilinear", align_corners=False)[0, 0] + hot = lat_heat >= threshold + if not bool(hot.any()): + _log(debug, f"no region above threshold {threshold}") + return None + ys, xs = torch.where(hot) + y0 = max(0, int(ys.min()) - TUBE_PAD) + y1 = min(latent_h, int(ys.max()) + 1 + TUBE_PAD) + x0 = max(0, int(xs.min()) - TUBE_PAD) + x1 = min(latent_w, int(xs.max()) + 1 + TUBE_PAD) + # Enforce a minimum edge (grow symmetrically, clamped to the frame). + while (y1 - y0) < min(MIN_TUBE_EDGE, latent_h): + y0, y1 = max(0, y0 - 1), min(latent_h, y1 + 1) + while (x1 - x0) < min(MIN_TUBE_EDGE, latent_w): + x0, x1 = max(0, x0 - 1), min(latent_w, x1 + 1) + area = (y1 - y0) * (x1 - x0) / float(latent_h * latent_w) + if area > area_cap: + _log(debug, f"tube area {area:.0%} exceeds cap {area_cap:.0%}; refusing " + "(a region that large is a re-render, not a detail pass)") + return None + # Soft mask: the heat inside the tube, floored at 0 outside the threshold and + # lightly blurred so the seam never lands on a hard edge. + m = lat_heat[y0:y1, x0:x1].clone() + m = ((m - threshold) / max(1e-6, 1.0 - threshold)).clamp(0.0, 1.0) + m = F.avg_pool2d(m[None, None], kernel_size=3, stride=1, padding=1)[0, 0] + return y0, y1, x0, x1, m[None, None, None] + + +# --------------------------------------------------------------------------- +# Refine +# --------------------------------------------------------------------------- + +def _strip_layout_conds(conds): + """Copy conditioning minus full-frame layout keys (anchors/guides). + + The crop refine is its own small denoise: text still applies, but keyframe + indices and guide latents describe positions in the full-frame latent and + would fight (or crash) the crop-sized pass. + """ + out = [] + for entry in conds or []: + if isinstance(entry, (list, tuple)) and len(entry) == 2 and isinstance(entry[1], dict): + meta = {k: v for k, v in entry[1].items() if k not in _LAYOUT_COND_KEYS} + out.append([entry[0], meta]) + else: + out.append(entry) + return out + + +def _per_channel_stats(vae): + return getattr(getattr(vae, "first_stage_model", None), "per_channel_statistics", None) + + +def _run_upsampler(upsampler, video_crop, vae, debug=False): + """un_normalize -> LatentUpsampler(2x spatial) -> normalize, on the crop. + + Same recipe as comfy's LTXVLatentUpsampler node: the model was trained on + raw (denormalized) VAE latents, so the trip through per_channel_statistics + is mandatory, not cosmetic. + """ + import comfy.model_management + + device = comfy.model_management.get_torch_device() + model_dtype = next(upsampler.parameters()).dtype + in_dtype, in_device = video_crop.dtype, video_crop.device + stats = _per_channel_stats(vae) + try: + upsampler.to(device) + x = video_crop.to(device=device, dtype=model_dtype) + if stats is not None: + x = stats.un_normalize(x) + with torch.no_grad(): + x = upsampler(x) + if stats is not None: + x = stats.normalize(x) + finally: + upsampler.cpu() + return x.to(device=in_device, dtype=in_dtype) + + +def _downscale_to(video, h, w): + """Area-downscale [B, C, F, H, W] spatially to (h, w); frames independent.""" + b, c, f, hh, ww = video.shape + x = video.permute(0, 2, 1, 3, 4).reshape(b * f, c, hh, ww) + x = F.interpolate(x.float(), size=(h, w), mode="area") + return x.reshape(b, f, c, h, w).permute(0, 2, 1, 3, 4).to(dtype=video.dtype) + + +def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, + detail_targets, upsampler, seed, cfg, + threshold=DEFAULT_THRESHOLD, strength=1.0, debug=False): + """Run the segmented-detail pass on one finished scene latent. + + Returns (latent, note): the refined latent dict plus a run_mechanisms note, + or (latent, None) untouched - same object - when disabled or nothing found. + `chain` is the FunPackLTXAVSceneChainSampler instance (reuses _latent_tensors + and _sample_chunk so the refine denoise goes through the exact same path as + the scene it is polishing). + """ + targets = parse_targets(detail_targets) + if not targets or upsampler is None or strength <= 0: + return latent, None + + tensors = chain._latent_tensors(latent) + video = tensors[0] + if video.dim() != 5: + _log(debug, f"unexpected video latent rank {video.dim()}; skipping") + return latent, None + b, c, f, lat_h, lat_w = video.shape + + # 1) Detect on a few decoded keyframes. + frames = _decode_detection_frames(vae, video, DETECTION_FRAMES, debug=debug) + heat = _clipseg_heat(frames, targets, debug=debug) + tube = find_tube(heat, lat_h, lat_w, threshold=threshold, debug=debug) + if tube is None: + return latent, None + y0, y1, x0, x1, mask = tube + _log(debug, f"tube y[{y0}:{y1}] x[{x0}:{x1}] of {lat_h}x{lat_w} for {targets}") + + # 2) Crop -> 2x latent upsample (Lightricks stage-2 model). + crop = video[:, :, :, y0:y1, x0:x1] + crop_up = _run_upsampler(upsampler, crop, vae, debug=debug) + + # 3) Re-denoise the crop over the official stage-2 tail. The clean crop goes + # in as the latent; CONST noise_scaling (sigma*noise + (1-sigma)*x) re-noises + # it to sigma=0.85 exactly like the official two-stage workflow's stage 2. + # The scene's audio stream rides along so joint attention sees a legal AV + # latent, but its refined output is discarded below - audio is protected by + # construction, not by masking. + import comfy.nested_tensor + + crop_latent = {"samples": crop_up} + if len(tensors) > 1: + crop_latent["samples"] = comfy.nested_tensor.NestedTensor( + [crop_up] + [t.detach().clone() for t in tensors[1:]]) + tail_sigmas = torch.tensor(STAGE2_SIGMAS, dtype=torch.float32) + refined = chain._sample_chunk( + model, sampler, tail_sigmas, int(seed) + 7777, cfg, + _strip_layout_conds(positive), _strip_layout_conds(negative), crop_latent) + refined_crop = chain._latent_tensors(refined)[0] + + # 4) Back to grid size, feathered paste through the CLIPSeg silhouette. + refined_down = _downscale_to(refined_crop, y1 - y0, x1 - x0) + m = (mask.to(device=video.device, dtype=video.dtype) * float(strength)).clamp(0.0, 1.0) + out_video = video.clone() + region = out_video[:, :, :, y0:y1, x0:x1] + out_video[:, :, :, y0:y1, x0:x1] = region + (refined_down.to(region.device) - region) * m + + out = dict(latent) + if len(tensors) > 1: + out["samples"] = comfy.nested_tensor.NestedTensor([out_video] + list(tensors[1:])) + else: + out["samples"] = out_video + note = (f"segmented_detail({','.join(targets)}, tube={y1 - y0}x{x1 - x0}" + f"/{lat_h}x{lat_w}, s={strength})") + return out, note diff --git a/samplers.py b/samplers.py index 49ddad3d..ce6242b7 100644 --- a/samplers.py +++ b/samplers.py @@ -2450,6 +2450,22 @@ class FunPackLTXAVSceneChainSampler: "default": False, "tooltip": "EXPERIMENTAL: source embed_guidance / score_slider from the taste direction learned on the prompts NEAREST this scene's prompt, instead of the single global liked-direction average. On every liked rating the Refiner records (prompt fingerprint -> that run's liked direction); with this on, each scene retrieves the similarity-weighted direction of its closest matches (a forest prompt pulls what worked on forests, not the mean across all prompts). Non-parametric retrieval — no extra model forward, just a cosine lookup + vector mean, and it can't collapse into a spurious attractor the way a value function can. Falls back to the global liked direction when no rated prompt is close enough (or the index is empty). Only affects embed_guidance / score_slider; needs refinement_key_input (or embed_guidance_source=absolute). UNVALIDATED LIVE.", }), + "segmented_detailing": ("BOOLEAN", { + "default": False, + "tooltip": "EXPERIMENTAL ADetailer-for-video: after each scene finishes denoising, CLIPSeg (text-prompted segmentation) locates the regions named in detail_targets on a few decoded keyframes; the matched region is cut out of the video latent as a spatiotemporal tube, pushed through Lightricks' trained latent upsampler (2x spatial — the official two-stage pipeline's stage-2 model), re-noised to the official stage-2 sigma (0.85) and re-denoised for the 3-step tail schedule, then downscaled back to its ORIGINAL latent size and pasted through the feathered CLIPSeg silhouette. Final resolution never changes — the upsample only lets the model resolve structure (fingers, small faces) at a higher working resolution. Cost ~= 4 x tube area fraction x 3 steps (hands ~+15%); tubes over 35% of the frame are refused (that's a re-render, not a detail pass). Audio untouched by construction. Requires detail_upsampler; silent no-op when nothing matches. UNVALIDATED LIVE.", + }), + "detail_targets": ("STRING", { + "default": "hands", + "tooltip": "Comma-separated regions to detail, in plain words ('hands', 'hands, feet', 'face'). Each becomes a CLIPSeg text query; matched regions merge into one tube per scene. CLIPSeg matches broad CLIP semantics, so malformed anatomy still lights up for its name.", + }), + "detail_upsampler": (cls._detail_upsampler_choices(), { + "default": "None", + "tooltip": "Latent upsampler checkpoint from models/latent_upscale_models (the LTX 2.3 spatial upsampler used by the official two-stage workflows). Required — segmented_detailing no-ops while this is None.", + }), + "detail_strength": ("FLOAT", { + "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05, + "tooltip": "Blend of the refined region into the frame at paste-back (through the feathered CLIPSeg mask). 1.0 = full replacement inside the silhouette; 0 disables the pass entirely.", + }), }, "hidden": { "unique_id": "UNIQUE_ID", @@ -2474,6 +2490,14 @@ class FunPackLTXAVSceneChainSampler: except Exception: return ["None"] + @classmethod + def _detail_upsampler_choices(cls): + try: + import folder_paths + return ["None"] + folder_paths.get_filename_list("latent_upscale_models") + except Exception: + return ["None"] + def _is_nested(self, samples): return bool(getattr(samples, "is_nested", False)) @@ -4637,6 +4661,8 @@ class FunPackLTXAVSceneChainSampler: carry_overlap_through_anchor=False, plateau_cache=False, plateau_cache_threshold=0.975, taste_nearest_prompt=False, + segmented_detailing=False, detail_targets="hands", + detail_upsampler="None", detail_strength=1.0, unique_id=None, prompt=None): if not isinstance(positive, list) or not positive: raise ValueError("positive conditioning must contain at least one scene entry.") @@ -4825,6 +4851,7 @@ class FunPackLTXAVSceneChainSampler: scene_runs: list = [] joyai_bank = _JoyAIMemoryBank(joyai_memory_size, joyai_fix_frames) if joyai_memory else None _identity_overlap_state: dict = {} + _detail_upsampler_model = None # lazy: loaded at the first detailed scene for scene_index, scene_cond in enumerate(scene_conditionings): scene_positive = [scene_cond] @@ -5083,6 +5110,29 @@ class FunPackLTXAVSceneChainSampler: sampled = self._crop_video_tail(sampled, guide_tail) if audio_tail > 0: sampled = self._crop_audio_tail(sampled, audio_tail) + # Segmented detailing runs on the clean, fully-cropped scene (guide/audio + # tails gone, overlap still present — the tube is spatial, so carried head + # frames detail together with the rest) and BEFORE the JoyAI harvest, so + # cross-shot memory banks the improved frames. + if segmented_detailing and detail_strength > 0 and detail_upsampler != "None": + try: + try: + from . import detailing as _detailing + except ImportError: + import detailing as _detailing + if _detail_upsampler_model is None: + _detail_upsampler_model = _detailing.load_latent_upsampler(detail_upsampler) + _t_detail0 = _time.perf_counter() + sampled, _detail_note = _detailing.detail_refine_scene( + self, model, vae, sampler, scene_positive, scene_negative, sampled, + detail_targets, _detail_upsampler_model, scene_seed, cfg, + strength=detail_strength, debug=debug_log) + if _detail_note: + run_mechanisms.append(_detail_note) + _phase_sampling += _time.perf_counter() - _t_detail0 + except Exception as _detail_exc: + # A failed detail pass must never cost the scene itself. + print(f"[FunPackSceneChain] segmented_detailing failed, scene kept as-is: {_detail_exc}") if joyai_bank is not None: # Harvest from the clean, fully-cropped scene so injected memory tails never re-enter # the bank. Scene 0 seeds the pinned anchor (num_fix); later scenes roll in. The audio diff --git a/tests/test_detailing.py b/tests/test_detailing.py new file mode 100644 index 00000000..d168f04f --- /dev/null +++ b/tests/test_detailing.py @@ -0,0 +1,248 @@ +"""Unit tests for segmented detailing (detailing.py). + +Covers the pure pieces (target parsing, tube geometry + area cap, layout-cond +stripping, spatial downscale) and the detail_refine_scene contract: bit-identical +no-op when disabled / nothing detected, and the crop -> refine -> feathered +paste round trip with the model-touching stages stubbed out. +""" +import sys +import types +from pathlib import Path + +import pytest +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +sys.modules.setdefault("comfy", types.ModuleType("comfy")) +_nested = types.ModuleType("comfy.nested_tensor") + + +class _FakeNested: + is_nested = True + + def __init__(self, tensors): + self._tensors = list(tensors) + + def unbind(self): + return list(self._tensors) + + +_nested.NestedTensor = _FakeNested +sys.modules.setdefault("comfy.nested_tensor", _nested) + + +@pytest.fixture(autouse=True) +def _nested_stub(monkeypatch): + """Re-install the usable NestedTensor stub around every test. + + Other test modules stub comfy.nested_tensor.NestedTensor = object on the SAME + shared sys.modules entry at collection time (whichever file is collected last + wins), and stubbed submodules never run real import machinery, so the parent + attribute must be bound by hand too.""" + monkeypatch.setattr(sys.modules["comfy.nested_tensor"], "NestedTensor", + _FakeNested, raising=False) + monkeypatch.setattr(sys.modules["comfy"], "nested_tensor", + sys.modules["comfy.nested_tensor"], raising=False) + +import detailing # noqa: E402 + + +# --------------------------------------------------------------------------- +# parse_targets +# --------------------------------------------------------------------------- + +def test_parse_targets_splits_and_strips(): + assert detailing.parse_targets(" hands , feet,,") == ["hands", "feet"] + assert detailing.parse_targets("") == [] + assert detailing.parse_targets(None) == [] + + +# --------------------------------------------------------------------------- +# find_tube +# --------------------------------------------------------------------------- + +def _heat_with_hot_square(h=64, w=64, y=(20, 30), x=(40, 50), value=0.9): + heat = torch.zeros(h, w) + heat[y[0]:y[1], x[0]:x[1]] = value + return heat + +def test_find_tube_boxes_hot_region_with_padding(): + heat = _heat_with_hot_square() + tube = detailing.find_tube(heat, 64, 64, threshold=0.35) + assert tube is not None + y0, y1, x0, x1, mask = tube + # Hot square plus TUBE_PAD on each side (interpolation is identity at same size). + assert y0 <= 20 - detailing.TUBE_PAD + 1 and y1 >= 30 + detailing.TUBE_PAD - 1 + assert x0 <= 40 - detailing.TUBE_PAD + 1 and x1 >= 50 + detailing.TUBE_PAD - 1 + assert mask.shape == (1, 1, 1, y1 - y0, x1 - x0) + assert float(mask.max()) <= 1.0 and float(mask.min()) >= 0.0 + # Mask peaks inside the hot region, feathers to ~0 at the padded border. + assert float(mask[..., (y1 - y0) // 2, (x1 - x0) // 2]) > 0.5 + assert float(mask[..., 0, 0]) < 0.2 + +def test_find_tube_none_below_threshold(): + heat = torch.full((64, 64), 0.1) + assert detailing.find_tube(heat, 64, 64, threshold=0.35) is None + assert detailing.find_tube(None, 64, 64) is None + +def test_find_tube_refuses_oversized_region(): + heat = torch.full((64, 64), 0.9) # whole frame hot -> a re-render, not a detail pass + assert detailing.find_tube(heat, 64, 64, threshold=0.35) is None + +def test_find_tube_enforces_minimum_edge(): + heat = _heat_with_hot_square(y=(31, 32), x=(31, 32)) # single hot cell + tube = detailing.find_tube(heat, 64, 64, threshold=0.35) + assert tube is not None + y0, y1, x0, x1, _ = tube + assert (y1 - y0) >= detailing.MIN_TUBE_EDGE + assert (x1 - x0) >= detailing.MIN_TUBE_EDGE + + +# --------------------------------------------------------------------------- +# _strip_layout_conds / _downscale_to +# --------------------------------------------------------------------------- + +def test_strip_layout_conds_removes_guide_keys_only(): + cond = [[torch.zeros(1, 4), {"keyframe_idxs": "kf", "guiding_latent": "g", + "pooled_output": "keep"}]] + out = detailing._strip_layout_conds(cond) + assert out[0][1] == {"pooled_output": "keep"} + # Original untouched (the scene's own conds must not lose their guides). + assert "keyframe_idxs" in cond[0][1] + +def test_downscale_to_shape_and_content(): + video = torch.arange(2 * 3 * 4 * 8 * 8, dtype=torch.float32).reshape(2, 3, 4, 8, 8) + down = detailing._downscale_to(video, 4, 4) + assert down.shape == (2, 3, 4, 4, 4) + # Area downscale of a constant tensor is exact. + const = torch.full((1, 2, 2, 6, 6), 3.5) + assert torch.allclose(detailing._downscale_to(const, 3, 3), torch.full((1, 2, 2, 3, 3), 3.5)) + + +# --------------------------------------------------------------------------- +# detail_refine_scene +# --------------------------------------------------------------------------- + +class _FakeChain: + """Just enough of FunPackLTXAVSceneChainSampler for detail_refine_scene.""" + + def __init__(self): + self.chunk_calls = [] + + def _latent_tensors(self, latent): + samples = latent["samples"] + if getattr(samples, "is_nested", False): + return list(samples.unbind()) + return [samples] + + def _sample_chunk(self, model, sampler, sigmas, seed, cfg, positive, negative, latent, + **kwargs): + self.chunk_calls.append({"sigmas": sigmas, "seed": seed, + "positive": positive, "latent": latent}) + return latent # identity "refine": returns the crop unchanged + + +def _scene_latent(nested=False): + video = torch.randn(1, 8, 5, 16, 24) + if not nested: + return {"samples": video}, video + audio = torch.randn(1, 4, 40) + return {"samples": _FakeNested([video, audio])}, video + +def test_refine_noop_without_targets_or_upsampler(): + chain = _FakeChain() + latent, _ = _scene_latent() + out, note = detailing.detail_refine_scene( + chain, None, None, None, [], [], latent, "", object(), 0, 1.0) + assert out is latent and note is None # same object: bit-identical guarantee + out, note = detailing.detail_refine_scene( + chain, None, None, None, [], [], latent, "hands", None, 0, 1.0) + assert out is latent and note is None + assert chain.chunk_calls == [] + +def test_refine_noop_when_nothing_detected(monkeypatch): + chain = _FakeChain() + latent, _ = _scene_latent() + monkeypatch.setattr(detailing, "_decode_detection_frames", lambda *a, **k: []) + monkeypatch.setattr(detailing, "_clipseg_heat", lambda *a, **k: None) + out, note = detailing.detail_refine_scene( + chain, None, None, None, [], [], latent, "hands", object(), 0, 1.0) + assert out is latent and note is None + assert chain.chunk_calls == [] + +def _patch_detection_and_upsampler(monkeypatch): + """Hot square in pixel space; upsampler = exact 2x nearest (model-free).""" + heat = torch.zeros(160, 240) + heat[40:80, 120:180] = 0.9 + monkeypatch.setattr(detailing, "_decode_detection_frames", + lambda *a, **k: [(0, torch.zeros(160, 240, 3))]) + monkeypatch.setattr(detailing, "_clipseg_heat", lambda *a, **k: heat) + monkeypatch.setattr( + detailing, "_run_upsampler", + lambda upsampler, crop, vae, debug=False: torch.nn.functional.interpolate( + crop, scale_factor=(1, 2, 2), mode="nearest")) + return heat + +def test_refine_round_trip_touches_only_the_tube(monkeypatch): + chain = _FakeChain() + latent, video = _scene_latent() + _patch_detection_and_upsampler(monkeypatch) + pos = [[torch.zeros(1, 4), {"keyframe_idxs": "kf", "pooled_output": "keep"}]] + out, note = detailing.detail_refine_scene( + chain, "model", "vae", "sampler", pos, [], latent, "hands", object(), + seed=11, cfg=1.0) + assert note is not None and "segmented_detail" in note + assert len(chain.chunk_calls) == 1 + call = chain.chunk_calls[0] + # Official stage-2 tail schedule and a detail-offset seed. + assert torch.allclose(call["sigmas"], torch.tensor(detailing.STAGE2_SIGMAS)) + assert call["seed"] == 11 + 7777 + # Layout conds stripped for the crop pass. + assert "keyframe_idxs" not in call["positive"][0][1] + assert call["positive"][0][1]["pooled_output"] == "keep" + # The crop passed to the refine is 2x the tube size. + crop = call["latent"]["samples"] + out_video = out["samples"] + assert out_video.shape == video.shape + # With an identity refine, upsample(2x)->downscale(area) reproduces the crop, so + # the paste changes (nearly) nothing — but crucially everything OUTSIDE the tube + # must be exactly untouched regardless of what the refine did. + diff = (out_video - video).abs().amax(dim=(0, 1, 2)) # [H, W] + hot = diff > 1e-6 + assert not hot[:2].any() and not hot[:, :2].any() # far corners untouched + +def test_refine_preserves_audio_stream(monkeypatch): + chain = _FakeChain() + latent, video = _scene_latent(nested=True) + audio = chain._latent_tensors(latent)[1] + _patch_detection_and_upsampler(monkeypatch) + out, note = detailing.detail_refine_scene( + chain, "model", "vae", "sampler", [], [], latent, "hands", object(), + seed=0, cfg=1.0) + assert note is not None + out_tensors = chain._latent_tensors(out) + assert len(out_tensors) == 2 + # Audio comes back as the SAME tensor the scene finished with: protected by + # construction, not by a lossy round trip. + assert out_tensors[1] is audio + # The crop pass carried an audio stream so joint attention saw a legal AV latent. + crop_samples = chain.chunk_calls[0]["latent"]["samples"] + assert getattr(crop_samples, "is_nested", False) + +def test_refine_strength_scales_paste(monkeypatch): + chain = _FakeChain() + latent, video = _scene_latent() + _patch_detection_and_upsampler(monkeypatch) + # Non-identity refine: chunk returns a shifted crop so the paste has signal. + def _shifted_chunk(model, sampler, sigmas, seed, cfg, positive, negative, crop_latent, **kw): + return {"samples": crop_latent["samples"] + 10.0} + chain._sample_chunk = _shifted_chunk + out_full, _ = detailing.detail_refine_scene( + chain, "m", "v", "s", [], [], latent, "hands", object(), 0, 1.0, strength=1.0) + out_half, _ = detailing.detail_refine_scene( + chain, "m", "v", "s", [], [], latent, "hands", object(), 0, 1.0, strength=0.5) + d_full = (out_full["samples"] - video).abs().sum() + d_half = (out_half["samples"] - video).abs().sum() + assert d_full > 0 + assert abs(float(d_half / d_full) - 0.5) < 1e-3 From db121973f15d456adfceaa6ff873193b6b65cab1 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sat, 18 Jul 2026 18:20:49 +0300 Subject: [PATCH 04/24] Editor UI for segmented detailing: Engine knobs + Composer targets field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine > Chain Sampler > Experimental: segmented_detailing toggle, detail_targets text knob (new 'text' kind: quiet patch while typing, commit on blur), detail_strength, and a live detail_upsampler combo fed by the chain sampler's own node spec (same lazy pattern as the ArcFace projector). Composer > Compose grows a Segmented detailing targets field, shown only while the feature is on — same store key as the Engine knob so the two can never disagree; Composer repaint fingerprint carries the toggle so flipping it in Engine repaints an open Composer. All wiring rides the existing sampler_inputs bridge — no backend change. Verified live: knob visibility/badge, upsampler list fetch, Composer block show/hide on toggle, typing sync, project state restored after test. Co-Authored-By: Claude Fable 5 --- movie_editor/frontend/composer.js | 38 ++++++++++++++++ movie_editor/frontend/engine_settings.js | 57 +++++++++++++++++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/movie_editor/frontend/composer.js b/movie_editor/frontend/composer.js index 189f8ef0..ec48cc59 100644 --- a/movie_editor/frontend/composer.js +++ b/movie_editor/frontend/composer.js @@ -217,8 +217,40 @@ // Variables ($name) — collapsible, same toggle pattern as Settings ▸ Built-in pipeline. wrap.append(composeVariables()); + // Segmented detailing targets — only while the Chain Sampler feature is enabled + // (Engine ▸ Chain Sampler ▸ Experimental), so prompt-craft and "what gets detailed" + // live side by side without adding a permanently-visible field. + const detailBlock = composeDetailTargets(st); + if (detailBlock) wrap.append(detailBlock); return wrap; } + + // ── segmented detailing targets (shown only when the feature is on) ───────────── + function composeDetailTargets(st) { + const p = st.project; + if (!p) return null; + const PC = window.PipelineCaps; + if (PC && PC.usesChainSampler && !PC.usesChainSampler(st)) return null; + const si = p.sampler_inputs || {}; + if (!si.segmented_detailing) return null; + const box = el("div", "compose-detail"); + const row = el("div", "compose-head"); + row.append(el("div", "lib-form-title", "Segmented detailing")); + box.append(row); + const input = el("input", "lib-in"); + input.type = "text"; + input.placeholder = "hands, feet"; + input.value = si.detail_targets != null ? String(si.detail_targets) : "hands"; + // Quiet while typing (no repaint under the caret), commit on blur/Enter — + // same contract as the Engine Settings text knob, same store key, so the + // two fields can never disagree. + input.oninput = () => S.setSamplerInput("detail_targets", input.value); + input.onchange = () => S.setSamplerInputNow("detail_targets", input.value); + box.append(input); + box.append(el("div", "insp-hint", + "Comma-separated regions to detail (CLIPSeg text queries — plain words work best). Strength and the upsampler model live in Engine ▸ Chain Sampler ▸ Experimental.")); + return box; + } let composeTextarea = null; let varsOpen = false; // Variables panel expanded? let varHintEl = null; // undeclared / cycle hint line under the prompt @@ -834,6 +866,12 @@ const fp = JSON.stringify({ s: st.shortcuts?.length, t: st.transitions?.length, pid: st.project?.id, c: (st.shortcutCategories || []).reduce((n, x) => n + 1 + (x.sub_categories?.length || 0), 0), + // Segmented detailing block visibility: toggling it in Engine Settings must + // repaint an open Composer or the targets field appears/disappears only on the + // next unrelated repaint ("store right, UI stale"). The targets TEXT is not + // fingerprinted — typing goes through quiet patches and the field itself is + // the source of those edits. + sd: !!(st.project?.sampler_inputs?.segmented_detailing), }); if (fp === lastFp) return; lastFp = fp; diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index a2c81538..29b39564 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -196,6 +196,12 @@ hint: "EXPERIMENTAL speed: the near-pure-noise early steps carry almost no signal, so the transformer output barely changes across them. Computes it once at the top of the plateau and reuses it for the rest, skipping ~3-4 of 8 transformer passes. Deterministic given seed (safe in Batch Training) but an approximation — A/B it before trusting on finals. Much of wall-clock time is outside the sampler, so total speedup is smaller than the forward count suggests. UNVALIDATED LIVE." }, { name: "plateau_cache_threshold", label: "Plateau threshold (sigma)", kind: "float", default: 0.975, min: 0.5, max: 0.999, step: 0.005, dependsOn: "plateau_cache", hint: "Steps with sigma at or above this count as the reusable plateau. Higher = fewer steps cached (safer); lower = more cached (faster, more approximation). 0.975 catches the documented noise plateau while leaving structure formation fully computed." }, + { name: "segmented_detailing", label: "Segmented detailing (region refine)", kind: "bool", default: false, + hint: "EXPERIMENTAL ADetailer-for-video: after each scene renders, CLIPSeg finds the regions named below (hands, feet, …), cuts them out of the latent as a tube, refines them at 2× working resolution through Lightricks' latent upsampler (3 extra steps on the crop only), and pastes them back through a feathered silhouette. Final resolution never changes. Cost ≈ 4 × region area × 3 steps (hands ~+15%); regions over 35% of the frame are refused. Needs the detail upsampler model below. UNVALIDATED LIVE." }, + { name: "detail_targets", label: "Detail targets", kind: "text", default: "hands", dependsOn: "segmented_detailing", placeholder: "hands, feet", + hint: "Comma-separated regions to detail, in plain words. Each becomes a CLIPSeg text query — malformed anatomy still matches its name. Also editable from Composer ▸ Compose while detailing is on." }, + { name: "detail_strength", label: "Detail strength", kind: "float", default: 1.0, min: 0, max: 1, step: 0.05, dependsOn: "segmented_detailing", + hint: "Blend of the refined region at paste-back. 1.0 = full replacement inside the silhouette; 0 disables the pass." }, ]; const SAMPLER_KNOB_MAP = Object.fromEntries(SAMPLER_KNOBS.map((k) => [k.name, k])); @@ -271,6 +277,14 @@ ctrl = el("select"); ctrl.dataset.k = "si-" + k.name; (k.choices || []).forEach((c) => { const o = el("option", null, c); o.value = c; if (c === val) o.selected = true; ctrl.append(o); }); ctrl.onchange = () => S.setSamplerInputNow(k.name, ctrl.value); + } else if (k.kind === "text") { + ctrl = el("input"); ctrl.type = "text"; + ctrl.value = val != null ? String(val) : ""; + if (k.placeholder) ctrl.placeholder = k.placeholder; + ctrl.dataset.k = "si-" + k.name; + // Quiet while typing (no repaint under the caret), commit on blur/Enter. + ctrl.oninput = () => S.setSamplerInput(k.name, ctrl.value); + ctrl.onchange = () => S.setSamplerInputNow(k.name, ctrl.value); } else { ctrl = el("input"); ctrl.type = "number"; if (k.step != null) ctrl.step = String(k.step); @@ -300,7 +314,7 @@ chain_timing: ["frame_overlap", "transition_duration", "use_same_seed"], chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "taste_nearest_prompt", "output_guidance", "output_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold"], chain_decode: ["decode_noise_scale", "decode_timestep", "decode_tile_size"], - chain_experimental: ["plateau_cache", "plateau_cache_threshold", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], + chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], }; function countChainView(p, id) { @@ -576,6 +590,46 @@ renderKnobList(g, st, CHAIN_VIEW_KNOBS[id]); } + // ── Segmented detailing: the upsampler model list is live (models/latent_upscale_models + // on the server), so like the ArcFace projector below it can't be a static SAMPLER_KNOBS + // combo. The chain sampler node's own spec already carries the choices (with "None"). + let _detailUpsamplerChoices = null; + + async function ensureDetailUpsamplerChoices() { + if (_detailUpsamplerChoices) return _detailUpsamplerChoices; + try { + const spec = await API.nodeSpec("FunPackLTXAVSceneChainSampler"); + const w = (spec?.inputs || []).find((i) => i.name === "detail_upsampler"); + _detailUpsamplerChoices = (w && w.choices) || ["None"]; + } catch (_) { + _detailUpsamplerChoices = ["None"]; + } + render(); + return _detailUpsamplerChoices; + } + + function renderDetailUpsampler(pane, si) { + if (!si.segmented_detailing) return; + const g = group(pane, "Segmented detailing: upsampler model"); + if (!_detailUpsamplerChoices) { + g.append(hintEl("Loading upsampler list…")); + ensureDetailUpsamplerChoices(); + return; + } + const sel = el("select"); sel.dataset.k = "si-detail_upsampler"; + _detailUpsamplerChoices.forEach((c) => { + const o = el("option", null, c); o.value = c; + if (c === (si.detail_upsampler || "None")) o.selected = true; + sel.append(o); + }); + sel.onchange = () => S.setSamplerInputNow("detail_upsampler", sel.value); + g.append(field("Latent upsampler", sel, + "The LTX 2.3 spatial upsampler from models/latent_upscale_models (the official two-stage workflows use the same file). Detailing no-ops while this is None.")); + if ((si.detail_upsampler || "None") === "None" && _detailUpsamplerChoices.length <= 1) { + g.append(hintEl("No models found in models/latent_upscale_models on the server — download the LTX 2.3 latent upsampler there, then reopen this pane.")); + } + } + // ── Best-FaceID ArcFace projector: the one identity-transfer field that needs a // live (server-fetched) choice list, so it can't live in the static SAMPLER_KNOBS combo // shape. Reuses the same loras folder listing LoraLoaderModelOnly exposes. @@ -598,6 +652,7 @@ renderChainKnobsView(pane, st, "chain_experimental", "Experimental", "Research techniques — off by default; expect quality/overhead trade-offs."); const si = st.project.sampler_inputs || {}; + renderDetailUpsampler(pane, si); if (!si.identity_transfer_enabled) return; const g = group(pane, "Best-FaceID: ArcFace projector"); if (!_loraChoices) { From c864f62480f36c99e227ddbf14acebd2da72e8c7 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sat, 18 Jul 2026 18:29:28 +0300 Subject: [PATCH 05/24] Segmented detailing: auto-resolve/download upsampler + loud skips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first live attempt silently no-oped: detail_upsampler defaulted to 'None' (nothing installed on the rental) and the guard skipped without a word. Two fixes: - detail_upsampler default is now 'auto': prefer the official installed file, else the highest-versioned spatial upscaler (version-aware compare — plain sort ranked ltx2_3_… over ltx-2.3-…-1.1), else hf_hub_download of Lightricks/LTX-2.3 ltx-2.3-spatial-upscaler-x2-1.1.safetensors (~1 GB, once) with an actionable error if the repo is gated. Legacy 'None' = auto. - With the toggle on, every outcome is visible: upsampler choice and skips print to console AND land in the scene report (segmented_detail(SKIPPED: …) / (no region matched)); resolve/load failure disables the pass for the run instead of retrying the download per scene. A failed pass still never costs the scene. Co-Authored-By: Claude Fable 5 --- detailing.py | 57 ++++++++++++++++++++++++ movie_editor/frontend/engine_settings.js | 11 ++--- samplers.py | 38 +++++++++++----- tests/test_detailing.py | 49 ++++++++++++++++++++ 4 files changed, 139 insertions(+), 16 deletions(-) diff --git a/detailing.py b/detailing.py index 832226f5..96adea14 100644 --- a/detailing.py +++ b/detailing.py @@ -66,6 +66,63 @@ def _log(debug, msg): # Models # --------------------------------------------------------------------------- +# Official Lightricks stage-2 spatial upscaler — the exact file the two-stage +# distilled workflows use. ~1 GB, fetched once into models/latent_upscale_models +# when nothing suitable is installed. +DEFAULT_UPSAMPLER_REPO = "Lightricks/LTX-2.3" +DEFAULT_UPSAMPLER_FILE = "ltx-2.3-spatial-upscaler-x2-1.1.safetensors" + + +def resolve_upsampler_name(model_name=None): + """Turn the node's detail_upsampler widget value into a loadable filename. + + Explicit filename -> returned as-is. "auto" (and the legacy "None" default) + -> prefer an installed spatial upscaler (highest version wins), else any + installed latent upscale model, else download the official file from HF. + Raises with a user-actionable message when nothing can be obtained — callers + surface it, they must NOT silently skip. + """ + import os + + import folder_paths + + name = str(model_name or "").strip() + if name and name not in ("None", "auto"): + return name + files = folder_paths.get_filename_list("latent_upscale_models") + if DEFAULT_UPSAMPLER_FILE in files: + return DEFAULT_UPSAMPLER_FILE + spatial = [f for f in files if "spatial" in f.lower() and "upscal" in f.lower()] + if spatial: + # Highest embedded version wins (…-1.1 over …-1.0); plain lexicographic would + # rank "ltx2_3_…" above "ltx-2.3-…-1.1" ('2' sorts after '-'). + import re + return max(spatial, key=lambda f: ([int(n) for n in re.findall(r"\d+", f)], f)) + if files: + return files[0] + try: + from huggingface_hub import hf_hub_download + except ImportError as exc: + raise RuntimeError( + "no model in models/latent_upscale_models and huggingface_hub is not " + f"installed — pip install huggingface_hub, or download {DEFAULT_UPSAMPLER_FILE} " + f"from {DEFAULT_UPSAMPLER_REPO} into that folder manually") from exc + dest_dir = folder_paths.get_folder_paths("latent_upscale_models")[0] + os.makedirs(dest_dir, exist_ok=True) + print(f"[FunPackDetail] no latent upscale model installed — downloading " + f"{DEFAULT_UPSAMPLER_FILE} (~1 GB) from {DEFAULT_UPSAMPLER_REPO} into {dest_dir} …") + try: + hf_hub_download(repo_id=DEFAULT_UPSAMPLER_REPO, filename=DEFAULT_UPSAMPLER_FILE, + local_dir=dest_dir) + except Exception as exc: + raise RuntimeError( + f"auto-download of {DEFAULT_UPSAMPLER_FILE} failed ({exc}) — if the repo is " + "gated, accept the license on huggingface.co and set HF_TOKEN, or place the " + f"file in {dest_dir} manually") from exc + print(f"[FunPackDetail] download complete: {DEFAULT_UPSAMPLER_FILE}") + return DEFAULT_UPSAMPLER_FILE + + def load_latent_upsampler(model_name): """Load Lightricks' LatentUpsampler from models/latent_upscale_models. diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 29b39564..8d166504 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -197,7 +197,7 @@ { name: "plateau_cache_threshold", label: "Plateau threshold (sigma)", kind: "float", default: 0.975, min: 0.5, max: 0.999, step: 0.005, dependsOn: "plateau_cache", hint: "Steps with sigma at or above this count as the reusable plateau. Higher = fewer steps cached (safer); lower = more cached (faster, more approximation). 0.975 catches the documented noise plateau while leaving structure formation fully computed." }, { name: "segmented_detailing", label: "Segmented detailing (region refine)", kind: "bool", default: false, - hint: "EXPERIMENTAL ADetailer-for-video: after each scene renders, CLIPSeg finds the regions named below (hands, feet, …), cuts them out of the latent as a tube, refines them at 2× working resolution through Lightricks' latent upsampler (3 extra steps on the crop only), and pastes them back through a feathered silhouette. Final resolution never changes. Cost ≈ 4 × region area × 3 steps (hands ~+15%); regions over 35% of the frame are refused. Needs the detail upsampler model below. UNVALIDATED LIVE." }, + hint: "EXPERIMENTAL ADetailer-for-video: after each scene renders, CLIPSeg finds the regions named below (hands, feet, …), cuts them out of the latent as a tube, refines them at 2× working resolution through Lightricks' latent upsampler (3 extra steps on the crop only), and pastes them back through a feathered silhouette. Final resolution never changes. Cost ≈ 4 × region area × 3 steps (hands ~+15%); regions over 35% of the frame are refused. The upsampler model below is found — or downloaded (~1 GB, once) — automatically; skips are reported in the scene report. UNVALIDATED LIVE." }, { name: "detail_targets", label: "Detail targets", kind: "text", default: "hands", dependsOn: "segmented_detailing", placeholder: "hands, feet", hint: "Comma-separated regions to detail, in plain words. Each becomes a CLIPSeg text query — malformed anatomy still matches its name. Also editable from Composer ▸ Compose while detailing is on." }, { name: "detail_strength", label: "Detail strength", kind: "float", default: 1.0, min: 0, max: 1, step: 0.05, dependsOn: "segmented_detailing", @@ -617,16 +617,17 @@ return; } const sel = el("select"); sel.dataset.k = "si-detail_upsampler"; + const cur = si.detail_upsampler && si.detail_upsampler !== "None" ? si.detail_upsampler : "auto"; _detailUpsamplerChoices.forEach((c) => { const o = el("option", null, c); o.value = c; - if (c === (si.detail_upsampler || "None")) o.selected = true; + if (c === cur) o.selected = true; sel.append(o); }); sel.onchange = () => S.setSamplerInputNow("detail_upsampler", sel.value); g.append(field("Latent upsampler", sel, - "The LTX 2.3 spatial upsampler from models/latent_upscale_models (the official two-stage workflows use the same file). Detailing no-ops while this is None.")); - if ((si.detail_upsampler || "None") === "None" && _detailUpsamplerChoices.length <= 1) { - g.append(hintEl("No models found in models/latent_upscale_models on the server — download the LTX 2.3 latent upsampler there, then reopen this pane.")); + "The LTX 2.3 spatial upsampler from models/latent_upscale_models (the official two-stage workflows use the same file). 'auto' picks the newest installed spatial upscaler — or downloads the official one (~1 GB, once) when the folder is empty.")); + if (_detailUpsamplerChoices.length <= 1) { + g.append(hintEl("Nothing installed in models/latent_upscale_models yet — the first detailed run downloads the official upsampler automatically (watch the ComfyUI console).")); } } diff --git a/samplers.py b/samplers.py index ce6242b7..58a78d93 100644 --- a/samplers.py +++ b/samplers.py @@ -2452,15 +2452,15 @@ class FunPackLTXAVSceneChainSampler: }), "segmented_detailing": ("BOOLEAN", { "default": False, - "tooltip": "EXPERIMENTAL ADetailer-for-video: after each scene finishes denoising, CLIPSeg (text-prompted segmentation) locates the regions named in detail_targets on a few decoded keyframes; the matched region is cut out of the video latent as a spatiotemporal tube, pushed through Lightricks' trained latent upsampler (2x spatial — the official two-stage pipeline's stage-2 model), re-noised to the official stage-2 sigma (0.85) and re-denoised for the 3-step tail schedule, then downscaled back to its ORIGINAL latent size and pasted through the feathered CLIPSeg silhouette. Final resolution never changes — the upsample only lets the model resolve structure (fingers, small faces) at a higher working resolution. Cost ~= 4 x tube area fraction x 3 steps (hands ~+15%); tubes over 35% of the frame are refused (that's a re-render, not a detail pass). Audio untouched by construction. Requires detail_upsampler; silent no-op when nothing matches. UNVALIDATED LIVE.", + "tooltip": "EXPERIMENTAL ADetailer-for-video: after each scene finishes denoising, CLIPSeg (text-prompted segmentation) locates the regions named in detail_targets on a few decoded keyframes; the matched region is cut out of the video latent as a spatiotemporal tube, pushed through Lightricks' trained latent upsampler (2x spatial — the official two-stage pipeline's stage-2 model), re-noised to the official stage-2 sigma (0.85) and re-denoised for the 3-step tail schedule, then downscaled back to its ORIGINAL latent size and pasted through the feathered CLIPSeg silhouette. Final resolution never changes — the upsample only lets the model resolve structure (fingers, small faces) at a higher working resolution. Cost ~= 4 x tube area fraction x 3 steps (hands ~+15%); tubes over 35% of the frame are refused (that's a re-render, not a detail pass). Audio untouched by construction. detail_upsampler 'auto' finds or downloads the official Lightricks upsampler (~1 GB, once); skips are reported loudly in console + scene report. UNVALIDATED LIVE.", }), "detail_targets": ("STRING", { "default": "hands", "tooltip": "Comma-separated regions to detail, in plain words ('hands', 'hands, feet', 'face'). Each becomes a CLIPSeg text query; matched regions merge into one tube per scene. CLIPSeg matches broad CLIP semantics, so malformed anatomy still lights up for its name.", }), "detail_upsampler": (cls._detail_upsampler_choices(), { - "default": "None", - "tooltip": "Latent upsampler checkpoint from models/latent_upscale_models (the LTX 2.3 spatial upsampler used by the official two-stage workflows). Required — segmented_detailing no-ops while this is None.", + "default": "auto", + "tooltip": "Latent upsampler checkpoint from models/latent_upscale_models (the LTX 2.3 spatial upsampler used by the official two-stage workflows). 'auto' picks the newest installed spatial upscaler, or downloads the official file (~1 GB, once) when the folder is empty. Pick a file explicitly to pin it.", }), "detail_strength": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05, @@ -2492,11 +2492,14 @@ class FunPackLTXAVSceneChainSampler: @classmethod def _detail_upsampler_choices(cls): + # "auto": prefer an installed spatial upscaler, else download the official + # Lightricks file on first use. ("None" from older saved workflows/projects is + # treated as auto too — the enable toggle is the only off switch.) try: import folder_paths - return ["None"] + folder_paths.get_filename_list("latent_upscale_models") + return ["auto"] + folder_paths.get_filename_list("latent_upscale_models") except Exception: - return ["None"] + return ["auto"] def _is_nested(self, samples): return bool(getattr(samples, "is_nested", False)) @@ -4851,7 +4854,8 @@ class FunPackLTXAVSceneChainSampler: scene_runs: list = [] joyai_bank = _JoyAIMemoryBank(joyai_memory_size, joyai_fix_frames) if joyai_memory else None _identity_overlap_state: dict = {} - _detail_upsampler_model = None # lazy: loaded at the first detailed scene + _detail_upsampler_model = None # lazy: resolved+loaded at the first detailed scene + _detail_disabled_reason = None # set on resolve/load failure: don't retry per scene for scene_index, scene_cond in enumerate(scene_conditionings): scene_positive = [scene_cond] @@ -5113,15 +5117,18 @@ class FunPackLTXAVSceneChainSampler: # Segmented detailing runs on the clean, fully-cropped scene (guide/audio # tails gone, overlap still present — the tube is spatial, so carried head # frames detail together with the rest) and BEFORE the JoyAI harvest, so - # cross-shot memory banks the improved frames. - if segmented_detailing and detail_strength > 0 and detail_upsampler != "None": + # cross-shot memory banks the improved frames. Every skip is LOUD (console + + # scene report): with the toggle on, silence must mean "ran and found nothing". + if segmented_detailing and detail_strength > 0 and not _detail_disabled_reason: try: try: from . import detailing as _detailing except ImportError: import detailing as _detailing if _detail_upsampler_model is None: - _detail_upsampler_model = _detailing.load_latent_upsampler(detail_upsampler) + _resolved = _detailing.resolve_upsampler_name(detail_upsampler) + _detail_upsampler_model = _detailing.load_latent_upsampler(_resolved) + print(f"[FunPackSceneChain] segmented detailing upsampler: {_resolved}") _t_detail0 = _time.perf_counter() sampled, _detail_note = _detailing.detail_refine_scene( self, model, vae, sampler, scene_positive, scene_negative, sampled, @@ -5130,9 +5137,18 @@ class FunPackLTXAVSceneChainSampler: if _detail_note: run_mechanisms.append(_detail_note) _phase_sampling += _time.perf_counter() - _t_detail0 + else: + run_mechanisms.append("segmented_detail(no region matched)") except Exception as _detail_exc: - # A failed detail pass must never cost the scene itself. - print(f"[FunPackSceneChain] segmented_detailing failed, scene kept as-is: {_detail_exc}") + # A failed detail pass must never cost the scene itself — but it must + # never fail silently either. Model resolution/load failures disable + # the pass for the rest of the run (no per-scene download retries). + if _detail_upsampler_model is None: + _detail_disabled_reason = str(_detail_exc) + print(f"[FunPackSceneChain] SEGMENTED DETAILING SKIPPED (scene kept as-is): {_detail_exc}") + run_mechanisms.append(f"segmented_detail(SKIPPED: {_detail_exc})") + elif segmented_detailing and _detail_disabled_reason: + run_mechanisms.append("segmented_detail(SKIPPED: upsampler unavailable, see first scene)") if joyai_bank is not None: # Harvest from the clean, fully-cropped scene so injected memory tails never re-enter # the bank. Scene 0 seeds the pinned anchor (num_fix); later scenes roll in. The audio diff --git a/tests/test_detailing.py b/tests/test_detailing.py index d168f04f..1430600f 100644 --- a/tests/test_detailing.py +++ b/tests/test_detailing.py @@ -99,6 +99,55 @@ def test_find_tube_enforces_minimum_edge(): assert (x1 - x0) >= detailing.MIN_TUBE_EDGE +# --------------------------------------------------------------------------- +# resolve_upsampler_name +# --------------------------------------------------------------------------- + +def _patch_folder_paths(monkeypatch, files, folder="/models/latent_upscale_models"): + fp = sys.modules["folder_paths"] + monkeypatch.setattr(fp, "get_filename_list", lambda kind: list(files), raising=False) + monkeypatch.setattr(fp, "get_folder_paths", lambda kind: [folder], raising=False) + +def test_resolve_explicit_name_passes_through(monkeypatch): + _patch_folder_paths(monkeypatch, []) + assert detailing.resolve_upsampler_name("my_upscaler.safetensors") == "my_upscaler.safetensors" + +def test_resolve_auto_prefers_newest_spatial_upscaler(monkeypatch): + _patch_folder_paths(monkeypatch, [ + "other_model.safetensors", + "ltx-2.3-spatial-upscaler-x2-1.0.safetensors", + "ltx-2.3-spatial-upscaler-x2-1.1.safetensors", + ]) + for alias in ("auto", "None", "", None): + assert detailing.resolve_upsampler_name(alias) == "ltx-2.3-spatial-upscaler-x2-1.1.safetensors" + +def test_resolve_auto_falls_back_to_any_installed(monkeypatch): + _patch_folder_paths(monkeypatch, ["something_else.safetensors"]) + assert detailing.resolve_upsampler_name("auto") == "something_else.safetensors" + +def test_resolve_auto_downloads_when_folder_empty(monkeypatch, tmp_path): + _patch_folder_paths(monkeypatch, [], folder=str(tmp_path / "lup")) + calls = {} + hub = types.ModuleType("huggingface_hub") + hub.hf_hub_download = lambda repo_id, filename, local_dir: calls.update( + repo=repo_id, file=filename, dest=local_dir) + monkeypatch.setitem(sys.modules, "huggingface_hub", hub) + name = detailing.resolve_upsampler_name("auto") + assert name == detailing.DEFAULT_UPSAMPLER_FILE + assert calls["repo"] == detailing.DEFAULT_UPSAMPLER_REPO + assert calls["file"] == detailing.DEFAULT_UPSAMPLER_FILE + +def test_resolve_download_failure_raises_actionable_error(monkeypatch, tmp_path): + _patch_folder_paths(monkeypatch, [], folder=str(tmp_path / "lup")) + hub = types.ModuleType("huggingface_hub") + def _boom(**kw): + raise OSError("401 gated") + hub.hf_hub_download = _boom + monkeypatch.setitem(sys.modules, "huggingface_hub", hub) + with pytest.raises(RuntimeError, match="manually"): + detailing.resolve_upsampler_name("auto") + + # --------------------------------------------------------------------------- # _strip_layout_conds / _downscale_to # --------------------------------------------------------------------------- From 5727c391892f2bbae69512eba32488d84e1edd1c Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sun, 19 Jul 2026 00:53:36 +0300 Subject: [PATCH 06/24] Fix CLIPSeg multi-prompt crash: missing padding/truncation on tokenizer call Live rental run hit it immediately on 'detail_targets' with more than one target: CLIPSegProcessor tokenizes each prompt to a different length, and without padding=True torch can't stack them into a batch ('excessive nesting... type list where int is expected'). Reproduced against the real CIDAS/clipseg-rd64-refined model before and after the fix. Single-target runs were unaffected (batch of 1 never needed padding), which is why the earlier probe steps didn't catch it. Co-Authored-By: Claude Fable 5 --- detailing.py | 7 ++++++- tests/test_detailing.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/detailing.py b/detailing.py index 96adea14..43918672 100644 --- a/detailing.py +++ b/detailing.py @@ -224,8 +224,13 @@ def _clipseg_heat(frames, prompts, debug=False): with torch.no_grad(): for _, img in frames: pil = Image.fromarray((img.numpy() * 255.0).astype(np.uint8)) + # padding=True is mandatory once len(prompts) > 1: CLIPSeg's tokenizer + # otherwise returns one variable-length id list per prompt and torch can't + # stack them into a batch ("excessive nesting... type list where int is + # expected"). truncation=True guards the rare very-long user-typed target. inputs = processor( - text=list(prompts), images=[pil] * len(prompts), return_tensors="pt") + text=list(prompts), images=[pil] * len(prompts), return_tensors="pt", + padding=True, truncation=True) logits = model(**inputs).logits # [P, 352, 352] (or [352, 352] for one prompt) if logits.dim() == 2: logits = logits[None] diff --git a/tests/test_detailing.py b/tests/test_detailing.py index 1430600f..b3705287 100644 --- a/tests/test_detailing.py +++ b/tests/test_detailing.py @@ -99,6 +99,41 @@ def test_find_tube_enforces_minimum_edge(): assert (x1 - x0) >= detailing.MIN_TUBE_EDGE +# --------------------------------------------------------------------------- +# _clipseg_heat +# --------------------------------------------------------------------------- + +def test_clipseg_heat_passes_padding_and_truncation(monkeypatch): + """Regression: a multi-prompt call without padding=True crashes CLIPSeg's + tokenizer ("excessive nesting... type list where int is expected") the moment + detail_targets has more than one entry — caught live on a real rental run. + truncation=True is the same call, cheap to require alongside it.""" + calls = [] + + class _FakeInputs(dict): + def to(self, *a, **k): + return self + + class _FakeProcessor: + def __call__(self, text, images, return_tensors, **kwargs): + calls.append(kwargs) + return _FakeInputs() + + class _FakeModel: + def __call__(self, **inputs): + n = 2 + return types.SimpleNamespace(logits=torch.zeros(n, 8, 8)) + + def eval(self): + return self + + monkeypatch.setattr(detailing, "_get_clipseg", lambda: (_FakeProcessor(), _FakeModel())) + frames = [(0, torch.zeros(16, 16, 3))] + detailing._clipseg_heat(frames, ["hands", "feet"]) + assert calls and calls[0].get("padding") is True + assert calls[0].get("truncation") is True + + # --------------------------------------------------------------------------- # resolve_upsampler_name # --------------------------------------------------------------------------- From 30b27b734a611277c8ac075f9b95ed90abf2094b Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sun, 19 Jul 2026 01:03:17 +0300 Subject: [PATCH 07/24] Instrument CLIPSeg misses + expose detail_threshold knob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live run hit 'no region matched' with zero diagnostics: CLIPSeg's raw sigmoid score for a real, correctly-named region is often well under a naive 0.5, so a miss against the hardcoded 0.35 threshold was indistinguishable from 'nothing there.' Two fixes: - find_tube now takes an optional diag dict and always records max_heat (plus area_cap vs below_threshold reason on a miss). - detail_refine_scene always returns a descriptive note once detection actually ran ('no match: max CLIPSeg score 0.XX < threshold 0.35 — try a lower detail_threshold...'), replacing the generic 'no region matched' the sampler used to fabricate. - detail_threshold is now a real widget/sampler_input (was a hardcoded constant with no UI — a hidden-setting violation once real runs started needing it), in Engine > Chain Sampler > Experimental next to the other detailing knobs. Co-Authored-By: Claude Fable 5 --- detailing.py | 44 +++++++++++++++++++--- movie_editor/frontend/engine_settings.js | 4 +- samplers.py | 13 +++++-- tests/test_detailing.py | 48 +++++++++++++++++++++++- 4 files changed, 96 insertions(+), 13 deletions(-) diff --git a/detailing.py b/detailing.py index 43918672..c175ac99 100644 --- a/detailing.py +++ b/detailing.py @@ -241,22 +241,33 @@ def _clipseg_heat(frames, prompts, debug=False): def find_tube(heat, latent_h, latent_w, threshold=DEFAULT_THRESHOLD, - area_cap=MAX_TUBE_AREA, debug=False): + area_cap=MAX_TUBE_AREA, debug=False, diag=None): """Threshold pixel-space heat into a latent-space tube box + soft paste mask. Returns (y0, y1, x0, x1, mask) in latent coords - mask is [1, 1, 1, th, tw] matching the tube, feathered from the CLIPSeg heat itself so the paste follows the actual silhouette instead of the rectangle. None when nothing crossed the threshold or the tube breached the area cap. + + `diag`, when given a dict, is filled with why a None came back (max_heat + always; reason + area on a miss) so a caller can report something more + useful than "nothing found" - CLIPSeg's raw sigmoid score for a real, + correctly-named region is often well under a naive 0.5, so a silent miss + is indistinguishable from "the threshold is simply too high" without it. """ if heat is None: return None # Heat -> latent grid first; box math happens where the crop happens. lat_heat = F.interpolate(heat[None, None], size=(latent_h, latent_w), mode="bilinear", align_corners=False)[0, 0] + max_heat = float(lat_heat.max()) + if diag is not None: + diag["max_heat"] = max_heat hot = lat_heat >= threshold if not bool(hot.any()): - _log(debug, f"no region above threshold {threshold}") + _log(debug, f"no region above threshold {threshold} (max heat seen: {max_heat:.3f})") + if diag is not None: + diag["reason"] = "below_threshold" return None ys, xs = torch.where(hot) y0 = max(0, int(ys.min()) - TUBE_PAD) @@ -272,6 +283,9 @@ def find_tube(heat, latent_h, latent_w, threshold=DEFAULT_THRESHOLD, if area > area_cap: _log(debug, f"tube area {area:.0%} exceeds cap {area_cap:.0%}; refusing " "(a region that large is a re-render, not a detail pass)") + if diag is not None: + diag["reason"] = "area_cap" + diag["area"] = area return None # Soft mask: the heat inside the tube, floored at 0 outside the threshold and # lightly blurred so the seam never lands on a hard edge. @@ -346,8 +360,14 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, threshold=DEFAULT_THRESHOLD, strength=1.0, debug=False): """Run the segmented-detail pass on one finished scene latent. - Returns (latent, note): the refined latent dict plus a run_mechanisms note, - or (latent, None) untouched - same object - when disabled or nothing found. + Returns (latent, note): the refined latent dict plus a run_mechanisms note. + Two different "nothing happened" shapes, both same-object no-ops: + - (latent, None): the pass is effectively OFF (no targets, no upsampler, + strength 0) - not worth a note on every scene. + - (latent, "segmented_detail(no match: ...)"): the pass RAN and found + nothing - always reported, with the max CLIPSeg score seen, because a + miss is otherwise indistinguishable from "nothing to detail here" when + it's actually just the threshold being wrong for this content. `chain` is the FunPackLTXAVSceneChainSampler instance (reuses _latent_tensors and _sample_chunk so the refine denoise goes through the exact same path as the scene it is polishing). @@ -362,13 +382,25 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, _log(debug, f"unexpected video latent rank {video.dim()}; skipping") return latent, None b, c, f, lat_h, lat_w = video.shape + target_str = ", ".join(targets) # 1) Detect on a few decoded keyframes. frames = _decode_detection_frames(vae, video, DETECTION_FRAMES, debug=debug) heat = _clipseg_heat(frames, targets, debug=debug) - tube = find_tube(heat, lat_h, lat_w, threshold=threshold, debug=debug) + if heat is None: + return latent, f"segmented_detail(no match: '{target_str}' - no keyframe decoded)" + diag = {} + tube = find_tube(heat, lat_h, lat_w, threshold=threshold, debug=debug, diag=diag) if tube is None: - return latent, None + if diag.get("reason") == "area_cap": + note = (f"segmented_detail(no match: '{target_str}' region covers " + f"{diag['area']:.0%} of frame > {MAX_TUBE_AREA:.0%} cap - refused, " + "not a detail pass at that size)") + else: + note = (f"segmented_detail(no match: '{target_str}', max CLIPSeg score " + f"{diag.get('max_heat', 0.0):.2f} < threshold {threshold:.2f} - " + "try a lower detail_threshold or a different target word)") + return latent, note y0, y1, x0, x1, mask = tube _log(debug, f"tube y[{y0}:{y1}] x[{x0}:{x1}] of {lat_h}x{lat_w} for {targets}") diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 8d166504..286db88a 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -202,6 +202,8 @@ hint: "Comma-separated regions to detail, in plain words. Each becomes a CLIPSeg text query — malformed anatomy still matches its name. Also editable from Composer ▸ Compose while detailing is on." }, { name: "detail_strength", label: "Detail strength", kind: "float", default: 1.0, min: 0, max: 1, step: 0.05, dependsOn: "segmented_detailing", hint: "Blend of the refined region at paste-back. 1.0 = full replacement inside the silhouette; 0 disables the pass." }, + { name: "detail_threshold", label: "Detail match threshold", kind: "float", default: 0.35, min: 0.05, max: 0.9, step: 0.05, dependsOn: "segmented_detailing", + hint: "CLIPSeg match confidence required before a region counts as found. Its raw score for a real region is often well under 0.5 — if the scene report shows 'no match: max CLIPSeg score X < threshold', lower this toward X rather than assuming nothing is there." }, ]; const SAMPLER_KNOB_MAP = Object.fromEntries(SAMPLER_KNOBS.map((k) => [k.name, k])); @@ -314,7 +316,7 @@ chain_timing: ["frame_overlap", "transition_duration", "use_same_seed"], chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "taste_nearest_prompt", "output_guidance", "output_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold"], chain_decode: ["decode_noise_scale", "decode_timestep", "decode_tile_size"], - chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], + chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], }; function countChainView(p, id) { diff --git a/samplers.py b/samplers.py index 58a78d93..4b88987f 100644 --- a/samplers.py +++ b/samplers.py @@ -2466,6 +2466,10 @@ class FunPackLTXAVSceneChainSampler: "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05, "tooltip": "Blend of the refined region into the frame at paste-back (through the feathered CLIPSeg mask). 1.0 = full replacement inside the silhouette; 0 disables the pass entirely.", }), + "detail_threshold": ("FLOAT", { + "default": 0.35, "min": 0.05, "max": 0.9, "step": 0.05, + "tooltip": "CLIPSeg match confidence (post-sigmoid) required before a region counts as found. CLIPSeg's raw score for a real, correctly-named region is often well under 0.5 — if the scene report shows 'no match: max CLIPSeg score X < threshold', lower this toward X (or just below it) rather than assuming nothing is there. Lower = more permissive (more false positives on unrelated regions); higher = stricter.", + }), }, "hidden": { "unique_id": "UNIQUE_ID", @@ -4665,7 +4669,7 @@ class FunPackLTXAVSceneChainSampler: plateau_cache=False, plateau_cache_threshold=0.975, taste_nearest_prompt=False, segmented_detailing=False, detail_targets="hands", - detail_upsampler="None", detail_strength=1.0, + detail_upsampler="None", detail_strength=1.0, detail_threshold=0.35, unique_id=None, prompt=None): if not isinstance(positive, list) or not positive: raise ValueError("positive conditioning must contain at least one scene entry.") @@ -5133,12 +5137,13 @@ class FunPackLTXAVSceneChainSampler: sampled, _detail_note = _detailing.detail_refine_scene( self, model, vae, sampler, scene_positive, scene_negative, sampled, detail_targets, _detail_upsampler_model, scene_seed, cfg, - strength=detail_strength, debug=debug_log) + threshold=detail_threshold, strength=detail_strength, debug=debug_log) + # detail_refine_scene always returns a diagnostic note when it actually + # ran detection (miss or hit) — None only means it never attempted + # detection at all (which can't happen on this branch). if _detail_note: run_mechanisms.append(_detail_note) _phase_sampling += _time.perf_counter() - _t_detail0 - else: - run_mechanisms.append("segmented_detail(no region matched)") except Exception as _detail_exc: # A failed detail pass must never cost the scene itself — but it must # never fail silently either. Model resolution/load failures disable diff --git a/tests/test_detailing.py b/tests/test_detailing.py index b3705287..b541902a 100644 --- a/tests/test_detailing.py +++ b/tests/test_detailing.py @@ -90,6 +90,19 @@ def test_find_tube_refuses_oversized_region(): heat = torch.full((64, 64), 0.9) # whole frame hot -> a re-render, not a detail pass assert detailing.find_tube(heat, 64, 64, threshold=0.35) is None +def test_find_tube_diag_reports_max_heat_and_reason(): + diag = {} + heat = torch.full((64, 64), 0.1) + assert detailing.find_tube(heat, 64, 64, threshold=0.35, diag=diag) is None + assert diag["reason"] == "below_threshold" + assert abs(diag["max_heat"] - 0.1) < 1e-6 + + diag2 = {} + heat2 = torch.full((64, 64), 0.9) # whole frame -> area cap, not threshold + assert detailing.find_tube(heat2, 64, 64, threshold=0.35, diag=diag2) is None + assert diag2["reason"] == "area_cap" + assert diag2["area"] > detailing.MAX_TUBE_AREA + def test_find_tube_enforces_minimum_edge(): heat = _heat_with_hot_square(y=(31, 32), x=(31, 32)) # single hot cell tube = detailing.find_tube(heat, 64, 64, threshold=0.35) @@ -245,14 +258,45 @@ def test_refine_noop_without_targets_or_upsampler(): assert out is latent and note is None assert chain.chunk_calls == [] -def test_refine_noop_when_nothing_detected(monkeypatch): +def test_refine_noop_when_nothing_decodable(monkeypatch): + """No keyframe could be decoded: same-object no-op, but WITH a diagnostic note — + a miss must never look identical to "the feature is off" once it actually ran.""" chain = _FakeChain() latent, _ = _scene_latent() monkeypatch.setattr(detailing, "_decode_detection_frames", lambda *a, **k: []) monkeypatch.setattr(detailing, "_clipseg_heat", lambda *a, **k: None) out, note = detailing.detail_refine_scene( chain, None, None, None, [], [], latent, "hands", object(), 0, 1.0) - assert out is latent and note is None + assert out is latent + assert note is not None and "no match" in note + assert chain.chunk_calls == [] + +def test_refine_reports_max_heat_when_below_threshold(monkeypatch): + """Regression: a real live run hit CLIPSeg-found-nothing with zero visibility into + WHY (was it a bad prompt, or just a threshold too strict for CLIPSeg's raw score?). + The note must carry the actual max score so the fix (lower detail_threshold) is + obvious without re-running with debug_log.""" + chain = _FakeChain() + latent, _ = _scene_latent() + monkeypatch.setattr(detailing, "_decode_detection_frames", + lambda *a, **k: [(0, torch.zeros(16, 16, 3))]) + monkeypatch.setattr(detailing, "_clipseg_heat", lambda *a, **k: torch.full((16, 16), 0.18)) + out, note = detailing.detail_refine_scene( + chain, None, None, None, [], [], latent, "hands", object(), 0, 1.0, threshold=0.35) + assert out is latent + assert "0.18" in note and "0.35" in note + assert chain.chunk_calls == [] + +def test_refine_reports_area_cap_reason(monkeypatch): + chain = _FakeChain() + latent, _ = _scene_latent() + monkeypatch.setattr(detailing, "_decode_detection_frames", + lambda *a, **k: [(0, torch.zeros(160, 240, 3))]) + monkeypatch.setattr(detailing, "_clipseg_heat", lambda *a, **k: torch.full((160, 240), 0.9)) + out, note = detailing.detail_refine_scene( + chain, None, None, None, [], [], latent, "hands", object(), 0, 1.0) + assert out is latent + assert "cap" in note.lower() assert chain.chunk_calls == [] def _patch_detection_and_upsampler(monkeypatch): From 0372ee82871f949b8ad15f271a7ccf0457013dcf Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sun, 19 Jul 2026 01:10:27 +0300 Subject: [PATCH 08/24] Make the area cap a user knob, not a silent content judgment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 35% area cap was refusing a user-requested 'hand' region (47% of frame) with no way to override it — the code was unilaterally deciding the request 'wasn't a detail pass' when the user never asked for that judgment. Fair complaint: cost guardrails are not the same as content decisions. detail_max_area is now threaded end-to-end (detailing.area_cap param -> sampler widget -> Editor knob in Chain Sampler > Experimental, 0.05-1.0, default 0.35 unchanged). Raising it lets a region of any size through; 1.0 = no cap. The refusal note now points at the actual setting name instead of citing a fixed constant. Co-Authored-By: Claude Fable 5 --- detailing.py | 21 ++++++++++++++------- movie_editor/frontend/engine_settings.js | 4 +++- samplers.py | 8 +++++++- tests/test_detailing.py | 22 ++++++++++++++++++++++ 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/detailing.py b/detailing.py index c175ac99..4d24ff4a 100644 --- a/detailing.py +++ b/detailing.py @@ -34,9 +34,10 @@ STAGE2_SIGMAS = (0.85, 0.725, 0.421875, 0.0) # CLIPSeg heat above this (post-sigmoid) counts as "the named thing is here". DEFAULT_THRESHOLD = 0.35 -# Tubes larger than this fraction of the frame are refused: at that size the pass -# stops being region detailing and becomes a second full render (cost ~= 4x area -# fraction x tail steps), which belongs to a different feature. +# Default ceiling on tube area, as a fraction of the frame - purely a COST guardrail +# (cost ~= 4x area fraction x tail steps, so a big tube starts to rival a second full +# render) exposed as detail_max_area so the user decides the cost/quality trade-off, +# not this constant. Never silently override what the user asked to detail. MAX_TUBE_AREA = 0.35 # Latent cells of context padding around the thresholded region, and the minimum @@ -357,7 +358,8 @@ def _downscale_to(video, h, w): def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, detail_targets, upsampler, seed, cfg, - threshold=DEFAULT_THRESHOLD, strength=1.0, debug=False): + threshold=DEFAULT_THRESHOLD, strength=1.0, area_cap=MAX_TUBE_AREA, + debug=False): """Run the segmented-detail pass on one finished scene latent. Returns (latent, note): the refined latent dict plus a run_mechanisms note. @@ -368,6 +370,10 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, nothing - always reported, with the max CLIPSeg score seen, because a miss is otherwise indistinguishable from "nothing to detail here" when it's actually just the threshold being wrong for this content. + `area_cap` is a COST guardrail, not a content judgment: the user named the + target, so a region crossing it is reported as a skip with the actual size + (never silently reinterpreted as "not found") and area_cap itself is a + user-facing knob (detail_max_area) - raise it, don't route around it here. `chain` is the FunPackLTXAVSceneChainSampler instance (reuses _latent_tensors and _sample_chunk so the refine denoise goes through the exact same path as the scene it is polishing). @@ -390,12 +396,13 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, if heat is None: return latent, f"segmented_detail(no match: '{target_str}' - no keyframe decoded)" diag = {} - tube = find_tube(heat, lat_h, lat_w, threshold=threshold, debug=debug, diag=diag) + tube = find_tube(heat, lat_h, lat_w, threshold=threshold, area_cap=area_cap, + debug=debug, diag=diag) if tube is None: if diag.get("reason") == "area_cap": note = (f"segmented_detail(no match: '{target_str}' region covers " - f"{diag['area']:.0%} of frame > {MAX_TUBE_AREA:.0%} cap - refused, " - "not a detail pass at that size)") + f"{diag['area']:.0%} of frame > {area_cap:.0%} cap - raise " + "detail_max_area if you want it detailed at this size anyway)") else: note = (f"segmented_detail(no match: '{target_str}', max CLIPSeg score " f"{diag.get('max_heat', 0.0):.2f} < threshold {threshold:.2f} - " diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 286db88a..701a071f 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -204,6 +204,8 @@ hint: "Blend of the refined region at paste-back. 1.0 = full replacement inside the silhouette; 0 disables the pass." }, { name: "detail_threshold", label: "Detail match threshold", kind: "float", default: 0.35, min: 0.05, max: 0.9, step: 0.05, dependsOn: "segmented_detailing", hint: "CLIPSeg match confidence required before a region counts as found. Its raw score for a real region is often well under 0.5 — if the scene report shows 'no match: max CLIPSeg score X < threshold', lower this toward X rather than assuming nothing is there." }, + { name: "detail_max_area", label: "Detail max area", kind: "float", default: 0.35, min: 0.05, max: 1.0, step: 0.05, dependsOn: "segmented_detailing", + hint: "Ceiling on how much of the frame the region may cover before it's refused, as a fraction of the frame. Cost-only guardrail (a bigger region costs more, roughly 4× its area × 3 steps) — never a judgment call about whether it's worth detailing. If the scene report shows a region refused at some %, raise this above that % to detail it anyway. 1.0 = no cap." }, ]; const SAMPLER_KNOB_MAP = Object.fromEntries(SAMPLER_KNOBS.map((k) => [k.name, k])); @@ -316,7 +318,7 @@ chain_timing: ["frame_overlap", "transition_duration", "use_same_seed"], chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "taste_nearest_prompt", "output_guidance", "output_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold"], chain_decode: ["decode_noise_scale", "decode_timestep", "decode_tile_size"], - chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], + chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "detail_max_area", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], }; function countChainView(p, id) { diff --git a/samplers.py b/samplers.py index 4b88987f..4bbf4e5f 100644 --- a/samplers.py +++ b/samplers.py @@ -2470,6 +2470,10 @@ class FunPackLTXAVSceneChainSampler: "default": 0.35, "min": 0.05, "max": 0.9, "step": 0.05, "tooltip": "CLIPSeg match confidence (post-sigmoid) required before a region counts as found. CLIPSeg's raw score for a real, correctly-named region is often well under 0.5 — if the scene report shows 'no match: max CLIPSeg score X < threshold', lower this toward X (or just below it) rather than assuming nothing is there. Lower = more permissive (more false positives on unrelated regions); higher = stricter.", }), + "detail_max_area": ("FLOAT", { + "default": 0.35, "min": 0.05, "max": 1.0, "step": 0.05, + "tooltip": "Ceiling on how much of the frame the detected region may cover before the pass refuses it, as a fraction of frame area. This is a COST guardrail only (cost ~= 4x area x 3 steps, so a large region starts to rival a second full render), never a judgment about whether the region is worth detailing — if the scene report shows a region refused at some %, raise this above that % to detail it anyway (up to 1.0 = no cap, full-frame allowed).", + }), }, "hidden": { "unique_id": "UNIQUE_ID", @@ -4670,6 +4674,7 @@ class FunPackLTXAVSceneChainSampler: taste_nearest_prompt=False, segmented_detailing=False, detail_targets="hands", detail_upsampler="None", detail_strength=1.0, detail_threshold=0.35, + detail_max_area=0.35, unique_id=None, prompt=None): if not isinstance(positive, list) or not positive: raise ValueError("positive conditioning must contain at least one scene entry.") @@ -5137,7 +5142,8 @@ class FunPackLTXAVSceneChainSampler: sampled, _detail_note = _detailing.detail_refine_scene( self, model, vae, sampler, scene_positive, scene_negative, sampled, detail_targets, _detail_upsampler_model, scene_seed, cfg, - threshold=detail_threshold, strength=detail_strength, debug=debug_log) + threshold=detail_threshold, strength=detail_strength, + area_cap=detail_max_area, debug=debug_log) # detail_refine_scene always returns a diagnostic note when it actually # ran detection (miss or hit) — None only means it never attempted # detection at all (which can't happen on this branch). diff --git a/tests/test_detailing.py b/tests/test_detailing.py index b541902a..5fdcc83c 100644 --- a/tests/test_detailing.py +++ b/tests/test_detailing.py @@ -299,6 +299,28 @@ def test_refine_reports_area_cap_reason(monkeypatch): assert "cap" in note.lower() assert chain.chunk_calls == [] +def test_refine_area_cap_is_a_cost_knob_not_a_judgment(monkeypatch): + """The cap must never override what the user explicitly asked to detail — it's a + cost guardrail the user can raise, not FunPack deciding a big region 'isn't worth + detailing'. Same whole-frame region: refused at the default cap, allowed once the + user raises detail_max_area (mirrors area_cap, threaded from the sampler node).""" + chain = _FakeChain() + latent, video = _scene_latent() + monkeypatch.setattr(detailing, "_decode_detection_frames", + lambda *a, **k: [(0, torch.zeros(160, 240, 3))]) + monkeypatch.setattr(detailing, "_clipseg_heat", lambda *a, **k: torch.full((160, 240), 0.9)) + monkeypatch.setattr( + detailing, "_run_upsampler", + lambda upsampler, crop, vae, debug=False: torch.nn.functional.interpolate( + crop, scale_factor=(1, 2, 2), mode="nearest")) + out_default, note_default = detailing.detail_refine_scene( + chain, "m", "v", "s", [], [], latent, "hands", object(), 0, 1.0) + assert out_default is latent and "cap" in note_default.lower() + out_raised, note_raised = detailing.detail_refine_scene( + chain, "m", "v", "s", [], [], latent, "hands", object(), 0, 1.0, area_cap=1.0) + assert note_raised is not None and "segmented_detail(" in note_raised and "cap" not in note_raised.lower() + assert out_raised is not latent + def _patch_detection_and_upsampler(monkeypatch): """Hot square in pixel space; upsampler = exact 2x nearest (model-free).""" heat = torch.zeros(160, 240) From 1644dde8296cb7cb8298f9167f039fb58f7731bc Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sun, 19 Jul 2026 01:19:34 +0300 Subject: [PATCH 09/24] Expose the crop re-noise strength: fixed 0.85 was sharpening, not repairing Live report: a hand was detected, masked, and upsampled, but the pasted result looked like a plain upscale, not an actually-corrected region. Root cause is mechanical, not a bug: 0.85 is Lightricks' own official re-entry sigma, but it was designed for their use case (sharpen an already-correct frame at higher resolution), not ours (repair a likely- malformed crop). At that re-entry point the tail mostly smooths the upsampled interpolation back down rather than giving the model enough noise/freedom to genuinely reconstruct structure. detail_denoise (0.3-0.99, default 0.85 = unchanged prior behavior) now controls the re-entry point; the tail's shape (ratio between its 3 steps) scales with it via the new stage2_sigmas() instead of a fixed tuple, so raising it doesn't just enlarge the first jump. Higher = more genuine repair freedom at some risk of drifting from the surrounding frame. Co-Authored-By: Claude Fable 5 --- detailing.py | 52 ++++++++++++++++++------ movie_editor/frontend/engine_settings.js | 4 +- samplers.py | 9 +++- tests/test_detailing.py | 34 ++++++++++++++++ 4 files changed, 84 insertions(+), 15 deletions(-) diff --git a/detailing.py b/detailing.py index 4d24ff4a..634ad719 100644 --- a/detailing.py +++ b/detailing.py @@ -25,11 +25,36 @@ import torch import torch.nn.functional as F # Official LTX-2.3 two-stage distilled workflow, stage-2 ManualSigmas: re-noise the -# upsampled latent to 0.85 and run the tail of the 8-step distilled schedule. Using -# Lightricks' published re-entry point instead of inventing one: too much noise and -# the region redraws (drifts from the surrounding frame), too little and no detail -# is gained. -STAGE2_SIGMAS = (0.85, 0.725, 0.421875, 0.0) +# upsampled latent to 0.85 and run the tail of the 8-step distilled schedule. +# Lightricks designed this re-entry point to SHARPEN already-correct structure at +# higher resolution, not to REPAIR a malformed region — our use case (a mangled +# hand) needs more genuine freedom to redraw than their use case (a clean shot +# that just needs finer detail). 0.85 is the built-in default (matches the +# official recipe / prior behavior exactly), but it is a real knob +# (detail_denoise on the sampler) precisely because the right value depends on +# how wrong the source region already is: too high and the region drifts from +# its surroundings; too low and the tail just smooths the upsampled +# interpolation back down without adding anything (upscaled-looking, not +# detailed - the failure mode this constant existed to describe from day one). +DEFAULT_RENOISE_SIGMA = 0.85 +# Shape of the tail preserved as ratios of the entry sigma, not fixed values, so +# raising/lowering the entry point scales the whole tail schedule consistently +# instead of only moving its first step. +_STAGE2_RATIOS = (1.0, 0.725 / 0.85, 0.421875 / 0.85, 0.0) + + +def stage2_sigmas(renoise_sigma=DEFAULT_RENOISE_SIGMA): + """The re-denoise tail schedule for a given re-entry sigma. + + stage2_sigmas(0.85) reproduces the exact official STAGE2_SIGMAS values. + """ + r = max(0.0, min(1.0, float(renoise_sigma))) + return tuple(r * ratio for ratio in _STAGE2_RATIOS) + + +# Kept for anyone importing the old name directly (tests, prior callers) - identical +# to stage2_sigmas() at the default re-entry point. +STAGE2_SIGMAS = stage2_sigmas(DEFAULT_RENOISE_SIGMA) # CLIPSeg heat above this (post-sigmoid) counts as "the named thing is here". DEFAULT_THRESHOLD = 0.35 @@ -359,7 +384,7 @@ def _downscale_to(video, h, w): def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, detail_targets, upsampler, seed, cfg, threshold=DEFAULT_THRESHOLD, strength=1.0, area_cap=MAX_TUBE_AREA, - debug=False): + renoise_sigma=DEFAULT_RENOISE_SIGMA, debug=False): """Run the segmented-detail pass on one finished scene latent. Returns (latent, note): the refined latent dict plus a run_mechanisms note. @@ -415,11 +440,14 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, crop = video[:, :, :, y0:y1, x0:x1] crop_up = _run_upsampler(upsampler, crop, vae, debug=debug) - # 3) Re-denoise the crop over the official stage-2 tail. The clean crop goes - # in as the latent; CONST noise_scaling (sigma*noise + (1-sigma)*x) re-noises - # it to sigma=0.85 exactly like the official two-stage workflow's stage 2. - # The scene's audio stream rides along so joint attention sees a legal AV - # latent, but its refined output is discarded below - audio is protected by + # 3) Re-denoise the crop over a stage-2 tail shaped like the official two-stage + # workflow's, but re-entering at `renoise_sigma` (default 0.85, their exact + # value) rather than a hardcoded one: how much genuine freedom the model needs + # to fix this crop depends on how wrong it already is, which only the person + # looking at the result can judge. CONST noise_scaling (sigma*noise + + # (1-sigma)*x) re-noises the clean crop to that sigma automatically. The + # scene's audio stream rides along so joint attention sees a legal AV latent, + # but its refined output is discarded below - audio is protected by # construction, not by masking. import comfy.nested_tensor @@ -427,7 +455,7 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, if len(tensors) > 1: crop_latent["samples"] = comfy.nested_tensor.NestedTensor( [crop_up] + [t.detach().clone() for t in tensors[1:]]) - tail_sigmas = torch.tensor(STAGE2_SIGMAS, dtype=torch.float32) + tail_sigmas = torch.tensor(stage2_sigmas(renoise_sigma), dtype=torch.float32) refined = chain._sample_chunk( model, sampler, tail_sigmas, int(seed) + 7777, cfg, _strip_layout_conds(positive), _strip_layout_conds(negative), crop_latent) diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 701a071f..3cb81293 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -206,6 +206,8 @@ hint: "CLIPSeg match confidence required before a region counts as found. Its raw score for a real region is often well under 0.5 — if the scene report shows 'no match: max CLIPSeg score X < threshold', lower this toward X rather than assuming nothing is there." }, { name: "detail_max_area", label: "Detail max area", kind: "float", default: 0.35, min: 0.05, max: 1.0, step: 0.05, dependsOn: "segmented_detailing", hint: "Ceiling on how much of the frame the region may cover before it's refused, as a fraction of the frame. Cost-only guardrail (a bigger region costs more, roughly 4× its area × 3 steps) — never a judgment call about whether it's worth detailing. If the scene report shows a region refused at some %, raise this above that % to detail it anyway. 1.0 = no cap." }, + { name: "detail_denoise", label: "Detail re-noise strength", kind: "float", default: 0.85, min: 0.3, max: 0.99, step: 0.05, dependsOn: "segmented_detailing", + hint: "How much noise the crop gets re-noised to before its 3-step refine (0.85 is the official LTX 2.3 recipe's own value). Higher = more freedom to genuinely reconstruct the region (fix bad anatomy), risking drift from the surrounding frame; lower = closer to a plain upscale — looks 'detailed' as interpolation but doesn't actually repair it. If the result looks upscaled but not corrected, raise this." }, ]; const SAMPLER_KNOB_MAP = Object.fromEntries(SAMPLER_KNOBS.map((k) => [k.name, k])); @@ -318,7 +320,7 @@ chain_timing: ["frame_overlap", "transition_duration", "use_same_seed"], chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "taste_nearest_prompt", "output_guidance", "output_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold"], chain_decode: ["decode_noise_scale", "decode_timestep", "decode_tile_size"], - chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "detail_max_area", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], + chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "detail_max_area", "detail_denoise", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], }; function countChainView(p, id) { diff --git a/samplers.py b/samplers.py index 4bbf4e5f..b978aeee 100644 --- a/samplers.py +++ b/samplers.py @@ -2474,6 +2474,10 @@ class FunPackLTXAVSceneChainSampler: "default": 0.35, "min": 0.05, "max": 1.0, "step": 0.05, "tooltip": "Ceiling on how much of the frame the detected region may cover before the pass refuses it, as a fraction of frame area. This is a COST guardrail only (cost ~= 4x area x 3 steps, so a large region starts to rival a second full render), never a judgment about whether the region is worth detailing — if the scene report shows a region refused at some %, raise this above that % to detail it anyway (up to 1.0 = no cap, full-frame allowed).", }), + "detail_denoise": ("FLOAT", { + "default": 0.85, "min": 0.3, "max": 0.99, "step": 0.05, + "tooltip": "How much noise the crop is re-noised to before the 3-step refine tail (the official LTX 2.3 two-stage recipe's own value, 0.85, is the default). Higher = more freedom for the model to genuinely reconstruct the region (fix bad anatomy) at the cost of possibly drifting from the surrounding frame; lower = closer to a plain upscale (looks 'detailed' as interpolation, but doesn't actually repair the region — if that's what you're seeing, raise this).", + }), }, "hidden": { "unique_id": "UNIQUE_ID", @@ -4674,7 +4678,7 @@ class FunPackLTXAVSceneChainSampler: taste_nearest_prompt=False, segmented_detailing=False, detail_targets="hands", detail_upsampler="None", detail_strength=1.0, detail_threshold=0.35, - detail_max_area=0.35, + detail_max_area=0.35, detail_denoise=0.85, unique_id=None, prompt=None): if not isinstance(positive, list) or not positive: raise ValueError("positive conditioning must contain at least one scene entry.") @@ -5143,7 +5147,8 @@ class FunPackLTXAVSceneChainSampler: self, model, vae, sampler, scene_positive, scene_negative, sampled, detail_targets, _detail_upsampler_model, scene_seed, cfg, threshold=detail_threshold, strength=detail_strength, - area_cap=detail_max_area, debug=debug_log) + area_cap=detail_max_area, renoise_sigma=detail_denoise, + debug=debug_log) # detail_refine_scene always returns a diagnostic note when it actually # ran detection (miss or hit) — None only means it never attempted # detection at all (which can't happen on this branch). diff --git a/tests/test_detailing.py b/tests/test_detailing.py index 5fdcc83c..ce210c73 100644 --- a/tests/test_detailing.py +++ b/tests/test_detailing.py @@ -58,6 +58,29 @@ def test_parse_targets_splits_and_strips(): assert detailing.parse_targets(None) == [] +# --------------------------------------------------------------------------- +# stage2_sigmas +# --------------------------------------------------------------------------- + +def test_stage2_sigmas_default_matches_official_recipe(): + # Must reproduce the exact official two-stage workflow values unless the + # user raises/lowers detail_denoise — no behavior change for existing runs. + assert torch.allclose(torch.tensor(detailing.stage2_sigmas(0.85)), + torch.tensor(detailing.STAGE2_SIGMAS)) + assert detailing.stage2_sigmas(0.85)[-1] == 0.0 + +def test_stage2_sigmas_scales_with_renoise_sigma(): + # Raising the re-entry point should raise every step in the tail proportionally + # (more genuine freedom to reconstruct, not just a bigger first jump). + low = detailing.stage2_sigmas(0.5) + high = detailing.stage2_sigmas(0.95) + assert high[0] > low[0] > 0 + assert high[1] > low[1] + assert low[-1] == high[-1] == 0.0 + # Shape (ratios between steps) preserved regardless of entry point. + assert abs(low[1] / low[0] - high[1] / high[0]) < 1e-9 + + # --------------------------------------------------------------------------- # find_tube # --------------------------------------------------------------------------- @@ -334,6 +357,17 @@ def _patch_detection_and_upsampler(monkeypatch): crop, scale_factor=(1, 2, 2), mode="nearest")) return heat +def test_refine_passes_renoise_sigma_to_the_tail_schedule(monkeypatch): + chain = _FakeChain() + latent, _ = _scene_latent() + _patch_detection_and_upsampler(monkeypatch) + detailing.detail_refine_scene( + chain, "model", "vae", "sampler", [], [], latent, "hands", object(), + seed=0, cfg=1.0, renoise_sigma=0.95) + sigmas = chain.chunk_calls[0]["sigmas"] + assert abs(float(sigmas[0]) - 0.95) < 1e-6 + assert torch.allclose(sigmas, torch.tensor(detailing.stage2_sigmas(0.95))) + def test_refine_round_trip_touches_only_the_tube(monkeypatch): chain = _FakeChain() latent, video = _scene_latent() From 726d88a5471470f94c6110cceeecf73ccf31ce72 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sun, 19 Jul 2026 01:26:34 +0300 Subject: [PATCH 10/24] Add detail_mode: sharpen (near-free, no diffusion) vs repair (current) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recovers the original pitch: 'cheap, on-the-fly sharpen' was quietly upgraded into a full re-denoise pass during design without ever offering the free version as an option. The trained latent upsampler net was already an intermediate step in the pipeline (crop_up) — sharpen mode just stops there and pastes it back, with ZERO video-model forward passes (only the small upsampler CNN runs). repair mode (default, unchanged, byte-identical behavior) keeps the existing re-noise + 3-step tail for cases that need actual structural correction, since only the generative model itself can decide a region should look structurally different — a super-res net can only sharpen what's already there. Also extends the Editor's knobVisible with a array (AND'd conditions) — detail_denoise needed to gate on BOTH segmented_detailing AND detail_mode=='repair', which the existing single dependsOn/dependsValue couldn't express without silently dropping one of the two checks. Co-Authored-By: Claude Fable 5 --- detailing.py | 71 ++++++++++++++++-------- movie_editor/frontend/engine_settings.js | 27 +++++++-- samplers.py | 12 ++-- tests/test_detailing.py | 24 ++++++++ 4 files changed, 102 insertions(+), 32 deletions(-) diff --git a/detailing.py b/detailing.py index 634ad719..fe01765a 100644 --- a/detailing.py +++ b/detailing.py @@ -384,9 +384,22 @@ def _downscale_to(video, h, w): def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, detail_targets, upsampler, seed, cfg, threshold=DEFAULT_THRESHOLD, strength=1.0, area_cap=MAX_TUBE_AREA, - renoise_sigma=DEFAULT_RENOISE_SIGMA, debug=False): + renoise_sigma=DEFAULT_RENOISE_SIGMA, mode="repair", debug=False): """Run the segmented-detail pass on one finished scene latent. + `mode`: + - "sharpen": crop -> upsampler forward -> downscale -> paste. NO video-model + calls at all - the trained upsampler net (small ResBlock/PixelShuffle CNN, + not naive interpolation) is itself doing the work, so this is close to + free next to a diffusion step. It can genuinely sharpen a region that's + blurry/under-resolved but already correctly shaped. It CANNOT fix wrong + structure (an extra finger stays an extra finger) - a super-res net adds + detail consistent with its input, it doesn't invent new correct content. + - "repair" (default): the above PLUS a `renoise_sigma`-entry, 3-step + re-denoise through the actual video model - genuinely expensive (3 extra + forwards on the crop) because only the generative model itself can decide + the region should look structurally different, not just sharper. + Returns (latent, note): the refined latent dict plus a run_mechanisms note. Two different "nothing happened" shapes, both same-object no-ops: - (latent, None): the pass is effectively OFF (no targets, no upsampler, @@ -414,6 +427,7 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, return latent, None b, c, f, lat_h, lat_w = video.shape target_str = ", ".join(targets) + import comfy.nested_tensor # 1) Detect on a few decoded keyframes. frames = _decode_detection_frames(vae, video, DETECTION_FRAMES, debug=debug) @@ -440,26 +454,39 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, crop = video[:, :, :, y0:y1, x0:x1] crop_up = _run_upsampler(upsampler, crop, vae, debug=debug) - # 3) Re-denoise the crop over a stage-2 tail shaped like the official two-stage - # workflow's, but re-entering at `renoise_sigma` (default 0.85, their exact - # value) rather than a hardcoded one: how much genuine freedom the model needs - # to fix this crop depends on how wrong it already is, which only the person - # looking at the result can judge. CONST noise_scaling (sigma*noise + - # (1-sigma)*x) re-noises the clean crop to that sigma automatically. The - # scene's audio stream rides along so joint attention sees a legal AV latent, - # but its refined output is discarded below - audio is protected by - # construction, not by masking. - import comfy.nested_tensor - - crop_latent = {"samples": crop_up} - if len(tensors) > 1: - crop_latent["samples"] = comfy.nested_tensor.NestedTensor( - [crop_up] + [t.detach().clone() for t in tensors[1:]]) - tail_sigmas = torch.tensor(stage2_sigmas(renoise_sigma), dtype=torch.float32) - refined = chain._sample_chunk( - model, sampler, tail_sigmas, int(seed) + 7777, cfg, - _strip_layout_conds(positive), _strip_layout_conds(negative), crop_latent) - refined_crop = chain._latent_tensors(refined)[0] + if mode == "sharpen": + # No re-denoise, no video-model forward at all: crop_up IS the result. The + # trained upsampler is itself a learned super-resolution net (ResBlocks + + # PixelShuffle), not naive interpolation, so it genuinely hallucinates + # plausible fine detail consistent with the crop's EXISTING structure - + # that's exactly the "cheap, on-the-fly sharpen" the feature was first + # pitched as. What it cannot do: invent correct structure the crop never + # had (an extra finger stays an extra finger, just a sharper one) - that + # needs the model to actually reconsider the region, which is "repair". + refined_crop = crop_up + cost_note = "sharpen: upsampler-only, no diffusion" + else: + # 3) Re-denoise the crop over a stage-2 tail shaped like the official two- + # stage workflow's, but re-entering at `renoise_sigma` (default 0.85, their + # exact value) rather than a hardcoded one: how much genuine freedom the + # model needs to fix this crop depends on how wrong it already is, which + # only the person looking at the result can judge. CONST noise_scaling + # (sigma*noise + (1-sigma)*x) re-noises the clean crop to that sigma + # automatically. The scene's audio stream rides along so joint attention + # sees a legal AV latent, but its refined output is discarded below - audio + # is protected by construction, not by masking. This is the expensive path + # (3 extra video-model forwards on the crop): actually changing WRONG + # structure requires the generative model to redo the region, no shortcut. + crop_latent = {"samples": crop_up} + if len(tensors) > 1: + crop_latent["samples"] = comfy.nested_tensor.NestedTensor( + [crop_up] + [t.detach().clone() for t in tensors[1:]]) + tail_sigmas = torch.tensor(stage2_sigmas(renoise_sigma), dtype=torch.float32) + refined = chain._sample_chunk( + model, sampler, tail_sigmas, int(seed) + 7777, cfg, + _strip_layout_conds(positive), _strip_layout_conds(negative), crop_latent) + refined_crop = chain._latent_tensors(refined)[0] + cost_note = f"repair: renoise={renoise_sigma:.2f}" # 4) Back to grid size, feathered paste through the CLIPSeg silhouette. refined_down = _downscale_to(refined_crop, y1 - y0, x1 - x0) @@ -473,6 +500,6 @@ def detail_refine_scene(chain, model, vae, sampler, positive, negative, latent, out["samples"] = comfy.nested_tensor.NestedTensor([out_video] + list(tensors[1:])) else: out["samples"] = out_video - note = (f"segmented_detail({','.join(targets)}, tube={y1 - y0}x{x1 - x0}" + note = (f"segmented_detail({cost_note}, '{target_str}', tube={y1 - y0}x{x1 - x0}" f"/{lat_h}x{lat_w}, s={strength})") return out, note diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 3cb81293..2b3d697e 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -206,8 +206,11 @@ hint: "CLIPSeg match confidence required before a region counts as found. Its raw score for a real region is often well under 0.5 — if the scene report shows 'no match: max CLIPSeg score X < threshold', lower this toward X rather than assuming nothing is there." }, { name: "detail_max_area", label: "Detail max area", kind: "float", default: 0.35, min: 0.05, max: 1.0, step: 0.05, dependsOn: "segmented_detailing", hint: "Ceiling on how much of the frame the region may cover before it's refused, as a fraction of the frame. Cost-only guardrail (a bigger region costs more, roughly 4× its area × 3 steps) — never a judgment call about whether it's worth detailing. If the scene report shows a region refused at some %, raise this above that % to detail it anyway. 1.0 = no cap." }, - { name: "detail_denoise", label: "Detail re-noise strength", kind: "float", default: 0.85, min: 0.3, max: 0.99, step: 0.05, dependsOn: "segmented_detailing", - hint: "How much noise the crop gets re-noised to before its 3-step refine (0.85 is the official LTX 2.3 recipe's own value). Higher = more freedom to genuinely reconstruct the region (fix bad anatomy), risking drift from the surrounding frame; lower = closer to a plain upscale — looks 'detailed' as interpolation but doesn't actually repair it. If the result looks upscaled but not corrected, raise this." }, + { name: "detail_mode", label: "Detail mode", kind: "combo", choices: ["repair", "sharpen"], default: "repair", dependsOn: "segmented_detailing", + hint: "'repair' (default): upsamples the crop, then re-denoises it through the video model — can genuinely fix wrong structure (bad anatomy) but costs real compute (~4× region area × 3 steps). 'sharpen': stops after the upsampler's own pass — no video-model calls at all, close to free — good for a region that's blurry/under-resolved but already correctly shaped; it cannot fix wrong structure (an extra finger stays an extra finger, just sharper)." }, + { name: "detail_denoise", label: "Detail re-noise strength", kind: "float", default: 0.85, min: 0.3, max: 0.99, step: 0.05, + deps: [{ name: "segmented_detailing" }, { name: "detail_mode", value: "repair" }], + hint: "How much noise the crop gets re-noised to before its 3-step refine (0.85 is the official LTX 2.3 recipe's own value). Higher = more freedom to genuinely reconstruct the region (fix bad anatomy), risking drift from the surrounding frame; lower = closer to a plain upscale — looks 'detailed' as interpolation but doesn't actually repair it. If the result looks upscaled but not corrected, raise this. Only used in 'repair' mode." }, ]; const SAMPLER_KNOB_MAP = Object.fromEntries(SAMPLER_KNOBS.map((k) => [k.name, k])); @@ -253,10 +256,22 @@ S.patchProject({ guide_settings: { ...(p.guide_settings || {}), ...patch } }); } + function _depSatisfied(name, value, si) { + const depVal = si[name] != null ? si[name] : SAMPLER_KNOB_MAP[name]?.default; + return value !== undefined ? depVal === value : !!depVal; + } + function knobVisible(k, si) { - if (k.dependsOn) { - const depVal = si[k.dependsOn] != null ? si[k.dependsOn] : SAMPLER_KNOB_MAP[k.dependsOn]?.default; - if (!depVal) return false; + // dependsOn/dependsValue: single condition (dependsValue absent = plain truthy + // gate, as every existing boolean dependsOn already relies on). + if (k.dependsOn && !_depSatisfied(k.dependsOn, k.dependsValue, si)) return false; + // deps: an AND'd list, for knobs gated by more than one condition (e.g. the + // feature toggle AND a mode combo equaling a specific option) — dependsOn + // alone can't express two conditions without one silently overriding the other. + if (Array.isArray(k.deps)) { + for (const d of k.deps) { + if (!_depSatisfied(d.name, d.value, si)) return false; + } } return true; } @@ -320,7 +335,7 @@ chain_timing: ["frame_overlap", "transition_duration", "use_same_seed"], chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "taste_nearest_prompt", "output_guidance", "output_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold"], chain_decode: ["decode_noise_scale", "decode_timestep", "decode_tile_size"], - chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "detail_max_area", "detail_denoise", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], + chain_experimental: ["plateau_cache", "plateau_cache_threshold", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "detail_max_area", "detail_mode", "detail_denoise", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], }; function countChainView(p, id) { diff --git a/samplers.py b/samplers.py index b978aeee..225db714 100644 --- a/samplers.py +++ b/samplers.py @@ -2452,7 +2452,7 @@ class FunPackLTXAVSceneChainSampler: }), "segmented_detailing": ("BOOLEAN", { "default": False, - "tooltip": "EXPERIMENTAL ADetailer-for-video: after each scene finishes denoising, CLIPSeg (text-prompted segmentation) locates the regions named in detail_targets on a few decoded keyframes; the matched region is cut out of the video latent as a spatiotemporal tube, pushed through Lightricks' trained latent upsampler (2x spatial — the official two-stage pipeline's stage-2 model), re-noised to the official stage-2 sigma (0.85) and re-denoised for the 3-step tail schedule, then downscaled back to its ORIGINAL latent size and pasted through the feathered CLIPSeg silhouette. Final resolution never changes — the upsample only lets the model resolve structure (fingers, small faces) at a higher working resolution. Cost ~= 4 x tube area fraction x 3 steps (hands ~+15%); tubes over 35% of the frame are refused (that's a re-render, not a detail pass). Audio untouched by construction. detail_upsampler 'auto' finds or downloads the official Lightricks upsampler (~1 GB, once); skips are reported loudly in console + scene report. UNVALIDATED LIVE.", + "tooltip": "EXPERIMENTAL ADetailer-for-video: after each scene finishes denoising, CLIPSeg (text-prompted segmentation) locates the regions named in detail_targets on a few decoded keyframes; the matched region is cut out of the video latent as a spatiotemporal tube, pushed through Lightricks' trained latent upsampler (2x spatial — the official two-stage pipeline's stage-2 model), then either used directly (detail_mode='sharpen', near-free) or re-noised + re-denoised for a 3-step tail (detail_mode='repair', default — costs ~4x tube area fraction x 3 steps, hands ~+15%). Downscaled back to its ORIGINAL latent size and pasted through the feathered CLIPSeg silhouette either way. Final resolution never changes. Tubes over detail_max_area (default 35%) are refused as a cost guard, not a content judgment. Audio untouched by construction. detail_upsampler 'auto' finds or downloads the official Lightricks upsampler (~1 GB, once); skips are reported loudly in console + scene report. UNVALIDATED LIVE.", }), "detail_targets": ("STRING", { "default": "hands", @@ -2476,7 +2476,11 @@ class FunPackLTXAVSceneChainSampler: }), "detail_denoise": ("FLOAT", { "default": 0.85, "min": 0.3, "max": 0.99, "step": 0.05, - "tooltip": "How much noise the crop is re-noised to before the 3-step refine tail (the official LTX 2.3 two-stage recipe's own value, 0.85, is the default). Higher = more freedom for the model to genuinely reconstruct the region (fix bad anatomy) at the cost of possibly drifting from the surrounding frame; lower = closer to a plain upscale (looks 'detailed' as interpolation, but doesn't actually repair the region — if that's what you're seeing, raise this).", + "tooltip": "Only used in 'repair' mode. How much noise the crop is re-noised to before the 3-step refine tail (the official LTX 2.3 two-stage recipe's own value, 0.85, is the default). Higher = more freedom for the model to genuinely reconstruct the region (fix bad anatomy) at the cost of possibly drifting from the surrounding frame; lower = closer to a plain upscale (looks 'detailed' as interpolation, but doesn't actually repair the region — if that's what you're seeing, raise this).", + }), + "detail_mode": (["repair", "sharpen"], { + "default": "repair", + "tooltip": "'repair' (default): upsample the crop, then re-denoise it through the video model for 3 extra steps — can genuinely fix wrong structure (bad anatomy) but costs real compute (~4x region area x 3 steps). 'sharpen': stop after the upsampler's own forward pass — no video-model calls at all, close to free — good for a region that's blurry/under-resolved but already correctly shaped; it CANNOT fix wrong structure (an extra finger stays an extra finger, just sharper), since a super-resolution net only adds detail consistent with what's already there.", }), }, "hidden": { @@ -4678,7 +4682,7 @@ class FunPackLTXAVSceneChainSampler: taste_nearest_prompt=False, segmented_detailing=False, detail_targets="hands", detail_upsampler="None", detail_strength=1.0, detail_threshold=0.35, - detail_max_area=0.35, detail_denoise=0.85, + detail_max_area=0.35, detail_denoise=0.85, detail_mode="repair", unique_id=None, prompt=None): if not isinstance(positive, list) or not positive: raise ValueError("positive conditioning must contain at least one scene entry.") @@ -5148,7 +5152,7 @@ class FunPackLTXAVSceneChainSampler: detail_targets, _detail_upsampler_model, scene_seed, cfg, threshold=detail_threshold, strength=detail_strength, area_cap=detail_max_area, renoise_sigma=detail_denoise, - debug=debug_log) + mode=detail_mode, debug=debug_log) # detail_refine_scene always returns a diagnostic note when it actually # ran detection (miss or hit) — None only means it never attempted # detection at all (which can't happen on this branch). diff --git a/tests/test_detailing.py b/tests/test_detailing.py index ce210c73..9345d5e7 100644 --- a/tests/test_detailing.py +++ b/tests/test_detailing.py @@ -368,6 +368,30 @@ def test_refine_passes_renoise_sigma_to_the_tail_schedule(monkeypatch): assert abs(float(sigmas[0]) - 0.95) < 1e-6 assert torch.allclose(sigmas, torch.tensor(detailing.stage2_sigmas(0.95))) +def test_refine_sharpen_mode_skips_the_diffusion_tail_entirely(monkeypatch): + """The whole point of sharpen mode: zero video-model forwards. It must use the + upsampler's own output directly, not merely a fast-looking diffusion call.""" + chain = _FakeChain() + latent, video = _scene_latent() + _patch_detection_and_upsampler(monkeypatch) + out, note = detailing.detail_refine_scene( + chain, "model", "vae", "sampler", [], [], latent, "hands", object(), + seed=0, cfg=1.0, mode="sharpen") + assert note is not None and "sharpen" in note + assert chain.chunk_calls == [] # no _sample_chunk call at all + assert out["samples"].shape == video.shape + diff = (out["samples"] - video).abs().amax(dim=(0, 1, 2)) + assert not diff[:2].any() and not diff[:, :2].any() # far corners still untouched + +def test_refine_repair_mode_is_still_the_default(monkeypatch): + chain = _FakeChain() + latent, _ = _scene_latent() + _patch_detection_and_upsampler(monkeypatch) + _, note = detailing.detail_refine_scene( + chain, "model", "vae", "sampler", [], [], latent, "hands", object(), 0, 1.0) + assert "repair" in note + assert len(chain.chunk_calls) == 1 + def test_refine_round_trip_touches_only_the_tube(monkeypatch): chain = _FakeChain() latent, video = _scene_latent() From 9791cb8ccde6aa04cb90b3fc842d41abd6baac98 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:37:26 +0300 Subject: [PATCH 11/24] Add FunPack Easy Gen (simplified single-scene UI) and a /funpack Hub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Easy Gen is a new Fooocus-style frontend at /funpack/easy — big preview, one prompt box, Upload, Generate, and an Advanced settings panel — built entirely on the Movie Editor's existing project format and API (no new backend endpoints), so projects stay interchangeable between the two UIs. The Hub at /funpack links to both UIs and surfaces version/branch/commit plus an Update/Restart action reusing the Editor's git_update.py flow. Startup now prints full host:port URLs for all three (Hub, Editor, Easy Gen) via a new config.comfy_display_url() helper. Also: settings_window.js's About section now labels itself via window.FunPackAppName instead of hardcoding "Cutting Room", and Easy Gen's boot-error banner accumulates distinct errors instead of overwriting, so a cascading failure (e.g. dom.js not loading) surfaces its root cause instead of just the last script in load order. --- __init__.py | 8 + easy_gen/__init__.py | 1 + easy_gen/frontend/api.js | 319 +++++++++++++++++++++++ easy_gen/frontend/app.js | 130 +++++++++ easy_gen/frontend/easy.css | 62 +++++ easy_gen/frontend/easy_maintenance.js | 104 ++++++++ easy_gen/frontend/easy_settings.js | 93 +++++++ easy_gen/frontend/easy_store.js | 109 ++++++++ easy_gen/frontend/index.html | 91 +++++++ easy_gen/frontend/preview.js | 107 ++++++++ easy_gen/frontend/project_menu.js | 104 ++++++++ easy_gen/server.py | 74 ++++++ hub.py | 253 ++++++++++++++++++ movie_editor/backend/config.py | 24 ++ movie_editor/frontend/settings_window.js | 5 +- movie_editor/server.py | 2 +- 16 files changed, 1484 insertions(+), 2 deletions(-) create mode 100644 easy_gen/__init__.py create mode 100644 easy_gen/frontend/api.js create mode 100644 easy_gen/frontend/app.js create mode 100644 easy_gen/frontend/easy.css create mode 100644 easy_gen/frontend/easy_maintenance.js create mode 100644 easy_gen/frontend/easy_settings.js create mode 100644 easy_gen/frontend/easy_store.js create mode 100644 easy_gen/frontend/index.html create mode 100644 easy_gen/frontend/preview.js create mode 100644 easy_gen/frontend/project_menu.js create mode 100644 easy_gen/server.py create mode 100644 hub.py diff --git a/__init__.py b/__init__.py index c3741389..3c55cb3b 100644 --- a/__init__.py +++ b/__init__.py @@ -26,10 +26,18 @@ if __package__: from . import batch_training # noqa: F401 registers /funpack/batch/* routes except Exception as _e: print(f"[FunPack] batch_training routes unavailable: {_e}") + try: + from . import hub # noqa: F401 registers /funpack/ (the UI picker) + except Exception as _e: + print(f"[FunPack] Hub route unavailable: {_e}") try: from .movie_editor import server as _movie_editor_server # noqa: F401 registers /funpack/movie/* routes except Exception as _e: print(f"[FunPack] Movie Editor routes unavailable: {_e}") + try: + from . import easy_gen # noqa: F401 registers /funpack/easy/* routes + except Exception as _e: + print(f"[FunPack] Easy Gen routes unavailable: {_e}") else: # Standalone tests may not have the full ComfyUI/CUDA runtime loaded. from conditioning import ( diff --git a/easy_gen/__init__.py b/easy_gen/__init__.py new file mode 100644 index 00000000..57b35744 --- /dev/null +++ b/easy_gen/__init__.py @@ -0,0 +1 @@ +from . import server # noqa: F401 registers /funpack/easy/* routes diff --git a/easy_gen/frontend/api.js b/easy_gen/frontend/api.js new file mode 100644 index 00000000..98a03763 --- /dev/null +++ b/easy_gen/frontend/api.js @@ -0,0 +1,319 @@ +// Thin backend client — FORKED from movie_editor/frontend/api.js on purpose: +// Easy Gen is served from /funpack/easy/ but there is no /funpack/easy/api/* +// backend (see easy_gen/server.py) — Easy Gen reuses the Movie Editor's +// project/generate/media/models API as-is, so BASE must stay pinned to +// /funpack/movie regardless of the page's own URL. Keep every other method +// here in sync with the Editor's api.js; only this constant differs. +(function () { + const BASE = "/funpack/movie"; + const API = (p) => `${BASE}/api${p}`; + const FUNPACK = "/funpack"; + + async function funpackFetch(method, path, body) { + const opts = { method, headers: {}, cache: "no-store" }; + if (body !== undefined) { + opts.headers["Content-Type"] = "application/json"; + opts.body = JSON.stringify(body); + } + const res = await fetch(path, opts); + if (!res.ok) { + let payload = null; + try { payload = await res.json(); } catch (_) {} + throw new Error(readApiError(res, payload)); + } + return res; + } + + function readApiError(res, payload) { + if (payload && typeof payload === "object") { + const detail = payload.detail; + if (typeof detail === "string" && detail.trim()) return detail.trim(); + if (Array.isArray(detail)) { + const parts = detail.map((item) => { + if (item && typeof item === "object" && item.msg) return String(item.msg); + return JSON.stringify(item); + }).filter(Boolean); + if (parts.length) return parts.join("; "); + } + if (payload.error) return String(payload.error); + if (payload.parse_errors && typeof payload.parse_errors === "object") { + return Object.entries(payload.parse_errors).map(([k, v]) => `${k}: ${v}`).join("; "); + } + } + const status = res && res.status ? `HTTP ${res.status}` : ""; + const statusText = (res && res.statusText ? res.statusText : "").trim(); + return statusText || status || "Request failed"; + } + + async function j(method, url, body) { + const opts = { method, headers: {} }; + if (body !== undefined) { + opts.headers["Content-Type"] = "application/json"; + opts.body = JSON.stringify(body); + } + const res = await fetch(url, opts); + if (!res.ok) { + let payload = null; + try { payload = await res.json(); } catch (_) {} + throw new Error(readApiError(res, payload)); + } + return res.status === 204 ? null : res.json(); + } + + const ClientAPI = { + health: () => j("GET", API("/health")), + + // projects + listProjects: () => j("GET", API("/projects")), + createProject: (name) => j("POST", API("/projects"), { name }), + getProject: (id) => j("GET", API(`/projects/${id}`)), + saveProject: (id, data) => j("PUT", API(`/projects/${id}`), data), + deleteProject: (id) => j("DELETE", API(`/projects/${id}`)), + downloadProjectUrl: (id) => API(`/projects/${id}/download`), + importProject: (data) => j("POST", API("/projects/import"), data), + + // timeline preview + preview: (id, includeExcluded, forGeneration = true) => + j("GET", API(`/projects/${id}/preview?include_excluded=${includeExcluded ? "true" : "false"}&for_generation=${forGeneration ? "true" : "false"}`)), + parsePrompt: (id, prompt) => j("POST", API(`/projects/${id}/parse`), { prompt }), + + // libraries + transitions: () => j("GET", API("/library/transitions")), + saveTransition: (item) => j("POST", API("/library/transitions"), item), + deleteTransition: (name) => j("DELETE", API(`/library/transitions/${encodeURIComponent(name)}`)), + exportTransitionsUrl: () => API("/library/transitions/export"), + importTransitions: (data, mode) => j("POST", API(`/library/transitions/import?mode=${mode || "merge"}`), data), + clearTransitions: () => j("POST", API("/library/transitions/clear"), {}), + shortcuts: () => j("GET", API("/library/shortcuts")), + suggestionStats: () => j("GET", API("/library/suggestion_stats")), + saveCategory: (payload) => j("POST", API("/library/categories"), payload), + saveShortcut: (item) => j("POST", API("/library/shortcuts"), item), + deleteShortcut: (name) => j("DELETE", API(`/library/shortcuts/${encodeURIComponent(name)}`)), + exportShortcutsUrl: () => API("/library/shortcuts/export"), + importShortcuts: (data, mode) => j("POST", API(`/library/shortcuts/import?mode=${mode || "merge"}`), data), + clearShortcuts: () => j("POST", API("/library/shortcuts/clear"), {}), + revolverSettings: () => j("GET", API("/library/revolver")), + setRevolverSettings: (payload) => j("POST", API("/library/revolver"), payload), + + // FunPack file manager (Composer ▸ Files) + listFiles: () => j("GET", API("/files")), + deleteFile: (group, name) => j("DELETE", API(`/files/${encodeURIComponent(group)}/${encodeURIComponent(name)}`)), + clearFiles: (group) => j("POST", API(`/files/${encodeURIComponent(group)}/clear`), {}), + nleLibrary: () => j("GET", API("/library/nle")), + + // media bin + listMedia: () => j("GET", API("/media")), + mediaUrl: (id) => API(`/media/${encodeURIComponent(id)}`), + deleteMedia: (id) => j("DELETE", API(`/media/${encodeURIComponent(id)}`)), + renameMedia: (id, name) => j("PATCH", API(`/media/${encodeURIComponent(id)}`), { name }), + importClipToMediaBin: (clip, name) => j("POST", API("/media/import-clip"), { clip, name: name || null }), + async uploadMedia(file) { + const fd = new FormData(); fd.append("file", file, file.name); + const res = await fetch(API("/media"), { method: "POST", body: fd }); + if (!res.ok) throw new Error((await res.json().catch(() => ({}))).detail || res.statusText); + return res.json(); + }, + + // models / node slots + nodeRoles: () => j("GET", API("/node-roles")), + nodeCandidates: (role, refresh) => j("GET", API(`/node-candidates/${role}${refresh ? "?refresh=true" : ""}`)), + allNodes: () => j("GET", API("/all-nodes")), + nodeSpec: (cls) => j("GET", API(`/node/${encodeURIComponent(cls)}`)), + pipelinePorts: () => j("GET", API("/pipeline-ports")), + pipelineDeps: () => j("GET", API("/pipeline-deps")), + pipelineDepsInstall: (packIds) => j("POST", API("/pipeline-deps/install"), { pack_ids: packIds }), + pipelineDepsInstallManager: () => j("POST", API("/pipeline-deps/install"), { install_manager: true }), + pipelineDepsInstallStatus: (jobId) => j("GET", API(`/pipeline-deps/install/${encodeURIComponent(jobId)}`)), + pipelineDepsInstallCancel: (jobId) => j("POST", API(`/pipeline-deps/install/${encodeURIComponent(jobId)}/cancel`), {}), + imageTargets: (pid) => j("GET", API("/image-targets" + (pid ? `?pid=${encodeURIComponent(pid)}` : ""))), + // Models config is per-project; pass the project id. Falls back to the global + // default route (used as the seed/template) when no project is given. + coreGraph: (pid) => j("GET", API("/core-graph" + (pid ? `?pid=${encodeURIComponent(pid)}` : ""))), + getModels: (pid) => j("GET", API(pid ? `/projects/${pid}/models` : "/models")), + saveModels: (pid, data) => j("PUT", API(pid ? `/projects/${pid}/models` : "/models"), data), + refreshModels: () => j("POST", API("/models/refresh")), + parseWorkflow: (workflow) => j("POST", API("/workflow/parse"), { workflow }), + applyWorkflow: (pid, workflow, bindings) => + j("POST", API(`/projects/${pid}/workflow/apply`), { workflow, bindings }), + restart: () => j("POST", API("/restart")), + gitStatus: () => j("GET", API("/git/status")), + gitUpdate: (branch) => j("POST", API("/git/update"), branch ? { branch } : {}), + gitCheckout: (branch) => j("POST", API("/git/checkout"), { branch }), + + // generate (a single scene, or an explicit run of scene ids = one chain request) + generate: (id, onlyScene, sceneIds, resetSession, nodeOverrides) => + j("POST", API(`/projects/${id}/generate`), { only_scene: onlyScene || null, scene_ids: sceneIds || null, reset_session: !!resetSession, node_overrides: nodeOverrides || null }), + status: (id, promptId) => j("GET", API(`/projects/${id}/status/${promptId}`)), + progress: () => j("GET", API("/progress")), + // The editor's own in-flight generation, recovered from ComfyUI's queue (survives a UI reload). + active: () => j("GET", API("/active")), + ratingLabels: () => j("GET", API("/rating-labels")), + log: (limit) => j("GET", API("/log" + (limit ? `?limit=${limit}` : ""))), + interrupt: () => j("POST", API("/interrupt")), + + // ComfyUI temp folder browser (scene previews & other transient outputs, wiped on restart). + listTemp: () => j("GET", API("/temp")), + // Served by ComfyUI's own same-origin /view endpoint — no editor route needed. + tempFileUrl: (f) => "/view?" + new URLSearchParams({ + filename: f.filename, subfolder: f.subfolder || "", type: "temp", t: String(f.mtime || Date.now()), + }).toString(), + async downloadTempFile(f) { + const res = await fetch(this.tempFileUrl(f), { cache: "no-store" }); + if (!res.ok) throw new Error(`Could not fetch ${f.filename} (HTTP ${res.status})`); + const blob = await res.blob(); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = f.filename; + document.body.appendChild(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); + }, + renderFinal: (id, clips) => j("POST", API(`/projects/${id}/render`), { clips }), + renderFinalStatus: (id, jobId) => j("GET", API(`/projects/${id}/render/${encodeURIComponent(jobId)}`)), + exportClip: (id, clip) => j("POST", API(`/projects/${id}/export-clip`), { clip }), + exportClipsCombined: (id, clips) => j("POST", API(`/projects/${id}/export-clips`), { clips }), + exportClipsStatus: (id, jobId) => j("GET", API(`/projects/${id}/export-clips/${encodeURIComponent(jobId)}`)), + resultUrl: (id, m) => + API(`/projects/${id}/result?filename=${encodeURIComponent(m.filename)}`) + + `&subfolder=${encodeURIComponent(m.subfolder || "")}&type=${encodeURIComponent(m.type || "output")}`, + previewSegmentUrl: (id, sceneId, spec) => { + let u = API(`/projects/${id}/preview-segment/${encodeURIComponent(sceneId)}`); + const m = spec?.media; + if (m?.filename) { + const q = new URLSearchParams({ + filename: m.filename, + subfolder: m.subfolder || "", + type: m.type || "output", + render_in: String(spec.renderIn != null ? spec.renderIn : 0), + }); + // dur is in the URL for two reasons: ghosts have no scene server-side (the trim + // window must travel in the query), and it versions the browser cache — segments + // are cached for an hour, so a timeline trim must produce a different URL. + if (spec.dur != null) q.set("dur", String(spec.dur)); + u += "?" + q.toString(); + } + return u; + }, + + // FunPack refinement keys (ComfyUI root routes, same as Studio / Refinement Key Loader) + refinementKeys: async () => { + const res = await funpackFetch("GET", `${FUNPACK}/refinement_keys?cache_bust=${Date.now()}`); + return res.json(); + }, + importRefinementKey: async (data, opts = {}) => { + const ow = opts.overwrite ? "&overwrite=true" : ""; + const json = JSON.stringify(data); + const bytes = new TextEncoder().encode(json); + // A 409 means a key of that name already exists — surface it as a tagged + // error so the caller can ask the user to overwrite (then retry). + const finish = async (res) => { + if (res.status === 409) { + let p = null; try { p = await res.json(); } catch (_) {} + const err = new Error((p && p.error) || "Refinement key already exists."); + err.exists = true; err.key = (p && p.key) || ""; + throw err; + } + if (!res.ok) { + let p = null; try { p = await res.json(); } catch (_) {} + throw new Error(readApiError(res, p)); + } + return res.json(); + }; + // Reverse proxies on Vast.ai / Runpod cap the request body well below + // ComfyUI's 100 MB, so a one-shot POST of a large key gets HTTP 413 at the + // proxy. Keep a fast single POST for small keys; stream big ones in chunks + // (each far under any proxy limit) so they can never be rejected. + const CHUNK = 256 * 1024; + if (bytes.length <= CHUNK) { + const res = await fetch(`${FUNPACK}/refinement_keys/import?_=${Date.now()}${ow}`, { + method: "POST", cache: "no-store", + headers: { "Content-Type": "application/json" }, body: json, + }); + if (res.status !== 413) return finish(res); // 413 → fall through to chunked + } + const uploadId = `${Date.now()}_${Math.random().toString(36).slice(2)}`; + const postChunk = async (path, body) => { + const res = await fetch(path, { + method: "POST", cache: "no-store", + headers: { "Content-Type": "application/octet-stream" }, body, + }); + if (!res.ok) { + let payload = null; + try { payload = await res.json(); } catch (_) {} + throw new Error(readApiError(res, payload)); + } + return res; + }; + let index = 0; + for (let off = 0; off < bytes.length; off += CHUNK, index++) { + // Slice by bytes (not string chars) so a multi-byte char is never split + // across the seam where the two halves get re-encoded independently. + await postChunk( + `${FUNPACK}/refinement_keys/import_chunk?upload_id=${encodeURIComponent(uploadId)}&index=${index}`, + bytes.slice(off, off + CHUNK), + ); + } + return finish(await fetch( + `${FUNPACK}/refinement_keys/import_finalize?upload_id=${encodeURIComponent(uploadId)}${ow}`, + { method: "POST", cache: "no-store" }, + )); + }, + async exportRefinementKeyFile(key) { + const res = await fetch( + `${FUNPACK}/refinement_keys/export?key=${encodeURIComponent(key)}&cache_bust=${Date.now()}`, + { cache: "no-store" }, + ); + if (!res.ok) { + let payload = null; + try { payload = await res.json(); } catch (_) {} + throw new Error(readApiError(res, payload)); + } + const blob = await res.blob(); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = `${key}.json`; + document.body.appendChild(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); + }, + + // Atomically delete a refinement key and ALL its sidecars (value function, + // blessed attention / K-V banks, creativity latent, velocity store). Deleting + // only the visible .json orphans those, and they keep steering future runs + // across restarts — the backend sweep is what makes "key deleted" mean it. + async deleteRefinementKey(key) { + const res = await funpackFetch( + "POST", `${FUNPACK}/refinement_keys/delete?key=${encodeURIComponent(key)}&_=${Date.now()}`, + ); + if (!res.ok) { + let payload = null; + try { payload = await res.json(); } catch (_) {} + throw new Error(readApiError(res, payload)); + } + return res.json(); + }, + + // The keyless Absolute "global taste" store — learns from every rated generation + // across all prompts, invisible in the key list, applied only in absolute/both + // steer mode, and otherwise only wiped by Session Reset. + async absoluteStoreInfo() { + const res = await funpackFetch("GET", `${FUNPACK}/refinement_keys/absolute?_=${Date.now()}`); + return res.json(); + }, + async clearAbsoluteStore() { + const res = await funpackFetch("POST", `${FUNPACK}/refinement_keys/clear_absolute?_=${Date.now()}`); + if (!res.ok) { + let payload = null; + try { payload = await res.json(); } catch (_) {} + throw new Error(readApiError(res, payload)); + } + return res.json(); + }, + }; + + window.MovieEditorAPI = ClientAPI; +})(); diff --git a/easy_gen/frontend/app.js b/easy_gen/frontend/app.js new file mode 100644 index 00000000..af5f3aef --- /dev/null +++ b/easy_gen/frontend/app.js @@ -0,0 +1,130 @@ +// Easy Gen entrypoint: wires the big-preview / prompt / upload / generate bar +// to the shared easy_store.js + preview.js, and opens the "FunPack Easy Gen" +// project picker on boot. No re-attach-on-reload support yet (the Editor's +// /api/active flow) — a reload while a run is in flight loses the progress +// view, though the run itself still completes server-side. +(function () { + const S = window.Store; + const API = window.MovieEditorAPI; + + const LAST_PROJECT_KEY = "funpack_easy_last_project"; + + const nameEl = document.getElementById("easy-project-name"); + const healthEl = document.getElementById("easy-health-chip"); + const promptEl = document.getElementById("easy-prompt"); + const uploadBtn = document.getElementById("easy-upload-btn"); + const uploadInput = document.getElementById("easy-upload-input"); + const generateBtn = document.getElementById("easy-generate-btn"); + const advancedBtn = document.getElementById("easy-advanced-btn"); + const projectsBtn = document.getElementById("easy-projects-btn"); + + const preview = window.EasyPreview.create(document.getElementById("easy-preview")); + + let saveTimer = null; + function scheduleSave() { + if (saveTimer) clearTimeout(saveTimer); + saveTimer = setTimeout(() => { S.save().catch(() => {}); }, 800); + } + + let promptFocused = false; + promptEl.addEventListener("focus", () => { promptFocused = true; }); + promptEl.addEventListener("blur", () => { promptFocused = false; }); + promptEl.addEventListener("input", () => { + S.setPromptText(promptEl.value); + scheduleSave(); + }); + + let lastProjectId = null; + function render(st) { + const p = st.project; + nameEl.textContent = p ? p.name : "No project"; + generateBtn.disabled = !p; + uploadBtn.disabled = !p; + if (p && !promptFocused) { + const text = p.scenes[0]?.text || ""; + if (promptEl.value !== text) promptEl.value = text; + } + uploadBtn.textContent = p && p.scenes[0]?.source?.type !== "empty" ? "⬆ Uploaded ✕" : "⬆ Upload"; + const pid = p ? p.id : null; + if (pid !== lastProjectId) { + lastProjectId = pid; + preview.stopPolling(); + preview.showEmpty(); + generateBtn.textContent = "Generate"; + } + } + S.subscribe(render); + + uploadBtn.onclick = () => { + const st = S.get(); + const attached = st.project && st.project.scenes[0]?.source?.type !== "empty"; + if (attached) { + S.setSceneMedia(null, null); + scheduleSave(); + return; + } + uploadInput.click(); + }; + + uploadInput.onchange = async () => { + const file = uploadInput.files?.[0]; + uploadInput.value = ""; + if (!file) return; + uploadBtn.disabled = true; + uploadBtn.textContent = "Uploading…"; + try { + const media = await API.uploadMedia(file); + const kind = file.type.startsWith("video") ? "video" : "image"; + S.setSceneMedia(media.id, kind); + await S.save(); + } catch (e) { + alert("Upload failed: " + (e.message || e)); + } finally { + render(S.get()); + } + }; + + generateBtn.onclick = async () => { + const st = S.get(); + if (!st.project) return; + if (!(st.project.scenes[0]?.text || "").trim()) { + alert("Type a prompt first."); + return; + } + generateBtn.disabled = true; + generateBtn.textContent = "Generating…"; + try { + await S.save(); + const resetSession = S.takeResetSessionFlag(); + const res = await API.generate(st.project.id, null, null, resetSession, null); + preview.watch(st.project.id, res.prompt_id, { + onDone: () => { generateBtn.disabled = false; generateBtn.textContent = "Generate"; }, + }); + } catch (e) { + preview.showError("Generate failed: " + (e.message || e)); + generateBtn.disabled = false; + generateBtn.textContent = "Generate"; + } + }; + + advancedBtn.onclick = () => window.SettingsWindow.open("generation"); + projectsBtn.onclick = () => window.ProjectMenu.open({ dismissable: !!S.get().project }); + + async function boot() { + S.refreshHealth(); + setInterval(() => S.refreshHealth(), 15000); + S.subscribe((st) => { + healthEl.textContent = st.health?.ok ? "● online" : "● offline"; + healthEl.classList.toggle("bad", !(st.health && st.health.ok)); + if (st.project) localStorage.setItem(LAST_PROJECT_KEY, st.project.id); + }); + + const lastId = localStorage.getItem(LAST_PROJECT_KEY); + if (lastId) { + try { await S.loadProject(lastId); return; } catch (_) { /* fall through to picker */ } + } + window.ProjectMenu.open({ dismissable: false }); + } + + boot(); +})(); diff --git a/easy_gen/frontend/easy.css b/easy_gen/frontend/easy.css new file mode 100644 index 00000000..da127435 --- /dev/null +++ b/easy_gen/frontend/easy.css @@ -0,0 +1,62 @@ +/* Easy Gen layout on top of the Editor's shared styles.css (design tokens, + .btn/.modal/.sw-*/.welcome-* rules) — big preview, prompt bar, nothing else. */ + +html, body.easy-body { + margin: 0; height: 100%; background: var(--ink-0); color: var(--text); + font-family: var(--font-ui); +} + +.easy-app { + height: 100vh; display: flex; flex-direction: column; +} + +.easy-top { + height: var(--menubar-h); flex: none; display: flex; align-items: center; gap: 14px; + padding: 0 16px; border-bottom: 1px solid var(--line); background: var(--ink-1); +} +.easy-project-name { color: var(--muted); font-size: 13px; } +.easy-top .easy-health-chip { margin-left: auto; } +a.brand { text-decoration: none; color: inherit; } +a.brand:hover .brand-sub { color: var(--amber-2); } + +.easy-preview { + flex: 1; min-height: 0; display: flex; align-items: center; justify-content: center; + background: var(--ink-0); position: relative; padding: 24px; +} +.easy-preview-inner { + width: 100%; height: 100%; display: flex; align-items: center; justify-content: center; +} +.easy-preview-inner img, .easy-preview-inner video { + max-width: 100%; max-height: 100%; border-radius: var(--radius); box-shadow: var(--shadow); + background: #000; +} +.easy-preview-empty { + color: var(--faint); font-size: 14px; font-family: var(--font-display); +} +.easy-preview-error { color: var(--danger); max-width: 60ch; text-align: center; white-space: pre-wrap; } + +.easy-progress { + position: absolute; left: 24px; right: 24px; bottom: 24px; + background: var(--ink-1); border: 1px solid var(--line); border-radius: var(--radius); + padding: 10px 14px; box-shadow: var(--shadow); +} +.easy-progress-bar { + height: 6px; border-radius: 4px; background: linear-gradient(90deg, var(--amber), var(--amber-2)); + width: 0%; transition: width .2s ease; margin-bottom: 6px; +} +.easy-progress-label { font-size: 12px; color: var(--muted); font-family: var(--font-mono); } + +.easy-bar { + flex: none; display: flex; align-items: flex-end; gap: 12px; + padding: 14px 16px; border-top: 1px solid var(--line); background: var(--ink-1); +} +.easy-upload-btn { flex: none; align-self: stretch; padding: 0 16px; } +.easy-prompt { + flex: 1; resize: none; background: var(--ink-2); border: 1px solid var(--line); + border-radius: var(--radius); color: var(--text); font: 14px/1.4 var(--font-ui); + padding: 10px 12px; +} +.easy-prompt:focus { outline: none; border-color: var(--amber); } +.easy-actions { flex: none; display: flex; flex-direction: column; align-items: stretch; gap: 6px; } +.easy-generate-btn { padding: 10px 26px; font-size: 14px; } +.easy-advanced-btn { align-self: center; } diff --git a/easy_gen/frontend/easy_maintenance.js b/easy_gen/frontend/easy_maintenance.js new file mode 100644 index 00000000..697547a0 --- /dev/null +++ b/easy_gen/frontend/easy_maintenance.js @@ -0,0 +1,104 @@ +// window.FunPackMaintenance shim — ports the five actions menubar.js exposes +// for the Settings window's built-in "Refinement & Taste" / "Updates & ComfyUI" +// sections (settings_window.js), without pulling in the other ~300 lines of +// menubar.js (menu rendering, keyboard shortcuts, timeline-specific state). +// Same API calls, same confirm/alert flows as the Editor. +(function () { + const API = window.MovieEditorAPI; + + function projectRefinementKey() { + return window.Store?.get().project?.refinement_key || ""; + } + + const refinementKeyImportInput = (function () { + const inp = document.createElement("input"); + inp.type = "file"; inp.accept = ".json,application/json"; inp.style.display = "none"; + inp.onchange = async () => { + const file = inp.files?.[0]; + inp.value = ""; + if (!file) return; + try { + const data = JSON.parse(await file.text()); + let res; + try { + res = await API.importRefinementKey(data); + } catch (e) { + if (!e.exists) throw e; + if (!confirm(`A refinement key named "${e.key}" already exists. Overwrite it?`)) return; + res = await API.importRefinementKey(data, { overwrite: true }); + } + alert(`Imported refinement key "${res.key}".`); + } catch (e) { + alert("Refinement key import failed: " + (e.message || e)); + } + }; + document.body.appendChild(inp); + return inp; + })(); + + async function exportRefinementKey() { + let payload; + try { payload = await API.refinementKeys(); } + catch (e) { alert("Could not list refinement keys: " + (e.message || e)); return; } + const names = (payload.keys || []).filter((k) => k && k !== "-None-"); + if (!names.length) { alert("No refinement keys on disk yet."); return; } + const cur = projectRefinementKey(); + if (!window.SlotPicker) return; + window.SlotPicker.open({ + title: "Export refinement key", + options: names.map((k) => ({ value: k, label: k, hint: k === cur ? "project session" : "" })), + onPick: async (key) => { + try { await API.exportRefinementKeyFile(key); } + catch (e) { alert("Refinement key export failed: " + (e.message || e)); } + }, + }); + } + + async function deleteRefinementKey() { + let payload; + try { payload = await API.refinementKeys(); } + catch (e) { alert("Could not list refinement keys: " + (e.message || e)); return; } + const names = (payload.keys || []).filter((k) => k && k !== "-None-"); + if (!names.length) { alert("No refinement keys on disk yet."); return; } + const cur = projectRefinementKey(); + if (!window.SlotPicker) return; + window.SlotPicker.open({ + title: "Delete refinement key", + options: names.map((k) => ({ value: k, label: k, hint: k === cur ? "project session" : "" })), + onPick: async (key) => { + if (!confirm(`Delete refinement key "${key}"?\n\nThis removes its learned state AND all sidecars (value function, blessed attention/K-V banks, creativity latent, velocity memory). This cannot be undone.`)) { + return; + } + try { await API.deleteRefinementKey(key); alert(`Deleted "${key}".`); } + catch (e) { alert("Delete failed: " + (e.message || e)); } + }, + }); + } + + async function clearGlobalTaste() { + let info; + try { info = await API.absoluteStoreInfo(); } + catch (e) { alert("Could not read the global-taste store: " + (e.message || e)); return; } + if (!info.exists) { alert("The Absolute global-taste store is already empty."); return; } + if (!confirm(`Clear the Absolute global-taste store?\n\nIt has pooled ${info.total_iterations} rated generation(s) (${info.liked_count} liked / ${info.bad_count} disliked directions) across all prompts. It learns from every rated run and is applied only in absolute/both steer mode. This cannot be undone.`)) { + return; + } + try { await API.clearAbsoluteStore(); alert("Cleared the Absolute global-taste store."); } + catch (e) { alert("Clear global taste failed: " + (e.message || e)); } + } + + async function restartComfy() { + if (!confirm("Restart ComfyUI now?\n\nThe server will be down for ~10-40s and any running generation will be lost. This page reloads automatically when it's back.")) return; + const msg = window.FunPackRestart?.showOverlay?.("Restarting ComfyUI…\nThis page will reload when it's back."); + try { await API.restart(); } catch (_) { /* connection drops as it execv's - expected */ } + window.FunPackRestart?.waitForReload?.(msg, Date.now()); + } + + window.FunPackMaintenance = { + exportRefinementKey, + importRefinementKeyFile: () => refinementKeyImportInput.click(), + deleteRefinementKey, + clearGlobalTaste, + restartComfy, + }; +})(); diff --git a/easy_gen/frontend/easy_settings.js b/easy_gen/frontend/easy_settings.js new file mode 100644 index 00000000..102a5fc5 --- /dev/null +++ b/easy_gen/frontend/easy_settings.js @@ -0,0 +1,93 @@ +// Easy Gen's own slimmed "Generation" Settings section — modeled on the +// field(label, control, hint) row pattern in movie_editor/frontend/engine_settings.js, +// but limited to plain Project-level fields (no scene-chain/continuity/overlay +// knobs, which only make sense with multiple scenes in the full Editor). +(function () { + const { el } = window.dom; + + function field(labelText, control, hint) { + const row = el("div", "sw-row eng-field"); + const main = el("div", "sw-row-main"); + main.append(el("div", "sw-row-title", labelText)); + if (hint) main.append(el("div", "sw-row-hint", hint)); + row.append(main, control); + return row; + } + + function numberInput(value, onChange, opts = {}) { + const inp = el("input", "sw-input"); + inp.type = "number"; + if (opts.min != null) inp.min = opts.min; + if (opts.step != null) inp.step = opts.step; + inp.value = value; + inp.style.width = "90px"; + inp.onchange = () => onChange(Number(inp.value)); + return inp; + } + + function textInput(value, onChange, placeholder) { + const inp = el("input", "sw-input"); + inp.type = "text"; + inp.value = value || ""; + if (placeholder) inp.placeholder = placeholder; + inp.style.width = "220px"; + inp.onchange = () => onChange(inp.value); + return inp; + } + + function mount(body) { + const S = window.Store; + const wrap = el("div", "sw-stack"); + body.append(wrap); + + function render() { + body.innerHTML = ""; + body.append(wrap); + wrap.innerHTML = ""; + const st = S.get(); + const p = st.project; + if (!p) { + wrap.append(el("div", "sw-hint", "Open or create a project first.")); + return; + } + const commit = async (mut) => { mut(p); await S.save(); }; + + wrap.append(el("div", "sw-rows-label", "Output")); + const out = el("div", "sw-rows"); + out.append(field("Width", numberInput(p.width, (v) => commit((pr) => pr.width = v), { min: 64, step: 16 }))); + out.append(field("Height", numberInput(p.height, (v) => commit((pr) => pr.height = v), { min: 64, step: 16 }))); + out.append(field("Frames", numberInput(p.num_frames_per_scene, (v) => commit((pr) => pr.num_frames_per_scene = v), { min: 9, step: 8 }), + "Length of the generated clip, in frames.")); + out.append(field("Frame rate", numberInput(p.frame_rate, (v) => commit((pr) => pr.frame_rate = v), { min: 1 }))); + wrap.append(out); + + wrap.append(el("div", "sw-rows-label", "Prompt")); + const pr = el("div", "sw-rows"); + pr.append(field("Negative prompt", textInput(p.negative_prompt, (v) => commit((proj) => proj.negative_prompt = v)), + "Passed to the negative conditioning for every run.")); + wrap.append(pr); + + wrap.append(el("div", "sw-rows-label", "Seed & learning")); + const sd = el("div", "sw-rows"); + sd.append(field("Seed", numberInput(p.seed, (v) => commit((proj) => proj.seed = v), { min: 0 }), + "Fixed seed for this project's runs.")); + sd.append(field("Refinement key", textInput(p.refinement_key, (v) => commit((proj) => proj.refinement_key = v || "default")), + "Named session FunPack learns from your ratings under. \"default\" = the keyless store. " + + "Manage keys under Refinement & Taste.")); + wrap.append(sd); + } + + const unsub = S.subscribe(render); + render(); + return () => unsub(); + } + + window.SettingsWindow.register({ + id: "generation", group: "Generation", order: 1, title: "Generation", + subtitle: "Output size, length, seed, and refinement key for this project.", + keywords: "width height frames frame rate seed negative prompt refinement key generation", + iconBg: "linear-gradient(180deg,#ffb64d,#e07f1f)", + icon: '', + mount, + }); +})(); diff --git a/easy_gen/frontend/easy_store.js b/easy_gen/frontend/easy_store.js new file mode 100644 index 00000000..fa4cbee1 --- /dev/null +++ b/easy_gen/frontend/easy_store.js @@ -0,0 +1,109 @@ +// Minimal state container for Easy Gen — NOT the Editor's store.js (that's a +// 4000+ line timeline/composer/overlay engine tightly coupled to the full NLE +// DOM). Easy Gen only ever holds one project with exactly one scene, so this +// is a small pub-sub wrapper around the same /funpack/movie/api/* endpoints. +// +// Exposed as window.Store (same global name the Editor uses) purely so +// reused sections — models.js, and the built-in "Refinement & Taste" / +// "Updates & ComfyUI" sections baked into settings_window.js — work +// unmodified: they all read window.Store.get().project/.health and call +// window.Store.subscribe(fn) without needing to know which app they're in. +(function () { + const API = window.MovieEditorAPI; + + const state = { + project: null, + health: null, + resetSessionArmed: false, + }; + const listeners = new Set(); + + function get() { return state; } + + function notify() { listeners.forEach((fn) => { try { fn(state); } catch (_) {} }); } + + function subscribe(fn) { + listeners.add(fn); + return () => listeners.delete(fn); + } + + function set(patch) { + Object.assign(state, patch); + notify(); + } + + async function refreshHealth() { + try { set({ health: await API.health() }); } catch (_) { set({ health: { ok: false } }); } + } + + function emptyScene() { + return { text: "", source: { type: "empty" } }; + } + + async function listProjects() { + return (await API.listProjects()).projects || []; + } + + async function createProject(name) { + const p = await API.createProject(name || "Untitled"); + p.scenes = [emptyScene()]; + const saved = await API.saveProject(p.id, p); + set({ project: saved }); + return saved; + } + + async function loadProject(id) { + const p = await API.getProject(id); + if (!p.scenes || !p.scenes.length) p.scenes = [emptyScene()]; + set({ project: p }); + return p; + } + + async function deleteProject(id) { + await API.deleteProject(id); + if (state.project && state.project.id === id) set({ project: null }); + } + + // Local-only edit — call save() to persist. Keeps the prompt box from + // round-tripping to the server on every keystroke. + function setPromptText(text) { + if (!state.project) return; + state.project.scenes[0].text = text; + notify(); + } + + // kind: "image" | "video" | null (null clears the upload → pure t2v). + function setSceneMedia(mediaId, kind) { + if (!state.project) return; + state.project.scenes[0].source = mediaId + ? { type: kind === "video" ? "v2v" : "image", media_ref: mediaId } + : { type: "empty" }; + notify(); + } + + async function save() { + if (!state.project) return; + const saved = await API.saveProject(state.project.id, state.project); + set({ project: saved }); + return saved; + } + + function resetStudioSession() { + set({ resetSessionArmed: !state.resetSessionArmed }); + } + + // Consumed by the Generate flow: reset_session applies to the NEXT run only. + function takeResetSessionFlag() { + const armed = state.resetSessionArmed; + if (armed) set({ resetSessionArmed: false }); + return armed; + } + + window.Store = { + get, subscribe, set, + refreshHealth, + listProjects, createProject, loadProject, deleteProject, + setPromptText, setSceneMedia, save, + resetStudioSession, takeResetSessionFlag, + }; +})(); diff --git a/easy_gen/frontend/index.html b/easy_gen/frontend/index.html new file mode 100644 index 00000000..064a677d --- /dev/null +++ b/easy_gen/frontend/index.html @@ -0,0 +1,91 @@ + + + + + + FunPack · Easy Gen + + + + + + + + +
+
+ ◉ FunPack Easy Gen +
No project
+ + +
+ +
+ +
+ + +
+ + +
+
+ +
+ + + + + + + + + + + + + + + + + + diff --git a/easy_gen/frontend/preview.js b/easy_gen/frontend/preview.js new file mode 100644 index 00000000..f38518e7 --- /dev/null +++ b/easy_gen/frontend/preview.js @@ -0,0 +1,107 @@ +// Big preview pane: plain