diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e4f3a5b..76c8d39a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,73 @@ # Changelog -## [Unreleased] +## [3.3.0] - 2026-07-15 + +### Added +- **Loop temporal style** - the "loop" option in Temporal style (inert since it was added) + now produces seamlessly looping clips, the LTX2.3 equivalent of WanVideoWrapper's Loop + Args: on eligible denoise steps the latent is cyclically rolled along time before the + model forward and the prediction unrolled after, so the model repeatedly smooths the + video's seam (last frame | first frame) as if it were an ordinary interior cut. Video and + audio roll by their own frame counts and stay time-aligned; i2v anchor masks roll in step. + Near-zero overhead (no extra forward passes). Also honored by the "auto" director when a + scene's prompt asks for a loop ("seamless loop", "cycling", "repeating"). On scenes that + carry guides (mid-scene guide, carry-i2v-guides, JoyAI memory, custom guide stacks) the + roll stays off - guide frames are pinned to absolute positions - and the run log shows + `temporal_loop_roll(inert: guides pin frames)`. +- **Plateau step-cache** (experimental, Engine ▸ Experimental) - reuses the base model's + forward across the near-noise plateau steps of the LTX 8-step schedule (sigma >= threshold, + default 0.975), skipping 3-4 of 8 forwards per scene for a negative-overhead speedup. + Guidance wrappers still post-process every step's prediction. +- **Wired widget locks** - node widget fields fed by an incoming wire are now locked in the + Models & Pipeline editor and show which node feeds them, instead of silently accepting + edits that the wire would overwrite on the next run. +- **Per-prompt taste direction** (`taste_nearest_prompt`, experimental, Engine ▸ Guidance) — + a retrieval layer over the learned taste steering. Instead of pushing Embed guidance / + Score slider along the single global liked-direction average, each liked rating now also + records a `(prompt fingerprint → that run's liked direction)` entry, and with this on each + scene retrieves the similarity-weighted direction of its nearest rated prompts (a forest + prompt pulls what worked on forests, not the mean across every prompt). Non-parametric + (a plain ring buffer keyed on pooled conditioning) — no extra model forward, just a cosine + lookup + vector mean, and no small value function to collapse into a spurious attractor. + Falls back to the global liked direction when nothing rated is close enough. Rides the same + refine_v2 clip-state file as `liked_dir`, so it's removed atomically with the key. + +### Changed +- **Best-FaceID identity transfer** (experimental, Engine ▸ Experimental) is now a full native + port of ComfyUI-BFSNodes' LTX Identity Transfer overlap+source_phase+ArcFace conditioning, + replacing 3.2.1's source-phase-only approximation. The reference face is now injected as + separate, non-rendered overlap tokens (never blended into a real frame) instead of tagging + Continuity's Identity pin keyframe, plus an optional ArcFace projector channel (`identity_projector`, + `id_strength`, `arcface_mode`) for a second identity signal. Still keyed off Continuity's + existing Identity pin image — no new picker. Requires the `insightface` package (added to + requirements.txt; downloads the buffalo_l model on first use) only when an ArcFace projector + is selected — the overlap tokens alone work without it. +- Frame overlap now carries through anchor changes (hard scene cuts with a new anchor image + used to drop the latent overlap; `carry_overlap_through_anchor`), and an Identity pin + configured on an anchor-swap scene resolves again instead of being skipped. + +### Fixed +- **Preview stability (rounds 3-5)** - the remaining causes of stale / black / undecodable + scene previews, especially "the second half stops loading": + - `/result` faststart-remuxes raw ComfyUI renders (VHS writes the MP4 index at the end of + the file; browsers cannot seek that) and never serves a video mid-write - a file whose + index isn't on disk yet answers 503 until the save finishes, instead of undecodable bytes. + - A failed remux answers 502 with rate-limited re-attempts instead of permanently falling + back to the raw file; temp-dir failures and the loopback fallback path can no longer + serve raw index-at-end video either (the fallback also gains Range/seek support). + - The player materializes video elements only in a window around the playhead instead of + one per timeline clip - a montage-scale timeline (100+ clips) used to create 100+ + elements and fire ~180 preview-segment encodes at once, starving the browser's + per-origin connections (and hitting Chrome's hard 75-media-players-per-page cap), which + left the back half of the timeline permanently "loading". Timeline filmstrips load + through a small shared queue for the same reason. + - The monitor shows honest readiness slates ("Video is processing…" / "Video is loading…" / + "Preview unavailable") instead of a black or blinking frame, and a long GIL-bound save + can no longer exhaust the retry budget. +- Exposed controls silently reverting: the autosave snapshot was taken before edited model + values synced back to the project, so an exposed-widget edit could be undone by its own + autosave. +- ALG crash on multi-scene runs (tensor truthiness in the `alg_latents` fallback). ## [3.2.1] - 2026-07-06 diff --git a/conditioning.py b/conditioning.py index 3cf89fc4..9d2baf29 100644 --- a/conditioning.py +++ b/conditioning.py @@ -9839,6 +9839,45 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): except Exception: pass + def _v2_store_prompt_keyed_direction(self, global_state, payload, session_mean_payload, + max_entries=32): + """Append a (prompt fingerprint -> liked direction) entry to a retrieval index. + + Stores the SAME unit-delta direction that feeds the global liked_dir slot + (pool(payload) minus the session source mean), paired with this run's pooled + conditioning as the prompt fingerprint, so the sampler can later retrieve the + direction learned on the NEAREST prompts instead of the one global average (see + samplers._resolve_prompt_keyed_direction). The fingerprint is pool(payload) — the + processed conditioning, which is exactly what the Chain Sampler pools from its + scene input at retrieval time, so cosine matching lines up on the same domain. + Non-parametric memory: a plain ring buffer, no model to train, so it can't collapse + into a spurious attractor the way a small value function can. Oldest rolls off; + rides the same clip-state file as liked_dir, so delete_refinement_key removes it + with the key. Silent no-op on any shape/None issue.""" + pooled = self._v2_pool_conditioning(payload) + session_mean = self._v2_pool_conditioning(session_mean_payload) + if pooled is None or session_mean is None: + return + try: + if list(session_mean.shape) != list(pooled.shape): + return + delta = pooled - session_mean.to(pooled.device) + magnitude = float(delta.norm().item()) + if magnitude < 1e-6: + return + unit = delta / magnitude + index = global_state.get("prompt_dir_index") + if not isinstance(index, list): + index = [] + index.append({ + "prompt": tensor_to_serializable(pooled.cpu()), + "direction": tensor_to_serializable(unit.cpu()), + "direction_magnitude": magnitude, + }) + global_state["prompt_dir_index"] = index[-int(max_entries):] + except Exception: + pass + def _v2_apply_direction(self, mixed, slot, strength, negate=False): """Apply stored unit direction to active tokens using NORM_SCALE calibration. Requires direction_count >= 3. Falls back silently if not ready.""" @@ -9936,6 +9975,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): if key in {"like", "loved_it", "nailed_it"}: liked_dir_slot = global_state.setdefault("liked_dir", {}) self._v2_store_direction(liked_dir_slot, payload, session_mean) + self._v2_store_prompt_keyed_direction(global_state, payload, session_mean) count = int(global_state.get("liked_conditioning_count", 0)) if count <= 0 or not isinstance(global_state.get("liked_conditioning"), dict): global_state["liked_conditioning"] = payload diff --git a/identity_transfer.py b/identity_transfer.py new file mode 100644 index 00000000..172c26a3 --- /dev/null +++ b/identity_transfer.py @@ -0,0 +1,145 @@ +"""Best-FaceID identity transfer — native port of ComfyUI-BFSNodes' LTX Identity Transfer +(alisson-anjos/ComfyUI-BFSNodes, LTXIdentityOverlapConditioning): the reference face is +injected as SEPARATE tokens that share the target's frame-0 RoPE grid (overlap) and are +tagged with a per-source RoPE phase (source_phase), exactly as ltx-trainer's flexible +strategy did at train/validation time. The ref tokens are clean (timestep 0), attend to the +target in self-attention, and are sliced off the output (never rendered) — unlike FunPack's +guide-attention path, they never blend into a real frame. Plus optional ArcFace IdentityProjector +tokens appended to the text context (as trained). + +Requires `insightface` (+ buffalo_l, auto-downloaded on first use, CPU) for the ArcFace half; +soft dependency — the overlap tokens alone (identity_projector="None") work without it. +""" +from __future__ import annotations + +import logging +import os + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from safetensors.torch import load_file + +log = logging.getLogger("FunPackIdentityTransfer") + +_USE_GPU = os.environ.get("FUNPACK_ARCFACE_GPU", "0") == "1" +_FACE_APP = None + + +def _get_face_app(): + global _FACE_APP + if _FACE_APP is None: + from insightface.app import FaceAnalysis + providers = (["CUDAExecutionProvider", "CPUExecutionProvider"] if _USE_GPU else ["CPUExecutionProvider"]) + app = FaceAnalysis(name="buffalo_l", providers=providers) + app.prepare(ctx_id=0 if _USE_GPU else -1, det_size=(640, 640)) + _FACE_APP = app + return _FACE_APP + + +def arcface_embed(image_bhwc, mode="auto_adjust"): + """Return the ArcFace embedding [512], or None if disabled / no face found. + mode: 'as_is' (detect on the image only), 'auto_adjust' (retry with border-pad + zoom-out + upscale when detection fails), 'disable' (skip ArcFace entirely).""" + if mode == "disable": + return None + import cv2 + app = _get_face_app() + img = np.ascontiguousarray( + (np.clip(image_bhwc[0].detach().cpu().numpy(), 0.0, 1.0) * 255.0).astype(np.uint8)[:, :, ::-1] + ) + attempts = [img] + if mode == "auto_adjust": + h, w = img.shape[:2] + pad = int(0.4 * max(h, w)) + attempts.append(cv2.copyMakeBorder(img, pad, pad, pad, pad, cv2.BORDER_REPLICATE)) + attempts.append(cv2.resize(img, None, fx=2.0, fy=2.0, interpolation=cv2.INTER_CUBIC)) + attempts.append(cv2.resize(img, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA)) + for a in attempts: + faces = app.get(a) + if faces: + f = max(faces, key=lambda x: (x.bbox[2] - x.bbox[0]) * (x.bbox[3] - x.bbox[1])) + return torch.from_numpy(f.normed_embedding.astype(np.float32)) + return None + + +class IdentityProjector(nn.Module): + """ArcFace embedding [in_dim] -> num_tokens context tokens [num_tokens, context_dim].""" + + def __init__(self, in_dim=512, context_dim=4096, num_tokens=4, proj=None): + super().__init__() + self.in_dim = in_dim + self.context_dim = context_dim + self.num_tokens = num_tokens + self.proj = proj if proj is not None else nn.Sequential( + nn.Linear(in_dim, 1024), nn.GELU(), nn.Linear(1024, num_tokens * context_dim)) + self.norm = nn.LayerNorm(context_dim) + + def forward(self, e): + return self.norm(self.proj(e).reshape(-1, self.num_tokens, self.context_dim)) + + +def load_identity_projector(path, device): + """Load an IdentityProjector of ANY depth by rebuilding proj.N Linears from the + state_dict (old = 2 linears proj.0/proj.2; enhanced = 3 linears proj.0/proj.2/proj.4).""" + sd = load_file(path) + context_dim = sd["norm.weight"].shape[0] + idxs = sorted({int(k.split(".")[1]) for k in sd if k.startswith("proj.") and k.endswith(".weight")}) + layers = [] + for j, i in enumerate(idxs): + w = sd[f"proj.{i}.weight"] + layers.append(nn.Linear(w.shape[1], w.shape[0])) + if j < len(idxs) - 1: + layers.append(nn.GELU()) + proj = nn.Sequential(*layers) + in_dim = sd["proj.0.weight"].shape[1] + num_tokens = sd[f"proj.{idxs[-1]}.weight"].shape[0] // context_dim + p = IdentityProjector(in_dim=in_dim, context_dim=context_dim, num_tokens=num_tokens, proj=proj) + p.load_state_dict(sd) + return p.to(device=device, dtype=torch.float32).eval() + + +def append_context_tokens(conditioning, tokens): + """Append `tokens` [1 or B, N, D'] onto every entry's text context, padding/truncating + the last dim to match and extending any attention_mask with all-ones for the new tokens.""" + out = [] + for ce, cd in conditioning: + t = tokens.to(device=ce.device, dtype=ce.dtype) + if t.shape[0] != ce.shape[0]: + t = t.expand(ce.shape[0], -1, -1) + if t.shape[-1] < ce.shape[-1]: + t = F.pad(t, (0, ce.shape[-1] - t.shape[-1])) + elif t.shape[-1] > ce.shape[-1]: + t = t[..., :ce.shape[-1]] + nd = cd.copy() + am = nd.get("attention_mask") + if am is not None: + nd["attention_mask"] = torch.cat( + [am, torch.ones((*am.shape[:-1], t.shape[1]), device=am.device, dtype=am.dtype)], dim=-1) + out.append([torch.cat([ce, t], dim=1), nd]) + return out + + +def rotate_overlap_freqs(pe, ref_len, seg_value, theta=10000.0): + """Rotate the LAST ref_len tokens' RoPE freqs by phase = seg_value * theta^(-d/L) + (port of ltx_core rope.apply_segment_phase). pe = (cos, sin, [split_flag]).""" + if ref_len <= 0 or seg_value == 0.0: + return pe + cos, sin = pe[0], pe[1] + rest = tuple(pe[2:]) + L = cos.shape[-1] + d = torch.arange(L, device=cos.device, dtype=torch.float32) + rate = theta ** (-d / float(L)) + phase = seg_value * rate + pc = phase.cos().to(cos.dtype) + ps = phase.sin().to(sin.dtype) + idx = [slice(None)] * cos.dim() + idx[-2] = slice(cos.shape[-2] - ref_len, cos.shape[-2]) + idx = tuple(idx) + c0, s0 = cos[idx], sin[idx] + cos = cos.clone() + sin = sin.clone() + cos[idx] = c0 * pc - s0 * ps + sin[idx] = s0 * pc + c0 * ps + return (cos, sin, *rest) diff --git a/ltx_enhancements.py b/ltx_enhancements.py index a8d7aeea..87d763c6 100644 --- a/ltx_enhancements.py +++ b/ltx_enhancements.py @@ -35,7 +35,7 @@ TEMPORAL_STYLES = ["natural", "auto", "accelerate", "decelerate", "loop", "freez TEMPORAL_STYLE_MULT = { "accelerate": 1.35, "decelerate": 0.72, - "loop": 1.0, # loop is a separate RoPE coordinate trick (not yet wired); mult is a no-op + "loop": 1.0, # loop = Mobius latent roll (make_loop_temporal_wrapper), not a frame_rate mult "freeze": 2.0, } @@ -188,6 +188,156 @@ def make_pulse_temporal_wrapper(old_wrapper, segment_count=None, peak_mult=None, return _pulse_wrapper +# --- Loop temporal style: Mobius-style latent roll (arXiv:2502.20307) --------------- +# Seamless looping without training or extra forward passes: on eligible denoise steps +# the latent is cyclically rolled along its temporal dim before the forward and the +# prediction unrolled after. The model then repeatedly sees the video's seam +# (last frame | first frame) as an INTERIOR frame pair at a step-varying position and +# smooths it like any other cut — after enough steps the sequence is consistent under +# rotation, i.e. it loops. WanVideoWrapper's "Loop Args" is the same trick for WAN. +# +# Roll starts below the near-noise plateau: content is ~pure noise up there (nothing to +# smooth) and the plateau step-cache reuses base forwards, which a per-step roll would +# invalidate. Guides pin absolute frame positions (keyframe_idxs / attention entries +# reference appended frames inside the same T range), so calls carrying them are left +# canonical rather than rolled. +LOOP_ROLL_MAX_SIGMA = 0.95 +LOOP_ROLL_MIN_FRAMES = 4 # below this many latent frames there is nothing to loop + + +def _van_der_corput(n): + """Base-2 bit-reversal sequence in (0,1): 0.5, 0.25, 0.75, 0.125, 0.625… — every + prefix covers the unit interval near-uniformly, so however many steps end up + eligible, the seam visits well-spread temporal positions.""" + v, denom = 0.0, 1.0 + n = int(n) + while n: + denom *= 2.0 + v += (n & 1) / denom + n >>= 1 + return v + + +def _loop_stream_shapes(args): + """Per-stream latent shapes for a packed [B,1,N] input, or None when the layout + can't be trusted. Mirrors the samplers' packed-latent handling (video first, then + audio; both streams keep their temporal dim at index 2).""" + c = args.get("c") or {} + shapes = c.get("latent_shapes") + if hasattr(shapes, "cond"): + shapes = shapes.cond + x = args.get("input") + if x is None: + return None + if not shapes: + # Single-stream (plain LTXV) fed unpacked: treat the input itself as one stream. + return [tuple(int(d) for d in x.shape)] if x.ndim == 5 else None + try: + shapes = [tuple(int(d) for d in s) for s in shapes] + import math as _math + if sum(_math.prod(s[1:]) for s in shapes) != int(x.shape[-1]): + return None # packed layout doesn't match — don't risk rolling blind + except (TypeError, ValueError): + return None + return shapes + + +def _loop_roll_packed(x, shapes, frac, direction): + """Cyclically roll every stream's temporal dim (dim 2) of a packed [B,1,N] tensor by + round(frac * T_stream) frames (video and audio each by their own frame count, so the + two stay time-aligned). direction=+1 rolls, -1 unrolls. Returns a new tensor.""" + import math as _math + if x.ndim == 5: # unpacked single stream + t = x.shape[2] + shift = int(round(frac * t)) % t + return torch.roll(x, shifts=direction * shift, dims=2) if shift else x + out = x.clone() + off = 0 + for dims in shapes: + sz = _math.prod(dims[1:]) + if len(dims) >= 3 and dims[2] >= 2: + t = dims[2] + shift = int(round(frac * t)) % t + if shift: + stream = x[..., off:off + sz].reshape([x.shape[0]] + list(dims[1:])) + out[..., off:off + sz] = torch.roll( + stream, shifts=direction * shift, dims=2).reshape(x.shape[0], 1, sz) + off += sz + return out + + +def _loop_roll_mask(mask, frac, direction): + """Roll a denoise mask ([B,1,T,H,W] video / [B,C,T,F] audio) in step with its stream.""" + if not isinstance(mask, torch.Tensor) or mask.ndim < 3 or mask.shape[2] < 2: + return mask + t = mask.shape[2] + shift = int(round(frac * t)) % t + return torch.roll(mask, shifts=direction * shift, dims=2) if shift else mask + + +def make_loop_temporal_wrapper(old_wrapper): + """Build the loop-style model_function_wrapper. Installed INNERMOST (closest to + apply_model): prediction-modifying wrappers layered above it (dynashift, output + guidance, …) must see canonical-orientation inputs and outputs — the roll exists + only for the duration of the base forward. The per-step shift follows a van der + Corput sequence, reset whenever sigma jumps back up (a new scene/run).""" + state = {"count": 0, "last_sigma": None, "logged": False} + + def _loop_wrapper(apply_fn, args, _old=old_wrapper): + def _call(a): + if _old is not None: + return _old(apply_fn, a) + return apply_fn(a["input"], a["timestep"], **a.get("c", {})) + + try: + ts = args.get("timestep") + sigma = float(ts.max().item()) if ts is not None else 1.0 + except Exception: + sigma = 1.0 + if state["last_sigma"] is not None and sigma > state["last_sigma"] + 1e-4: + state["count"] = 0 # sigma went back up: new scene/run + state["last_sigma"] = sigma + + c = args.get("c") or {} + if ( + sigma > LOOP_ROLL_MAX_SIGMA + or c.get("keyframe_idxs") is not None + or c.get("guide_attention_entries") is not None + ): + return _call(args) + shapes = _loop_stream_shapes(args) + if not shapes: + return _call(args) + # Gate on the VIDEO stream's frame count (the 5-dim shape); audio rides along. + t_video = max((s[2] for s in shapes if len(s) == 5), + default=max((s[2] for s in shapes if len(s) >= 3), default=0)) + if t_video < LOOP_ROLL_MIN_FRAMES: + return _call(args) + + state["count"] += 1 + frac = _van_der_corput(state["count"]) + if not state["logged"]: + state["logged"] = True + print(f"[FunPack] loop temporal style: Mobius latent roll active (T={t_video})") + try: + new_c = dict(c) + for key in ("denoise_mask", "audio_denoise_mask"): + if isinstance(new_c.get(key), torch.Tensor): + new_c[key] = _loop_roll_mask(new_c[key], frac, 1) + rolled = dict(args) + rolled["input"] = _loop_roll_packed(args["input"], shapes, frac, 1) + rolled["c"] = new_c + out = _call(rolled) + return _loop_roll_packed(out, shapes, frac, -1) + except Exception as e: + if state.get("roll_error") is None: + state["roll_error"] = True + print(f"[FunPack] loop roll failed, running canonical: {e}") + return _call(args) + + return _loop_wrapper + + # Confirmed semantic focal points (PAG default=14, STG defaults=14,19) ANCHOR_BLOCKS = [14, 19] @@ -1206,11 +1356,16 @@ def build_enhancements(model, rating_profile, temporal_style, refinement_key, re # "auto" and "pulse" are per-scene and owned by the chain sampler (auto bakes a # per-scene multiplier; pulse installs a segmented ease-down wrapper per scene). # The global path here fires for concrete manual styles and pulse on single-scene - # workflows. "loop" stays a no-op until the RoPE coordinate trick is wired. + # workflows. "loop" = Mobius latent roll (not a frame_rate mult); installed here it + # sits innermost relative to the chain sampler's per-scene wrappers, which is + # required — prediction-modifying wrappers must see canonical orientation. if temporal_style == "pulse": wrapper = make_pulse_temporal_wrapper(model.model_options.get("model_function_wrapper")) if wrapper is not None: model.model_options["model_function_wrapper"] = wrapper + elif temporal_style == "loop": + model.model_options["model_function_wrapper"] = make_loop_temporal_wrapper( + model.model_options.get("model_function_wrapper")) elif temporal_style not in ("natural", "auto"): mult = TEMPORAL_STYLE_MULT.get(temporal_style, 1.0) wrapper = make_temporal_wrapper(model.model_options.get("model_function_wrapper"), mult) diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 6eea3300..763cf600 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -5,6 +5,7 @@ (function () { const { el, clear } = window.dom; const S = window.Store; + const API = window.MovieEditorAPI; let _mounted = null; // { scroller (pane, set per render), content (shell root) } let unsub = null; @@ -158,6 +159,8 @@ { name: "embed_guidance_strength", label: "Embed strength", kind: "float", default: 0.02, min: 0.005, max: 0.1, step: 0.005, dependsOn: "embed_guidance" }, { name: "score_slider", label: "Score slider", kind: "bool", default: false }, { name: "score_slider_strength", label: "Slider strength (eta)", kind: "float", default: 1.0, min: 0, max: 3, step: 0.25, dependsOn: "score_slider" }, + { name: "taste_nearest_prompt", label: "Per-prompt taste direction", kind: "bool", default: false, + hint: "EXPERIMENTAL: source Embed guidance / Score slider from the taste direction learned on the prompts NEAREST this scene's prompt, instead of one global liked-direction average. Each liked rating records (prompt → its liked direction); this retrieves the closest matches per scene (a forest prompt pulls what worked on forests). No extra model pass — a cosine lookup + vector mean. Falls back to the global direction when nothing rated is close. Needs Embed guidance or Score slider on. UNVALIDATED LIVE." }, { name: "output_guidance", label: "Output guidance", kind: "bool", default: false }, { name: "output_guidance_strength", label: "Output guidance strength", kind: "float", default: 0.02, min: 0.005, max: 0.1, step: 0.005, dependsOn: "output_guidance" }, { name: "decode_noise_scale", label: "Decode noise scale", kind: "float", default: 0.0, min: 0, max: 1, step: 0.01 }, @@ -180,9 +183,19 @@ { name: "dynashift_strength", label: "DynaShift strength", kind: "float", default: 0.3, min: 0.05, max: 1.0, step: 0.05, dependsOn: "dynashift" }, { name: "dynashift_threshold", label: "DynaShift match threshold", kind: "float", default: 0.6, min: 0.3, max: 0.95, step: 0.05, dependsOn: "dynashift" }, { name: "identity_transfer_enabled", label: "Best-FaceID compatibility", kind: "bool", default: false, - hint: "Tags Continuity's Identity pin guide (Engine → Continuity) with the source-phase RoPE rotation Best-FaceID-style identity LoRAs were trained on. Load the LoRA itself the normal way — Models → add a LoRA loader onto the model path. No effect without an Identity pin image set." }, - { name: "identity_transfer_phase", label: "Source-phase id", kind: "float", default: 2.0, min: 0.0, max: 8.0, step: 0.5, dependsOn: "identity_transfer_enabled", - hint: "Matches the LoRA's training convention (ltx-trainer used 2)." }, + hint: "Full native port of the overlap+source_phase+ArcFace conditioning Best-FaceID-style identity LoRAs were trained on. Replaces Continuity's Identity pin guide (Engine → Continuity) with separate, non-rendered reference tokens plus an optional ArcFace projector below. Load the LoRA itself the normal way — Models → add a LoRA loader onto the model path. No effect without an Identity pin image set." }, + { name: "source_id", label: "Source-phase id", kind: "float", default: 2.0, min: 0.0, max: 8.0, step: 1.0, dependsOn: "identity_transfer_enabled", + hint: "Matches the LoRA's training convention (ltx-trainer used 2). 0 disables the RoPE rotation." }, + { name: "phase_scale", label: "Phase scale", kind: "float", default: 1.0, min: 0.0, max: 4.0, step: 0.1, dependsOn: "identity_transfer_enabled" }, + { name: "id_strength", label: "ArcFace token strength", kind: "float", default: 1.0, min: 0.0, max: 50.0, step: 0.5, dependsOn: "identity_transfer_enabled", + hint: "Only applies when an ArcFace projector is selected below — weak channel, push high (5-20) to test." }, + { name: "arcface_mode", label: "ArcFace detection mode", kind: "combo", choices: ["auto_adjust", "as_is", "disable"], default: "auto_adjust", dependsOn: "identity_transfer_enabled" }, + { name: "debug_log", label: "Debug log", kind: "bool", default: false, dependsOn: "identity_transfer_enabled", + hint: "Print per-scene identity-transfer shape/status logs to the ComfyUI console." }, + { name: "plateau_cache", label: "Plateau step-cache (speed)", kind: "bool", default: false, + hint: "EXPERIMENTAL speed: the near-pure-noise early steps carry almost no signal, so the transformer output barely changes across them. Computes it once at the top of the plateau and reuses it for the rest, skipping ~3-4 of 8 transformer passes. Deterministic given seed (safe in Batch Training) but an approximation — A/B it before trusting on finals. Much of wall-clock time is outside the sampler, so total speedup is smaller than the forward count suggests. UNVALIDATED LIVE." }, + { name: "plateau_cache_threshold", label: "Plateau threshold (sigma)", kind: "float", default: 0.975, min: 0.5, max: 0.999, step: 0.005, dependsOn: "plateau_cache", + hint: "Steps with sigma at or above this count as the reusable plateau. Higher = fewer steps cached (safer); lower = more cached (faster, more approximation). 0.975 catches the documented noise plateau while leaving structure formation fully computed." }, ]; const SAMPLER_KNOB_MAP = Object.fromEntries(SAMPLER_KNOBS.map((k) => [k.name, k])); @@ -285,9 +298,9 @@ const CHAIN_VIEW_KNOBS = { chain_continuity: ["carry_i2v_guides"], chain_timing: ["frame_overlap", "transition_duration", "use_same_seed"], - chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "output_guidance", "output_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold"], + chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "taste_nearest_prompt", "output_guidance", "output_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold"], chain_decode: ["decode_noise_scale", "decode_timestep", "decode_tile_size"], - chain_experimental: ["mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "identity_transfer_phase"], + chain_experimental: ["plateau_cache", "plateau_cache_threshold", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], }; function countChainView(p, id) { @@ -563,6 +576,49 @@ renderKnobList(g, st, CHAIN_VIEW_KNOBS[id]); } + // ── Best-FaceID ArcFace projector: the one identity-transfer field that needs a + // live (server-fetched) choice list, so it can't live in the static SAMPLER_KNOBS combo + // shape. Reuses the same loras folder listing LoraLoaderModelOnly exposes. + let _loraChoices = null; + + async function ensureLoraChoices() { + if (_loraChoices) return _loraChoices; + try { + const spec = await API.nodeSpec("LoraLoaderModelOnly"); + const w = (spec?.inputs || []).find((i) => i.name === "lora_name"); + _loraChoices = (w && w.choices) || []; + } catch (_) { + _loraChoices = []; + } + render(); + return _loraChoices; + } + + function renderChainExperimental(pane, st) { + renderChainKnobsView(pane, st, "chain_experimental", "Experimental", + "Research techniques — off by default; expect quality/overhead trade-offs."); + const si = st.project.sampler_inputs || {}; + if (!si.identity_transfer_enabled) return; + const g = group(pane, "Best-FaceID: ArcFace projector"); + if (!_loraChoices) { + g.append(hintEl("Loading projector list…")); + ensureLoraChoices(); + return; + } + const sel = el("select"); sel.dataset.k = "si-identity_projector"; + const noneOpt = el("option", null, "None"); noneOpt.value = "None"; + if (!si.identity_projector || si.identity_projector === "None") noneOpt.selected = true; + sel.append(noneOpt); + _loraChoices.forEach((c) => { + const o = el("option", null, c); o.value = c; + if (c === si.identity_projector) o.selected = true; + sel.append(o); + }); + sel.onchange = () => S.setSamplerInputNow("identity_projector", sel.value); + g.append(field("ArcFace projector", sel, + "Optional secondary identity channel — the overlap reference tokens above carry the bulk of identity even with this set to None.")); + } + function renderPane(pane, st) { const p = st.project; if (!p) { pane.append(el("div", "pj-meta", "No project open.")); return; } @@ -582,8 +638,7 @@ case "chain_timing": return renderChainTiming(pane, st); case "chain_guidance": return renderChainKnobsView(pane, st, "chain_guidance", "Guidance"); case "chain_decode": return renderChainKnobsView(pane, st, "chain_decode", "Decode"); - case "chain_experimental": return renderChainKnobsView(pane, st, "chain_experimental", "Experimental", - "Research techniques — off by default; expect quality/overhead trade-offs."); + case "chain_experimental": return renderChainExperimental(pane, st); default: return renderOverview(pane, st); } } diff --git a/movie_editor/frontend/inspector.js b/movie_editor/frontend/inspector.js index b2d04cf2..1a3db6ad 100644 --- a/movie_editor/frontend/inspector.js +++ b/movie_editor/frontend/inspector.js @@ -685,7 +685,29 @@ wrap.append(field(`🔗 ${l.name} (${(l.members || []).length})`, ctrl)); } }); + // A widget input wired from another node's output is replaced by that connection at + // generation — an exposed control for it would edit a value generation ignores, so + // show the wire source instead (mirrors the locked field on the node's own page). + const wireInto = (slot, name) => { + const target = `node:${slot.id}:${name}`; + for (const s2 of (m.slots || [])) { + if (s2.id === slot.id) continue; + for (const [outName, raw] of Object.entries(s2.wires || {})) { + const arr = Array.isArray(raw) ? raw : (raw ? [raw] : []); + if (arr.includes(target)) return { slot: s2, out: outName }; + } + } + return null; + }; slotItems.forEach(([slot, d]) => { + const iw = d.isBypass ? null : wireInto(slot, d.name); + if (iw) { + const note = el("div", "lib-sub"); + note.textContent = `⇐ ${slotLabel(iw.slot)} · ${iw.out}`; + note.title = `This value comes from ${slotLabel(iw.slot)}'s "${iw.out}" output at generation.`; + wrap.append(field(`${slotLabel(slot)} · ${d.label || d.name}`, note)); + return; + } // Bypass is a slot-level flag (slot.bypassed), not a real node input — route it // separately so it never gets sent to ComfyUI as a fake widget value. const ctrl = d.isBypass diff --git a/movie_editor/frontend/models.js b/movie_editor/frontend/models.js index d96e2442..d2d6e5c0 100644 --- a/movie_editor/frontend/models.js +++ b/movie_editor/frontend/models.js @@ -459,13 +459,24 @@ const grid = el("div", "slot-fields"); cand.inputs.forEach((spec) => { const lk = linkOf(slot.id, spec.name); + const iw = incomingWidgetWire(slot, spec.name); const f = widgetField(spec, slot.inputs[spec.name], async (v) => { slot.inputs[spec.name] = v; const l2 = linkOf(slot.id, spec.name); if (l2) applyLinkValue(l2, v); // keep group in sync await persist(); }); f.classList.add("with-eye"); - if (lk) { + if (iw) { + // Wired from another node's output: the widget value is replaced by the + // connection at generation — lock the field and say where it comes from + // (same treatment as a linked control; the eye button is skipped because an + // exposed control would edit a value generation ignores). + f.classList.add("linked"); + const ctrl = f.querySelector("input,select"); if (ctrl) ctrl.disabled = true; + const tag = el("span", "link-tag", `⇐ ${slotName(iw.slot)} · ${iw.out}`); + tag.title = `This value comes from ${slotFullLabel(iw.slot)}'s "${iw.out}" output at generation — the local field is ignored while wired.`; + f.append(tag); + } else if (lk) { f.classList.add("linked"); const ctrl = f.querySelector("input,select"); if (ctrl) ctrl.disabled = true; f.append(el("span", "link-tag", "🔗 " + lk.name)); @@ -571,6 +582,21 @@ return Array.isArray(raw) ? raw : [raw]; } + // The wire (if any) feeding a WIDGET input of `slot` from another slot's output — + // e.g. ImageTransform.width → EmptyLatent.width. The builder replaces the widget's + // value with that connection at generation, so the local field is dead while wired + // and must render locked (same treatment as a linked control). + function incomingWidgetWire(slot, inputName) { + const target = `node:${slot.id}:${inputName}`; + for (const s2 of config.slots) { + if (s2.id === slot.id) continue; + for (const [outName, raw] of Object.entries(s2.wires || {})) { + if (wireTargets(raw).includes(target)) return { slot: s2, out: outName }; + } + } + return null; + } + // ── edge mirroring ──────────────────────────────────────────────────────────── // An output→input edge can be authored from either side: a slot's wires // ("node::") OR the destination's input_sources ("out::"). diff --git a/movie_editor/frontend/player.js b/movie_editor/frontend/player.js index 85f15b80..4c307bb0 100644 --- a/movie_editor/frontend/player.js +++ b/movie_editor/frontend/player.js @@ -463,7 +463,7 @@ // when previewing run 1's video while ComfyUI was GIL-bound saving run 2). Bounded // backoff, cache-busted so a poisoned browser cache entry can't be replayed. const MAX_VIDEO_RETRIES = 4; - function _scheduleVideoRetry(url, v) { + function _scheduleVideoRetry(url, v, delayMs) { const n = (v._pmRetries || 0) + 1; if (n > MAX_VIDEO_RETRIES) return; v._pmRetries = n; @@ -474,7 +474,37 @@ const sep = url.includes("?") ? "&" : "?"; v.src = url + sep + "r=" + Date.now(); v.load(); - }, 1500 * n); + }, delayMs != null ? delayMs : 1500 * n); + } + + // If the active clip's element changed readiness, repaint the slate/badge over it. + function _refreshClipStatusUi(url) { + if (_currentClip && !_currentClip.pending && _clipUrl(_currentClip) === url) { + _applyClipUi(_currentClip); + } + } + + // Failed element → ask the server WHY (HEAD /result now answers 503 while the render is + // still being written to disk) so the slate can say "processing" vs "loading", and so a + // long GIL-bound save can't exhaust the retry budget: server-confirmed "processing" + // retries on a steady cadence with the budget refunded, everything else backs off. + function _classifyVideoError(url, v) { + const probe = url.split("&r=")[0].split("?r=")[0]; + fetch(probe, { method: "HEAD", cache: "no-store" }) + .then((r) => (r.status === 503 ? "processing" : "loading")) + .catch(() => "loading") + .then((status) => { + if (pool.get(url) !== v || !_urlClips.has(url)) return; + v._pmStatus = status; + if (status === "processing") { + v._pmRetries = 0; + _scheduleVideoRetry(url, v, 3000); + } else { + if ((v._pmRetries || 0) >= MAX_VIDEO_RETRIES) v._pmStatus = "failed"; + _scheduleVideoRetry(url, v); + } + _refreshClipStatusUi(url); + }); } function _ensureVideo(url, clip) { @@ -494,10 +524,13 @@ // _updatePreloadHints() promotes the current clip and its neighbours to auto. v.className = "pm-video"; v.preload = "metadata"; v.playsInline = true; v.src = url; v._pmFx = { viewport, zoom }; + v._pmStatus = "loading"; // → "ready" on loadedmetadata; "processing"/"failed" from _classifyVideoError zoom.append(v); viewport.append(zoom); v.addEventListener("loadedmetadata", () => { v._pmRetries = 0; // healthy again — restore the full retry budget + v._pmStatus = "ready"; + _refreshClipStatusUi(url); const pending = v._pmSeekPending != null ? v._pmSeekPending : (v === _active ? _seekPending : null); if (pending != null) { _applyVideoTime(v, pending, !_playing); @@ -516,6 +549,9 @@ // to 0 and would restart the clip from its beginning. Stash the playhead position so // loadedmetadata seeks back instead; _onVideoSeeked then resumes playback if playing. v.addEventListener("emptied", () => { + // A media reset means the bytes are gone — downgrade so the slate covers the element + // instead of it blinking black while the retry reloads. + if (v._pmStatus === "ready") { v._pmStatus = "loading"; _refreshClipStatusUi(url); } if (v !== _active || !_currentClip || _currentClip.pending) return; if (_clipUrl(_currentClip) !== url) return; if (v._pmSeekPending == null) { @@ -526,8 +562,13 @@ const clips = _urlClips.get(url); if (clips) [...clips].forEach((c) => _fallbackSegmentClip(c)); // Segment clips fell back to their base URL above (and unregistered); anything still - // registered here has no alternative URL — retry this one instead of staying dead. - if (_urlClips.has(url)) _scheduleVideoRetry(url, v); + // registered here has no alternative URL — classify the failure (processing vs dead) + // and retry this one instead of staying dead. + if (_urlClips.has(url)) { + v._pmStatus = "loading"; + _refreshClipStatusUi(url); + _classifyVideoError(url, v); + } }); v.addEventListener("ended", () => { if (v !== _active || _isSeeking() || _clipBoundaryToken) return; @@ -538,33 +579,87 @@ return v; } + // Materializing a