Add embed_guidance: per-step conditioning nudge toward liked quality direction
Extends the Refiner's one-shot conditioning steering to apply at every denoising step. Reads the liked conditioning direction from the Refiner's state file via refinement_key_input. At each step, nudges c_crossattn toward the learned liked direction with sigma-scaling (no effect at sigma=1, full effect below sigma=0.5 where detail refinement happens). Requires 3+ liked generations to have a meaningful direction. No-ops silently if direction is unavailable. New inputs: embed_guidance (BOOLEAN), embed_guidance_strength (FLOAT 0.005-0.1). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
0ed6922fdc
commit
2c41209f2a
+77
@@ -802,6 +802,14 @@ class FunPackLTXAVSceneChainSampler:
|
||||
"default": 0.25, "min": 0.25, "max": 0.5, "step": 0.05,
|
||||
"tooltip": "Guide attention strength for mid-scene anchor. 0.25 is the minimum — below that audio degrades and character appearance drifts. Above 0.35 causes spatial conflicts when scene composition shifts.",
|
||||
}),
|
||||
"embed_guidance": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "Apply the Refiner's learned quality direction at each denoising step, not just once before sampling. Requires refinement_key_input and enough liked generations to have a direction. Adds ~20-30% inference overhead.",
|
||||
}),
|
||||
"embed_guidance_strength": ("FLOAT", {
|
||||
"default": 0.02, "min": 0.005, "max": 0.1, "step": 0.005,
|
||||
"tooltip": "Per-step nudge strength toward the liked conditioning direction. Keep small — the direction is applied at every step so it compounds. 0.01-0.03 is typical.",
|
||||
}),
|
||||
"transition_duration": ("INT", {
|
||||
"default": 16, "min": 0, "max": 128, "step": 2,
|
||||
"tooltip": "Extra pixel frames of fade beyond the blend zone on each side of a scene boundary. 0 = disable all transition effects.",
|
||||
@@ -1279,6 +1287,60 @@ class FunPackLTXAVSceneChainSampler:
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _load_liked_direction(self, refinement_key):
|
||||
"""Read the liked conditioning direction from the Refiner's state file."""
|
||||
try:
|
||||
try:
|
||||
from .conditioning import refinement_state_path, serializable_to_tensor
|
||||
except ImportError:
|
||||
from conditioning import refinement_state_path, serializable_to_tensor
|
||||
import json as _json
|
||||
path = refinement_state_path(refinement_key, "clip", prefix="refine_v2")
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
state = _json.load(f)
|
||||
liked_dir_slot = state.get("liked_dir", {})
|
||||
if int(liked_dir_slot.get("direction_count", 0)) < 3:
|
||||
return None
|
||||
raw = liked_dir_slot.get("direction")
|
||||
if raw is None:
|
||||
return None
|
||||
return serializable_to_tensor(raw)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _build_embed_guidance_wrapper(self, model, liked_dir, strength):
|
||||
"""Register a model_function_wrapper that nudges conditioning toward the
|
||||
liked quality direction at each denoising step, scaling with sigma so
|
||||
the effect is stronger during detail refinement (low sigma) than structure
|
||||
formation (high sigma)."""
|
||||
old_wrapper = model.model_options.get("model_function_wrapper")
|
||||
direction = torch.nn.functional.normalize(liked_dir.float(), dim=-1)
|
||||
|
||||
def _embed_wrapper(apply_fn, args, _ew=old_wrapper, _dir=direction, _s=strength):
|
||||
c = args.get("c") or {}
|
||||
cond = c.get("c_crossattn")
|
||||
if cond is not None:
|
||||
ts = args.get("timestep")
|
||||
try:
|
||||
sigma = float(ts.max().item()) if ts is not None else 1.0
|
||||
except Exception:
|
||||
sigma = 1.0
|
||||
# Scale: no effect at sigma=1 (pure noise), full effect below sigma=0.5
|
||||
scale = max(0.0, 1.0 - sigma * 2.0)
|
||||
if scale > 0:
|
||||
d = _dir.to(cond.device, cond.dtype)
|
||||
new_cond = cond + (_s * scale) * d.expand_as(cond)
|
||||
new_c = dict(c)
|
||||
new_c["c_crossattn"] = new_cond
|
||||
args = dict(args)
|
||||
args["c"] = new_c
|
||||
if _ew is not None:
|
||||
return _ew(apply_fn, args)
|
||||
return apply_fn(args["input"], args["timestep"], **args.get("c", {}))
|
||||
|
||||
model.model_options["model_function_wrapper"] = _embed_wrapper
|
||||
return old_wrapper
|
||||
|
||||
def _append_mid_scene_guide(self, chunk, previous_output, positive, negative, vae, strength):
|
||||
"""Append the middle frame of the previous scene as a guide for the current chunk
|
||||
using LTX's guide attention mechanism (keyframe_idxs + guide_attention_entries).
|
||||
@@ -1350,6 +1412,7 @@ class FunPackLTXAVSceneChainSampler:
|
||||
num_frames_per_scene, frame_overlap, cfg, max_scenes, use_same_seed=False,
|
||||
carry_i2v_guides=False,
|
||||
mid_scene_guide=False, mid_scene_guide_strength=0.4,
|
||||
embed_guidance=False, embed_guidance_strength=0.02,
|
||||
transition_duration=16, decode_tile_size=0,
|
||||
refinement_key_input="", unique_id=None, prompt=None):
|
||||
if not isinstance(positive, list) or not positive:
|
||||
@@ -1370,6 +1433,13 @@ class FunPackLTXAVSceneChainSampler:
|
||||
boundary_entries = []
|
||||
cumulative_latent_frames = 0
|
||||
|
||||
# Load liked direction once for embed_guidance
|
||||
_liked_dir = None
|
||||
if embed_guidance and refinement_key_input:
|
||||
_liked_dir = self._load_liked_direction(refinement_key_input)
|
||||
if _liked_dir is None:
|
||||
print("[FunPackSceneChain] embed_guidance: no liked direction found (need 3+ liked generations)")
|
||||
|
||||
first_scene_seed = self._scene_seed(scene_conditionings[0])
|
||||
if first_scene_seed is None:
|
||||
first_scene_seed = int(seed)
|
||||
@@ -1413,9 +1483,16 @@ class FunPackLTXAVSceneChainSampler:
|
||||
else:
|
||||
guide_tail = 0
|
||||
|
||||
if embed_guidance and _liked_dir is not None:
|
||||
_eg_old_wrapper = self._build_embed_guidance_wrapper(model, _liked_dir, embed_guidance_strength)
|
||||
sampled = self._sample_chunk(
|
||||
model, sampler, sigmas, scene_seed, cfg, scene_positive, scene_negative, chunk,
|
||||
)
|
||||
if embed_guidance and _liked_dir is not None:
|
||||
if _eg_old_wrapper is not None:
|
||||
model.model_options["model_function_wrapper"] = _eg_old_wrapper
|
||||
elif "model_function_wrapper" in model.model_options:
|
||||
del model.model_options["model_function_wrapper"]
|
||||
if carried + soft_carried > 0:
|
||||
sampled = self._crop_video_head(sampled, carried + soft_carried)
|
||||
if guide_tail > 0:
|
||||
|
||||
Reference in New Issue
Block a user