Move reference image conditioning to Studio, where the image already lives
_v2_transition_scene_conditionings was encoding per-scene texts without the reference image — only the main single-scene conditioning path passed it through. Now passes reference_image=source_image to all per-scene encodings so every scene conditioning includes the character reference via Gemma3 vision. Removes the redundant reference_conditioning option from the chain sampler (which required a wasteful VAE decode mid-sampling to get the pixel image that Studio already has). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
f00ca83151
commit
7d1a6f7f77
+3
-2
@@ -6397,7 +6397,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
|
||||
scene_effects.append(effect)
|
||||
return scene_texts, scene_effects
|
||||
|
||||
def _v2_transition_scene_conditionings(self, clip, scene_texts, scene_effects=None, encode_cache=None):
|
||||
def _v2_transition_scene_conditionings(self, clip, scene_texts, scene_effects=None, encode_cache=None, reference_image=None):
|
||||
if clip is None:
|
||||
return None
|
||||
scene_texts = list(scene_texts or [])
|
||||
@@ -6412,7 +6412,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
|
||||
seen_count = seen_texts.get(scene_text, 0)
|
||||
seen_texts[scene_text] = seen_count + 1
|
||||
encode_text = f"Returning to an earlier scene: {scene_text}" if seen_count > 0 else scene_text
|
||||
cond, meta, _ = self._v2_encode_prompt(clip, encode_text, encode_cache=encode_cache)
|
||||
cond, meta, _ = self._v2_encode_prompt(clip, encode_text, encode_cache=encode_cache, reference_image=reference_image)
|
||||
if not isinstance(cond, torch.Tensor):
|
||||
return None
|
||||
scene_meta = dict(meta) if isinstance(meta, dict) else {"pooled_output": None}
|
||||
@@ -11839,6 +11839,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
|
||||
if scene_texts:
|
||||
scene_conditionings = self._v2_transition_scene_conditionings(
|
||||
clip, scene_texts, scene_effects=split_scene_effects, encode_cache=encode_cache,
|
||||
reference_image=source_image,
|
||||
)
|
||||
if scene_conditionings is not None:
|
||||
output_conditioning = self._v2_apply_scene_seed_metadata(
|
||||
|
||||
+2
-55
@@ -794,10 +794,6 @@ class FunPackLTXAVSceneChainSampler:
|
||||
"default": False,
|
||||
"tooltip": "Carry protected frames from latent_template noise_mask into each continuation chunk as a style guide.",
|
||||
}),
|
||||
"reference_conditioning": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "Re-encode every scene's conditioning with the original reference image as visual context (requires CLIP). Gives the model character identity information directly in the text conditioning for every scene.",
|
||||
}),
|
||||
"self_consistency": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "Experimental: capture hidden states from the middle of the video at an early denoising step and inject them back in later steps to reduce within-scene character drift.",
|
||||
@@ -813,7 +809,7 @@ class FunPackLTXAVSceneChainSampler:
|
||||
},
|
||||
"optional": {
|
||||
"clip": ("CLIP", {
|
||||
"tooltip": "Connect the same CLIP as your Studio node. Enables reference_conditioning and per-scene vision re-encoding from the previous frame.",
|
||||
"tooltip": "Connect the same CLIP as your Studio node. Enables per-scene vision re-encoding from the previous frame.",
|
||||
}),
|
||||
"decode_tile_size": ("INT", {
|
||||
"default": 0, "min": 0, "max": 4096, "step": 64,
|
||||
@@ -1286,32 +1282,6 @@ class FunPackLTXAVSceneChainSampler:
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _reference_reencoded_conditioning(self, clip, ref_pixel, scene_cond, scene_index):
|
||||
"""Re-encode scene conditioning with the pre-decoded reference image as visual context."""
|
||||
try:
|
||||
t = clip.cond_stage_model.gemma3_12b.transformer
|
||||
if not (hasattr(t, "vision_model") and hasattr(t, "multi_modal_projector")):
|
||||
return None
|
||||
existing_meta = scene_cond[1] if isinstance(scene_cond, (list, tuple)) and len(scene_cond) >= 2 and isinstance(scene_cond[1], dict) else {}
|
||||
text = existing_meta.get("funpack_encode_text") or self._scene_text(scene_cond, scene_index)
|
||||
if not text:
|
||||
return None
|
||||
tokens = clip.tokenize(text, image=ref_pixel, skip_template=False)
|
||||
encoded = clip.encode_from_tokens_scheduled(tokens)
|
||||
if not isinstance(encoded, list) or not encoded:
|
||||
return None
|
||||
item = encoded[0]
|
||||
if not isinstance(item, (list, tuple)) or len(item) < 2:
|
||||
return None
|
||||
new_cond = item[0]
|
||||
new_meta = item[1] if isinstance(item[1], dict) else {"pooled_output": None}
|
||||
if not isinstance(new_cond, torch.Tensor):
|
||||
return None
|
||||
merged = {**new_meta, **{k: v for k, v in existing_meta.items() if k.startswith("funpack_")}}
|
||||
return (new_cond, merged)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _build_self_consistency_hooks(self, model, video_shape, strength):
|
||||
"""Register hooks that capture middle-frame hidden states at early sigma
|
||||
and inject them back in later steps to reduce within-scene drift."""
|
||||
@@ -1420,7 +1390,7 @@ class FunPackLTXAVSceneChainSampler:
|
||||
|
||||
def sample(self, model, vae, positive, negative, sampler, sigmas, seed, latent_template,
|
||||
num_frames_per_scene, frame_overlap, cfg, max_scenes, use_same_seed=False,
|
||||
carry_i2v_guides=False, reference_conditioning=False,
|
||||
carry_i2v_guides=False,
|
||||
self_consistency=False, self_consistency_strength=0.05,
|
||||
transition_duration=16, decode_tile_size=0,
|
||||
refinement_key_input="", clip=None, unique_id=None, prompt=None):
|
||||
@@ -1442,21 +1412,6 @@ class FunPackLTXAVSceneChainSampler:
|
||||
boundary_entries = []
|
||||
cumulative_latent_frames = 0
|
||||
|
||||
# Decode reference frame once for reference_conditioning
|
||||
_ref_pixel = None
|
||||
if reference_conditioning and clip is not None:
|
||||
try:
|
||||
ref_tensors = self._latent_tensors(latent_template)
|
||||
if ref_tensors:
|
||||
ref_lat = self._time_slice(ref_tensors[0], 0, 1)
|
||||
decoded = vae.decode(ref_lat)
|
||||
if decoded is not None:
|
||||
if decoded.dim() == 5:
|
||||
decoded = decoded[:, 0]
|
||||
_ref_pixel = decoded.clamp(0.0, 1.0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
first_scene_seed = self._scene_seed(scene_conditionings[0])
|
||||
if first_scene_seed is None:
|
||||
first_scene_seed = int(seed)
|
||||
@@ -1464,14 +1419,6 @@ class FunPackLTXAVSceneChainSampler:
|
||||
scene_positive = [scene_cond]
|
||||
scene_negative = negative
|
||||
|
||||
# Apply reference image to this scene's conditioning
|
||||
if _ref_pixel is not None:
|
||||
updated = self._reference_reencoded_conditioning(
|
||||
clip, _ref_pixel, scene_cond, scene_index
|
||||
)
|
||||
if updated is not None:
|
||||
scene_positive = [updated]
|
||||
|
||||
provided_seed = self._scene_seed(scene_cond)
|
||||
if use_same_seed:
|
||||
scene_seed = first_scene_seed
|
||||
|
||||
Reference in New Issue
Block a user