Fix mixed source: Img2Video anchors, not guide tokens

i2v anchors are starting latents via LTXVImgToVideoInplace; i2v
guides remain separate (carry_i2v_guides / custom guide stack).
Removed the incorrect anchor-as-guide path; mixed scenes now build
their chunk from Img2Video then apply overlap and carried guides.
This commit is contained in:
digital-garbage
2026-06-10 15:13:14 +03:00
parent 31785600c0
commit e6767d00d9
5 changed files with 185 additions and 97 deletions
+34 -60
View File
@@ -72,20 +72,20 @@ def is_mixed_source(scene: Scene) -> bool:
return (scene.source.type or "carry") == "mixed"
def mixed_anchor_entry(scene: Scene, scene_index: int) -> Optional[dict]:
"""Per-scene i2v anchor for mixed source (applied at frame 0 in the sampler)."""
if not is_mixed_source(scene) or not scene.source.media_ref:
return None
return {
"enabled": True,
"source": "anchor",
"scene_id": scene.id,
"scene_index": scene_index,
"media_ref": scene.source.media_ref,
"frame_idx": 0,
"apply_at": 0,
"strength": 0.35,
}
def build_scene_anchors_payload(project: Project) -> Optional[dict]:
"""Per-scene i2v anchors for mixed sources — image → LTXVImgToVideoInplace starting
latent, NOT guide attention. Scene 0 uses the graph-level Studio/img2video path."""
active = [s for s in project.scenes if not s.excluded]
anchors: dict[str, dict] = {}
for i, sc in enumerate(active):
if i == 0 or not is_mixed_source(sc) or not sc.source.media_ref:
continue
anchors[str(i)] = {
"scene_id": sc.id,
"media_ref": sc.source.media_ref,
"strength": 1.0,
}
return anchors if anchors else None
@dataclass
@@ -93,7 +93,7 @@ class SceneSource:
"""How a scene's latent is born. V1 ignores this (uniform chain); Phase 2 maps
it onto EmptyLTXVLatent (empty/t2v) or LTXV Image to Video (image/i2v)."""
type: str = "carry" # "carry" | "empty" | "image" | "generated_frame" | "mixed"
media_ref: Optional[str] = None # asset id, for type == "image"
media_ref: Optional[str] = None # asset id — image/mixed anchor (Img2Video)
frame_ref: Optional[dict[str, Any]] = None # {scene_id, frame_idx}, for "generated_frame"
target: Optional[str] = None # wire dest for the image: "port:<id>" | "node:<slotId>:<input>"
@@ -377,63 +377,37 @@ def build_combined_prompt(project: Project, include_excluded: bool = False,
def build_scene_guides_payload(project: Project) -> Optional[dict]:
"""Per-scene guide lists for the chain sampler.
None when stack_enabled is off AND no mixed sources (Studio default:
carry_i2v_guides from scene 1 template at frame 0).
Mixed sources always emit a payload for continuation scenes that need both
the Studio prior guide and the scene's own anchor at frame 0.
None when stack_enabled is off (Studio default: carry_i2v_guides from scene 1
template at frame 0). i2v anchors for mixed scenes are separate — see
build_scene_anchors_payload().
"""
gs = normalize_guide_settings(project.guide_settings)
active = [s for s in project.scenes if not s.excluded]
has_mixed = any(is_mixed_source(s) and s.source.media_ref for s in active)
if not gs["stack_enabled"] and not has_mixed:
if not gs["stack_enabled"]:
return None
active = [s for s in project.scenes if not s.excluded]
per_scene: list[Optional[list[dict]]] = []
uses_custom = False
for i, sc in enumerate(active):
if i == 0:
per_scene.append(None)
continue
entries: list[dict] = []
own_anchor = mixed_anchor_entry(sc, i)
if gs["stack_enabled"]:
for raw in (sc.guides or []):
g = GuideEntry.from_dict(raw if isinstance(raw, dict) else {})
if g.enabled:
entries.append(g.to_dict())
if gs["accumulate_prior"]:
for j in range(i):
ref = active[j]
acc = dict(STUDIO_DEFAULT_GUIDE)
acc["source"] = "scene"
acc["scene_id"] = ref.id
acc["scene_index"] = j
entries.append(acc)
if own_anchor:
has_prior = any(e.get("source") in ("template", "scene") for e in entries)
if not has_prior:
entries.insert(0, dict(STUDIO_DEFAULT_GUIDE))
entries.append(own_anchor)
uses_custom = True
elif gs["stack_enabled"] and not entries:
for raw in (sc.guides or []):
g = GuideEntry.from_dict(raw if isinstance(raw, dict) else {})
if g.enabled:
entries.append(g.to_dict())
if gs["accumulate_prior"]:
for j in range(i):
ref = active[j]
acc = dict(STUDIO_DEFAULT_GUIDE)
acc["source"] = "scene"
acc["scene_id"] = ref.id
acc["scene_index"] = j
entries.append(acc)
if not entries:
entries = [dict(STUDIO_DEFAULT_GUIDE)]
uses_custom = True
elif not entries:
per_scene.append(None)
continue
else:
uses_custom = True
per_scene.append(entries)
if not uses_custom:
return None
return {
"stack_enabled": bool(gs["stack_enabled"]),
"stack_enabled": True,
"accumulate_prior": gs["accumulate_prior"],
"has_mixed": has_mixed,
"scenes": per_scene,
}
+4 -4
View File
@@ -10,7 +10,7 @@
["image", "Image · i2v anchor"],
["generated_frame", "From generated frame"],
["carry", "Carry i2v guide · continue previous"],
["mixed", "Mixed · anchor + prior guides"],
["mixed", "Mixed · Img2Video anchor + prior guides"],
];
function transitionSelect(value, onChange) {
@@ -56,7 +56,7 @@
(st.mediaBin || []).forEach((m) => { const o = new Option(m.name, m.id); if (m.id === ref) o.selected = true; pick.append(o); });
pick.onchange = () => S.patchScene(scene.id, { source: { ...(scene.source || {}), type: "image", media_ref: pick.value || null } });
body.append(field("Media asset", pick));
body.append(el("div", "insp-hint", "Feeds the i2v anchor for this scene. Routing is automatic (→ Studio source image); to send it through a node first, set that node's Input source to Timeline in Models."));
body.append(el("div", "insp-hint", "Feeds the i2v anchor for this scene (image → LTX Img2Video → starting latent). Routing is automatic (→ Studio source image); to send it through a node first, set that node's Input source to Timeline in Models."));
}
function renderGeneratedFrameSource(st, scene) {
@@ -168,7 +168,7 @@
if ((root.source?.type) === "mixed") {
const hint = el("div", "insp-block");
hint.append(el("div", "insp-hint",
"Stays in the same chain run (overlap + prior guides at frame 0) while also applying this scene's image anchor at frame 0. Default: Studio guide from scene 1 plus this anchor — enable Custom guide stack in Project for more control."));
"Stays in the same chain run: this image builds a new starting latent via LTX Img2Video (i2v anchor), while prior-scene i2v guides are carried separately (Studio default: scene 1 template guide at frame 0). Enable Custom guide stack in Project to manage guides explicitly."));
body.append(hint);
}
@@ -518,7 +518,7 @@
const hint = el("div", "insp-hint");
hint.textContent = gs.stack_enabled
? "Custom guide stack — per-scene lists in the Scene inspector. Scenes without entries use the Studio default (scene 1 template · frame 0 · apply 0)."
: "Studio default: one i2v guide from scene 1, carried to later scenes at frame 0 (same as FunPack Studio).";
: "Studio default: one i2v guide (not anchor) from scene 1's template, carried to later scenes at frame 0. Anchors are separate — image → Img2Video starting latent.";
parent.append(hint);
const stackLbl = el("label", "chk");
const stackCb = el("input"); stackCb.type = "checkbox"; stackCb.checked = gs.stack_enabled;
+25 -4
View File
@@ -203,10 +203,31 @@ def _run_sampler_inputs(target: Project, scene_count: int) -> dict:
return samp
def _attach_scene_media(samp: dict, media_pack: Optional[dict]) -> dict:
def _attach_scene_anchors(samp: dict, media_pack: Optional[dict], target: Project) -> dict:
"""Mixed-source i2v anchors (Img2Video starting latents), separate from guides."""
import json
if media_pack and media_pack.get("by_scene"):
samp["funpack_scene_media"] = json.dumps(media_pack["by_scene"])
from .backend.timeline import build_scene_anchors_payload
anchors = build_scene_anchors_payload(target)
if not anchors or not media_pack:
return samp
by_scene = media_pack.get("by_scene") or {}
enriched: dict = {}
for idx, meta in anchors.items():
sid = meta.get("scene_id")
file_entry = by_scene.get(sid) if sid else None
fn = (file_entry or {}).get("filename")
if fn:
enriched[idx] = {**meta, "filename": fn}
if enriched:
samp["funpack_scene_anchors"] = json.dumps(enriched)
by_ref = {
entry["media_ref"]: entry["filename"]
for entry in by_scene.values()
if isinstance(entry, dict) and entry.get("media_ref") and entry.get("filename")
}
if by_ref:
samp["funpack_scene_media_refs"] = json.dumps(by_ref)
return samp
@@ -643,7 +664,7 @@ if web is not None and PromptServer is not None:
)
media_pack = _prepare_media(target)
sampler_inputs = _run_sampler_inputs(target, active_scene_count)
_attach_scene_media(sampler_inputs, media_pack)
_attach_scene_anchors(sampler_inputs, media_pack, target)
graph, report = builder.build(oi, _project_models(target), {
"prompt": prompt, "seed": target.seed,
"num_frames_per_scene": effective_frames,
+7 -10
View File
@@ -16,6 +16,7 @@ from movie_editor.backend.timeline import ( # noqa: E402
Scene,
STUDIO_DEFAULT_GUIDE,
build_combined_prompt,
build_scene_anchors_payload,
build_scene_guides_payload,
collapse_generative_units,
group_generative_units,
@@ -130,7 +131,7 @@ def test_guide_stack_studio_default_per_continuation_scene():
assert payload["scenes"][2] == [STUDIO_DEFAULT_GUIDE]
def test_mixed_source_emits_anchor_and_studio_default():
def test_mixed_source_anchors_separate_from_guides():
p = _project(
scenes=[
{"id": "s1", "text": "a", "source": {"type": "image", "media_ref": "img1"}},
@@ -138,15 +139,11 @@ def test_mixed_source_emits_anchor_and_studio_default():
],
)
assert is_mixed_source(p.scenes[1])
payload = build_scene_guides_payload(p)
assert payload is not None
assert payload["has_mixed"] is True
scene2 = payload["scenes"][1]
assert len(scene2) == 2
assert scene2[0] == STUDIO_DEFAULT_GUIDE
assert scene2[1]["source"] == "anchor"
assert scene2[1]["media_ref"] == "img2"
assert scene2[1]["apply_at"] == 0
assert build_scene_guides_payload(p) is None
anchors = build_scene_anchors_payload(p)
assert anchors["1"]["scene_id"] == "s2"
assert anchors["1"]["media_ref"] == "img2"
assert anchors["1"]["strength"] == 1.0
def test_guide_stack_accumulate_prior():
+115 -19
View File
@@ -2085,10 +2085,15 @@ class FunPackLTXAVSceneChainSampler:
"multiline": True,
"tooltip": "Optional JSON from Movie Editor when guide_settings.stack_enabled: per-scene guide lists with source, frame_idx, apply_at, strength. When empty, carry_i2v_guides uses the Studio default (scene 1 template at frame 0).",
}),
"funpack_scene_media": ("STRING", {
"funpack_scene_anchors": ("STRING", {
"default": "",
"multiline": True,
"tooltip": "Optional JSON map of scene_id → {filename} for mixed-source per-scene i2v anchors (Movie Editor).",
"tooltip": "Optional JSON map of scene_index → {filename, strength} for mixed-source i2v anchors (LTXVImgToVideoInplace starting latent). Distinct from i2v guides.",
}),
"funpack_scene_media_refs": ("STRING", {
"default": "",
"multiline": True,
"tooltip": "Optional media_ref → filename map for image-type i2v guides in custom guide stacks.",
}),
},
"hidden": {
@@ -2835,13 +2840,11 @@ class FunPackLTXAVSceneChainSampler:
data = json.loads(str(raw))
except Exception:
return None
if not isinstance(data, dict):
return None
if not data.get("scenes") and not data.get("has_mixed"):
if not isinstance(data, dict) or not data.get("stack_enabled"):
return None
return data
def _parse_scene_media(self, raw):
def _parse_scene_anchors(self, raw):
if not raw or not str(raw).strip():
return {}
try:
@@ -2851,6 +2854,89 @@ class FunPackLTXAVSceneChainSampler:
return {}
return data if isinstance(data, dict) else {}
def _load_image_tensor(self, filename):
import os
try:
import folder_paths
import numpy as np
from PIL import Image
except ImportError:
return None
path = os.path.join(folder_paths.get_input_directory(), filename)
if not os.path.isfile(path):
return None
try:
img = Image.open(path).convert("RGB")
arr = np.array(img).astype(np.float32) / 255.0
return torch.from_numpy(arr)[None,]
except Exception:
return None
def _invoke_img2video_inplace(self, vae, image, latent, strength=1.0):
"""Run LTXVImgToVideoInplace — image → starting latent (i2v anchor, not a guide)."""
try:
from comfy_extras.nodes_lt import LTXVImgToVideoInplace
except ImportError:
return None
for method in ("execute", "generate"):
fn = getattr(LTXVImgToVideoInplace, method, None)
if fn is None:
continue
for args in (
(vae, image, latent, strength, False),
(vae, image, latent, strength),
):
try:
out = fn(*args)
if hasattr(out, "args") and out.args:
out = out.args[0]
elif isinstance(out, tuple) and out:
out = out[0]
if isinstance(out, dict) and "samples" in out:
return out
except (TypeError, ValueError):
continue
return None
def _apply_img2video_to_video_latent(self, vae, image, chunk, strength=1.0):
"""Apply Img2Video inplace to the video stream of an (optionally nested AV) latent."""
if self._is_nested(chunk.get("samples")):
tensors = self._latent_tensors(chunk)
masks = self._latent_masks(chunk, len(tensors))
video_in = {"samples": tensors[0]}
if masks[0] is not None:
video_in["noise_mask"] = masks[0]
video_out = self._invoke_img2video_inplace(vae, image, video_in, strength)
if video_out is None:
return chunk
result = self._clone_latent(chunk)
out_tensors = list(tensors)
out_masks = list(masks)
out_tensors[0] = video_out["samples"]
if "noise_mask" in video_out:
out_masks[0] = video_out["noise_mask"]
result["samples"] = comfy.nested_tensor.NestedTensor(out_tensors)
if any(m is not None for m in out_masks):
result["noise_mask"] = comfy.nested_tensor.NestedTensor(out_masks)
return result
out = self._invoke_img2video_inplace(vae, image, chunk, strength)
return out if out is not None else chunk
def _build_mixed_anchor_chunk(self, vae, anchor_meta, latent_template, previous, video_overlap):
"""Mixed source: Img2Video anchor latent, then chain overlap from previous scene."""
filename = (anchor_meta or {}).get("filename")
strength = float((anchor_meta or {}).get("strength", 1.0))
image = self._load_image_tensor(filename) if filename else None
if image is None:
if previous is None:
return self._clone_latent(latent_template)
return self._build_continuation_chunk(latent_template, previous, video_overlap)
base = self._clone_latent(latent_template)
i2v_chunk = self._apply_img2video_to_video_latent(vae, image, base, strength)
if previous is None:
return i2v_chunk
return self._build_continuation_chunk(i2v_chunk, previous, video_overlap)
def _encode_image_guide_frame(self, filename, vae, ref_tensor):
import os
try:
@@ -2904,7 +2990,7 @@ class FunPackLTXAVSceneChainSampler:
) + (0,)
def _apply_configured_guides(self, chunk, scene_index, guide_list, latent_template,
scene_outputs, scene_media, positive, negative, vae):
scene_outputs, scene_media_by_ref, positive, negative, vae):
head_crop = 0
tail_crop = 0
for g in guide_list or []:
@@ -2931,15 +3017,9 @@ class FunPackLTXAVSceneChainSampler:
chunk, positive, negative, head, tail = self._append_scene_guide_at(
chunk, scene_outputs[si], frame_idx, apply_at, strength, positive, negative, vae,
)
elif source in ("anchor", "image"):
sid = g.get("scene_id")
meta = (scene_media or {}).get(sid) if sid else None
if not meta and g.get("media_ref"):
for _k, m in (scene_media or {}).items():
if isinstance(m, dict) and m.get("media_ref") == g.get("media_ref"):
meta = m
break
fn = (meta or {}).get("filename") if isinstance(meta, dict) else None
elif source == "image":
ref = g.get("media_ref")
fn = (scene_media_by_ref or {}).get(ref) if ref else None
if not fn:
continue
chunk, positive, negative, head, tail = self._append_media_guide_at(
@@ -3046,7 +3126,8 @@ class FunPackLTXAVSceneChainSampler:
transition_duration=16, decode_tile_size=0,
decode_noise_scale=0.0, decode_timestep=0.05,
refinement_key_input="", funpack_scene_guides="",
funpack_scene_media="",
funpack_scene_anchors="",
funpack_scene_media_refs="",
unique_id=None, prompt=None):
if not isinstance(positive, list) or not positive:
raise ValueError("positive conditioning must contain at least one scene entry.")
@@ -3119,8 +3200,9 @@ class FunPackLTXAVSceneChainSampler:
scene_guides_cfg = self._parse_scene_guides(funpack_scene_guides)
per_scene_guides = (scene_guides_cfg or {}).get("scenes") if scene_guides_cfg else None
scene_media = self._parse_scene_media(funpack_scene_media)
scene_anchors = self._parse_scene_anchors(funpack_scene_anchors)
scene_outputs: list = []
scene_media_by_ref = self._parse_scene_anchors(funpack_scene_media_refs)
for scene_index, scene_cond in enumerate(scene_conditionings):
scene_positive = [scene_cond]
@@ -3134,8 +3216,22 @@ class FunPackLTXAVSceneChainSampler:
carried = 0
soft_carried = 0
guide_tail = 0
anchor_meta = (scene_anchors or {}).get(str(scene_index))
if output is None:
chunk = self._clone_latent(latent_template)
elif anchor_meta:
effect = self._scene_transition_effect(scene_cond)
if effect and transition_duration > 0:
boundary_latent = cumulative_latent_frames
boundary_pixel = int((boundary_latent - 1) * time_scale + 1) if time_scale > 1 else boundary_latent
boundary_entries.append({
"boundary_latent": boundary_latent,
"pixel_frame": max(0, boundary_pixel),
"effect": effect,
})
chunk = self._build_mixed_anchor_chunk(
vae, anchor_meta, latent_template, output, video_overlap,
)
else:
# Record boundary before blending
effect = self._scene_transition_effect(scene_cond)
@@ -3155,7 +3251,7 @@ class FunPackLTXAVSceneChainSampler:
custom_guides = per_scene_guides[scene_index]
if custom_guides:
chunk, scene_positive, scene_negative, carried, guide_tail = self._apply_configured_guides(
chunk, scene_index, custom_guides, latent_template, scene_outputs, scene_media,
chunk, scene_index, custom_guides, latent_template, scene_outputs, scene_media_by_ref,
scene_positive, scene_negative, vae,
)
carried_guide_frames = max(carried_guide_frames, carried)