Fix mixed source: Img2Video anchors, not guide tokens
i2v anchors are starting latents via LTXVImgToVideoInplace; i2v guides remain separate (carry_i2v_guides / custom guide stack). Removed the incorrect anchor-as-guide path; mixed scenes now build their chunk from Img2Video then apply overlap and carried guides.
This commit is contained in:
@@ -72,20 +72,20 @@ def is_mixed_source(scene: Scene) -> bool:
|
||||
return (scene.source.type or "carry") == "mixed"
|
||||
|
||||
|
||||
def mixed_anchor_entry(scene: Scene, scene_index: int) -> Optional[dict]:
|
||||
"""Per-scene i2v anchor for mixed source (applied at frame 0 in the sampler)."""
|
||||
if not is_mixed_source(scene) or not scene.source.media_ref:
|
||||
return None
|
||||
return {
|
||||
"enabled": True,
|
||||
"source": "anchor",
|
||||
"scene_id": scene.id,
|
||||
"scene_index": scene_index,
|
||||
"media_ref": scene.source.media_ref,
|
||||
"frame_idx": 0,
|
||||
"apply_at": 0,
|
||||
"strength": 0.35,
|
||||
}
|
||||
def build_scene_anchors_payload(project: Project) -> Optional[dict]:
|
||||
"""Per-scene i2v anchors for mixed sources — image → LTXVImgToVideoInplace starting
|
||||
latent, NOT guide attention. Scene 0 uses the graph-level Studio/img2video path."""
|
||||
active = [s for s in project.scenes if not s.excluded]
|
||||
anchors: dict[str, dict] = {}
|
||||
for i, sc in enumerate(active):
|
||||
if i == 0 or not is_mixed_source(sc) or not sc.source.media_ref:
|
||||
continue
|
||||
anchors[str(i)] = {
|
||||
"scene_id": sc.id,
|
||||
"media_ref": sc.source.media_ref,
|
||||
"strength": 1.0,
|
||||
}
|
||||
return anchors if anchors else None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -93,7 +93,7 @@ class SceneSource:
|
||||
"""How a scene's latent is born. V1 ignores this (uniform chain); Phase 2 maps
|
||||
it onto EmptyLTXVLatent (empty/t2v) or LTXV Image to Video (image/i2v)."""
|
||||
type: str = "carry" # "carry" | "empty" | "image" | "generated_frame" | "mixed"
|
||||
media_ref: Optional[str] = None # asset id, for type == "image"
|
||||
media_ref: Optional[str] = None # asset id — image/mixed anchor (Img2Video)
|
||||
frame_ref: Optional[dict[str, Any]] = None # {scene_id, frame_idx}, for "generated_frame"
|
||||
target: Optional[str] = None # wire dest for the image: "port:<id>" | "node:<slotId>:<input>"
|
||||
|
||||
@@ -377,63 +377,37 @@ def build_combined_prompt(project: Project, include_excluded: bool = False,
|
||||
def build_scene_guides_payload(project: Project) -> Optional[dict]:
|
||||
"""Per-scene guide lists for the chain sampler.
|
||||
|
||||
None when stack_enabled is off AND no mixed sources (Studio default:
|
||||
carry_i2v_guides from scene 1 template at frame 0).
|
||||
|
||||
Mixed sources always emit a payload for continuation scenes that need both
|
||||
the Studio prior guide and the scene's own anchor at frame 0.
|
||||
None when stack_enabled is off (Studio default: carry_i2v_guides from scene 1
|
||||
template at frame 0). i2v anchors for mixed scenes are separate — see
|
||||
build_scene_anchors_payload().
|
||||
"""
|
||||
gs = normalize_guide_settings(project.guide_settings)
|
||||
active = [s for s in project.scenes if not s.excluded]
|
||||
has_mixed = any(is_mixed_source(s) and s.source.media_ref for s in active)
|
||||
if not gs["stack_enabled"] and not has_mixed:
|
||||
if not gs["stack_enabled"]:
|
||||
return None
|
||||
|
||||
active = [s for s in project.scenes if not s.excluded]
|
||||
per_scene: list[Optional[list[dict]]] = []
|
||||
uses_custom = False
|
||||
for i, sc in enumerate(active):
|
||||
if i == 0:
|
||||
per_scene.append(None)
|
||||
continue
|
||||
entries: list[dict] = []
|
||||
own_anchor = mixed_anchor_entry(sc, i)
|
||||
|
||||
if gs["stack_enabled"]:
|
||||
for raw in (sc.guides or []):
|
||||
g = GuideEntry.from_dict(raw if isinstance(raw, dict) else {})
|
||||
if g.enabled:
|
||||
entries.append(g.to_dict())
|
||||
if gs["accumulate_prior"]:
|
||||
for j in range(i):
|
||||
ref = active[j]
|
||||
acc = dict(STUDIO_DEFAULT_GUIDE)
|
||||
acc["source"] = "scene"
|
||||
acc["scene_id"] = ref.id
|
||||
acc["scene_index"] = j
|
||||
entries.append(acc)
|
||||
|
||||
if own_anchor:
|
||||
has_prior = any(e.get("source") in ("template", "scene") for e in entries)
|
||||
if not has_prior:
|
||||
entries.insert(0, dict(STUDIO_DEFAULT_GUIDE))
|
||||
entries.append(own_anchor)
|
||||
uses_custom = True
|
||||
elif gs["stack_enabled"] and not entries:
|
||||
for raw in (sc.guides or []):
|
||||
g = GuideEntry.from_dict(raw if isinstance(raw, dict) else {})
|
||||
if g.enabled:
|
||||
entries.append(g.to_dict())
|
||||
if gs["accumulate_prior"]:
|
||||
for j in range(i):
|
||||
ref = active[j]
|
||||
acc = dict(STUDIO_DEFAULT_GUIDE)
|
||||
acc["source"] = "scene"
|
||||
acc["scene_id"] = ref.id
|
||||
acc["scene_index"] = j
|
||||
entries.append(acc)
|
||||
if not entries:
|
||||
entries = [dict(STUDIO_DEFAULT_GUIDE)]
|
||||
uses_custom = True
|
||||
elif not entries:
|
||||
per_scene.append(None)
|
||||
continue
|
||||
else:
|
||||
uses_custom = True
|
||||
|
||||
per_scene.append(entries)
|
||||
|
||||
if not uses_custom:
|
||||
return None
|
||||
return {
|
||||
"stack_enabled": bool(gs["stack_enabled"]),
|
||||
"stack_enabled": True,
|
||||
"accumulate_prior": gs["accumulate_prior"],
|
||||
"has_mixed": has_mixed,
|
||||
"scenes": per_scene,
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
["image", "Image · i2v anchor"],
|
||||
["generated_frame", "From generated frame"],
|
||||
["carry", "Carry i2v guide · continue previous"],
|
||||
["mixed", "Mixed · anchor + prior guides"],
|
||||
["mixed", "Mixed · Img2Video anchor + prior guides"],
|
||||
];
|
||||
|
||||
function transitionSelect(value, onChange) {
|
||||
@@ -56,7 +56,7 @@
|
||||
(st.mediaBin || []).forEach((m) => { const o = new Option(m.name, m.id); if (m.id === ref) o.selected = true; pick.append(o); });
|
||||
pick.onchange = () => S.patchScene(scene.id, { source: { ...(scene.source || {}), type: "image", media_ref: pick.value || null } });
|
||||
body.append(field("Media asset", pick));
|
||||
body.append(el("div", "insp-hint", "Feeds the i2v anchor for this scene. Routing is automatic (→ Studio source image); to send it through a node first, set that node's Input source to Timeline in Models."));
|
||||
body.append(el("div", "insp-hint", "Feeds the i2v anchor for this scene (image → LTX Img2Video → starting latent). Routing is automatic (→ Studio source image); to send it through a node first, set that node's Input source to Timeline in Models."));
|
||||
}
|
||||
|
||||
function renderGeneratedFrameSource(st, scene) {
|
||||
@@ -168,7 +168,7 @@
|
||||
if ((root.source?.type) === "mixed") {
|
||||
const hint = el("div", "insp-block");
|
||||
hint.append(el("div", "insp-hint",
|
||||
"Stays in the same chain run (overlap + prior guides at frame 0) while also applying this scene's image anchor at frame 0. Default: Studio guide from scene 1 plus this anchor — enable Custom guide stack in Project for more control."));
|
||||
"Stays in the same chain run: this image builds a new starting latent via LTX Img2Video (i2v anchor), while prior-scene i2v guides are carried separately (Studio default: scene 1 template guide at frame 0). Enable Custom guide stack in Project to manage guides explicitly."));
|
||||
body.append(hint);
|
||||
}
|
||||
|
||||
@@ -518,7 +518,7 @@
|
||||
const hint = el("div", "insp-hint");
|
||||
hint.textContent = gs.stack_enabled
|
||||
? "Custom guide stack — per-scene lists in the Scene inspector. Scenes without entries use the Studio default (scene 1 template · frame 0 · apply 0)."
|
||||
: "Studio default: one i2v guide from scene 1, carried to later scenes at frame 0 (same as FunPack Studio).";
|
||||
: "Studio default: one i2v guide (not anchor) from scene 1's template, carried to later scenes at frame 0. Anchors are separate — image → Img2Video starting latent.";
|
||||
parent.append(hint);
|
||||
const stackLbl = el("label", "chk");
|
||||
const stackCb = el("input"); stackCb.type = "checkbox"; stackCb.checked = gs.stack_enabled;
|
||||
|
||||
+25
-4
@@ -203,10 +203,31 @@ def _run_sampler_inputs(target: Project, scene_count: int) -> dict:
|
||||
return samp
|
||||
|
||||
|
||||
def _attach_scene_media(samp: dict, media_pack: Optional[dict]) -> dict:
|
||||
def _attach_scene_anchors(samp: dict, media_pack: Optional[dict], target: Project) -> dict:
|
||||
"""Mixed-source i2v anchors (Img2Video starting latents), separate from guides."""
|
||||
import json
|
||||
if media_pack and media_pack.get("by_scene"):
|
||||
samp["funpack_scene_media"] = json.dumps(media_pack["by_scene"])
|
||||
from .backend.timeline import build_scene_anchors_payload
|
||||
|
||||
anchors = build_scene_anchors_payload(target)
|
||||
if not anchors or not media_pack:
|
||||
return samp
|
||||
by_scene = media_pack.get("by_scene") or {}
|
||||
enriched: dict = {}
|
||||
for idx, meta in anchors.items():
|
||||
sid = meta.get("scene_id")
|
||||
file_entry = by_scene.get(sid) if sid else None
|
||||
fn = (file_entry or {}).get("filename")
|
||||
if fn:
|
||||
enriched[idx] = {**meta, "filename": fn}
|
||||
if enriched:
|
||||
samp["funpack_scene_anchors"] = json.dumps(enriched)
|
||||
by_ref = {
|
||||
entry["media_ref"]: entry["filename"]
|
||||
for entry in by_scene.values()
|
||||
if isinstance(entry, dict) and entry.get("media_ref") and entry.get("filename")
|
||||
}
|
||||
if by_ref:
|
||||
samp["funpack_scene_media_refs"] = json.dumps(by_ref)
|
||||
return samp
|
||||
|
||||
|
||||
@@ -643,7 +664,7 @@ if web is not None and PromptServer is not None:
|
||||
)
|
||||
media_pack = _prepare_media(target)
|
||||
sampler_inputs = _run_sampler_inputs(target, active_scene_count)
|
||||
_attach_scene_media(sampler_inputs, media_pack)
|
||||
_attach_scene_anchors(sampler_inputs, media_pack, target)
|
||||
graph, report = builder.build(oi, _project_models(target), {
|
||||
"prompt": prompt, "seed": target.seed,
|
||||
"num_frames_per_scene": effective_frames,
|
||||
|
||||
@@ -16,6 +16,7 @@ from movie_editor.backend.timeline import ( # noqa: E402
|
||||
Scene,
|
||||
STUDIO_DEFAULT_GUIDE,
|
||||
build_combined_prompt,
|
||||
build_scene_anchors_payload,
|
||||
build_scene_guides_payload,
|
||||
collapse_generative_units,
|
||||
group_generative_units,
|
||||
@@ -130,7 +131,7 @@ def test_guide_stack_studio_default_per_continuation_scene():
|
||||
assert payload["scenes"][2] == [STUDIO_DEFAULT_GUIDE]
|
||||
|
||||
|
||||
def test_mixed_source_emits_anchor_and_studio_default():
|
||||
def test_mixed_source_anchors_separate_from_guides():
|
||||
p = _project(
|
||||
scenes=[
|
||||
{"id": "s1", "text": "a", "source": {"type": "image", "media_ref": "img1"}},
|
||||
@@ -138,15 +139,11 @@ def test_mixed_source_emits_anchor_and_studio_default():
|
||||
],
|
||||
)
|
||||
assert is_mixed_source(p.scenes[1])
|
||||
payload = build_scene_guides_payload(p)
|
||||
assert payload is not None
|
||||
assert payload["has_mixed"] is True
|
||||
scene2 = payload["scenes"][1]
|
||||
assert len(scene2) == 2
|
||||
assert scene2[0] == STUDIO_DEFAULT_GUIDE
|
||||
assert scene2[1]["source"] == "anchor"
|
||||
assert scene2[1]["media_ref"] == "img2"
|
||||
assert scene2[1]["apply_at"] == 0
|
||||
assert build_scene_guides_payload(p) is None
|
||||
anchors = build_scene_anchors_payload(p)
|
||||
assert anchors["1"]["scene_id"] == "s2"
|
||||
assert anchors["1"]["media_ref"] == "img2"
|
||||
assert anchors["1"]["strength"] == 1.0
|
||||
|
||||
|
||||
def test_guide_stack_accumulate_prior():
|
||||
|
||||
+115
-19
@@ -2085,10 +2085,15 @@ class FunPackLTXAVSceneChainSampler:
|
||||
"multiline": True,
|
||||
"tooltip": "Optional JSON from Movie Editor when guide_settings.stack_enabled: per-scene guide lists with source, frame_idx, apply_at, strength. When empty, carry_i2v_guides uses the Studio default (scene 1 template at frame 0).",
|
||||
}),
|
||||
"funpack_scene_media": ("STRING", {
|
||||
"funpack_scene_anchors": ("STRING", {
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": "Optional JSON map of scene_id → {filename} for mixed-source per-scene i2v anchors (Movie Editor).",
|
||||
"tooltip": "Optional JSON map of scene_index → {filename, strength} for mixed-source i2v anchors (LTXVImgToVideoInplace starting latent). Distinct from i2v guides.",
|
||||
}),
|
||||
"funpack_scene_media_refs": ("STRING", {
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": "Optional media_ref → filename map for image-type i2v guides in custom guide stacks.",
|
||||
}),
|
||||
},
|
||||
"hidden": {
|
||||
@@ -2835,13 +2840,11 @@ class FunPackLTXAVSceneChainSampler:
|
||||
data = json.loads(str(raw))
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
if not data.get("scenes") and not data.get("has_mixed"):
|
||||
if not isinstance(data, dict) or not data.get("stack_enabled"):
|
||||
return None
|
||||
return data
|
||||
|
||||
def _parse_scene_media(self, raw):
|
||||
def _parse_scene_anchors(self, raw):
|
||||
if not raw or not str(raw).strip():
|
||||
return {}
|
||||
try:
|
||||
@@ -2851,6 +2854,89 @@ class FunPackLTXAVSceneChainSampler:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
def _load_image_tensor(self, filename):
|
||||
import os
|
||||
try:
|
||||
import folder_paths
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
return None
|
||||
path = os.path.join(folder_paths.get_input_directory(), filename)
|
||||
if not os.path.isfile(path):
|
||||
return None
|
||||
try:
|
||||
img = Image.open(path).convert("RGB")
|
||||
arr = np.array(img).astype(np.float32) / 255.0
|
||||
return torch.from_numpy(arr)[None,]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _invoke_img2video_inplace(self, vae, image, latent, strength=1.0):
|
||||
"""Run LTXVImgToVideoInplace — image → starting latent (i2v anchor, not a guide)."""
|
||||
try:
|
||||
from comfy_extras.nodes_lt import LTXVImgToVideoInplace
|
||||
except ImportError:
|
||||
return None
|
||||
for method in ("execute", "generate"):
|
||||
fn = getattr(LTXVImgToVideoInplace, method, None)
|
||||
if fn is None:
|
||||
continue
|
||||
for args in (
|
||||
(vae, image, latent, strength, False),
|
||||
(vae, image, latent, strength),
|
||||
):
|
||||
try:
|
||||
out = fn(*args)
|
||||
if hasattr(out, "args") and out.args:
|
||||
out = out.args[0]
|
||||
elif isinstance(out, tuple) and out:
|
||||
out = out[0]
|
||||
if isinstance(out, dict) and "samples" in out:
|
||||
return out
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return None
|
||||
|
||||
def _apply_img2video_to_video_latent(self, vae, image, chunk, strength=1.0):
|
||||
"""Apply Img2Video inplace to the video stream of an (optionally nested AV) latent."""
|
||||
if self._is_nested(chunk.get("samples")):
|
||||
tensors = self._latent_tensors(chunk)
|
||||
masks = self._latent_masks(chunk, len(tensors))
|
||||
video_in = {"samples": tensors[0]}
|
||||
if masks[0] is not None:
|
||||
video_in["noise_mask"] = masks[0]
|
||||
video_out = self._invoke_img2video_inplace(vae, image, video_in, strength)
|
||||
if video_out is None:
|
||||
return chunk
|
||||
result = self._clone_latent(chunk)
|
||||
out_tensors = list(tensors)
|
||||
out_masks = list(masks)
|
||||
out_tensors[0] = video_out["samples"]
|
||||
if "noise_mask" in video_out:
|
||||
out_masks[0] = video_out["noise_mask"]
|
||||
result["samples"] = comfy.nested_tensor.NestedTensor(out_tensors)
|
||||
if any(m is not None for m in out_masks):
|
||||
result["noise_mask"] = comfy.nested_tensor.NestedTensor(out_masks)
|
||||
return result
|
||||
out = self._invoke_img2video_inplace(vae, image, chunk, strength)
|
||||
return out if out is not None else chunk
|
||||
|
||||
def _build_mixed_anchor_chunk(self, vae, anchor_meta, latent_template, previous, video_overlap):
|
||||
"""Mixed source: Img2Video anchor latent, then chain overlap from previous scene."""
|
||||
filename = (anchor_meta or {}).get("filename")
|
||||
strength = float((anchor_meta or {}).get("strength", 1.0))
|
||||
image = self._load_image_tensor(filename) if filename else None
|
||||
if image is None:
|
||||
if previous is None:
|
||||
return self._clone_latent(latent_template)
|
||||
return self._build_continuation_chunk(latent_template, previous, video_overlap)
|
||||
base = self._clone_latent(latent_template)
|
||||
i2v_chunk = self._apply_img2video_to_video_latent(vae, image, base, strength)
|
||||
if previous is None:
|
||||
return i2v_chunk
|
||||
return self._build_continuation_chunk(i2v_chunk, previous, video_overlap)
|
||||
|
||||
def _encode_image_guide_frame(self, filename, vae, ref_tensor):
|
||||
import os
|
||||
try:
|
||||
@@ -2904,7 +2990,7 @@ class FunPackLTXAVSceneChainSampler:
|
||||
) + (0,)
|
||||
|
||||
def _apply_configured_guides(self, chunk, scene_index, guide_list, latent_template,
|
||||
scene_outputs, scene_media, positive, negative, vae):
|
||||
scene_outputs, scene_media_by_ref, positive, negative, vae):
|
||||
head_crop = 0
|
||||
tail_crop = 0
|
||||
for g in guide_list or []:
|
||||
@@ -2931,15 +3017,9 @@ class FunPackLTXAVSceneChainSampler:
|
||||
chunk, positive, negative, head, tail = self._append_scene_guide_at(
|
||||
chunk, scene_outputs[si], frame_idx, apply_at, strength, positive, negative, vae,
|
||||
)
|
||||
elif source in ("anchor", "image"):
|
||||
sid = g.get("scene_id")
|
||||
meta = (scene_media or {}).get(sid) if sid else None
|
||||
if not meta and g.get("media_ref"):
|
||||
for _k, m in (scene_media or {}).items():
|
||||
if isinstance(m, dict) and m.get("media_ref") == g.get("media_ref"):
|
||||
meta = m
|
||||
break
|
||||
fn = (meta or {}).get("filename") if isinstance(meta, dict) else None
|
||||
elif source == "image":
|
||||
ref = g.get("media_ref")
|
||||
fn = (scene_media_by_ref or {}).get(ref) if ref else None
|
||||
if not fn:
|
||||
continue
|
||||
chunk, positive, negative, head, tail = self._append_media_guide_at(
|
||||
@@ -3046,7 +3126,8 @@ class FunPackLTXAVSceneChainSampler:
|
||||
transition_duration=16, decode_tile_size=0,
|
||||
decode_noise_scale=0.0, decode_timestep=0.05,
|
||||
refinement_key_input="", funpack_scene_guides="",
|
||||
funpack_scene_media="",
|
||||
funpack_scene_anchors="",
|
||||
funpack_scene_media_refs="",
|
||||
unique_id=None, prompt=None):
|
||||
if not isinstance(positive, list) or not positive:
|
||||
raise ValueError("positive conditioning must contain at least one scene entry.")
|
||||
@@ -3119,8 +3200,9 @@ class FunPackLTXAVSceneChainSampler:
|
||||
|
||||
scene_guides_cfg = self._parse_scene_guides(funpack_scene_guides)
|
||||
per_scene_guides = (scene_guides_cfg or {}).get("scenes") if scene_guides_cfg else None
|
||||
scene_media = self._parse_scene_media(funpack_scene_media)
|
||||
scene_anchors = self._parse_scene_anchors(funpack_scene_anchors)
|
||||
scene_outputs: list = []
|
||||
scene_media_by_ref = self._parse_scene_anchors(funpack_scene_media_refs)
|
||||
|
||||
for scene_index, scene_cond in enumerate(scene_conditionings):
|
||||
scene_positive = [scene_cond]
|
||||
@@ -3134,8 +3216,22 @@ class FunPackLTXAVSceneChainSampler:
|
||||
carried = 0
|
||||
soft_carried = 0
|
||||
guide_tail = 0
|
||||
anchor_meta = (scene_anchors or {}).get(str(scene_index))
|
||||
if output is None:
|
||||
chunk = self._clone_latent(latent_template)
|
||||
elif anchor_meta:
|
||||
effect = self._scene_transition_effect(scene_cond)
|
||||
if effect and transition_duration > 0:
|
||||
boundary_latent = cumulative_latent_frames
|
||||
boundary_pixel = int((boundary_latent - 1) * time_scale + 1) if time_scale > 1 else boundary_latent
|
||||
boundary_entries.append({
|
||||
"boundary_latent": boundary_latent,
|
||||
"pixel_frame": max(0, boundary_pixel),
|
||||
"effect": effect,
|
||||
})
|
||||
chunk = self._build_mixed_anchor_chunk(
|
||||
vae, anchor_meta, latent_template, output, video_overlap,
|
||||
)
|
||||
else:
|
||||
# Record boundary before blending
|
||||
effect = self._scene_transition_effect(scene_cond)
|
||||
@@ -3155,7 +3251,7 @@ class FunPackLTXAVSceneChainSampler:
|
||||
custom_guides = per_scene_guides[scene_index]
|
||||
if custom_guides:
|
||||
chunk, scene_positive, scene_negative, carried, guide_tail = self._apply_configured_guides(
|
||||
chunk, scene_index, custom_guides, latent_template, scene_outputs, scene_media,
|
||||
chunk, scene_index, custom_guides, latent_template, scene_outputs, scene_media_by_ref,
|
||||
scene_positive, scene_negative, vae,
|
||||
)
|
||||
carried_guide_frames = max(carried_guide_frames, carried)
|
||||
|
||||
Reference in New Issue
Block a user