Mixed mode: independent i2v runs with prior-scene guides

Each mixed scene is its own queue request (max_scenes=1), not continuous
chain overlap. Scene 2+ uses its anchor via graph Img2Video and borrows
hidden guides from the previous scene. Sampler applies guide stacks on
the first chunk for solo mixed runs.
This commit is contained in:
digital-garbage
2026-06-10 15:36:15 +03:00
parent 682fe95bb3
commit 995f65e215
8 changed files with 197 additions and 32 deletions
+27 -2
View File
@@ -72,9 +72,34 @@ def is_mixed_source(scene: Scene) -> bool:
return (scene.source.type or "carry") == "mixed"
def build_mixed_solo_guides_payload(project: Project, mixed_scene: Scene) -> Optional[dict]:
"""Guides for one independent mixed-scene i2v run (max_scenes=1).
The mixed scene's own anchor image goes through the graph-level Img2Video path;
this payload only supplies hidden guides borrowed from the prior timeline scene."""
active = [s for s in project.scenes if not s.excluded]
idx = next((i for i, sc in enumerate(active) if sc.id == mixed_scene.id), -1)
if idx <= 0:
return None
prior = active[idx - 1]
guide = dict(STUDIO_DEFAULT_GUIDE)
prior_src = prior.source
prior_ref = getattr(prior_src, "media_ref", None) if prior_src else None
prior_type = (prior_src.type or "carry") if prior_src else "carry"
if prior_ref and prior_type in ("image", "mixed", "generated_frame"):
guide["source"] = "image"
guide["media_ref"] = prior_ref
return {
"stack_enabled": True,
"accumulate_prior": False,
"scenes": [[guide]],
}
def build_scene_anchors_payload(project: Project) -> Optional[dict]:
"""Per-scene i2v anchors for mixed sources — image → LTXVImgToVideoInplace starting
latent, NOT guide attention. Scene 0 uses the graph-level Studio/img2video path."""
"""Legacy multi-scene chain anchors (unused by Movie Editor mixed runs).
Mixed timeline scenes are generated solo via the graph-level Img2Video path."""
active = [s for s in project.scenes if not s.excluded]
anchors: dict[str, dict] = {}
for i, sc in enumerate(active):
+1 -1
View File
@@ -173,7 +173,7 @@
if ((root.source?.type) === "mixed") {
const hint = el("div", "insp-block");
hint.append(el("div", "insp-hint",
"Stays in the same chain run: this image builds a new starting latent via LTX Img2Video (i2v anchor), while prior-scene i2v guides are carried separately (Studio default: scene 1 template guide at frame 0). Enable Custom guide stack in Project to manage guides explicitly."));
"Generates as its own i2v run (one sampler call): this image is the starting latent via LTX Img2Video, while hidden guides are borrowed from the previous scene (its anchor image when set, else template frame 0). Not continuous chain overlap — use Carry for that."));
body.append(hint);
}
+6 -7
View File
@@ -606,9 +606,9 @@
return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m ${s % 60}s`;
}
// Split the active scenes into chain runs. A run starts at an anchored scene
// (empty / image / generated_frame); "mixed" and "carry" continue the current run.
// so it overlaps the previous scene (one chain-sampler request per run).
// Split the active scenes into generation runs. Each anchored scene (empty / image /
// generated_frame / mixed) is its own queue request. Only "carry" continues the
// previous run (continuous chain overlap). Mixed = solo i2v + prior-scene guides.
// True if a scene's image anchor still exists in the media bin (deleted image → fall
// back to carry / i2v guides, not a broken anchor).
function _anchorAvailable(s) {
@@ -625,8 +625,7 @@
// Editorial subclips always continue the current run; carry / broken anchors too.
const isCarry = isGenSubclip(s)
|| t === "carry"
|| t === "mixed"
|| ((t === "image" || t === "generated_frame") && !_anchorAvailable(s));
|| ((t === "image" || t === "generated_frame" || t === "mixed") && !_anchorAvailable(s));
if (isCarry && runs.length) runs[runs.length - 1].push(s.id);
else runs.push([s.id]);
}
@@ -665,9 +664,9 @@
state.sceneGhosts = (state.sceneGhosts || []).filter((g) => !ids.has(g.afterSceneId) && !ids.has(g.id));
}
// Pixel overlap between two consecutive chain scenes (0 when the next scene is mixed).
// Pixel overlap between consecutive scenes inside one carry chain run.
function _overlapBetweenScenes(prev, curr) {
if (!prev || !curr || (curr.source?.type) === "mixed") return 0;
if (!prev || !curr) return 0;
const frames = +(state.project?.sampler_inputs?.frame_overlap ?? 16);
const fps = state.project?.frame_rate || 25;
return Math.max(0, frames / fps);
+59 -18
View File
@@ -94,11 +94,12 @@ def _copy_scene_media(ref: str, indir: str) -> Optional[str]:
return fn
def _prepare_media(proj: Project) -> Optional[dict]:
def _prepare_media(proj: Project, extra_refs: Optional[list] = None) -> Optional[dict]:
"""Copy scene image assets into ComfyUI's input folder.
Returns {"primary": {filename, target}, "by_scene": {scene_id: {filename, media_ref, target}}}
for the builder (first run anchor) and mixed per-scene anchors in the sampler.
for the builder (run anchor) and sampler guide lookups. `extra_refs` copies additional
media-bin ids (e.g. prior-scene guide images for solo mixed runs).
"""
import os
try:
@@ -108,6 +109,24 @@ def _prepare_media(proj: Project) -> Optional[dict]:
return None
primary = None
by_scene: dict = {}
seen_refs: set = set()
def _add_ref(ref: str, tgt=None, sid: Optional[str] = None) -> None:
nonlocal primary
if not ref or ref in seen_refs:
return
fn = _copy_scene_media(ref, indir)
if not fn:
return
seen_refs.add(ref)
entry = {"filename": fn, "media_ref": ref, "target": tgt}
if sid:
by_scene[sid] = entry
else:
by_scene[f"_ref_{ref}"] = entry
if primary is None:
primary = {"filename": fn, "target": tgt}
for sc in proj.scenes:
if sc.excluded:
continue
@@ -117,13 +136,9 @@ def _prepare_media(proj: Project) -> Optional[dict]:
stype = getattr(src, "type", "") if src else ""
if stype not in ("image", "generated_frame", "mixed") or not ref:
continue
fn = _copy_scene_media(ref, indir)
if not fn:
continue
entry = {"filename": fn, "media_ref": ref, "target": tgt}
by_scene[sc.id] = entry
if primary is None:
primary = {"filename": fn, "target": tgt}
_add_ref(ref, tgt, sc.id)
for ref in (extra_refs or []):
_add_ref(str(ref))
if not by_scene:
return None
return {"primary": primary, "by_scene": by_scene}
@@ -180,21 +195,46 @@ def _run_studio_inputs(target: Project, active_scenes: list) -> dict:
return si
def _run_sampler_inputs(target: Project, scene_count: int) -> dict:
"""Sampler widget overrides for one chain run.
def _mixed_solo_guide_refs(full: Project, target: Project) -> list:
"""Media-bin ids for prior-scene guide images on a solo mixed run."""
from .backend.timeline import build_mixed_solo_guides_payload, is_mixed_source
Default (guide_settings.stack_enabled off): multi-scene runs force carry_i2v_guides
— one guide from scene 1's template at frame 0, same as Studio always did.
active = [s for s in target.scenes if not s.excluded]
if len(active) != 1 or not is_mixed_source(active[0]):
return []
payload = build_mixed_solo_guides_payload(full, active[0])
if not payload:
return []
refs = []
for g in (payload.get("scenes") or [[]])[0] or []:
if g.get("source") == "image" and g.get("media_ref"):
refs.append(g["media_ref"])
return refs
Custom guide stack (stack_enabled on): per-scene guide JSON is passed instead;
carry_i2v_guides is not auto-forced (avoids double-applying the template guide).
def _run_sampler_inputs(target: Project, scene_count: int, full: Optional[Project] = None) -> dict:
"""Sampler widget overrides for one generation run.
Mixed timeline scenes are solo i2v (max_scenes=1): anchor via Img2Video, guides from
the prior scene — not continuous chain overlap.
Carry runs (multi-scene): default carry_i2v_guides unless a custom guide stack is on.
"""
if target.sampler_slot != "funpack":
return {}
import json
from .backend.timeline import build_scene_guides_payload
from .backend.timeline import build_mixed_solo_guides_payload, build_scene_guides_payload, is_mixed_source
samp = dict(target.sampler_inputs or {})
active = [s for s in target.scenes if not s.excluded]
if scene_count == 1 and len(active) == 1 and is_mixed_source(active[0]):
guides = build_mixed_solo_guides_payload(full or target, active[0])
if guides:
samp["funpack_scene_guides"] = json.dumps(guides)
samp["frame_overlap"] = 0
samp["carry_i2v_guides"] = False
return samp
guides = build_scene_guides_payload(target)
if guides:
samp["funpack_scene_guides"] = json.dumps(guides)
@@ -662,8 +702,9 @@ if web is not None and PromptServer is not None:
if trimmed and all(f == trimmed[0] for f in trimmed)
else target.num_frames_per_scene
)
media_pack = _prepare_media(target)
sampler_inputs = _run_sampler_inputs(target, active_scene_count)
guide_refs = _mixed_solo_guide_refs(p, target)
media_pack = _prepare_media(target, guide_refs)
sampler_inputs = _run_sampler_inputs(target, active_scene_count, full=p)
_attach_scene_anchors(sampler_inputs, media_pack, target)
graph, report = builder.build(oi, _project_models(target), {
"prompt": prompt, "seed": target.seed,
+26 -4
View File
@@ -3,17 +3,24 @@
from pathlib import Path
from movie_editor.backend.timeline import Project, Scene
from movie_editor.server import _run_studio_inputs
from movie_editor.server import _run_sampler_inputs, _run_studio_inputs
CONTINUE = "__funpack_continue__"
def _project(scenes, rating=""):
return Project(
def _project(scenes, rating="", **kw):
p = Project(
name="t",
scenes=[Scene(id="s1", text="a", rating=rating)],
scenes=[Scene.from_dict(s) for s in scenes],
conditioning_slot="funpack",
sampler_slot="funpack",
**{k: v for k, v in kw.items() if k != "scenes"},
)
if not scenes:
p.scenes = [Scene(id="s1", text="a", rating=rating)]
elif rating:
p.scenes[0].rating = rating
return p
def test_studio_inputs_user_rating():
@@ -36,6 +43,21 @@ def test_continue_rating_is_valid_studio_combo_value():
assert "MOVIE_EDITOR_CONTINUE_RATING]" in src
def test_mixed_solo_sampler_inputs():
full = _project(scenes=[
{"id": "s1", "text": "a", "source": {"type": "image", "media_ref": "img1"}},
{"id": "s2", "text": "b", "source": {"type": "mixed", "media_ref": "img2"}},
])
solo = Project.from_dict(full.to_dict())
solo.scenes = [full.scenes[1]]
samp = _run_sampler_inputs(solo, 1, full=full)
assert samp["frame_overlap"] == 0
assert samp["carry_i2v_guides"] is False
import json
guides = json.loads(samp["funpack_scene_guides"])
assert guides["scenes"][0][0]["media_ref"] == "img1"
def test_studio_inputs_skips_custom_conditioning():
p = _project([])
p.conditioning_slot = "custom"
+26
View File
@@ -16,6 +16,7 @@ from movie_editor.backend.timeline import ( # noqa: E402
Scene,
STUDIO_DEFAULT_GUIDE,
build_combined_prompt,
build_mixed_solo_guides_payload,
build_scene_anchors_payload,
build_scene_guides_payload,
collapse_generative_units,
@@ -131,6 +132,31 @@ def test_guide_stack_studio_default_per_continuation_scene():
assert payload["scenes"][2] == [STUDIO_DEFAULT_GUIDE]
def test_mixed_solo_guides_borrow_prior_anchor():
p = _project(
scenes=[
{"id": "s1", "text": "a", "source": {"type": "image", "media_ref": "img1"}},
{"id": "s2", "text": "b", "source": {"type": "mixed", "media_ref": "img2"}},
],
)
payload = build_mixed_solo_guides_payload(p, p.scenes[1])
assert payload is not None
guide = payload["scenes"][0][0]
assert guide["source"] == "image"
assert guide["media_ref"] == "img1"
def test_mixed_solo_guides_template_when_prior_is_carry():
p = _project(
scenes=[
{"id": "s1", "text": "a", "source": {"type": "carry"}},
{"id": "s2", "text": "b", "source": {"type": "mixed", "media_ref": "img2"}},
],
)
payload = build_mixed_solo_guides_payload(p, p.scenes[1])
assert payload["scenes"][0][0]["source"] == "template"
def test_mixed_source_anchors_separate_from_guides():
p = _project(
scenes=[
+10
View File
@@ -3375,6 +3375,16 @@ class FunPackLTXAVSceneChainSampler:
anchor_meta = (scene_anchors or {}).get(str(scene_index))
if output is None:
chunk = self._clone_latent(latent_template)
custom_guides = None
if per_scene_guides and scene_index < len(per_scene_guides):
custom_guides = per_scene_guides[scene_index]
if custom_guides:
run_mechanisms.append("custom_guide_stack")
chunk, scene_positive, scene_negative, carried, guide_tail = self._apply_configured_guides(
chunk, scene_index, custom_guides, latent_template, scene_outputs, scene_media_by_ref,
scene_positive, scene_negative, vae,
)
carried_guide_frames = max(carried_guide_frames, carried)
elif anchor_meta:
run_mechanisms.append("mixed_i2v_anchor")
effect = self._scene_transition_effect(scene_cond)
+42
View File
@@ -478,6 +478,48 @@ def test_overlap_diagnostics_report_latent_blend_zone():
assert "overlap_blend=2px" in status
def test_mixed_solo_applies_guides_on_first_chunk():
import json
sample_calls.clear()
node = FunPackLTXAVSceneChainSampler()
guides = json.dumps({
"stack_enabled": True,
"accumulate_prior": False,
"scenes": [[{"enabled": True, "source": "template", "frame_idx": 0, "apply_at": 0, "strength": 0.35}]],
})
latent_template = {
"samples": torch.zeros(1, 2, 5, 3, 3),
"noise_mask": torch.cat([torch.zeros(1, 1, 2, 1, 1), torch.ones(1, 1, 3, 1, 1)], dim=2),
}
positive = [scene_cond(0)]
latent, _images, status, scene_count, _report, boundaries_json = node.sample(
model=object(),
vae=FakeVAE(),
positive=positive,
negative=[],
sampler=object(),
sigmas=torch.tensor([1.0, 0.0]),
seed=90,
latent_template=latent_template,
num_frames_per_scene=5,
frame_overlap=0,
cfg=1.0,
max_scenes=1,
funpack_scene_guides=guides,
prompt={"n1": {"inputs": {}}},
unique_id="test-node",
)
assert scene_count == 1
chunk = sample_calls[0]["latent_image"]
frames = chunk["samples"].shape[2] if isinstance(chunk, dict) else chunk.shape[2]
assert frames == 7
parsed = json.loads(boundaries_json)
assert "custom_guide_stack" in parsed["scenes"][0]["mechanisms"]
def test_mixed_anchor_skips_frame_overlap():
import json