Mixed mode: independent i2v runs with prior-scene guides
Each mixed scene is its own queue request (max_scenes=1), not continuous chain overlap. Scene 2+ uses its anchor via graph Img2Video and borrows hidden guides from the previous scene. Sampler applies guide stacks on the first chunk for solo mixed runs.
This commit is contained in:
@@ -72,9 +72,34 @@ def is_mixed_source(scene: Scene) -> bool:
|
||||
return (scene.source.type or "carry") == "mixed"
|
||||
|
||||
|
||||
def build_mixed_solo_guides_payload(project: Project, mixed_scene: Scene) -> Optional[dict]:
|
||||
"""Guides for one independent mixed-scene i2v run (max_scenes=1).
|
||||
|
||||
The mixed scene's own anchor image goes through the graph-level Img2Video path;
|
||||
this payload only supplies hidden guides borrowed from the prior timeline scene."""
|
||||
active = [s for s in project.scenes if not s.excluded]
|
||||
idx = next((i for i, sc in enumerate(active) if sc.id == mixed_scene.id), -1)
|
||||
if idx <= 0:
|
||||
return None
|
||||
prior = active[idx - 1]
|
||||
guide = dict(STUDIO_DEFAULT_GUIDE)
|
||||
prior_src = prior.source
|
||||
prior_ref = getattr(prior_src, "media_ref", None) if prior_src else None
|
||||
prior_type = (prior_src.type or "carry") if prior_src else "carry"
|
||||
if prior_ref and prior_type in ("image", "mixed", "generated_frame"):
|
||||
guide["source"] = "image"
|
||||
guide["media_ref"] = prior_ref
|
||||
return {
|
||||
"stack_enabled": True,
|
||||
"accumulate_prior": False,
|
||||
"scenes": [[guide]],
|
||||
}
|
||||
|
||||
|
||||
def build_scene_anchors_payload(project: Project) -> Optional[dict]:
|
||||
"""Per-scene i2v anchors for mixed sources — image → LTXVImgToVideoInplace starting
|
||||
latent, NOT guide attention. Scene 0 uses the graph-level Studio/img2video path."""
|
||||
"""Legacy multi-scene chain anchors (unused by Movie Editor mixed runs).
|
||||
|
||||
Mixed timeline scenes are generated solo via the graph-level Img2Video path."""
|
||||
active = [s for s in project.scenes if not s.excluded]
|
||||
anchors: dict[str, dict] = {}
|
||||
for i, sc in enumerate(active):
|
||||
|
||||
@@ -173,7 +173,7 @@
|
||||
if ((root.source?.type) === "mixed") {
|
||||
const hint = el("div", "insp-block");
|
||||
hint.append(el("div", "insp-hint",
|
||||
"Stays in the same chain run: this image builds a new starting latent via LTX Img2Video (i2v anchor), while prior-scene i2v guides are carried separately (Studio default: scene 1 template guide at frame 0). Enable Custom guide stack in Project to manage guides explicitly."));
|
||||
"Generates as its own i2v run (one sampler call): this image is the starting latent via LTX Img2Video, while hidden guides are borrowed from the previous scene (its anchor image when set, else template frame 0). Not continuous chain overlap — use Carry for that."));
|
||||
body.append(hint);
|
||||
}
|
||||
|
||||
|
||||
@@ -606,9 +606,9 @@
|
||||
return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m ${s % 60}s`;
|
||||
}
|
||||
|
||||
// Split the active scenes into chain runs. A run starts at an anchored scene
|
||||
// (empty / image / generated_frame); "mixed" and "carry" continue the current run.
|
||||
// so it overlaps the previous scene (one chain-sampler request per run).
|
||||
// Split the active scenes into generation runs. Each anchored scene (empty / image /
|
||||
// generated_frame / mixed) is its own queue request. Only "carry" continues the
|
||||
// previous run (continuous chain overlap). Mixed = solo i2v + prior-scene guides.
|
||||
// True if a scene's image anchor still exists in the media bin (deleted image → fall
|
||||
// back to carry / i2v guides, not a broken anchor).
|
||||
function _anchorAvailable(s) {
|
||||
@@ -625,8 +625,7 @@
|
||||
// Editorial subclips always continue the current run; carry / broken anchors too.
|
||||
const isCarry = isGenSubclip(s)
|
||||
|| t === "carry"
|
||||
|| t === "mixed"
|
||||
|| ((t === "image" || t === "generated_frame") && !_anchorAvailable(s));
|
||||
|| ((t === "image" || t === "generated_frame" || t === "mixed") && !_anchorAvailable(s));
|
||||
if (isCarry && runs.length) runs[runs.length - 1].push(s.id);
|
||||
else runs.push([s.id]);
|
||||
}
|
||||
@@ -665,9 +664,9 @@
|
||||
state.sceneGhosts = (state.sceneGhosts || []).filter((g) => !ids.has(g.afterSceneId) && !ids.has(g.id));
|
||||
}
|
||||
|
||||
// Pixel overlap between two consecutive chain scenes (0 when the next scene is mixed).
|
||||
// Pixel overlap between consecutive scenes inside one carry chain run.
|
||||
function _overlapBetweenScenes(prev, curr) {
|
||||
if (!prev || !curr || (curr.source?.type) === "mixed") return 0;
|
||||
if (!prev || !curr) return 0;
|
||||
const frames = +(state.project?.sampler_inputs?.frame_overlap ?? 16);
|
||||
const fps = state.project?.frame_rate || 25;
|
||||
return Math.max(0, frames / fps);
|
||||
|
||||
+59
-18
@@ -94,11 +94,12 @@ def _copy_scene_media(ref: str, indir: str) -> Optional[str]:
|
||||
return fn
|
||||
|
||||
|
||||
def _prepare_media(proj: Project) -> Optional[dict]:
|
||||
def _prepare_media(proj: Project, extra_refs: Optional[list] = None) -> Optional[dict]:
|
||||
"""Copy scene image assets into ComfyUI's input folder.
|
||||
|
||||
Returns {"primary": {filename, target}, "by_scene": {scene_id: {filename, media_ref, target}}}
|
||||
for the builder (first run anchor) and mixed per-scene anchors in the sampler.
|
||||
for the builder (run anchor) and sampler guide lookups. `extra_refs` copies additional
|
||||
media-bin ids (e.g. prior-scene guide images for solo mixed runs).
|
||||
"""
|
||||
import os
|
||||
try:
|
||||
@@ -108,6 +109,24 @@ def _prepare_media(proj: Project) -> Optional[dict]:
|
||||
return None
|
||||
primary = None
|
||||
by_scene: dict = {}
|
||||
seen_refs: set = set()
|
||||
|
||||
def _add_ref(ref: str, tgt=None, sid: Optional[str] = None) -> None:
|
||||
nonlocal primary
|
||||
if not ref or ref in seen_refs:
|
||||
return
|
||||
fn = _copy_scene_media(ref, indir)
|
||||
if not fn:
|
||||
return
|
||||
seen_refs.add(ref)
|
||||
entry = {"filename": fn, "media_ref": ref, "target": tgt}
|
||||
if sid:
|
||||
by_scene[sid] = entry
|
||||
else:
|
||||
by_scene[f"_ref_{ref}"] = entry
|
||||
if primary is None:
|
||||
primary = {"filename": fn, "target": tgt}
|
||||
|
||||
for sc in proj.scenes:
|
||||
if sc.excluded:
|
||||
continue
|
||||
@@ -117,13 +136,9 @@ def _prepare_media(proj: Project) -> Optional[dict]:
|
||||
stype = getattr(src, "type", "") if src else ""
|
||||
if stype not in ("image", "generated_frame", "mixed") or not ref:
|
||||
continue
|
||||
fn = _copy_scene_media(ref, indir)
|
||||
if not fn:
|
||||
continue
|
||||
entry = {"filename": fn, "media_ref": ref, "target": tgt}
|
||||
by_scene[sc.id] = entry
|
||||
if primary is None:
|
||||
primary = {"filename": fn, "target": tgt}
|
||||
_add_ref(ref, tgt, sc.id)
|
||||
for ref in (extra_refs or []):
|
||||
_add_ref(str(ref))
|
||||
if not by_scene:
|
||||
return None
|
||||
return {"primary": primary, "by_scene": by_scene}
|
||||
@@ -180,21 +195,46 @@ def _run_studio_inputs(target: Project, active_scenes: list) -> dict:
|
||||
return si
|
||||
|
||||
|
||||
def _run_sampler_inputs(target: Project, scene_count: int) -> dict:
|
||||
"""Sampler widget overrides for one chain run.
|
||||
def _mixed_solo_guide_refs(full: Project, target: Project) -> list:
|
||||
"""Media-bin ids for prior-scene guide images on a solo mixed run."""
|
||||
from .backend.timeline import build_mixed_solo_guides_payload, is_mixed_source
|
||||
|
||||
Default (guide_settings.stack_enabled off): multi-scene runs force carry_i2v_guides
|
||||
— one guide from scene 1's template at frame 0, same as Studio always did.
|
||||
active = [s for s in target.scenes if not s.excluded]
|
||||
if len(active) != 1 or not is_mixed_source(active[0]):
|
||||
return []
|
||||
payload = build_mixed_solo_guides_payload(full, active[0])
|
||||
if not payload:
|
||||
return []
|
||||
refs = []
|
||||
for g in (payload.get("scenes") or [[]])[0] or []:
|
||||
if g.get("source") == "image" and g.get("media_ref"):
|
||||
refs.append(g["media_ref"])
|
||||
return refs
|
||||
|
||||
Custom guide stack (stack_enabled on): per-scene guide JSON is passed instead;
|
||||
carry_i2v_guides is not auto-forced (avoids double-applying the template guide).
|
||||
|
||||
def _run_sampler_inputs(target: Project, scene_count: int, full: Optional[Project] = None) -> dict:
|
||||
"""Sampler widget overrides for one generation run.
|
||||
|
||||
Mixed timeline scenes are solo i2v (max_scenes=1): anchor via Img2Video, guides from
|
||||
the prior scene — not continuous chain overlap.
|
||||
|
||||
Carry runs (multi-scene): default carry_i2v_guides unless a custom guide stack is on.
|
||||
"""
|
||||
if target.sampler_slot != "funpack":
|
||||
return {}
|
||||
import json
|
||||
from .backend.timeline import build_scene_guides_payload
|
||||
from .backend.timeline import build_mixed_solo_guides_payload, build_scene_guides_payload, is_mixed_source
|
||||
|
||||
samp = dict(target.sampler_inputs or {})
|
||||
active = [s for s in target.scenes if not s.excluded]
|
||||
if scene_count == 1 and len(active) == 1 and is_mixed_source(active[0]):
|
||||
guides = build_mixed_solo_guides_payload(full or target, active[0])
|
||||
if guides:
|
||||
samp["funpack_scene_guides"] = json.dumps(guides)
|
||||
samp["frame_overlap"] = 0
|
||||
samp["carry_i2v_guides"] = False
|
||||
return samp
|
||||
|
||||
guides = build_scene_guides_payload(target)
|
||||
if guides:
|
||||
samp["funpack_scene_guides"] = json.dumps(guides)
|
||||
@@ -662,8 +702,9 @@ if web is not None and PromptServer is not None:
|
||||
if trimmed and all(f == trimmed[0] for f in trimmed)
|
||||
else target.num_frames_per_scene
|
||||
)
|
||||
media_pack = _prepare_media(target)
|
||||
sampler_inputs = _run_sampler_inputs(target, active_scene_count)
|
||||
guide_refs = _mixed_solo_guide_refs(p, target)
|
||||
media_pack = _prepare_media(target, guide_refs)
|
||||
sampler_inputs = _run_sampler_inputs(target, active_scene_count, full=p)
|
||||
_attach_scene_anchors(sampler_inputs, media_pack, target)
|
||||
graph, report = builder.build(oi, _project_models(target), {
|
||||
"prompt": prompt, "seed": target.seed,
|
||||
|
||||
@@ -3,17 +3,24 @@
|
||||
from pathlib import Path
|
||||
|
||||
from movie_editor.backend.timeline import Project, Scene
|
||||
from movie_editor.server import _run_studio_inputs
|
||||
from movie_editor.server import _run_sampler_inputs, _run_studio_inputs
|
||||
|
||||
CONTINUE = "__funpack_continue__"
|
||||
|
||||
|
||||
def _project(scenes, rating=""):
|
||||
return Project(
|
||||
def _project(scenes, rating="", **kw):
|
||||
p = Project(
|
||||
name="t",
|
||||
scenes=[Scene(id="s1", text="a", rating=rating)],
|
||||
scenes=[Scene.from_dict(s) for s in scenes],
|
||||
conditioning_slot="funpack",
|
||||
sampler_slot="funpack",
|
||||
**{k: v for k, v in kw.items() if k != "scenes"},
|
||||
)
|
||||
if not scenes:
|
||||
p.scenes = [Scene(id="s1", text="a", rating=rating)]
|
||||
elif rating:
|
||||
p.scenes[0].rating = rating
|
||||
return p
|
||||
|
||||
|
||||
def test_studio_inputs_user_rating():
|
||||
@@ -36,6 +43,21 @@ def test_continue_rating_is_valid_studio_combo_value():
|
||||
assert "MOVIE_EDITOR_CONTINUE_RATING]" in src
|
||||
|
||||
|
||||
def test_mixed_solo_sampler_inputs():
|
||||
full = _project(scenes=[
|
||||
{"id": "s1", "text": "a", "source": {"type": "image", "media_ref": "img1"}},
|
||||
{"id": "s2", "text": "b", "source": {"type": "mixed", "media_ref": "img2"}},
|
||||
])
|
||||
solo = Project.from_dict(full.to_dict())
|
||||
solo.scenes = [full.scenes[1]]
|
||||
samp = _run_sampler_inputs(solo, 1, full=full)
|
||||
assert samp["frame_overlap"] == 0
|
||||
assert samp["carry_i2v_guides"] is False
|
||||
import json
|
||||
guides = json.loads(samp["funpack_scene_guides"])
|
||||
assert guides["scenes"][0][0]["media_ref"] == "img1"
|
||||
|
||||
|
||||
def test_studio_inputs_skips_custom_conditioning():
|
||||
p = _project([])
|
||||
p.conditioning_slot = "custom"
|
||||
|
||||
@@ -16,6 +16,7 @@ from movie_editor.backend.timeline import ( # noqa: E402
|
||||
Scene,
|
||||
STUDIO_DEFAULT_GUIDE,
|
||||
build_combined_prompt,
|
||||
build_mixed_solo_guides_payload,
|
||||
build_scene_anchors_payload,
|
||||
build_scene_guides_payload,
|
||||
collapse_generative_units,
|
||||
@@ -131,6 +132,31 @@ def test_guide_stack_studio_default_per_continuation_scene():
|
||||
assert payload["scenes"][2] == [STUDIO_DEFAULT_GUIDE]
|
||||
|
||||
|
||||
def test_mixed_solo_guides_borrow_prior_anchor():
|
||||
p = _project(
|
||||
scenes=[
|
||||
{"id": "s1", "text": "a", "source": {"type": "image", "media_ref": "img1"}},
|
||||
{"id": "s2", "text": "b", "source": {"type": "mixed", "media_ref": "img2"}},
|
||||
],
|
||||
)
|
||||
payload = build_mixed_solo_guides_payload(p, p.scenes[1])
|
||||
assert payload is not None
|
||||
guide = payload["scenes"][0][0]
|
||||
assert guide["source"] == "image"
|
||||
assert guide["media_ref"] == "img1"
|
||||
|
||||
|
||||
def test_mixed_solo_guides_template_when_prior_is_carry():
|
||||
p = _project(
|
||||
scenes=[
|
||||
{"id": "s1", "text": "a", "source": {"type": "carry"}},
|
||||
{"id": "s2", "text": "b", "source": {"type": "mixed", "media_ref": "img2"}},
|
||||
],
|
||||
)
|
||||
payload = build_mixed_solo_guides_payload(p, p.scenes[1])
|
||||
assert payload["scenes"][0][0]["source"] == "template"
|
||||
|
||||
|
||||
def test_mixed_source_anchors_separate_from_guides():
|
||||
p = _project(
|
||||
scenes=[
|
||||
|
||||
+10
@@ -3375,6 +3375,16 @@ class FunPackLTXAVSceneChainSampler:
|
||||
anchor_meta = (scene_anchors or {}).get(str(scene_index))
|
||||
if output is None:
|
||||
chunk = self._clone_latent(latent_template)
|
||||
custom_guides = None
|
||||
if per_scene_guides and scene_index < len(per_scene_guides):
|
||||
custom_guides = per_scene_guides[scene_index]
|
||||
if custom_guides:
|
||||
run_mechanisms.append("custom_guide_stack")
|
||||
chunk, scene_positive, scene_negative, carried, guide_tail = self._apply_configured_guides(
|
||||
chunk, scene_index, custom_guides, latent_template, scene_outputs, scene_media_by_ref,
|
||||
scene_positive, scene_negative, vae,
|
||||
)
|
||||
carried_guide_frames = max(carried_guide_frames, carried)
|
||||
elif anchor_meta:
|
||||
run_mechanisms.append("mixed_i2v_anchor")
|
||||
effect = self._scene_transition_effect(scene_cond)
|
||||
|
||||
@@ -478,6 +478,48 @@ def test_overlap_diagnostics_report_latent_blend_zone():
|
||||
assert "overlap_blend=2px" in status
|
||||
|
||||
|
||||
def test_mixed_solo_applies_guides_on_first_chunk():
|
||||
import json
|
||||
|
||||
sample_calls.clear()
|
||||
node = FunPackLTXAVSceneChainSampler()
|
||||
guides = json.dumps({
|
||||
"stack_enabled": True,
|
||||
"accumulate_prior": False,
|
||||
"scenes": [[{"enabled": True, "source": "template", "frame_idx": 0, "apply_at": 0, "strength": 0.35}]],
|
||||
})
|
||||
latent_template = {
|
||||
"samples": torch.zeros(1, 2, 5, 3, 3),
|
||||
"noise_mask": torch.cat([torch.zeros(1, 1, 2, 1, 1), torch.ones(1, 1, 3, 1, 1)], dim=2),
|
||||
}
|
||||
positive = [scene_cond(0)]
|
||||
|
||||
latent, _images, status, scene_count, _report, boundaries_json = node.sample(
|
||||
model=object(),
|
||||
vae=FakeVAE(),
|
||||
positive=positive,
|
||||
negative=[],
|
||||
sampler=object(),
|
||||
sigmas=torch.tensor([1.0, 0.0]),
|
||||
seed=90,
|
||||
latent_template=latent_template,
|
||||
num_frames_per_scene=5,
|
||||
frame_overlap=0,
|
||||
cfg=1.0,
|
||||
max_scenes=1,
|
||||
funpack_scene_guides=guides,
|
||||
prompt={"n1": {"inputs": {}}},
|
||||
unique_id="test-node",
|
||||
)
|
||||
|
||||
assert scene_count == 1
|
||||
chunk = sample_calls[0]["latent_image"]
|
||||
frames = chunk["samples"].shape[2] if isinstance(chunk, dict) else chunk.shape[2]
|
||||
assert frames == 7
|
||||
parsed = json.loads(boundaries_json)
|
||||
assert "custom_guide_stack" in parsed["scenes"][0]["mechanisms"]
|
||||
|
||||
|
||||
def test_mixed_anchor_skips_frame_overlap():
|
||||
import json
|
||||
|
||||
|
||||
Reference in New Issue
Block a user