diff --git a/movie_editor/backend/timeline.py b/movie_editor/backend/timeline.py index 2c857802..e5063b91 100644 --- a/movie_editor/backend/timeline.py +++ b/movie_editor/backend/timeline.py @@ -72,20 +72,20 @@ def is_mixed_source(scene: Scene) -> bool: return (scene.source.type or "carry") == "mixed" -def mixed_anchor_entry(scene: Scene, scene_index: int) -> Optional[dict]: - """Per-scene i2v anchor for mixed source (applied at frame 0 in the sampler).""" - if not is_mixed_source(scene) or not scene.source.media_ref: - return None - return { - "enabled": True, - "source": "anchor", - "scene_id": scene.id, - "scene_index": scene_index, - "media_ref": scene.source.media_ref, - "frame_idx": 0, - "apply_at": 0, - "strength": 0.35, - } +def build_scene_anchors_payload(project: Project) -> Optional[dict]: + """Per-scene i2v anchors for mixed sources — image → LTXVImgToVideoInplace starting + latent, NOT guide attention. Scene 0 uses the graph-level Studio/img2video path.""" + active = [s for s in project.scenes if not s.excluded] + anchors: dict[str, dict] = {} + for i, sc in enumerate(active): + if i == 0 or not is_mixed_source(sc) or not sc.source.media_ref: + continue + anchors[str(i)] = { + "scene_id": sc.id, + "media_ref": sc.source.media_ref, + "strength": 1.0, + } + return anchors if anchors else None @dataclass @@ -93,7 +93,7 @@ class SceneSource: """How a scene's latent is born. V1 ignores this (uniform chain); Phase 2 maps it onto EmptyLTXVLatent (empty/t2v) or LTXV Image to Video (image/i2v).""" type: str = "carry" # "carry" | "empty" | "image" | "generated_frame" | "mixed" - media_ref: Optional[str] = None # asset id, for type == "image" + media_ref: Optional[str] = None # asset id — image/mixed anchor (Img2Video) frame_ref: Optional[dict[str, Any]] = None # {scene_id, frame_idx}, for "generated_frame" target: Optional[str] = None # wire dest for the image: "port:" | "node::" @@ -377,63 +377,37 @@ def build_combined_prompt(project: Project, include_excluded: bool = False, def build_scene_guides_payload(project: Project) -> Optional[dict]: """Per-scene guide lists for the chain sampler. - None when stack_enabled is off AND no mixed sources (Studio default: - carry_i2v_guides from scene 1 template at frame 0). - - Mixed sources always emit a payload for continuation scenes that need both - the Studio prior guide and the scene's own anchor at frame 0. + None when stack_enabled is off (Studio default: carry_i2v_guides from scene 1 + template at frame 0). i2v anchors for mixed scenes are separate — see + build_scene_anchors_payload(). """ gs = normalize_guide_settings(project.guide_settings) - active = [s for s in project.scenes if not s.excluded] - has_mixed = any(is_mixed_source(s) and s.source.media_ref for s in active) - if not gs["stack_enabled"] and not has_mixed: + if not gs["stack_enabled"]: return None - + active = [s for s in project.scenes if not s.excluded] per_scene: list[Optional[list[dict]]] = [] - uses_custom = False for i, sc in enumerate(active): if i == 0: per_scene.append(None) continue entries: list[dict] = [] - own_anchor = mixed_anchor_entry(sc, i) - - if gs["stack_enabled"]: - for raw in (sc.guides or []): - g = GuideEntry.from_dict(raw if isinstance(raw, dict) else {}) - if g.enabled: - entries.append(g.to_dict()) - if gs["accumulate_prior"]: - for j in range(i): - ref = active[j] - acc = dict(STUDIO_DEFAULT_GUIDE) - acc["source"] = "scene" - acc["scene_id"] = ref.id - acc["scene_index"] = j - entries.append(acc) - - if own_anchor: - has_prior = any(e.get("source") in ("template", "scene") for e in entries) - if not has_prior: - entries.insert(0, dict(STUDIO_DEFAULT_GUIDE)) - entries.append(own_anchor) - uses_custom = True - elif gs["stack_enabled"] and not entries: + for raw in (sc.guides or []): + g = GuideEntry.from_dict(raw if isinstance(raw, dict) else {}) + if g.enabled: + entries.append(g.to_dict()) + if gs["accumulate_prior"]: + for j in range(i): + ref = active[j] + acc = dict(STUDIO_DEFAULT_GUIDE) + acc["source"] = "scene" + acc["scene_id"] = ref.id + acc["scene_index"] = j + entries.append(acc) + if not entries: entries = [dict(STUDIO_DEFAULT_GUIDE)] - uses_custom = True - elif not entries: - per_scene.append(None) - continue - else: - uses_custom = True - per_scene.append(entries) - - if not uses_custom: - return None return { - "stack_enabled": bool(gs["stack_enabled"]), + "stack_enabled": True, "accumulate_prior": gs["accumulate_prior"], - "has_mixed": has_mixed, "scenes": per_scene, } diff --git a/movie_editor/frontend/inspector.js b/movie_editor/frontend/inspector.js index 82a41099..a78d9798 100644 --- a/movie_editor/frontend/inspector.js +++ b/movie_editor/frontend/inspector.js @@ -10,7 +10,7 @@ ["image", "Image · i2v anchor"], ["generated_frame", "From generated frame"], ["carry", "Carry i2v guide · continue previous"], - ["mixed", "Mixed · anchor + prior guides"], + ["mixed", "Mixed · Img2Video anchor + prior guides"], ]; function transitionSelect(value, onChange) { @@ -56,7 +56,7 @@ (st.mediaBin || []).forEach((m) => { const o = new Option(m.name, m.id); if (m.id === ref) o.selected = true; pick.append(o); }); pick.onchange = () => S.patchScene(scene.id, { source: { ...(scene.source || {}), type: "image", media_ref: pick.value || null } }); body.append(field("Media asset", pick)); - body.append(el("div", "insp-hint", "Feeds the i2v anchor for this scene. Routing is automatic (→ Studio source image); to send it through a node first, set that node's Input source to Timeline in Models.")); + body.append(el("div", "insp-hint", "Feeds the i2v anchor for this scene (image → LTX Img2Video → starting latent). Routing is automatic (→ Studio source image); to send it through a node first, set that node's Input source to Timeline in Models.")); } function renderGeneratedFrameSource(st, scene) { @@ -168,7 +168,7 @@ if ((root.source?.type) === "mixed") { const hint = el("div", "insp-block"); hint.append(el("div", "insp-hint", - "Stays in the same chain run (overlap + prior guides at frame 0) while also applying this scene's image anchor at frame 0. Default: Studio guide from scene 1 plus this anchor — enable Custom guide stack in Project for more control.")); + "Stays in the same chain run: this image builds a new starting latent via LTX Img2Video (i2v anchor), while prior-scene i2v guides are carried separately (Studio default: scene 1 template guide at frame 0). Enable Custom guide stack in Project to manage guides explicitly.")); body.append(hint); } @@ -518,7 +518,7 @@ const hint = el("div", "insp-hint"); hint.textContent = gs.stack_enabled ? "Custom guide stack — per-scene lists in the Scene inspector. Scenes without entries use the Studio default (scene 1 template · frame 0 · apply 0)." - : "Studio default: one i2v guide from scene 1, carried to later scenes at frame 0 (same as FunPack Studio)."; + : "Studio default: one i2v guide (not anchor) from scene 1's template, carried to later scenes at frame 0. Anchors are separate — image → Img2Video starting latent."; parent.append(hint); const stackLbl = el("label", "chk"); const stackCb = el("input"); stackCb.type = "checkbox"; stackCb.checked = gs.stack_enabled; diff --git a/movie_editor/server.py b/movie_editor/server.py index 2adca928..7ca29fe7 100644 --- a/movie_editor/server.py +++ b/movie_editor/server.py @@ -203,10 +203,31 @@ def _run_sampler_inputs(target: Project, scene_count: int) -> dict: return samp -def _attach_scene_media(samp: dict, media_pack: Optional[dict]) -> dict: +def _attach_scene_anchors(samp: dict, media_pack: Optional[dict], target: Project) -> dict: + """Mixed-source i2v anchors (Img2Video starting latents), separate from guides.""" import json - if media_pack and media_pack.get("by_scene"): - samp["funpack_scene_media"] = json.dumps(media_pack["by_scene"]) + from .backend.timeline import build_scene_anchors_payload + + anchors = build_scene_anchors_payload(target) + if not anchors or not media_pack: + return samp + by_scene = media_pack.get("by_scene") or {} + enriched: dict = {} + for idx, meta in anchors.items(): + sid = meta.get("scene_id") + file_entry = by_scene.get(sid) if sid else None + fn = (file_entry or {}).get("filename") + if fn: + enriched[idx] = {**meta, "filename": fn} + if enriched: + samp["funpack_scene_anchors"] = json.dumps(enriched) + by_ref = { + entry["media_ref"]: entry["filename"] + for entry in by_scene.values() + if isinstance(entry, dict) and entry.get("media_ref") and entry.get("filename") + } + if by_ref: + samp["funpack_scene_media_refs"] = json.dumps(by_ref) return samp @@ -643,7 +664,7 @@ if web is not None and PromptServer is not None: ) media_pack = _prepare_media(target) sampler_inputs = _run_sampler_inputs(target, active_scene_count) - _attach_scene_media(sampler_inputs, media_pack) + _attach_scene_anchors(sampler_inputs, media_pack, target) graph, report = builder.build(oi, _project_models(target), { "prompt": prompt, "seed": target.seed, "num_frames_per_scene": effective_frames, diff --git a/movie_editor/tests/test_timeline.py b/movie_editor/tests/test_timeline.py index 599074dd..1141b6b6 100644 --- a/movie_editor/tests/test_timeline.py +++ b/movie_editor/tests/test_timeline.py @@ -16,6 +16,7 @@ from movie_editor.backend.timeline import ( # noqa: E402 Scene, STUDIO_DEFAULT_GUIDE, build_combined_prompt, + build_scene_anchors_payload, build_scene_guides_payload, collapse_generative_units, group_generative_units, @@ -130,7 +131,7 @@ def test_guide_stack_studio_default_per_continuation_scene(): assert payload["scenes"][2] == [STUDIO_DEFAULT_GUIDE] -def test_mixed_source_emits_anchor_and_studio_default(): +def test_mixed_source_anchors_separate_from_guides(): p = _project( scenes=[ {"id": "s1", "text": "a", "source": {"type": "image", "media_ref": "img1"}}, @@ -138,15 +139,11 @@ def test_mixed_source_emits_anchor_and_studio_default(): ], ) assert is_mixed_source(p.scenes[1]) - payload = build_scene_guides_payload(p) - assert payload is not None - assert payload["has_mixed"] is True - scene2 = payload["scenes"][1] - assert len(scene2) == 2 - assert scene2[0] == STUDIO_DEFAULT_GUIDE - assert scene2[1]["source"] == "anchor" - assert scene2[1]["media_ref"] == "img2" - assert scene2[1]["apply_at"] == 0 + assert build_scene_guides_payload(p) is None + anchors = build_scene_anchors_payload(p) + assert anchors["1"]["scene_id"] == "s2" + assert anchors["1"]["media_ref"] == "img2" + assert anchors["1"]["strength"] == 1.0 def test_guide_stack_accumulate_prior(): diff --git a/samplers.py b/samplers.py index cf34f226..4afbc4c7 100644 --- a/samplers.py +++ b/samplers.py @@ -2085,10 +2085,15 @@ class FunPackLTXAVSceneChainSampler: "multiline": True, "tooltip": "Optional JSON from Movie Editor when guide_settings.stack_enabled: per-scene guide lists with source, frame_idx, apply_at, strength. When empty, carry_i2v_guides uses the Studio default (scene 1 template at frame 0).", }), - "funpack_scene_media": ("STRING", { + "funpack_scene_anchors": ("STRING", { "default": "", "multiline": True, - "tooltip": "Optional JSON map of scene_id → {filename} for mixed-source per-scene i2v anchors (Movie Editor).", + "tooltip": "Optional JSON map of scene_index → {filename, strength} for mixed-source i2v anchors (LTXVImgToVideoInplace starting latent). Distinct from i2v guides.", + }), + "funpack_scene_media_refs": ("STRING", { + "default": "", + "multiline": True, + "tooltip": "Optional media_ref → filename map for image-type i2v guides in custom guide stacks.", }), }, "hidden": { @@ -2835,13 +2840,11 @@ class FunPackLTXAVSceneChainSampler: data = json.loads(str(raw)) except Exception: return None - if not isinstance(data, dict): - return None - if not data.get("scenes") and not data.get("has_mixed"): + if not isinstance(data, dict) or not data.get("stack_enabled"): return None return data - def _parse_scene_media(self, raw): + def _parse_scene_anchors(self, raw): if not raw or not str(raw).strip(): return {} try: @@ -2851,6 +2854,89 @@ class FunPackLTXAVSceneChainSampler: return {} return data if isinstance(data, dict) else {} + def _load_image_tensor(self, filename): + import os + try: + import folder_paths + import numpy as np + from PIL import Image + except ImportError: + return None + path = os.path.join(folder_paths.get_input_directory(), filename) + if not os.path.isfile(path): + return None + try: + img = Image.open(path).convert("RGB") + arr = np.array(img).astype(np.float32) / 255.0 + return torch.from_numpy(arr)[None,] + except Exception: + return None + + def _invoke_img2video_inplace(self, vae, image, latent, strength=1.0): + """Run LTXVImgToVideoInplace — image → starting latent (i2v anchor, not a guide).""" + try: + from comfy_extras.nodes_lt import LTXVImgToVideoInplace + except ImportError: + return None + for method in ("execute", "generate"): + fn = getattr(LTXVImgToVideoInplace, method, None) + if fn is None: + continue + for args in ( + (vae, image, latent, strength, False), + (vae, image, latent, strength), + ): + try: + out = fn(*args) + if hasattr(out, "args") and out.args: + out = out.args[0] + elif isinstance(out, tuple) and out: + out = out[0] + if isinstance(out, dict) and "samples" in out: + return out + except (TypeError, ValueError): + continue + return None + + def _apply_img2video_to_video_latent(self, vae, image, chunk, strength=1.0): + """Apply Img2Video inplace to the video stream of an (optionally nested AV) latent.""" + if self._is_nested(chunk.get("samples")): + tensors = self._latent_tensors(chunk) + masks = self._latent_masks(chunk, len(tensors)) + video_in = {"samples": tensors[0]} + if masks[0] is not None: + video_in["noise_mask"] = masks[0] + video_out = self._invoke_img2video_inplace(vae, image, video_in, strength) + if video_out is None: + return chunk + result = self._clone_latent(chunk) + out_tensors = list(tensors) + out_masks = list(masks) + out_tensors[0] = video_out["samples"] + if "noise_mask" in video_out: + out_masks[0] = video_out["noise_mask"] + result["samples"] = comfy.nested_tensor.NestedTensor(out_tensors) + if any(m is not None for m in out_masks): + result["noise_mask"] = comfy.nested_tensor.NestedTensor(out_masks) + return result + out = self._invoke_img2video_inplace(vae, image, chunk, strength) + return out if out is not None else chunk + + def _build_mixed_anchor_chunk(self, vae, anchor_meta, latent_template, previous, video_overlap): + """Mixed source: Img2Video anchor latent, then chain overlap from previous scene.""" + filename = (anchor_meta or {}).get("filename") + strength = float((anchor_meta or {}).get("strength", 1.0)) + image = self._load_image_tensor(filename) if filename else None + if image is None: + if previous is None: + return self._clone_latent(latent_template) + return self._build_continuation_chunk(latent_template, previous, video_overlap) + base = self._clone_latent(latent_template) + i2v_chunk = self._apply_img2video_to_video_latent(vae, image, base, strength) + if previous is None: + return i2v_chunk + return self._build_continuation_chunk(i2v_chunk, previous, video_overlap) + def _encode_image_guide_frame(self, filename, vae, ref_tensor): import os try: @@ -2904,7 +2990,7 @@ class FunPackLTXAVSceneChainSampler: ) + (0,) def _apply_configured_guides(self, chunk, scene_index, guide_list, latent_template, - scene_outputs, scene_media, positive, negative, vae): + scene_outputs, scene_media_by_ref, positive, negative, vae): head_crop = 0 tail_crop = 0 for g in guide_list or []: @@ -2931,15 +3017,9 @@ class FunPackLTXAVSceneChainSampler: chunk, positive, negative, head, tail = self._append_scene_guide_at( chunk, scene_outputs[si], frame_idx, apply_at, strength, positive, negative, vae, ) - elif source in ("anchor", "image"): - sid = g.get("scene_id") - meta = (scene_media or {}).get(sid) if sid else None - if not meta and g.get("media_ref"): - for _k, m in (scene_media or {}).items(): - if isinstance(m, dict) and m.get("media_ref") == g.get("media_ref"): - meta = m - break - fn = (meta or {}).get("filename") if isinstance(meta, dict) else None + elif source == "image": + ref = g.get("media_ref") + fn = (scene_media_by_ref or {}).get(ref) if ref else None if not fn: continue chunk, positive, negative, head, tail = self._append_media_guide_at( @@ -3046,7 +3126,8 @@ class FunPackLTXAVSceneChainSampler: transition_duration=16, decode_tile_size=0, decode_noise_scale=0.0, decode_timestep=0.05, refinement_key_input="", funpack_scene_guides="", - funpack_scene_media="", + funpack_scene_anchors="", + funpack_scene_media_refs="", unique_id=None, prompt=None): if not isinstance(positive, list) or not positive: raise ValueError("positive conditioning must contain at least one scene entry.") @@ -3119,8 +3200,9 @@ class FunPackLTXAVSceneChainSampler: scene_guides_cfg = self._parse_scene_guides(funpack_scene_guides) per_scene_guides = (scene_guides_cfg or {}).get("scenes") if scene_guides_cfg else None - scene_media = self._parse_scene_media(funpack_scene_media) + scene_anchors = self._parse_scene_anchors(funpack_scene_anchors) scene_outputs: list = [] + scene_media_by_ref = self._parse_scene_anchors(funpack_scene_media_refs) for scene_index, scene_cond in enumerate(scene_conditionings): scene_positive = [scene_cond] @@ -3134,8 +3216,22 @@ class FunPackLTXAVSceneChainSampler: carried = 0 soft_carried = 0 guide_tail = 0 + anchor_meta = (scene_anchors or {}).get(str(scene_index)) if output is None: chunk = self._clone_latent(latent_template) + elif anchor_meta: + effect = self._scene_transition_effect(scene_cond) + if effect and transition_duration > 0: + boundary_latent = cumulative_latent_frames + boundary_pixel = int((boundary_latent - 1) * time_scale + 1) if time_scale > 1 else boundary_latent + boundary_entries.append({ + "boundary_latent": boundary_latent, + "pixel_frame": max(0, boundary_pixel), + "effect": effect, + }) + chunk = self._build_mixed_anchor_chunk( + vae, anchor_meta, latent_template, output, video_overlap, + ) else: # Record boundary before blending effect = self._scene_transition_effect(scene_cond) @@ -3155,7 +3251,7 @@ class FunPackLTXAVSceneChainSampler: custom_guides = per_scene_guides[scene_index] if custom_guides: chunk, scene_positive, scene_negative, carried, guide_tail = self._apply_configured_guides( - chunk, scene_index, custom_guides, latent_template, scene_outputs, scene_media, + chunk, scene_index, custom_guides, latent_template, scene_outputs, scene_media_by_ref, scene_positive, scene_negative, vae, ) carried_guide_frames = max(carried_guide_frames, carried)