From aacb35ce4ae135c953459177b5935d10470cd7e2 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:54:30 +0300 Subject: [PATCH] Movie Editor: generative units for editorial timeline cuts Timeline splits now share gen_unit_id so NLE cuts stay one generative scene. Generate collapses subclips to the uncut prompt and frame total; output is mapped back across cuts. Subclips show shared prompt in the inspector and distinct timeline styling. --- movie_editor/backend/timeline.py | 62 +++++++++++++++++++++++++---- movie_editor/frontend/inspector.js | 26 ++++++++---- movie_editor/frontend/store.js | 56 +++++++++++++++++++++++--- movie_editor/frontend/styles.css | 2 + movie_editor/frontend/timeline.js | 10 ++++- movie_editor/server.py | 18 ++++++--- movie_editor/tests/test_timeline.py | 28 ++++++++++++- 7 files changed, 172 insertions(+), 30 deletions(-) diff --git a/movie_editor/backend/timeline.py b/movie_editor/backend/timeline.py index 70d29020..e69a0cb7 100644 --- a/movie_editor/backend/timeline.py +++ b/movie_editor/backend/timeline.py @@ -82,6 +82,11 @@ class Scene: # scenes are skipped by a full Generate/Render; "generate this scene only" is a # transient request handled at the API/route level, not stored here. excluded: bool = False + # Editorial timeline cuts of one generative scene share `gen_unit_id` (the root + # scene's id). Only the root (cut_offset_frames == 0) owns prompt/rating/source + # for generation; subclips are NLE trims of the same output. + gen_unit_id: Optional[str] = None + cut_offset_frames: int = 0 @staticmethod def from_dict(d: dict) -> "Scene": @@ -102,6 +107,8 @@ class Scene: source=SceneSource.from_dict(d.get("source")), rating=str(d.get("rating", "")), excluded=bool(d.get("excluded", False)), + gen_unit_id=d.get("gen_unit_id"), + cut_offset_frames=int(d.get("cut_offset_frames", 0) or 0), ) def to_dict(self) -> dict: @@ -192,6 +199,45 @@ class Project: return d +def gen_unit_id(scene: Scene) -> str: + return scene.gen_unit_id or scene.id + + +def group_generative_units(scenes: list[Scene]) -> list[tuple[str, list[Scene]]]: + """Consecutive timeline scenes that share a gen_unit_id are one generative unit.""" + units: list[tuple[str, list[Scene]]] = [] + for scene in scenes: + uid = gen_unit_id(scene) + if units and units[-1][0] == uid: + units[-1][1].append(scene) + else: + units.append((uid, [scene])) + return units + + +def gen_unit_root(group: list[Scene]) -> Scene: + return min(group, key=lambda s: (int(s.cut_offset_frames or 0), s.id)) + + +def collapse_generative_units(project: Project) -> Project: + """Merge editorial subclips into one Scene per generative unit for ComfyUI runs.""" + clone = Project.from_dict(project.to_dict()) + active = [s for s in clone.scenes if not s.excluded] + collapsed: list[Scene] = [] + for uid, group in group_generative_units(active): + root = gen_unit_root(group) + total_frames = sum(s.eff_frames(clone) for s in group) + merged = Scene.from_dict(root.to_dict()) + merged.id = uid + merged.gen_unit_id = None + merged.cut_offset_frames = 0 + merged.frames = total_frames + merged.frames_mode = "timeline" + collapsed.append(merged) + clone.scenes = collapsed + return clone + + _TRIGGER_RE = None _TRIGGER_RE_BUILT = False @@ -233,20 +279,20 @@ def build_combined_prompt(project: Project, include_excluded: bool = False, only what's sent to Studio, never the displayed global prompt. """ scenes = [s for s in project.scenes if include_excluded or not s.excluded] + units = group_generative_units(scenes) parts: list[str] = [] anchor = (project.anchor or "").strip() if anchor: parts.append(anchor) - # Only inject separators for a MULTI-scene run: Studio treats text before the first - # transition as the (prepended) anchor, so every scene — including the first — needs a - # leading trigger or it merges. A single-scene run needs none (the whole prompt is that - # one scene), avoiding a leaked 'scene N' marker. - inject = for_generation and len(scenes) > 1 + # Only inject separators for a MULTI-unit run: editorial subclips count as one unit. + inject = for_generation and len(units) > 1 trig_re = _leading_trigger_re() if inject else None - for i, scene in enumerate(scenes): - text = (scene.text or "").strip() + for i, (_uid, group) in enumerate(units): + root = gen_unit_root(group) + text = (root.text or "").strip() if inject and not (text and trig_re and trig_re.match(text)): - marker = (project.intro_transition or "").strip() if i == 0 else (scenes[i - 1].transition_to_next or "").strip() + prev_root = gen_unit_root(units[i - 1][1]) if i > 0 else None + marker = (project.intro_transition or "").strip() if i == 0 else ((prev_root.transition_to_next if prev_root else "") or "").strip() if not marker: marker = f"scene {i + 1}" parts.append(marker) diff --git a/movie_editor/frontend/inspector.js b/movie_editor/frontend/inspector.js index 4f137389..b9306f1b 100644 --- a/movie_editor/frontend/inspector.js +++ b/movie_editor/frontend/inspector.js @@ -126,21 +126,33 @@ } function renderScene(st, scene) { - title.textContent = `Scene · ${st.project.scenes.indexOf(scene) + 1}`; + const root = S.genUnitRoot(S.genUnitId(scene)) || scene; + const unitScenes = (st.project.scenes || []).filter((s) => S.genUnitId(s) === S.genUnitId(scene)) + .sort((a, b) => (a.cut_offset_frames || 0) - (b.cut_offset_frames || 0)); + const cutNo = unitScenes.indexOf(scene) + 1; + title.textContent = `Scene · ${st.project.scenes.indexOf(scene) + 1}` + + (unitScenes.length > 1 ? ` · cut ${cutNo}/${unitScenes.length}` : ""); const tag = el("div", "insp-tag"); tag.textContent = "Clip properties"; body.append(tag); + if (unitScenes.length > 1) { + const hint = el("div", "insp-hint"); + hint.textContent = S.isGenSubclip(scene) + ? "Editorial cut of the same generative scene — prompt and source are shared; Generate regens the whole uncut scene, then maps it back to these cuts." + : "This scene has editorial cuts on the timeline — Generate collapses them into one uncut scene."; + body.append(hint); + } - const ta = el("textarea"); ta.rows = 5; ta.value = scene.text || ""; ta.placeholder = "Describe this scene…"; ta.dataset.k = "sc-text"; + const ta = el("textarea"); ta.rows = 5; ta.value = root.text || ""; ta.placeholder = "Describe this scene…"; ta.dataset.k = "sc-text"; ta.oninput = () => S.patchSceneQuiet(scene.id, { text: ta.value }); body.append(field("Prompt", ta)); const src = el("select"); - SRC.forEach(([v, label]) => { const o = el("option", null, label); o.value = v; if ((scene.source?.type) === v) o.selected = true; src.append(o); }); - src.onchange = () => S.patchScene(scene.id, { source: { ...(scene.source || {}), type: src.value } }); + SRC.forEach(([v, label]) => { const o = el("option", null, label); o.value = v; if ((root.source?.type) === v) o.selected = true; src.append(o); }); + src.onchange = () => S.patchScene(scene.id, { source: { ...(root.source || {}), type: src.value } }); body.append(field("Source", src)); - if ((scene.source?.type) === "image") renderImageSource(st, scene); - if ((scene.source?.type) === "generated_frame") renderGeneratedFrameSource(st, scene); - if ((scene.source?.type) === "carry") { + if ((root.source?.type) === "image") renderImageSource(st, root); + if ((root.source?.type) === "generated_frame") renderGeneratedFrameSource(st, root); + if ((root.source?.type) === "carry") { const hint = el("div", "insp-block"); hint.append(el("div", "insp-hint", "No start frame of its own — this scene continues from the previous scene's i2v guide and overlaps with it (chain-sampler carry behaviour). Use this for a continuous shot; use an Image / generated frame to hard-cut to a new anchor.")); diff --git a/movie_editor/frontend/store.js b/movie_editor/frontend/store.js index c493d5b6..3c958197 100644 --- a/movie_editor/frontend/store.js +++ b/movie_editor/frontend/store.js @@ -168,9 +168,23 @@ function scene(id) { return state.project?.scenes.find((s) => s.id === id) || null; } + function genUnitId(sc) { return (sc && sc.gen_unit_id) || (sc && sc.id) || ""; } + function isGenSubclip(sc) { return !!((sc && sc.cut_offset_frames) > 0); } + function genUnitRoot(unitId) { + const group = (state.project?.scenes || []).filter((s) => genUnitId(s) === unitId); + return group.find((s) => !isGenSubclip(s)) || group[0] || null; + } + function genUnitSceneIds(unitId) { + return (state.project?.scenes || []).filter((s) => genUnitId(s) === unitId && !s.excluded).map((s) => s.id); + } + function patchScene(id, patch) { const s = scene(id); if (!s) return; - Object.assign(s, patch); notify(); scheduleSave(); + const root = isGenSubclip(s) ? genUnitRoot(genUnitId(s)) : s; + const targetId = (root && isGenSubclip(s) && (patch.text != null || patch.rating != null || patch.source != null)) + ? root.id : id; + const t = scene(targetId); if (!t) return; + Object.assign(t, patch); notify(); scheduleSave(); } function patchSceneQuiet(id, patch) { const s = scene(id); if (!s) return; @@ -195,6 +209,22 @@ const idx = arr.findIndex((s) => s.id === id); if (idx < 0) return; const sc = arr[idx]; + const unitId = genUnitId(sc); + const unitMates = arr.filter((s) => genUnitId(s) === unitId); + if (!isGenSubclip(sc) && unitMates.length > 1) { + const next = unitMates.find((s) => s.id !== id); + if (next) { + next.cut_offset_frames = 0; + next.text = sc.text || next.text; + next.rating = sc.rating || next.rating; + next.source = JSON.parse(JSON.stringify(sc.source || next.source || {})); + const removedOffset = sc.frames || 0; + unitMates.filter((s) => s.id !== id).forEach((s) => { + if ((s.cut_offset_frames || 0) > (sc.cut_offset_frames || 0)) + s.cut_offset_frames = Math.max(0, (s.cut_offset_frames || 0) - removedOffset); + }); + } + } const r = state.sceneRenders[id]; if (sc && r && r.media) { state.sceneGhosts = (state.sceneGhosts || []).filter((g) => g.id !== id); @@ -290,7 +320,8 @@ scheduleSaveSilent(); } - // Split a clip in two at `atFrames` (defaults to the midpoint). + // Split a clip in two at `atFrames` (defaults to the midpoint). Both halves stay one + // generative unit — Generate collapses them back into a single uncut scene. function splitScene(id, atFrames) { if (!state.project) return; const arr = state.project.scenes; @@ -300,13 +331,20 @@ const frames = s.frames != null ? s.frames : state.project.num_frames_per_scene; const cut = snapFrames(atFrames != null ? atFrames : frames / 2); if (cut <= 9 || cut >= frames) return; + const unitId = genUnitId(s); + const baseOffset = s.cut_offset_frames || 0; const second = JSON.parse(JSON.stringify(s)); // Assign a client id up-front (server honors it) so the rendered video can be mapped // to BOTH halves immediately — like an NLE, a split yields two clips of one source. second.id = "c" + Math.random().toString(36).slice(2, 10) + Date.now().toString(36); + second.gen_unit_id = unitId; + second.cut_offset_frames = baseOffset + cut; second.frames = snapFrames(frames - cut); + second.text = ""; + second.rating = ""; second.transition_to_next = s.transition_to_next || ""; second.transition_frames = s.transition_frames || null; + s.gen_unit_id = unitId; s.frames = cut; s.transition_to_next = ""; s.transition_frames = null; arr.splice(i + 1, 0, second); // Keep the render across the cut: the second half plays the SAME source video starting @@ -497,8 +535,10 @@ const runs = []; for (const s of active) { const t = (s.source && s.source.type) || "empty"; - // carry OR an anchor whose image is gone -> continue the previous run (i2v guides). - const isCarry = t === "carry" || ((t === "image" || t === "generated_frame") && !_anchorAvailable(s)); + // Editorial subclips always continue the current run; carry / broken anchors too. + const isCarry = isGenSubclip(s) + || t === "carry" + || ((t === "image" || t === "generated_frame") && !_anchorAvailable(s)); if (isCarry && runs.length) runs[runs.length - 1].push(s.id); else runs.push([s.id]); } @@ -523,7 +563,8 @@ state.sceneRenders[id] = { media: primary, inSec: inAcc }; // New render invalidates the prior RLHF rating — UI shows "— rate —"; server // uses continue-refinement (session memory, no new rating) on the next Studio run. - if (sc.rating) { sc.rating = ""; clearedRating = true; } + const root = genUnitRoot(genUnitId(sc)); + if (root && root.rating) { root.rating = ""; clearedRating = true; } inAcc += sceneDurationSec(sc); } if (clearedRating) scheduleSaveSilent(); @@ -621,7 +662,9 @@ if (!onlyScene) return generateMontage(); const reset = _resetSessionPending; _resetSessionPending = false; if (reset) state.resetSessionArmed = false; - await _generateRun([onlyScene], onlyScene, "Generating scene", reset); + const sc = scene(onlyScene); + const ids = sc ? genUnitSceneIds(genUnitId(sc)) : [onlyScene]; + await _generateRun(ids, onlyScene, "Generating scene", reset); } // Generate the whole montage: one chain request per run, fired sequentially @@ -838,6 +881,7 @@ get, set, subscribe, init, refreshProjectList, loadProject, newProject, deleteProject, downloadProject, importProject, patchProject, patchProjectQuiet, patchScene, patchSceneQuiet, flushSave, selectScene, addScene, removeScene, dismissGhost, moveScene, moveSceneTo, scene, + genUnitId, isGenSubclip, genUnitRoot, genUnitSceneIds, buildPreviewSegments, previewTotalSec, segmentDurationSec, addAudioTrack, updateAudioTrack, removeAudioTrack, resizeScene, splitScene, snapFrames, diff --git a/movie_editor/frontend/styles.css b/movie_editor/frontend/styles.css index 420bb61d..e89b2afb 100644 --- a/movie_editor/frontend/styles.css +++ b/movie_editor/frontend/styles.css @@ -499,6 +499,8 @@ body.tl-reordering { cursor: grabbing; user-select: none; } .clip.ghost { border-style: dashed; border-color: color-mix(in srgb, var(--amber) 45%, var(--line)); opacity: .72; cursor: default; } .clip.ghost::before { background: var(--amber); opacity: .35; } .clip.ghost .ghost-label { font-style: italic; color: var(--amber-2); } +.clip.gen-cut::before { background: color-mix(in srgb, var(--teal) 55%, var(--amber)); } +.clip.subclip { border-left: 2px solid color-mix(in srgb, var(--teal) 70%, transparent); } /* a clip that has a rendered video available (in temp) */ .clip.rendered::after { content: "●"; position: absolute; top: 4px; right: 6px; font-size: 7px; color: #57c98e; text-shadow: 0 0 5px rgba(87,201,142,.7); pointer-events: none; z-index: 4; } diff --git a/movie_editor/frontend/timeline.js b/movie_editor/frontend/timeline.js index cc3058c1..d40c2edc 100644 --- a/movie_editor/frontend/timeline.js +++ b/movie_editor/frontend/timeline.js @@ -114,7 +114,9 @@ // ── clip ─────────────────────────────────────────────────────────────────────── function clipEl(st, p, scene, index, leftPx, widthPx) { - const clip = el("div", "clip" + (scene.id === st.selectedSceneId ? " selected" : "") + (scene.excluded ? " excluded" : "") + (hasRender(st, scene.id) ? " rendered" : (!scene.excluded ? " pending" : ""))); + const unitCuts = (p.scenes || []).filter((s) => (s.gen_unit_id || s.id) === (scene.gen_unit_id || scene.id)).length; + const subclip = (scene.cut_offset_frames || 0) > 0; + const clip = el("div", "clip" + (scene.id === st.selectedSceneId ? " selected" : "") + (scene.excluded ? " excluded" : "") + (hasRender(st, scene.id) ? " rendered" : (!scene.excluded ? " pending" : "")) + (unitCuts > 1 ? " gen-cut" : "") + (subclip ? " subclip" : "")); clip.style.left = leftPx + "px"; clip.style.width = Math.max(widthPx, 8) + "px"; clip.onclick = () => S.selectScene(scene.id); @@ -183,7 +185,11 @@ head.append(el("span", "clip-dur", timecode(sDur(scene, p), sFps(scene, p)))); clip.append(head); - clip.append(el("div", "clip-text" + (scene.text ? "" : " empty"), scene.text || "empty scene")); + const root = unitCuts > 1 + ? (p.scenes || []).find((s) => (s.gen_unit_id || s.id) === (scene.gen_unit_id || scene.id) && !(s.cut_offset_frames > 0)) + : null; + const label = scene.text || (root && root.text) || (subclip ? "cut" : "empty scene"); + clip.append(el("div", "clip-text" + (label && label !== "empty scene" && label !== "cut" ? "" : " empty"), label)); const actions = el("div", "clip-actions"); const mk = (label, title, cls, fn) => { const b = el("button", "ic-btn" + (cls ? " " + cls : ""), label); b.title = title; b.onclick = (e) => { e.stopPropagation(); fn(); }; return b; }; diff --git a/movie_editor/server.py b/movie_editor/server.py index a9235e17..5309bfa4 100644 --- a/movie_editor/server.py +++ b/movie_editor/server.py @@ -20,7 +20,13 @@ except Exception: # pragma: no cover - only available inside ComfyUI PromptServer = None from .backend import bridge, builder, config, media, nodes, projects -from .backend.timeline import Project, build_combined_prompt +from .backend.timeline import ( + Project, + build_combined_prompt, + collapse_generative_units, + gen_unit_id, + group_generative_units, +) UI_PREFIX = "/funpack/movie" @@ -121,13 +127,13 @@ def _project_models(p: Optional[Project]) -> dict: def _solo(p: Project, only_scene: Optional[str]) -> Project: if not only_scene: - return p + return collapse_generative_units(p) scene = next((s for s in p.scenes if s.id == only_scene), None) if scene is None: raise web.HTTPNotFound(reason=f"Scene {only_scene} not found") - clone = Project.from_dict(p.to_dict()) - clone.scenes = [scene] - return clone + uid = gen_unit_id(scene) + ids = [s.id for s in p.scenes if gen_unit_id(s) == uid and not s.excluded] + return collapse_generative_units(_segment(p, ids)) def _run_studio_inputs(target: Project, active_scenes: list) -> dict: @@ -297,7 +303,7 @@ def _segment(p: Project, scene_ids: list) -> Project: clone.scenes = [s for s in clone.scenes if s.id in ids] if not clone.scenes: raise web.HTTPNotFound(reason="No scenes matched the requested segment") - return clone + return collapse_generative_units(clone) def _serve_static(tail: str) -> "web.Response": diff --git a/movie_editor/tests/test_timeline.py b/movie_editor/tests/test_timeline.py index 5f0c3a63..03669c94 100644 --- a/movie_editor/tests/test_timeline.py +++ b/movie_editor/tests/test_timeline.py @@ -11,7 +11,13 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) -from movie_editor.backend.timeline import Project, Scene, build_combined_prompt # noqa: E402 +from movie_editor.backend.timeline import ( # noqa: E402 + Project, + Scene, + build_combined_prompt, + collapse_generative_units, + group_generative_units, +) def _project(**kw): @@ -80,6 +86,26 @@ def test_project_roundtrip_preserves_forward_compat_fields(): assert p2.scenes[0].excluded is True +def test_split_subclips_collapse_to_one_generative_unit(): + p = _project( + scenes=[ + {"id": "u1", "text": "one long shot", "frames": 49, "gen_unit_id": "u1", "cut_offset_frames": 0}, + {"id": "u1b", "text": "", "frames": 49, "gen_unit_id": "u1", "cut_offset_frames": 49}, + {"id": "s2", "text": "second scene", "frames": 97}, + ], + ) + units = group_generative_units(p.scenes) + assert len(units) == 2 + assert len(units[0][1]) == 2 + prompt = build_combined_prompt(p, for_generation=True) + assert prompt.count("one long shot") == 1 + assert "second scene" in prompt + collapsed = collapse_generative_units(p) + assert len(collapsed.scenes) == 2 + assert collapsed.scenes[0].frames == 98 + assert collapsed.scenes[0].text == "one long shot" + + def test_projects_store_crud(tmp_path, monkeypatch): from movie_editor.backend import config, projects monkeypatch.setattr(config, "PROJECTS_DIR", tmp_path / "projects")