From cd9f4dfb81017983993313b97eb5b92b5b87f59a Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sun, 7 Jun 2026 09:11:06 +0300 Subject: [PATCH] Movie Editor: run-based montage generation + final video stitch (video+audio) Generate Whole Montage now splits active scenes into chain runs bounded by anchored scenes: a run starts at an empty/image/generated_frame scene and a 'carry' scene appends to it (overlapping via carry_i2v_guides). One ComfyUI chain-sampler request is fired per run, sequentially; each run's first scene supplies its i2v anchor. No sampler changes needed. New 'Render Final Video' button hard-cut concatenates the rendered runs in timeline order (skipping ungenerated gaps) via ffmpeg, preserving the LTXAV audio track, and plays the stitched result in the program monitor. - server.py: generate accepts scene_ids (a run); _segment() clone; carry_i2v_guides forced on for multi-scene runs; POST /render ffmpeg concat (-c:a aac). - store.js: _runs() segmentation, promise-based polling, generateMontage(), renderFinal(). - api.js: generate(scene_ids), renderFinal(); menubar: Render Final Video item. Co-Authored-By: Claude Opus 4.8 --- movie_editor/frontend/api.js | 6 +- movie_editor/frontend/menubar.js | 4 +- movie_editor/frontend/store.js | 168 +++++++++++++++++++++---------- movie_editor/server.py | 93 ++++++++++++++++- 4 files changed, 211 insertions(+), 60 deletions(-) diff --git a/movie_editor/frontend/api.js b/movie_editor/frontend/api.js index 752ffee7..dfefa892 100644 --- a/movie_editor/frontend/api.js +++ b/movie_editor/frontend/api.js @@ -71,9 +71,11 @@ refreshModels: () => j("POST", API("/models/refresh")), restart: () => j("POST", API("/restart")), - // generate - generate: (id, onlyScene) => j("POST", API(`/projects/${id}/generate`), { only_scene: onlyScene || null }), + // generate (a single scene, or an explicit run of scene ids = one chain request) + generate: (id, onlyScene, sceneIds) => + j("POST", API(`/projects/${id}/generate`), { only_scene: onlyScene || null, scene_ids: sceneIds || null }), status: (id, promptId) => j("GET", API(`/projects/${id}/status/${promptId}`)), + renderFinal: (id, clips) => j("POST", API(`/projects/${id}/render`), { clips }), resultUrl: (id, m) => API(`/projects/${id}/result?filename=${encodeURIComponent(m.filename)}`) + `&subfolder=${encodeURIComponent(m.subfolder || "")}&type=${encodeURIComponent(m.type || "output")}`, diff --git a/movie_editor/frontend/menubar.js b/movie_editor/frontend/menubar.js index 5094196b..00ca5778 100644 --- a/movie_editor/frontend/menubar.js +++ b/movie_editor/frontend/menubar.js @@ -75,12 +75,12 @@ { label: "Generate Whole Montage", hint: "▶", disabled: !hasProject(), action: () => S.generate(null) }, { label: "Generate Selected Scene", disabled: !sel(), action: () => S.generate(sel()) }, { sep: true }, + { label: "Render Final Video", hint: "⧉", disabled: !hasProject(), action: () => S.renderFinal() }, + { sep: true }, { label: `Conditioning: ${_roleLabel(st.project?.conditioning_slot, "FunPack Studio")}`, disabled: !hasProject(), action: () => _pickRole("conditioning_slot", "Conditioning node", st) }, { label: `Sampler: ${_roleLabel(st.project?.sampler_slot, "FunPack Chain Sampler")}`, disabled: !hasProject(), action: () => _pickRole("sampler_slot", "Sampler node", st) }, - { sep: true }, - { label: "Render / Export…", soon: true, disabled: true }, ], Models: [ { label: "Settings…", action: () => window.ModelsModal.open() }, diff --git a/movie_editor/frontend/store.js b/movie_editor/frontend/store.js index cf374dd9..b1390d5a 100644 --- a/movie_editor/frontend/store.js +++ b/movie_editor/frontend/store.js @@ -309,68 +309,128 @@ let pollTimer = null; let pollStart = 0; - async function generate(onlyScene) { - if (!state.project) return; - // Capture which scene ids this generation covers so the player can map - // the output video to the correct timeline positions. - const activeScenes = state.project.scenes.filter((s) => !s.excluded); - const targetSceneIds = onlyScene ? [onlyScene] : activeScenes.map((s) => s.id); - set({ gen: { state: "queuing", promptId: null, media: [], msg: onlyScene ? "Queuing scene…" : "Queuing montage…" } }); - try { - const r = await API.generate(state.project.id, onlyScene); - if (!r.prompt_id) { set({ gen: { ...state.gen, state: "error", msg: "No prompt id returned." } }); return; } - pollStart = Date.now(); - set({ gen: { state: "running", promptId: r.prompt_id, media: [], msg: "Generating…" } }); - poll(r.prompt_id, targetSceneIds); - } catch (e) { - set({ gen: { state: "error", promptId: null, media: [], msg: _friendlyGenError(e.message) } }); - } - } function _elapsed() { const s = Math.floor((Date.now() - pollStart) / 1000); return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m ${s % 60}s`; } - function poll(promptId, targetSceneIds) { - clearInterval(pollTimer); - let pendingStreak = 0; - pollTimer = setInterval(async () => { - try { - const s = await API.status(state.project.id, promptId); - if (s.state === "error") { - clearInterval(pollTimer); - const msg = s.error ? `ComfyUI error: ${s.error}` : "Generation failed inside ComfyUI — check the ComfyUI terminal for details."; - set({ gen: { state: "error", promptId, media: [], msg } }); - } else if (s.state === "completed") { - clearInterval(pollTimer); - // Record the segment so the player can map it to timeline positions. - if (s.media.length && targetSceneIds?.length) { - const keep = (state.renderedSegments || []).filter( - (seg) => !(seg.sceneIds || []).some((id) => targetSceneIds.includes(id)) - ); - // Primary media (first video/gif) is the playable output. - const primary = s.media.find((m) => m.kind === "videos" || m.kind === "gifs") || s.media[0]; - state.renderedSegments = [...keep, { sceneIds: targetSceneIds, media: primary }]; - } - set({ gen: { state: "done", promptId, media: s.media, msg: s.media.length ? "" : "Completed but no output media found — check ComfyUI terminal." } }); - } else { - // "pending" after being "running" means the job left the queue without a - // history entry — it likely crashed or was interrupted by ComfyUI. - if (s.state === "pending") pendingStreak++; - else pendingStreak = 0; - if (pendingStreak >= 3) { + // Split the active scenes into chain runs. A run starts at an anchored scene + // (empty / image / generated_frame); a "carry" scene appends to the current run + // so it overlaps the previous scene (one chain-sampler request per run). + function _runs() { + const active = state.project.scenes.filter((s) => !s.excluded); + const runs = []; + for (const s of active) { + const t = (s.source && s.source.type) || "empty"; + if (t === "carry" && runs.length) runs[runs.length - 1].push(s.id); + else runs.push([s.id]); + } + return runs; + } + + // Record a completed run's output so the player can map it onto the timeline. + function _recordSegment(mediaList, targetSceneIds) { + if (!mediaList || !mediaList.length || !targetSceneIds || !targetSceneIds.length) return; + const keep = (state.renderedSegments || []).filter( + (seg) => !(seg.sceneIds || []).some((id) => targetSceneIds.includes(id)) + ); + const primary = mediaList.find((m) => m.kind === "videos" || m.kind === "gifs") || mediaList[0]; + state.renderedSegments = [...keep, { sceneIds: targetSceneIds, media: primary }]; + } + + // Poll a single queued prompt to completion. Resolves true on success, false on error. + function _pollPromise(promptId, targetSceneIds, prefix) { + prefix = prefix || "Generating…"; + return new Promise((resolve) => { + clearInterval(pollTimer); + let pendingStreak = 0; + pollTimer = setInterval(async () => { + try { + const s = await API.status(state.project.id, promptId); + if (s.state === "error") { clearInterval(pollTimer); - set({ gen: { state: "error", promptId, media: [], msg: "Job disappeared from ComfyUI queue — it may have crashed or been interrupted. Check the ComfyUI terminal." } }); - return; + const msg = s.error ? `ComfyUI error: ${s.error}` : "Generation failed inside ComfyUI — check the ComfyUI terminal for details."; + set({ gen: { state: "error", promptId, media: [], msg } }); + resolve(false); + } else if (s.state === "completed") { + clearInterval(pollTimer); + _recordSegment(s.media, targetSceneIds); + set({ gen: { state: "done", promptId, media: s.media, msg: s.media.length ? "" : "Completed but no output media found — check ComfyUI terminal." } }); + resolve(true); + } else { + // "pending" after "running" means the job left the queue without a history + // entry — it likely crashed or was interrupted by ComfyUI. + if (s.state === "pending") pendingStreak++; else pendingStreak = 0; + if (pendingStreak >= 3) { + clearInterval(pollTimer); + set({ gen: { state: "error", promptId, media: [], msg: "Job disappeared from ComfyUI queue — it may have crashed or been interrupted. Check the ComfyUI terminal." } }); + resolve(false); + return; + } + set({ gen: { ...state.gen, state: s.state, msg: `${prefix} ${_elapsed()}` } }); } - set({ gen: { ...state.gen, state: s.state, msg: `Generating… ${_elapsed()}` } }); + } catch (e) { + clearInterval(pollTimer); + set({ gen: { ...state.gen, state: "error", msg: e.message } }); + resolve(false); } - } catch (e) { - clearInterval(pollTimer); - set({ gen: { ...state.gen, state: "error", msg: e.message } }); - } - }, 2000); + }, 2000); + }); + } + + // Generate one run (single scene, or an explicit list of scene ids). Returns success. + async function _generateRun(sceneIds, onlyScene, prefix) { + set({ gen: { state: "queuing", promptId: null, media: [], msg: `${prefix}: queuing…` } }); + try { + const r = await API.generate(state.project.id, onlyScene || null, onlyScene ? null : sceneIds); + if (!r.prompt_id) { set({ gen: { ...state.gen, state: "error", msg: "No prompt id returned." } }); return false; } + pollStart = Date.now(); + set({ gen: { state: "running", promptId: r.prompt_id, media: [], msg: `${prefix}: generating…` } }); + return await _pollPromise(r.prompt_id, sceneIds, prefix); + } catch (e) { + set({ gen: { state: "error", promptId: null, media: [], msg: _friendlyGenError(e.message) } }); + return false; + } + } + + async function generate(onlyScene) { + if (!state.project) return; + if (!onlyScene) return generateMontage(); + await _generateRun([onlyScene], onlyScene, "Generating scene"); + } + + // Generate the whole montage: one chain request per run, fired sequentially + // (one GPU at a time). Each run's first scene supplies its i2v anchor. + async function generateMontage() { + if (!state.project) return; + const runs = _runs(); + if (!runs.length) { set({ gen: { state: "error", promptId: null, media: [], msg: "No active scenes to generate." } }); return; } + for (let i = 0; i < runs.length; i++) { + const ok = await _generateRun(runs[i], null, `Run ${i + 1}/${runs.length}`); + if (!ok) return; // error already surfaced + } + set({ gen: { state: "done", promptId: null, media: state.gen.media, msg: `${runs.length} run(s) generated — use Render Final Video to stitch them.` } }); + } + + // Stitch the rendered runs (hard cut, video + audio) into one final file. + async function renderFinal() { + if (!state.project) return; + const order = state.project.scenes.filter((s) => !s.excluded).map((s) => s.id); + const pos = (ids) => Math.min(...ids.map((id) => { const i = order.indexOf(id); return i < 0 ? 1e9 : i; })); + const segs = (state.renderedSegments || []).filter((s) => s.media) + .slice().sort((a, b) => pos(a.sceneIds || []) - pos(b.sceneIds || [])); + if (!segs.length) { alert("No generated runs to stitch yet — generate first."); return; } + const clips = segs.map((s) => ({ filename: s.media.filename, subfolder: s.media.subfolder || "", type: s.media.type || "output" })); + set({ gen: { state: "running", promptId: null, media: [], msg: `Stitching ${clips.length} run(s)…` } }); + try { + const r = await API.renderFinal(state.project.id, clips); + // Show the stitched file across the whole timeline in the program monitor. + state.renderedSegments = [{ sceneIds: order, media: r.media }]; + set({ gen: { state: "done", promptId: null, media: [r.media], msg: `Final video rendered from ${r.clips} run(s).` } }); + } catch (e) { + set({ gen: { state: "error", promptId: null, media: [], msg: "Render failed: " + e.message } }); + } } // ── pluggable models / exposed controls ────────────────────────────────────── @@ -475,7 +535,7 @@ refreshProjectList, loadProject, newProject, deleteProject, downloadProject, importProject, patchProject, patchProjectQuiet, patchScene, patchSceneQuiet, selectScene, addScene, removeScene, moveScene, scene, resizeScene, splitScene, snapFrames, - refreshPreview, syncFromPreview, applyGlobalPrompt, generate, loadModels, loadImageTargets, setModelInput, setModelLink, + refreshPreview, syncFromPreview, applyGlobalPrompt, generate, generateMontage, renderFinal, loadModels, loadImageTargets, setModelInput, setModelLink, setConditioningSlot, setSamplerSlot, setSamplerInput, setSamplerInputNow, setStudioInput, setStudioInputNow, loadMedia, uploadMedia, deleteMedia, assignMediaToScene, loadShortcuts, saveShortcut, deleteShortcut, importShortcuts, loadTransitions, saveTransition, deleteTransition, importTransitions, diff --git a/movie_editor/server.py b/movie_editor/server.py index 1566f471..3daf686f 100644 --- a/movie_editor/server.py +++ b/movie_editor/server.py @@ -120,6 +120,29 @@ def _solo(p: Project, only_scene: Optional[str]) -> Project: return clone +def _run_sampler_inputs(target: Project, scene_count: int) -> dict: + """Sampler widget overrides for one chain run. Carries inside a multi-scene run + must overlap, so carry_i2v_guides is forced on; a 1-scene run leaves it alone.""" + if target.sampler_slot != "funpack": + return {} + samp = dict(target.sampler_inputs or {}) + if scene_count > 1: + samp["carry_i2v_guides"] = True + return samp + + +def _segment(p: Project, scene_ids: list) -> Project: + """Project clone holding only `scene_ids` (in their project order). Used to + generate one chain run: its first scene supplies the i2v anchor, the rest are + carries that overlap inside the run (one ComfyUI chain-sampler request).""" + ids = set(scene_ids) + clone = Project.from_dict(p.to_dict()) + clone.scenes = [s for s in clone.scenes if s.id in ids] + if not clone.scenes: + raise web.HTTPNotFound(reason="No scenes matched the requested segment") + return clone + + def _serve_static(tail: str) -> "web.Response": root = config.FRONTEND_DIR.resolve() rel = (tail or "index.html").lstrip("/") @@ -373,7 +396,11 @@ if web is not None and PromptServer is not None: async def _generate(req): p = _project_or_404(req.match_info["pid"]) body = await req.json() if req.can_read_body else {} - target = _solo(p, body.get("only_scene")) + scene_ids = body.get("scene_ids") + if scene_ids: + target = _segment(p, scene_ids) + else: + target = _solo(p, body.get("only_scene")) prompt = build_combined_prompt(target) if not prompt.strip(): return web.json_response({"detail": "Nothing to generate — no active scene text."}, status=400) @@ -405,7 +432,7 @@ if web is not None and PromptServer is not None: "negative_prompt": target.negative_prompt or None, "max_scenes": active_scene_count, "studio_inputs": target.studio_inputs if target.conditioning_slot == "funpack" else {}, - "sampler_inputs": target.sampler_inputs if target.sampler_slot == "funpack" else {}, + "sampler_inputs": _run_sampler_inputs(target, active_scene_count), }, media=_prepare_media(target)) if report["blocking"]: detail = "Generation blocked — " + "; ".join(report["blocking"]) @@ -460,6 +487,68 @@ if web is not None and PromptServer is not None: raise web.HTTPBadGateway(reason=f"could not fetch result: {e}") return web.Response(body=data, content_type=ctype.split(";")[0]) + # --- API: final render (hard-cut concat of generated runs, video + audio) --- + @routes.post(UI_PREFIX + "/api/projects/{pid}/render") + async def _render_final(req): + _project_or_404(req.match_info["pid"]) + body = await req.json() if req.can_read_body else {} + clips = body.get("clips") or [] # ordered [{filename, subfolder, type}] + if not clips: + return web.json_response({"detail": "Nothing to render — no generated runs."}, status=400) + import os + import shutil + import subprocess + import tempfile + import time as _time + ff = shutil.which("ffmpeg") + if not ff: + return web.json_response({"detail": "ffmpeg not found on PATH — install it to render the final video."}, status=503) + try: + import folder_paths + outdir = folder_paths.get_output_directory() + tempdir = folder_paths.get_temp_directory() + except Exception as e: # noqa: BLE001 + return web.json_response({"detail": f"Output directory unavailable: {e}"}, status=500) + + def _resolve(c): + base = outdir if c.get("type", "output") == "output" else tempdir + return os.path.join(base, c.get("subfolder", ""), c.get("filename", "")) + + paths = [_resolve(c) for c in clips] + missing = [p for p in paths if not os.path.isfile(p)] + if missing: + return web.json_response({"detail": f"{len(missing)} clip file(s) not found on disk — regenerate then render."}, status=400) + + out_name = f"funpack_final_{int(_time.time())}.mp4" + out_path = os.path.join(outdir, out_name) + # concat demuxer + re-encode → tolerates differing codecs/params between runs + # and keeps BOTH the video and the LTXAV audio stream. + list_fd, list_path = tempfile.mkstemp(suffix=".txt") + try: + with os.fdopen(list_fd, "w") as lf: + for pth in paths: + safe = pth.replace("'", "'\\''") + lf.write(f"file '{safe}'\n") + cmd = [ + ff, "-y", "-f", "concat", "-safe", "0", "-i", list_path, + "-c:v", "libx264", "-pix_fmt", "yuv420p", + "-c:a", "aac", "-b:a", "192k", + "-movflags", "+faststart", out_path, + ] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + tail = (proc.stderr or "")[-800:] + return web.json_response({"detail": f"ffmpeg concat failed: {tail}"}, status=500) + finally: + try: + os.unlink(list_path) + except OSError: + pass + return web.json_response({ + "media": {"filename": out_name, "subfolder": "", "type": "output", "kind": "videos"}, + "clips": len(paths), + }) + # --- API: models / pluggable node slots --- @routes.get(UI_PREFIX + "/api/node-roles") async def _node_roles(_req):