nodes/olivv-cs-ComfyUI-FunPack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Movie Editor: run-based montage generation + final video stitch (video+audio)

Browse Source 
Generate Whole Montage now splits active scenes into chain runs bounded by
anchored scenes: a run starts at an empty/image/generated_frame scene and a
'carry' scene appends to it (overlapping via carry_i2v_guides). One ComfyUI
chain-sampler request is fired per run, sequentially; each run's first scene
supplies its i2v anchor. No sampler changes needed.

New 'Render Final Video' button hard-cut concatenates the rendered runs in
timeline order (skipping ungenerated gaps) via ffmpeg, preserving the LTXAV
audio track, and plays the stitched result in the program monitor.

- server.py: generate accepts scene_ids (a run); _segment() clone; carry_i2v_guides
  forced on for multi-scene runs; POST /render ffmpeg concat (-c:a aac).
- store.js: _runs() segmentation, promise-based polling, generateMontage(), renderFinal().
- api.js: generate(scene_ids), renderFinal(); menubar: Render Final Video item.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
digital-garbage
2026-06-07 09:11:06 +03:00
co-authored by Claude Opus 4.8
parent ff2f3c65d8
commit cd9f4dfb81
4 changed files with 211 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
movie_editor/frontend/api.js
+4 -2
View File
@@ -71,9 +71,11 @@
refreshModels: () => j("POST", API("/models/refresh")),
restart: () => j("POST", API("/restart")),
// generate
generate: (id, onlyScene) => j("POST", API(`/projects/${id}/generate`), { only_scene: onlyScene || null }),
// generate (a single scene, or an explicit run of scene ids = one chain request)
generate: (id, onlyScene, sceneIds) =>
j("POST", API(`/projects/${id}/generate`), { only_scene: onlyScene || null, scene_ids: sceneIds || null }),
status: (id, promptId) => j("GET", API(`/projects/${id}/status/${promptId}`)),
renderFinal: (id, clips) => j("POST", API(`/projects/${id}/render`), { clips }),
resultUrl: (id, m) =>
API(`/projects/${id}/result?filename=${encodeURIComponent(m.filename)}`) +
`&subfolder=${encodeURIComponent(m.subfolder || "")}&type=${encodeURIComponent(m.type || "output")}`,
movie_editor/frontend/menubar.js
+2 -2
View File
@@ -75,12 +75,12 @@
{ label: "Generate Whole Montage", hint: "▶", disabled: !hasProject(), action: () => S.generate(null) },
{ label: "Generate Selected Scene", disabled: !sel(), action: () => S.generate(sel()) },
{ sep: true },
{ label: "Render Final Video", hint: "⧉", disabled: !hasProject(), action: () => S.renderFinal() },
{ sep: true },
{ label: `Conditioning: ${_roleLabel(st.project?.conditioning_slot, "FunPack Studio")}`, disabled: !hasProject(),
action: () => _pickRole("conditioning_slot", "Conditioning node", st) },
{ label: `Sampler: ${_roleLabel(st.project?.sampler_slot, "FunPack Chain Sampler")}`, disabled: !hasProject(),
action: () => _pickRole("sampler_slot", "Sampler node", st) },
{ sep: true },
{ label: "Render / Export…", soon: true, disabled: true },
],
Models: [
{ label: "Settings…", action: () => window.ModelsModal.open() },
movie_editor/frontend/store.js
+114 -54
View File
@@ -309,68 +309,128 @@
let pollTimer = null;
let pollStart = 0;
async function generate(onlyScene) {
if (!state.project) return;
// Capture which scene ids this generation covers so the player can map
// the output video to the correct timeline positions.
const activeScenes = state.project.scenes.filter((s) => !s.excluded);
const targetSceneIds = onlyScene ? [onlyScene] : activeScenes.map((s) => s.id);
set({ gen: { state: "queuing", promptId: null, media: [], msg: onlyScene ? "Queuing scene…" : "Queuing montage…" } });
try {
const r = await API.generate(state.project.id, onlyScene);
if (!r.prompt_id) { set({ gen: { ...state.gen, state: "error", msg: "No prompt id returned." } }); return; }
pollStart = Date.now();
set({ gen: { state: "running", promptId: r.prompt_id, media: [], msg: "Generating…" } });
poll(r.prompt_id, targetSceneIds);
} catch (e) {
set({ gen: { state: "error", promptId: null, media: [], msg: _friendlyGenError(e.message) } });
}
}
function _elapsed() {
const s = Math.floor((Date.now() - pollStart) / 1000);
return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m ${s % 60}s`;
}
function poll(promptId, targetSceneIds) {
clearInterval(pollTimer);
let pendingStreak = 0;
pollTimer = setInterval(async () => {
try {
const s = await API.status(state.project.id, promptId);
if (s.state === "error") {
clearInterval(pollTimer);
const msg = s.error ? `ComfyUI error: ${s.error}` : "Generation failed inside ComfyUI — check the ComfyUI terminal for details.";
set({ gen: { state: "error", promptId, media: [], msg } });
} else if (s.state === "completed") {
clearInterval(pollTimer);
// Record the segment so the player can map it to timeline positions.
if (s.media.length && targetSceneIds?.length) {
const keep = (state.renderedSegments || []).filter(
(seg) => !(seg.sceneIds || []).some((id) => targetSceneIds.includes(id))
);
// Primary media (first video/gif) is the playable output.
const primary = s.media.find((m) => m.kind === "videos" || m.kind === "gifs") || s.media[0];
state.renderedSegments = [...keep, { sceneIds: targetSceneIds, media: primary }];
}
set({ gen: { state: "done", promptId, media: s.media, msg: s.media.length ? "" : "Completed but no output media found — check ComfyUI terminal." } });
} else {
// "pending" after being "running" means the job left the queue without a
// history entry — it likely crashed or was interrupted by ComfyUI.
if (s.state === "pending") pendingStreak++;
else pendingStreak = 0;
if (pendingStreak >= 3) {
// Split the active scenes into chain runs. A run starts at an anchored scene
// (empty / image / generated_frame); a "carry" scene appends to the current run
// so it overlaps the previous scene (one chain-sampler request per run).
function _runs() {
const active = state.project.scenes.filter((s) => !s.excluded);
const runs = [];
for (const s of active) {
const t = (s.source && s.source.type) || "empty";
if (t === "carry" && runs.length) runs[runs.length - 1].push(s.id);
else runs.push([s.id]);
}
return runs;
}
// Record a completed run's output so the player can map it onto the timeline.
function _recordSegment(mediaList, targetSceneIds) {
if (!mediaList || !mediaList.length || !targetSceneIds || !targetSceneIds.length) return;
const keep = (state.renderedSegments || []).filter(
(seg) => !(seg.sceneIds || []).some((id) => targetSceneIds.includes(id))
);
const primary = mediaList.find((m) => m.kind === "videos" || m.kind === "gifs") || mediaList[0];
state.renderedSegments = [...keep, { sceneIds: targetSceneIds, media: primary }];
}
// Poll a single queued prompt to completion. Resolves true on success, false on error.
function _pollPromise(promptId, targetSceneIds, prefix) {
prefix = prefix || "Generating…";
return new Promise((resolve) => {
clearInterval(pollTimer);
let pendingStreak = 0;
pollTimer = setInterval(async () => {
try {
const s = await API.status(state.project.id, promptId);
if (s.state === "error") {
clearInterval(pollTimer);
set({ gen: { state: "error", promptId, media: [], msg: "Job disappeared from ComfyUI queue — it may have crashed or been interrupted. Check the ComfyUI terminal." } });
return;
const msg = s.error ? `ComfyUI error: ${s.error}` : "Generation failed inside ComfyUI — check the ComfyUI terminal for details.";
set({ gen: { state: "error", promptId, media: [], msg } });
resolve(false);
} else if (s.state === "completed") {
clearInterval(pollTimer);
_recordSegment(s.media, targetSceneIds);
set({ gen: { state: "done", promptId, media: s.media, msg: s.media.length ? "" : "Completed but no output media found — check ComfyUI terminal." } });
resolve(true);
} else {
// "pending" after "running" means the job left the queue without a history
// entry — it likely crashed or was interrupted by ComfyUI.
if (s.state === "pending") pendingStreak++; else pendingStreak = 0;
if (pendingStreak >= 3) {
clearInterval(pollTimer);
set({ gen: { state: "error", promptId, media: [], msg: "Job disappeared from ComfyUI queue — it may have crashed or been interrupted. Check the ComfyUI terminal." } });
resolve(false);
return;
}
set({ gen: { ...state.gen, state: s.state, msg: `${prefix} ${_elapsed()}` } });
}
set({ gen: { ...state.gen, state: s.state, msg: `Generating… ${_elapsed()}` } });
} catch (e) {
clearInterval(pollTimer);
set({ gen: { ...state.gen, state: "error", msg: e.message } });
resolve(false);
}
} catch (e) {
clearInterval(pollTimer);
set({ gen: { ...state.gen, state: "error", msg: e.message } });
}
}, 2000);
}, 2000);
});
}
// Generate one run (single scene, or an explicit list of scene ids). Returns success.
async function _generateRun(sceneIds, onlyScene, prefix) {
set({ gen: { state: "queuing", promptId: null, media: [], msg: `${prefix}: queuing…` } });
try {
const r = await API.generate(state.project.id, onlyScene || null, onlyScene ? null : sceneIds);
if (!r.prompt_id) { set({ gen: { ...state.gen, state: "error", msg: "No prompt id returned." } }); return false; }
pollStart = Date.now();
set({ gen: { state: "running", promptId: r.prompt_id, media: [], msg: `${prefix}: generating…` } });
return await _pollPromise(r.prompt_id, sceneIds, prefix);
} catch (e) {
set({ gen: { state: "error", promptId: null, media: [], msg: _friendlyGenError(e.message) } });
return false;
}
}
async function generate(onlyScene) {
if (!state.project) return;
if (!onlyScene) return generateMontage();
await _generateRun([onlyScene], onlyScene, "Generating scene");
}
// Generate the whole montage: one chain request per run, fired sequentially
// (one GPU at a time). Each run's first scene supplies its i2v anchor.
async function generateMontage() {
if (!state.project) return;
const runs = _runs();
if (!runs.length) { set({ gen: { state: "error", promptId: null, media: [], msg: "No active scenes to generate." } }); return; }
for (let i = 0; i < runs.length; i++) {
const ok = await _generateRun(runs[i], null, `Run ${i + 1}/${runs.length}`);
if (!ok) return; // error already surfaced
}
set({ gen: { state: "done", promptId: null, media: state.gen.media, msg: `${runs.length} run(s) generated — use Render Final Video to stitch them.` } });
}
// Stitch the rendered runs (hard cut, video + audio) into one final file.
async function renderFinal() {
if (!state.project) return;
const order = state.project.scenes.filter((s) => !s.excluded).map((s) => s.id);
const pos = (ids) => Math.min(...ids.map((id) => { const i = order.indexOf(id); return i < 0 ? 1e9 : i; }));
const segs = (state.renderedSegments || []).filter((s) => s.media)
.slice().sort((a, b) => pos(a.sceneIds || []) - pos(b.sceneIds || []));
if (!segs.length) { alert("No generated runs to stitch yet — generate first."); return; }
const clips = segs.map((s) => ({ filename: s.media.filename, subfolder: s.media.subfolder || "", type: s.media.type || "output" }));
set({ gen: { state: "running", promptId: null, media: [], msg: `Stitching ${clips.length} run(s)…` } });
try {
const r = await API.renderFinal(state.project.id, clips);
// Show the stitched file across the whole timeline in the program monitor.
state.renderedSegments = [{ sceneIds: order, media: r.media }];
set({ gen: { state: "done", promptId: null, media: [r.media], msg: `Final video rendered from ${r.clips} run(s).` } });
} catch (e) {
set({ gen: { state: "error", promptId: null, media: [], msg: "Render failed: " + e.message } });
}
}
// ── pluggable models / exposed controls ──────────────────────────────────────
@@ -475,7 +535,7 @@
refreshProjectList, loadProject, newProject, deleteProject, downloadProject, importProject,
patchProject, patchProjectQuiet, patchScene, patchSceneQuiet, selectScene, addScene, removeScene, moveScene, scene,
resizeScene, splitScene, snapFrames,
refreshPreview, syncFromPreview, applyGlobalPrompt, generate, loadModels, loadImageTargets, setModelInput, setModelLink,
refreshPreview, syncFromPreview, applyGlobalPrompt, generate, generateMontage, renderFinal, loadModels, loadImageTargets, setModelInput, setModelLink,
setConditioningSlot, setSamplerSlot, setSamplerInput, setSamplerInputNow, setStudioInput, setStudioInputNow,
loadMedia, uploadMedia, deleteMedia, assignMediaToScene,
loadShortcuts, saveShortcut, deleteShortcut, importShortcuts, loadTransitions, saveTransition, deleteTransition, importTransitions,
movie_editor/server.py
+91 -2
View File
@@ -120,6 +120,29 @@ def _solo(p: Project, only_scene: Optional[str]) -> Project:
return clone
def _run_sampler_inputs(target: Project, scene_count: int) -> dict:
"""Sampler widget overrides for one chain run. Carries inside a multi-scene run
must overlap, so carry_i2v_guides is forced on; a 1-scene run leaves it alone."""
if target.sampler_slot != "funpack":
return {}
samp = dict(target.sampler_inputs or {})
if scene_count > 1:
samp["carry_i2v_guides"] = True
return samp
def _segment(p: Project, scene_ids: list) -> Project:
"""Project clone holding only `scene_ids` (in their project order). Used to
generate one chain run: its first scene supplies the i2v anchor, the rest are
carries that overlap inside the run (one ComfyUI chain-sampler request)."""
ids = set(scene_ids)
clone = Project.from_dict(p.to_dict())
clone.scenes = [s for s in clone.scenes if s.id in ids]
if not clone.scenes:
raise web.HTTPNotFound(reason="No scenes matched the requested segment")
return clone
def _serve_static(tail: str) -> "web.Response":
root = config.FRONTEND_DIR.resolve()
rel = (tail or "index.html").lstrip("/")
@@ -373,7 +396,11 @@ if web is not None and PromptServer is not None:
async def _generate(req):
p = _project_or_404(req.match_info["pid"])
body = await req.json() if req.can_read_body else {}
target = _solo(p, body.get("only_scene"))
scene_ids = body.get("scene_ids")
if scene_ids:
target = _segment(p, scene_ids)
else:
target = _solo(p, body.get("only_scene"))
prompt = build_combined_prompt(target)
if not prompt.strip():
return web.json_response({"detail": "Nothing to generate — no active scene text."}, status=400)
@@ -405,7 +432,7 @@ if web is not None and PromptServer is not None:
"negative_prompt": target.negative_prompt or None,
"max_scenes": active_scene_count,
"studio_inputs": target.studio_inputs if target.conditioning_slot == "funpack" else {},
"sampler_inputs": target.sampler_inputs if target.sampler_slot == "funpack" else {},
"sampler_inputs": _run_sampler_inputs(target, active_scene_count),
}, media=_prepare_media(target))
if report["blocking"]:
detail = "Generation blocked — " + "; ".join(report["blocking"])
@@ -460,6 +487,68 @@ if web is not None and PromptServer is not None:
raise web.HTTPBadGateway(reason=f"could not fetch result: {e}")
return web.Response(body=data, content_type=ctype.split(";")[0])
# --- API: final render (hard-cut concat of generated runs, video + audio) ---
@routes.post(UI_PREFIX + "/api/projects/{pid}/render")
async def _render_final(req):
_project_or_404(req.match_info["pid"])
body = await req.json() if req.can_read_body else {}
clips = body.get("clips") or [] # ordered [{filename, subfolder, type}]
if not clips:
return web.json_response({"detail": "Nothing to render — no generated runs."}, status=400)
import os
import shutil
import subprocess
import tempfile
import time as _time
ff = shutil.which("ffmpeg")
if not ff:
return web.json_response({"detail": "ffmpeg not found on PATH — install it to render the final video."}, status=503)
try:
import folder_paths
outdir = folder_paths.get_output_directory()
tempdir = folder_paths.get_temp_directory()
except Exception as e: # noqa: BLE001
return web.json_response({"detail": f"Output directory unavailable: {e}"}, status=500)
def _resolve(c):
base = outdir if c.get("type", "output") == "output" else tempdir
return os.path.join(base, c.get("subfolder", ""), c.get("filename", ""))
paths = [_resolve(c) for c in clips]
missing = [p for p in paths if not os.path.isfile(p)]
if missing:
return web.json_response({"detail": f"{len(missing)} clip file(s) not found on disk — regenerate then render."}, status=400)
out_name = f"funpack_final_{int(_time.time())}.mp4"
out_path = os.path.join(outdir, out_name)
# concat demuxer + re-encode → tolerates differing codecs/params between runs
# and keeps BOTH the video and the LTXAV audio stream.
list_fd, list_path = tempfile.mkstemp(suffix=".txt")
try:
with os.fdopen(list_fd, "w") as lf:
for pth in paths:
safe = pth.replace("'", "'\\''")
lf.write(f"file '{safe}'\n")
cmd = [
ff, "-y", "-f", "concat", "-safe", "0", "-i", list_path,
"-c:v", "libx264", "-pix_fmt", "yuv420p",
"-c:a", "aac", "-b:a", "192k",
"-movflags", "+faststart", out_path,
]
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
tail = (proc.stderr or "")[-800:]
return web.json_response({"detail": f"ffmpeg concat failed: {tail}"}, status=500)
finally:
try:
os.unlink(list_path)
except OSError:
pass
return web.json_response({
"media": {"filename": out_name, "subfolder": "", "type": "output", "kind": "videos"},
"clips": len(paths),
})
# --- API: models / pluggable node slots ---
@routes.get(UI_PREFIX + "/api/node-roles")
async def _node_roles(_req):