feat: Continuity — a 'From generated frame' scene starts from the last picture of the clip before it (render module's last-frame route, dark-frame guard), identity picture for every scene, Engine ▸ Continuity settings

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
digital-garbage
2026-10-04 04:03:13 +03:00
co-authored by Claude Sonnet 5.5
parent 5300b19bda
commit 4d76f4c6e0
16 changed files with 280 additions and 5 deletions
+1 -1
View File
@@ -29,7 +29,7 @@ linkPipeline({ project, pipeline, onOpen: (fn) => on((what) => { if (what === "o
const frame = build(document.getElementById("app"));
const selection = createSelection({ project });
selection.on(() => say("select"));
const app = { editorSettings: [], timelineLanes: [], timelineView: {}, mediaPeek: {}, sceneSections: [], clipTags: [], has: new Set(), actions: [], inputHooks: [], lastRun: { typed: undefined, n: 0 }, project, pipeline, generate, api, on, say, title: frame.setZoneTitle, selection, playhead: createPlayhead(), theme: window.ComposerTheme, mode: createMode(), panel: frame.togglePanel, closePanels: frame.closePanels, dock: createDock(), maintenance: createMaintenance({ api, flush: async () => { await project.flush(); if (project.unsaved) throw new Error("Your latest edits could not be saved, so nothing was changed. Try again in a moment."); } }), keys: createKeys(), resetLayout: () => { frame.resetLayout(); app.dock.reset(); } };
const app = { editorSettings: [], timelineLanes: [], timelineView: {}, mediaMenu: [], mediaPeek: {}, sceneSections: [], clipTags: [], has: new Set(), actions: [], inputHooks: [], lastRun: { typed: undefined, n: 0 }, project, pipeline, generate, api, on, say, title: frame.setZoneTitle, selection, playhead: createPlayhead(), theme: window.ComposerTheme, mode: createMode(), panel: frame.togglePanel, closePanels: frame.closePanels, dock: createDock(), maintenance: createMaintenance({ api, flush: async () => { await project.flush(); if (project.unsaved) throw new Error("Your latest edits could not be saved, so nothing was changed. Try again in a moment."); } }), keys: createKeys(), resetLayout: () => { frame.resetLayout(); app.dock.reset(); } };
const settings = createSettings({ menubar: hostFor("menubar.menus") });
app.openSettings = (id) => settings.open(id);
const result = await loadFeatures(paths, { app });
+51
View File
@@ -0,0 +1,51 @@
import test from "node:test";
import assert from "node:assert/strict";
import hub from "./index.js";
const media = { filename: "a.mp4", subfolder: "", type: "output" };
const slots = [{ id: "7", roles: [{ at: "assets.source_image", input: "media_id" }] }];
const project = (extra = {}) => ({ id: "pj", num_frames_per_scene: 97, frame_rate: 25, scenes: [{ id: "a" }, { id: "b", source: { type: "carry" } }], scene_renders: { a: { media, inSec: 0 } }, ...extra });
const make = (values, lastFrame) => { const app = { api: { lastFrame }, pipeline: { currentValues: () => ({ continuity: values }) }, inputHooks: [], mediaMenu: [], project: { project: null, edit: () => true } }; hub.setup({ app }); return app.inputHooks[0]; };
test("a carry shot gets the previous clip's last frame as its start picture", async () => {
const calls = [];
const hook = make({ carry: true, dark_guard: true }, async (pid, body) => { calls.push([pid, body.scene_id, body.dur]); return { media_id: "m9", dark: false }; });
const p = project();
assert.deepEqual((await hook({ project: p, scene: p.scenes[1], slots })).inputs, { 7: { media_id: "m9" } });
assert.deepEqual([calls[0][0], calls[0][1], Math.round(calls[0][2] * 25)], ["pj", "a", 97]);
});
test("nothing happens when it is off, the shot has its own picture, the source is another kind, or the pipeline has no start picture", async () => {
const hook = make({ carry: false }, async () => assert.fail("must not ask"));
const p = project();
assert.deepEqual(await hook({ project: p, scene: p.scenes[1], slots }), { inputs: {}, notes: [] });
const on = make({}, async () => assert.fail("must not ask"));
p.scenes[1].source_image = "mine";
assert.deepEqual(await on({ project: p, scene: p.scenes[1], slots }), { inputs: {}, notes: [] });
p.scenes[1].source_image = "";
p.scenes[1].source = { type: "image" };
assert.deepEqual(await on({ project: p, scene: p.scenes[1], slots }), { inputs: {}, notes: [] });
p.scenes[1].source = { type: "carry" };
assert.deepEqual(await on({ project: p, scene: p.scenes[1], slots: [] }), { inputs: {}, notes: [] });
});
test("a dark last frame, a failed extraction and a missing predecessor each say why the shot starts without a picture", async () => {
const p = project();
const dark = make({ dark_guard: true }, async () => ({ media_id: "m", dark: true }));
assert.match((await dark({ project: p, scene: p.scenes[1], slots })).notes[0], /fade to black/);
const allowed = make({ dark_guard: false }, async () => ({ media_id: "m", dark: true }));
assert.equal((await allowed({ project: p, scene: p.scenes[1], slots })).inputs[7].media_id, "m");
const broken = make({}, async () => { throw new Error("no ffmpeg"); });
assert.match((await broken({ project: p, scene: p.scenes[1], slots })).notes[0], /no ffmpeg/);
assert.match((await broken({ project: p, scene: p.scenes[0], slots })).notes[0], /first clip/);
});
test("an identity picture rides along as the last reference of every scene, and says so when the pipeline has no room for it", async () => {
const refSlots = [...slots, { id: "9", roles: [{ at: "assets.reference_1", input: "media_id" }] }];
const hook = make({ carry: false }, async () => assert.fail("must not ask"));
const p = project({ continuity_settings: { identity_pin_ref: "face" } });
assert.deepEqual((await hook({ project: p, scene: p.scenes[1], slots: refSlots })).inputs, { 9: { media_id: "face" } });
assert.match((await hook({ project: p, scene: p.scenes[1], slots })).notes[0], /no free reference input/);
p.scenes[1].references = ["face"];
assert.deepEqual((await hook({ project: p, scene: p.scenes[1], slots: refSlots })).inputs, {}); // already one of the scene's own
});
+47
View File
@@ -0,0 +1,47 @@
// Continuity: a scene set to "From generated frame" starts from the last picture of the clip before it. This puts that picture into the
// run's start-picture input (a generate input hook); the choices live in Settings ▸ Engine ▸ Continuity.
import { previousClip } from "../../shell/continuity.js";
import { wireReferences } from "../../shell/reference_wiring.js";
const startSlot = (slots) => (slots || []).find((s) => (s.roles || []).some((r) => r.at === "assets.source_image"));
export default {
id: "continuity",
mount: "menubar.menus",
needs: ["project", "api", "pipeline", "inputHooks"],
setup({ app }) {
const pin = (p) => (p.continuity_settings || {}).identity_pin_ref || "";
const setPin = (ref) => app.project.edit((pr) => { pr.continuity_settings = { ...(pr.continuity_settings || {}), identity_pin_ref: ref || null }; return true; });
const menu = (it, item) => (item && item.kind === "image" && app.project.project ? [pin(app.project.project) === it.id
? { id: "unpin", label: "Unpin identity picture", run: () => setPin("") }
: { id: "pin", label: "Use as the identity picture for every scene", run: () => setPin(it.id) }] : []);
app.mediaMenu.push(menu);
// The identity picture goes in after the scene's own references, through the pipeline's own reference inputs.
const withPin = ({ project, scene, slots }) => {
const ref = pin(project);
if (!ref || (scene.references || []).includes(ref)) return {};
const { overrides, unwired } = wireReferences([...(scene.references || []), ref], slots, startSlot(slots) ? [startSlot(slots).id] : []);
return { inputs: overrides, notes: unwired > (scene.references || []).length ? ["The identity picture was not used: this pipeline has no free reference input for it."] : [] };
};
const carryHook = async ({ project, scene, slots }) => {
const mine = ((app.pipeline.currentValues() || {}).continuity) || {};
const carry = mine.carry !== false, guard = mine.dark_guard !== false;
const slot = startSlot(slots);
if (!slot || !carry || project.generation_mode === "t2v" || ((scene.source || {}).type || "carry") !== "carry" || scene.source_image) return {};
const prev = previousClip(project, scene);
if (!prev.sceneId) return { notes: [`Starts without a picture: ${prev.why}.`] };
let got;
try { got = await app.api.lastFrame(project.id, { scene_id: prev.sceneId, render: prev.render, dur: prev.dur, src_in: prev.srcIn }); }
catch (err) { return { notes: [`Starts without a picture: the last frame of the clip before it could not be taken (${err.message}).`] }; }
if (guard && got.dark) return { notes: ["Starts without a picture: the clip before it ends in a fade to black (turn off “Don't continue from a fade to black” in Engine settings to use it anyway)."] };
return { inputs: { [slot.id]: { media_id: got.media_id } } };
};
const hook = async (ctx) => {
const a = withPin(ctx), b = await carryHook(ctx);
return { inputs: { ...a.inputs, ...b.inputs }, notes: [...(a.notes || []), ...(b.notes || [])] };
};
app.inputHooks.push(hook);
return () => { for (const [list, x] of [[app.inputHooks, hook], [app.mediaMenu, menu]]) { const i = list.indexOf(x); if (i >= 0) list.splice(i, 1); } };
},
};
+1 -1
View File
@@ -74,7 +74,7 @@ export default {
const images = g.run.state.images;
if (end !== g.DONE) { tell("Generation failed. The log has ComfyUI's message."); break; }
if (!images.length) { tell("ComfyUI finished without a result. Try again."); break; }
record(pid, unit, images[images.length - 1], frames, g.run.state.promptId);
await record(pid, unit, images[images.length - 1], frames, g.run.state.promptId); // the next shot may continue from this one: it must be on the clip first
}
if (!made && !stopped && !said) tell("Nothing to generate: every scene is left out or is a video clip.");
} finally { busy = false; draw(); }
+1 -1
View File
@@ -31,7 +31,7 @@ export async function buildInputs({ project, scene, slots, expand, frames, hooks
const notes = [];
for (const hook of hooks) {
let out;
try { out = hook({ project, scene, slots }) || {}; } catch (err) { notes.push(`An add-on's input could not be added to this run: ${err.message}`); continue; }
try { out = (await hook({ project, scene, slots })) || {}; } catch (err) { notes.push(`An add-on's input could not be added to this run: ${err.message}`); continue; }
for (const [id, fields] of Object.entries(out.inputs || {})) raw[id] = { ...raw[id], ...fields };
notes.push(...(out.notes || []));
}
+3 -2
View File
@@ -31,9 +31,10 @@ export default {
p.setScene(sc.id, "source_image", item.id);
draw();
};
const context = (it, e) => c.menu.context({ x: e.clientX, y: e.clientY, onPick: (id) => act[id](it),
const extras = (it) => (app.mediaMenu || []).flatMap((f) => { try { return f(it, items.find((m) => m.id === it.id)) || []; } catch { return []; } }); // entries other features add to a tile's menu: { id, label, run }
const context = (it, e) => c.menu.context({ x: e.clientX, y: e.clientY, onPick: (id) => (act[id] ? act[id](it) : extras(it).find((x) => x.id === id)?.run()),
items: [{ id: "look", label: "Look at it on the monitor" }, { id: "res", label: "Use as the selected scene's resolution source", disabled: !p.selected || it.kind === "audio" || !items.some((m) => m.id === it.id && m.kind === "image") }, { id: "ref", label: "Use as a reference for the selected scene", disabled: !p.selected || (p.selected.references || []).includes(it.id) }, { id: "rename", label: "Rename…" },
{ id: "export", label: "Save to your computer", disabled: !items.some((m) => m.id === it.id && m.kind !== "audio") }, { separator: true }, { id: "delete", label: "Delete from the bin", danger: true }] });
{ id: "export", label: "Save to your computer", disabled: !items.some((m) => m.id === it.id && m.kind !== "audio") }, ...extras(it).map(({ id, label, disabled }) => ({ id, label, disabled })), { separator: true }, { id: "delete", label: "Delete from the bin", danger: true }] });
/** Delete these from the bin, and nothing in this project may keep pointing at them. */
async function gone(ids) {
const dead = new Set();
+1
View File
@@ -35,6 +35,7 @@ export default [
"./features/montage/index.js",
"./features/generate/warn.js",
"./features/generate/progress.js",
"./features/continuity/index.js",
"./features/composer/index.js",
"./features/settings/index.js",
"./features/welcome/index.js",
+1
View File
@@ -88,6 +88,7 @@ export const api = {
return videos.length ? { videos } : { error: "no video came out, check the ComfyUI terminal" };
},
renderLibrary: () => call("GET", "/api/m/render/library"),
lastFrame: (pid, body) => call("POST", `/api/m/render/projects/${encodeURIComponent(pid)}/last-frame`, body),
suggestionStats: () => call("GET", "/api/shortcuts/suggestion_stats"),
expandPrompt: (body) => call("POST", "/api/prompt/expand", { ...body, seed: 1 }),
packs: () => call("GET", "/api/packs"),
+15
View File
@@ -0,0 +1,15 @@
// Which clip a shot continues from: the one right before its generation unit on the timeline.
import { segments, genUnitId, isVideoClip } from "./scenes.js";
/** -> { sceneId, render, dur, srcIn } for the clip a shot should continue from, or { why } when there is nothing to continue from. */
export function previousClip(p, scene) {
const unit = genUnitId(scene), segs = segments(p).filter((s) => s.kind === "scene");
const first = segs.findIndex((s) => genUnitId(s.scene) === unit);
if (first <= 0) return { why: "it is the first clip" };
const prev = segs[first - 1];
if (prev.scene.excluded) return { why: "the clip before it is left out" };
if (isVideoClip(prev.scene)) return { why: "the clip before it is an imported video" };
const render = (p.scene_renders || {})[prev.scene.id];
if (!(render && render.media)) return { why: "the clip before it has no render yet" };
return { sceneId: prev.scene.id, render: { media: render.media, inSec: render.inSec || 0 }, dur: prev.dur, srcIn: prev.scene.source_in || 0 };
}
+23
View File
@@ -0,0 +1,23 @@
import test from "node:test";
import assert from "node:assert/strict";
import { previousClip } from "../continuity.js";
const media = { filename: "a.mp4", subfolder: "", type: "output" };
const p = () => ({ num_frames_per_scene: 97, frame_rate: 25, scenes: [{ id: "a" }, { id: "b" }, { id: "c", excluded: true }, { id: "d" }, { id: "v", source: { type: "video", media_ref: "m" } }, { id: "e" }],
scene_renders: { a: { media, inSec: 0.5 } } });
test("a shot continues from the clip right before its unit, if that one has a render", () => {
const q = p();
assert.equal(previousClip(q, q.scenes[0]).why, "it is the first clip");
const got = previousClip(q, q.scenes[1]);
assert.deepEqual([got.sceneId, got.render.inSec, Math.round(got.dur * 25)], ["a", 0.5, 97]);
assert.match(previousClip(q, q.scenes[3]).why, /left out/);
assert.match(previousClip(q, q.scenes[5]).why, /imported video/);
assert.match(previousClip(q, q.scenes[4]).why, /no render/);
});
test("the second half of a cut unit looks past its own unit", () => {
const q = p();
q.scenes = [{ id: "a" }, { id: "b" }, { id: "b2", gen_unit_id: "b", cut_offset_frames: 40 }];
assert.equal(previousClip(q, q.scenes[2]).sceneId, "a");
});
@@ -0,0 +1,31 @@
"""Continuity: a shot that follows another starts from where the other ended.
A scene whose Source is "From generated frame" starts from the last picture of the clip before it, so
the shot carries on instead of starting from nothing. The picture is taken from the finished render
(never invented), saved in the media bin, and sent through the pipeline's own start-picture input; this
module only holds the two choices. The work is the editor's: it asks the render module for the frame.
A clip that ends in a fade to black has no picture to continue from (v4's i2i lesson: 3 of 4 anchors were
solid black), so by default such a shot starts without one and says so, rather than starting black.
"""
ID = "continuity"
TITLE = "Continuity"
MOUNT = "settings.general"
STAGE = "conditioning"
CATEGORY = "continuity"
STATUS = "experimental"
SETTINGS = {
"carry": {
"type": "bool", "default": True,
"label": "Continue from the previous clip",
"hint": "A scene set to 'From generated frame' starts from the last picture of the clip before it. Off: such a scene starts without a picture.",
},
"dark_guard": {
"type": "bool", "default": True,
"label": "Don't continue from a fade to black",
"hint": "A clip that ends dark gives the next one a black start. With this on, that shot starts without a picture and the run says so.",
"when": {"carry": True},
},
}
@@ -0,0 +1,8 @@
from core import registry
def test_continuity_is_announced_in_its_own_category_with_the_two_choices():
spec = registry.scan().specs["continuity"]
assert spec.category == "continuity"
assert set(spec.settings) == {"carry", "dark_guard"}
assert spec.settings["carry"]["default"] is True and spec.settings["dark_guard"]["default"] is True
+16
View File
@@ -55,6 +55,22 @@ def routes(table, base, web):
async def _segment(req):
return await serving.segment(req, project_of(req))
@table.post(base + "/projects/{pid}/last-frame")
async def _last_frame(req):
proj = project_of(req)
body = await body_of(req)
scene_id = body.get("scene_id")
if not isinstance(scene_id, str) or not scene_id:
return bad("Send {scene_id}.")
render = body.get("render") if isinstance(body.get("render"), dict) and isinstance((body["render"].get("media") or None), dict) else None
window = serving.query_window({"dur": body.get("dur"), "src_in": body.get("src_in")})
try:
return web.json_response(await asyncio.to_thread(serving.last_frame, proj, scene_id, render, window))
except KeyError as exc:
return bad(f"No render for this scene ({exc}).", 404)
except (files.ClipError, ValueError, OSError) as exc:
return bad(str(exc), 502)
@table.get(base + "/upscale_models")
async def _upscale_models(_req):
return web.json_response({"models": upscale.models()})
+14
View File
@@ -154,6 +154,20 @@ def trim(src: str, out: str, start=None, dur=None, *, fast=False, reverse=False,
run(cmd)
def last_frame(src: str, out_png: str, start=0.0, dur=None, fps=25.0) -> float:
"""Write the last picture of [start, start+dur) of `src` as a PNG; -> its average brightness, 0 (black) .. 255.
One frame before the window's end, not at it: a seek to the very end of a file finds nothing."""
end = float(start) + float(dur) if dur else None
at = max(float(start), end - 1.5 / (fps or 25.0)) if end is not None else float(start)
run([ffmpeg(), "-y", "-ss", f"{at:.3f}", "-i", src, "-frames:v", "1", out_png])
if not os.path.isfile(out_png) or os.path.getsize(out_png) == 0:
raise ClipError("the clip has no picture at its end (is the window past the end of the render?)")
from PIL import Image, ImageStat
with Image.open(out_png) as im:
return float(ImageStat.Stat(im.convert("L")).mean[0])
def has_audio(path: str) -> bool:
"""True when the file has a sound stream. ffprobe if there is one, else a probe run of ffmpeg."""
probe = shutil.which("ffprobe")
+31
View File
@@ -209,6 +209,37 @@ def ghost_clip(q) -> dict | None:
"type": q.get("type") or "output", "in": start, "dur": dur}
DARK_BELOW = 20.0 # average brightness (0..255) under which a last frame is a fade-out, not a picture to continue from
_frames: dict = {} # what a window's last frame was saved as: key -> (media id, brightness)
def last_frame(project, scene_id: str, render: dict | None = None, window: dict | None = None) -> dict:
"""The last picture of a scene's render, saved in the media bin: {media_id, brightness, dark}.
Dark means the clip ends in a fade to black -- starting the next shot from it gives a black start
(v4's i2i lesson), so the caller may refuse it. The same window answers from the bin it saved to."""
from ..._core import media
clip = scene_clip(project, scene_id, render, window) # a render just made, and the trim the page shows, win over the saved project
src = files.clip_path(clip)
sig = files.signature(src)
key = (clip["filename"], clip["subfolder"], clip["type"], clip["in"], clip["dur"], sig)
hit = _frames.get(key)
if hit and media.path_for(hit[0]):
return {"media_id": hit[0], "brightness": hit[1], "dark": hit[1] < DARK_BELOW}
out = files.temp_file(f"funpack_last_{projects.safe_part(project.id)[:8]}_{projects.safe_part(scene_id)}_{int(time.time() * 1000)}.png")
try:
brightness = files.last_frame(src, out, clip["in"], clip["dur"], clip["fps"])
with open(out, "rb") as fh:
entry = media.save_upload(f"last_frame_{projects.safe_part(scene_id)[:12]}.png", fh.read())
finally:
try:
os.remove(out)
except OSError:
pass
_frames[key] = (entry["id"], brightness)
return {"media_id": entry["id"], "brightness": brightness, "dark": brightness < DARK_BELOW}
async def segment(request, project) -> "web.StreamResponse":
q = request.query
scene_id = request.match_info["scene_id"]
@@ -203,3 +203,39 @@ def test_a_window_past_the_end_of_the_render_is_refused_not_served_empty(comfy,
"&render_in=5&dur=1")
status, _h, body = get(server, url)
assert status == 400 and b"past the end" in body
@needs_ffmpeg
def test_last_frame_is_a_picture_with_its_brightness(comfy):
src = make_clip(comfy.output / "lf.mp4", 2.0, audio=False)
out = str(comfy.temp / "lf.png")
bright = files.last_frame(str(src), out, 0.0, 2.0, 25.0)
assert (comfy.temp / "lf.png").stat().st_size > 0 and bright > 20
def _post(port, path, body):
import json
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=30)
conn.request("POST", path, body=json.dumps(body), headers={"Content-Type": "application/json"})
resp = conn.getresponse()
out = (resp.status, json.loads(resp.read() or b"{}"))
conn.close()
return out
@needs_ffmpeg
def test_last_frame_route_saves_a_picture_to_the_bin_once_and_flags_a_fade_to_black(comfy, server):
from core import media
moov_at_end(comfy.output / "a.mp4", 4.0)
proj = _project(comfy, frames=50, frames_mode="timeline")
serving._frames.clear()
status, a = _post(server, f"{BASE}/projects/{proj.id}/last-frame", {"scene_id": "s1"})
assert status == 200 and a["dark"] is False and media.path_for(a["media_id"])
assert _post(server, f"{BASE}/projects/{proj.id}/last-frame", {"scene_id": "s1"})[1]["media_id"] == a["media_id"] # same window: same file
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "color=c=black:s=160x120:d=2:r=25", "-c:v", "libx264", "-pix_fmt", "yuv420p", str(comfy.output / "black.mp4")], check=True, capture_output=True)
dark = _project(comfy, frames=50, frames_mode="timeline")
dark.scene_renders["s1"] = {"media": {"filename": "black.mp4", "subfolder": "", "type": "output"}, "inSec": 0}
projects.save(dark)
assert _post(server, f"{BASE}/projects/{dark.id}/last-frame", {"scene_id": "s1"})[1]["dark"] is True
assert _post(server, f"{BASE}/projects/{dark.id}/last-frame", {"scene_id": "nope"})[0] == 404
assert _post(server, f"{BASE}/projects/{dark.id}/last-frame", {})[0] == 400