Shot Planner: continuity 'history' - the last 17/34/51 frames of the previous result placed just before frame 0 on H3's timeline (sliding history, after Akatz's H3 Relay / Ethanfel's Context Loop), with every mask mode, crop and the duet canvas; cuts reset it

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
alisson-anjos
2026-10-09 01:37:36 +00:00
co-authored by Claude Opus 5.5
parent 78aa214584
commit a03e0be342
7 changed files with 274 additions and 12 deletions
+104
View File
@@ -0,0 +1,104 @@
"""Sliding history for MiniMax H3: the end of the previous shot's result as clean frames placed just BEFORE the video
on H3's timeline, so a long continuous take is generated window by window with motion (and look) carried across.
Idea from Akatz's H3 Relay windowed edit and Ethanfel's ComfyUI-MiniMaxH3-Context-Loop (both GPL-3.0); this is an
independent implementation. A guide keyframe marked ``anchor = "history"`` is packed like any H3 guide, then its rows
are kept at the target's origin and everything else on the target timeline (the video, its audio, the other guides)
moves forward by the history's span. Text and reference rows are untouched.
The layout is patched in memory only (ComfyUI files are never edited), once, and only changes layouts that carry a
history keyframe; when ComfyUI's own H3 layout already understands history anchors the patch stands down.
"""
from __future__ import annotations
import logging
import torch
LOG = logging.getLogger("bfs.h3_history")
HISTORY_FRAMES = (17, 34, 51) # whole H3 VAE groups: 1+4+4+4+4 frames per 5 latent steps
_PATCH = "_bfs_history_patch"
def history_span(latent_t: int) -> float:
"""Timeline span of `latent_t` video latent steps (FRAME_RESCALE per pixel frame)."""
import comfy.ldm.minimax.model as mm
return float(sum(mm._video_t_spans(int(latent_t))))
def relocate(layout, keyframes) -> bool:
"""Moves the target timeline past the history keyframes. Returns whether anything changed."""
hist = [i for i, kf in enumerate(keyframes or ()) if kf.get("anchor") == "history"]
if not hist:
return False
# keyframe -> its packed segments: every guide with a video latent owns the next "cond" segment, every guide
# with audio the next "cond_audio" one, in keyframe order (the order PackedLayout packs them)
cond = [s for s in layout.segments if s[2] == "cond"]
cond_a = [s for s in layout.segments if s[2] == "cond_audio"]
own, ci, ai = {}, 0, 0
for i, kf in enumerate(keyframes):
segs = []
if kf.get("latent") is not None and ci < len(cond):
segs.append(cond[ci]); ci += 1
if kf.get("audio_latent") is not None and ai < len(cond_a):
segs.append(cond_a[ai]); ai += 1
own[i] = segs
span = sum(history_span(keyframes[i]["latent"].shape[2]) for i in hist if keyframes[i].get("latent") is not None)
if span <= 0:
return False
keep = torch.zeros(layout.position_ids.shape[0], dtype=torch.bool)
for i in hist:
for a, b, _ in own[i]:
keep[a:b] = True
vid = next(((a, b) for a, b, k in layout.segments if k == "video"), None)
if vid is not None and keep.any():
# another history implementation (e.g. H3 Relay's, or a future ComfyUI) already moved the target: skip
if float(layout.position_ids[vid[0]:vid[1], 0].min()) - float(layout.position_ids[keep, 0].min()) >= span - 1e-6:
return False
move = torch.zeros_like(keep)
for a, b, kind in layout.segments:
if kind in ("video", "audio", "cond", "cond_audio"):
move[a:b] = True
move &= ~keep
pos = layout.position_ids.clone()
pos[move, 0] += span
layout.position_ids = pos
LOG.info("H3 history: %d rows before the video, target timeline moved by %.2f", int(keep.sum()), span)
return True
def install() -> bool:
"""Wraps PackedLayout.__init__ once (process-local). False when ComfyUI is missing or already supports it."""
try:
import comfy.ldm.minimax.model as mm
except ImportError:
return False
cls = mm.PackedLayout
if getattr(cls.__init__, _PATCH, False):
return True
import inspect
if "history" in inspect.getsource(cls): # native history anchors: nothing to do
return False
orig = cls.__init__
def __init__(self, text_len, latent_t, latent_h, latent_w, audio_t, keyframes=None, refs=None):
orig(self, text_len, latent_t, latent_h, latent_w, audio_t, keyframes=keyframes, refs=refs)
relocate(self, keyframes)
setattr(__init__, _PATCH, True)
cls.__init__ = __init__
return True
def add_history(positive, vae, frames: torch.Tensor):
"""Appends `frames` ([N,H,W,3] at the video's size, N = 17k: the end of the previous result) as a history keyframe.
Returns (positive, frames used)."""
import node_helpers
n = (int(frames.shape[0]) // 17) * 17
if n < 17:
return positive, 0
install()
lat = vae.encode(frames[-n:, ..., :3].float())
kfs = list(positive[0][1].get("minimax_keyframes", []))
kfs.append({"resolved_frame_index": 0, "latent": lat, "anchor": "history"})
return node_helpers.conditioning_set_values(positive, {"minimax_keyframes": kfs}), n
+82 -1
View File
@@ -474,6 +474,7 @@ def resolve_plan(plan: dict, analysis: dict, path: str | None = None) -> list[di
s["prompt"] = m.get("prompt") or ""
s["chain"] = m.get("chain") or "off"
s["chain_frame"] = m.get("chain_frame") or "first"
s["chain_history"] = int(m.get("chain_history") or 17)
s["crop"] = bool(m.get("crop"))
s["inpaint"] = bool(m.get("inpaint"))
s["paste"] = bool(m.get("paste")) and not s["crop"] and not s["inpaint"]
@@ -1955,7 +1956,7 @@ def duet_prompt(vlm, prompt: str, frames: torch.Tensor | None, refs: list, task:
# ---------------------------------------------------------------------------- continuity between shots
CHAIN_MODES = ("off", "reference", "first frame")
CHAIN_MODES = ("off", "reference", "first frame", "history")
_LAST_RESULT: dict[str, Any] = {} # auto loop: the last shot a render node produced (index, count, frames)
@@ -1983,6 +1984,44 @@ def chain_image(shot: dict) -> torch.Tensor | None:
return None
def history_for(path, analysis, src, seg, i, prev_len, queue, rid, W, H, crop, ft):
"""Continuity 'history': the end of the previous shot's result (17/34/51 frames) at this shot's size, plus the
source frames under it (the duet panel shows them). Same take only: a camera cut has nothing to carry over."""
tag = f"[BFS Shot Planner] shot {i + 1}: history"
if seg["cut_before"]:
print(f"{tag} skipped: a camera cut starts this shot", flush=True)
return None, None
want = int(seg.get("chain_history") or 17)
prev = None
if queue:
fp = os.path.join(run_dir(rid), f"shot_{i - 1:04d}.pt")
if os.path.exists(fp):
prev = torch.load(fp)["frames"].float() / 255.0
if prev is None:
print(f"{tag}: the previous shot's result is not there yet (queue loop: shots run in order; auto loop: "
"it is taken from the previous render if it matches)", flush=True)
return None, None
avail = min(int(prev_len), prev.shape[0])
n = min(want, (avail // 17) * 17)
if n < 17:
print(f"{tag} skipped: the previous shot has only {avail} frames (needs 17)", flush=True)
return None, None
hf = prev[avail - n:avail, ..., :3]
lo = max(0, int(seg["start"]) - n)
sf = _read_frames(path, src[np.arange(lo, int(seg["start"]))], (W, H))
hs = torch.from_numpy(np.stack(sf).astype(np.float32) / 255.0)
if hs.shape[0] < n:
hs = torch.cat([hs[:1].expand(n - hs.shape[0], -1, -1, -1), hs], 0)
if hf.shape[1:3] != (H, W):
hf = _fit_to(hf, W, H)
if crop is not None: # the same box as this shot's crop, at the crop's generation size
x0, y0, x1, y1 = box_px(crop["box"], W, H)
hf = _fit_to(hf[:, y0:y1, x0:x1], ft.shape[2], ft.shape[1])
hs = _fit_to(hs[:, y0:y1, x0:x1], ft.shape[2], ft.shape[1])
print(f"{tag}: the last {n} frames of shot {i} go before this one on H3's timeline", flush=True)
return hf, hs
def _fit_to(img: torch.Tensor, w: int, h: int) -> torch.Tensor:
x = img[..., :3].movedim(-1, 1).float()
return torch.nn.functional.interpolate(x, size=(h, w), mode="bilinear", align_corners=False).movedim(1, -1)
@@ -2264,6 +2303,8 @@ class BFSShotPlanner:
bx0, by0, bx1, by1 = box_px(r["box"], r["size"][0], r["size"][1])
crop["crop_mask"] = torch.nn.functional.interpolate(
r["masks"][:, None, by0:by1, bx0:bx1].float(), size=(chh, cw), mode="nearest")[:, 0]
hist, hist_src = (history_for(path, a, src, s, i, prev_len, queue, rid, W, H, crop, ft)
if s["chain"] == "history" and i > 0 else (None, None))
shots.append({
"index": i, "count": len(segs), "start": s["start"], "end": s["end"],
"length": s["end"] - s["start"], "gen_length": s["gen_len"], "fps": fps,
@@ -2279,6 +2320,7 @@ class BFSShotPlanner:
"audio": _slice_audio(audio, s["start"] / fps, s["gen_len"] / fps),
"run_id": rid, "queue": queue,
"chain": s["chain"], "chain_frame": s["chain_frame"], "chain_image": chain_img,
"history": hist, "history_src": hist_src,
"prev_length": prev_len,
"source": {"path": path, "frames": [int(x) for x in src[idx]]},
})
@@ -2895,8 +2937,47 @@ class BFSShotH3Conditioning:
if model is None:
raise ValueError("duet 'shifted RoPE' needs the model input (and its model output in the sampler)")
model = patch_model_rope(model, info, rope_gap)
positive = self._history(shot, positive, vae, gtag, on_canvas, duet, panel_position, panel_size)
return (positive, latent, model, text)
@staticmethod
def _history(shot, positive, vae, tag, on_canvas, duet, panel_position, panel_size):
"""Continuity 'history': the end of the previous result as clean frames just before this shot on H3's
timeline (added last, so the duet panel and the other guides leave it alone)."""
if shot.get("chain") != "history" or int(shot.get("index", 0)) == 0:
return positive
hist, hsrc = shot.get("history"), shot.get("history_src")
if hist is None and not shot.get("crop") and _LAST_RESULT.get("index") == int(shot["index"]) - 1:
prev_len = int(shot.get("prev_length") or _LAST_RESULT["frames"].shape[0])
f = _LAST_RESULT["frames"][:prev_len].float()
f = f / 255.0 if f.max() > 1.5 else f
n = (min(int(shot.get("chain_history") or 17), f.shape[0]) // 17) * 17
hist = f[f.shape[0] - n:] if n >= 17 else None
if hist is None:
return positive
if duet == "shifted RoPE":
print(f"{tag}: history is not used with duet 'shifted RoPE' (use canvas)", flush=True)
return positive
W, H = int(shot["width"]), int(shot["height"])
hist = _fit_to(hist, W, H) if tuple(hist.shape[1:3]) != (H, W) else hist
if on_canvas: # the canvas holds the panel too: the source frames under the history go in its strip
try:
from .bfs_h3_side_panel import compose, strip_frames
except ImportError:
from bfs_h3_side_panel import compose, strip_frames
panel = hsrc if hsrc is not None and hsrc.shape[0] >= hist.shape[0] else hist
strip = strip_frames(panel[-hist.shape[0]:], list(range(hist.shape[0])), W, H, panel_position, panel_size,
0, "contain")
hist = compose(hist.cpu(), strip, panel_position)
try:
from .bfs_h3_history import add_history
except ImportError:
from bfs_h3_history import add_history
positive, n = add_history(positive, vae, hist)
if n:
print(f"{tag}: history = the last {n} frames of the previous shot, before frame 0 on H3's timeline", flush=True)
return positive
class BFSShotJoin:
"""Put the decoded shots back together in order, trimmed to their true lengths."""
+1 -1
View File
@@ -1,7 +1,7 @@
[project]
name = "bfsnodes"
description = "Custom ComfyUI nodes for reference-guided video: BFS Sampler (guide-driven IC-LoRAs, auto crop, mask inpainting, temporal chunking), LTX Multiple Controls / Identity Transfer / Edit Anything, Head-Swap and Anime2Real conditioning, MiniMax-H3 latent guides, RoPE layouts and loaders, LoRA Surgery, and Shot Planner / Shot Loop (tabbed panel): split long videos into shots at camera cuts, face-tracked cast, SAM 3 mask per shot with four generation modes (full frame, mask only, crop, crop + mask), optional Qwen3-VL to suggest settings, describe references ({details}) and the person to replace ({target}), continuity between shots, H3 conditioning with an optional duet and prompt writer, join with the original timing, audio and a side-by-side comparison. H3 Duet: pin a source clip beside the video, training-free."
version = "1.86.2"
version = "1.87.0"
license = {file = "LICENSE"}
dependencies = [
"torch",
+48
View File
@@ -0,0 +1,48 @@
import importlib.util
import types
import unittest
from pathlib import Path
import torch
ROOT = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location("bfs_h3_history", ROOT / "bfs_h3_history.py")
HI = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(HI)
class Layout:
"""text 4 rows | guide cond 6 rows | history cond 4 rows | audio 3 | video 6, all at the target origin 4.0"""
def __init__(self):
self.segments = [(0, 4, "text"), (4, 10, "cond"), (10, 14, "cond"), (14, 17, "audio"), (17, 23, "video")]
p = torch.zeros(23, 3)
p[:4, 0] = torch.arange(4.)
p[4:, 0] = 4.0
self.position_ids = p
class HistoryTest(unittest.TestCase):
def setUp(self):
HI.history_span = lambda t: 10.0 * t
def test_target_moves_past_the_history_only(self):
lay = Layout()
kfs = [{"latent": torch.zeros(1, 24, 3, 2, 2)}, {"latent": torch.zeros(1, 24, 2, 2, 2), "anchor": "history"}]
self.assertTrue(HI.relocate(lay, kfs))
t = lay.position_ids[:, 0]
self.assertTrue(torch.equal(t[:4], torch.arange(4.))) # text untouched
self.assertTrue(bool((t[10:14] == 4.0).all())) # history stays at the origin
self.assertTrue(bool((t[4:10] == 24.0).all())) # the other guide moves with the video
self.assertTrue(bool((t[14:] == 24.0).all())) # audio and video move by the span (2 steps)
self.assertFalse(HI.relocate(lay, kfs)) # already moved: no double shift
def test_no_history_no_change(self):
lay = Layout()
before = lay.position_ids.clone()
self.assertFalse(HI.relocate(lay, [{"latent": torch.zeros(1, 24, 3, 2, 2)}]))
self.assertTrue(torch.equal(before, lay.position_ids))
if __name__ == "__main__":
unittest.main()
+3 -3
View File
@@ -108,7 +108,7 @@ function Panel(io) {
const m = plan.segs[out.length] || {};
out.push({ start: b[i], end: b[i + 1], len: b[i + 1] - b[i], gen: snapUp(b[i + 1] - b[i], plan.grid),
cut: cs.has(b[i]), enabled: m.enabled !== false, ref: m.ref || "", ref2: m.ref2 || "", prompt: m.prompt || "",
force: m.force || "auto", chain: m.chain || "off", chainFrame: m.chain_frame || "first",
force: m.force || "auto", chain: m.chain || "off", chainFrame: m.chain_frame || "first", chainHistory: m.chain_history || 17,
crop: !!m.crop, inpaint: !!m.inpaint, paste: !!m.paste, pasteText: m.paste_text || "", strength: m.strength ?? 1, mask: m.mask || {}, target: m.target || "",
extMask: !!plan.mask_video || !!(plan.track?.points?.length), trackMask: !!(plan.track?.points?.length) });
}
@@ -212,7 +212,7 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
if (apply) {
plan.bounds = j.segs.slice(1).map(s => s.start);
const keep = plan.segs; plan.segs = j.segs.map((_, i) => ({ enabled: true, ref: keep[i]?.ref || "", ref2: keep[i]?.ref2 || "", prompt: keep[i]?.prompt || "",
chain: keep[i]?.chain || "off", chain_frame: keep[i]?.chain_frame || "first",
chain: keep[i]?.chain || "off", chain_frame: keep[i]?.chain_frame || "first", chain_history: keep[i]?.chain_history || 17,
crop: !!keep[i]?.crop, inpaint: !!keep[i]?.inpaint, paste: !!keep[i]?.paste, paste_text: keep[i]?.paste_text || "", strength: keep[i]?.strength ?? 1, mask: keep[i]?.mask || {}, target: keep[i]?.target || "" }));
sel.value = 0; save();
}
@@ -334,7 +334,7 @@ const meta = i => { while (plan.segs.length <= i) plan.segs.push({}); return pla
if (fields.mask_video) m.mask = { ...(m.mask || {}), video: src.mask.video || "" };
if (fields.crop && (hasMask(x) || src.mask.text || fields.mask_text)) { m.crop = src.crop; m.inpaint = src.inpaint; m.paste = src.paste; m.paste_text = src.pasteText; m.strength = src.strength; }
if (fields.target) m.target = src.target;
if (fields.chain && i > 0) { m.chain = src.chain; m.chain_frame = src.chainFrame; }
if (fields.chain && i > 0) { m.chain = src.chain; m.chain_frame = src.chainFrame; m.chain_history = src.chainHistory; }
});
if (fields.points && (src.mask.points || []).length) pointsTo(src.mask.points, src.mask.key ?? Math.floor(src.len / 2), undefined, from, scope);
if (fields.mask_text || fields.points || fields.mask_video) maskPrev.value = {};
+20
View File
@@ -296,6 +296,26 @@ the **auto loop with BFS Shot H3 Duet** (it renders shot by shot and keeps the l
outputs it as `previous_result`. This is different from H3 Conditioning's *first_frame*, which anchors the shot's
own frame from the source video.
### History: a long take, window by window (1.87.0)
*Continuity* = **previous shot's end as history** carries the **last 17, 34 or 51 frames of the previous shot's
result** into the next shot as clean frames placed **just before frame 0 on H3's timeline**: the model sees them as
the immediate past of the video it is generating, so motion, camera, light and the new look continue across the join
(instead of one still frame). Idea from Akatz's [H3 Relay](https://github.com/akatz-ai/h3-relay) windowed edit and
Ethanfel's MiniMax H3 Context Loop; this is BFS's own implementation (a process-local patch of H3's packed layout
that moves the video, its audio and the other guides past the history; ComfyUI files are not edited).
- **Same take only.** A shot that starts at a camera cut skips it (nothing to continue). For one long take, split it
with *Mode = fixed length* (or let *Camera cuts* split long shots) and set history on every shot after the first
with **Copy to other shots → continuity**. Shots still exist: they are the windows, and cuts reset the history.
- **Every mode:** full frame, *Frame + paste*, *Mask only* (the mask regenerates the target; the history is a
condition outside it), *Crop* (the history is cut with this shot's box) and the **duet canvas** (the source frames
under the history go into the panel strip). Not with duet *shifted RoPE*. The aligned guide still drives this shot's
own frames, so sync stays frame-exact.
- **Queue loop:** shots run in order and each reads the stored result of the one before. Auto loop: taken from the
previous render when it matches.
- Video only for now (the previous audio is not carried). Costs one more 17/34/51-frame condition per shot.
## MiniMax H3 with an aligned guide (example)
```
+16 -7
View File
@@ -21,7 +21,7 @@ function shotCard(c, s, i) {
sam: bdg("🎯 " + (s.mask.text ? s.mask.text.slice(0, 14) : `${(s.mask.points || []).length} pts`), "", s.mask.text || `${(s.mask.points || []).length} points`) }[maskSource(s)] || null,
s.crop || s.inpaint || s.paste ? bdg({ paste: "🧩 frame+paste", mask: "🎭 mask only", crop: "✂ crop", cropmask: "✂🎭 crop+mask" }[c.modeOf(s)] + (s.inpaint && s.strength < 1 ? ` ${Number(s.strength).toFixed(2)}` : ""), "on", c.modeName(s)) : null,
s.target ? bdg("🧑 target", "on", s.target) : null,
i > 0 && s.chain !== "off" ? bdg("⛓", "on", `continues from the previous shot's ${s.chainFrame} frame as ${s.chain}`) : null,
i > 0 && s.chain !== "off" ? bdg("⛓", "on", s.chain === "history" ? `continues the previous shot's last ${s.chainHistory} frames (history)` : `continues from the previous shot's ${s.chainFrame} frame as ${s.chain}`) : null,
warnN ? bdg(`⚠ ${warnN}`, "warn", issues.map(x => x.text).join("\n")) : null,
]),
who ? h("div", { class: "who" }, who.people.length ? who.people.map(id => h("img", {
@@ -203,14 +203,23 @@ function editor(c, cur) {
const cont = section("Continuity", h("div", { class: "row" }, [
h("select", { value: cur.chain, style: "width:auto", disabled: i === 0,
title: "reference: the previous result's frame becomes one more <Picture n> after this shot's own references. first frame: it is anchored at frame 0 of this shot.",
title: "reference: the previous result's frame becomes one more <Picture n> after this shot's own references. first frame: it is anchored at frame 0 of this shot. history: the END of the previous result (17/34/51 frames) goes just before this shot on H3's timeline, so a long take continues its motion and look (same take only, not across a cut).",
onChange: e => c.setMeta(i, "chain", e.target.value) },
[h("option", { value: "off" }, "off"), h("option", { value: "reference" }, "previous shot as reference"),
h("option", { value: "first frame" }, "previous shot as first frame")]),
h("select", { value: cur.chainFrame, style: "width:auto", disabled: i === 0 || cur.chain === "off",
title: "Which frame of the previous shot's result", onChange: e => c.setMeta(i, "chain_frame", e.target.value) },
[h("option", { value: "first" }, "its first frame"), h("option", { value: "middle" }, "its middle frame"), h("option", { value: "last" }, "its last frame")]),
hint(i === 0 ? "the first shot uses only its references" : "uses a frame of the PREVIOUS shot's result (queue loop, or auto loop with BFS Shot H3 Duet)"),
h("option", { value: "first frame" }, "previous shot as first frame"),
h("option", { value: "history" }, "previous shot's end as history (long take)")]),
cur.chain === "history"
? h("select", { value: String(cur.chainHistory), style: "width:auto", disabled: i === 0,
title: "How much of the previous result's end goes before this shot (whole H3 VAE groups). More carries more motion, costs more memory.",
onChange: e => c.setMeta(i, "chain_history", Number(e.target.value)) },
[17, 34, 51].map(n => h("option", { value: String(n) }, `last ${n} frames (${(n / 24).toFixed(1)} s)`)))
: h("select", { value: cur.chainFrame, style: "width:auto", disabled: i === 0 || cur.chain === "off",
title: "Which frame of the previous shot's result", onChange: e => c.setMeta(i, "chain_frame", e.target.value) },
[h("option", { value: "first" }, "its first frame"), h("option", { value: "middle" }, "its middle frame"), h("option", { value: "last" }, "its last frame")]),
hint(i === 0 ? "the first shot uses only its references"
: cur.chain === "history" ? (cur.cut ? "⚠ a camera cut starts this shot: history is skipped (it only carries a continuing take)"
: "the end of the PREVIOUS shot's result continues into this one (queue loop: shots run in order). Works with every mask mode, crop and the duet canvas.")
: "uses a frame of the PREVIOUS shot's result (queue loop, or auto loop with BFS Shot H3 Duet)"),
]), { open: cur.chain !== "off", sub: cur.chain !== "off" ? cur.chain : "off" });
const copy = section("Copy to other shots", [