BFS Shot H3 Duet: continuity 'history' too (the previous end before frame 0, panel strip filled with the source under it); shared shot_history() pops it from the shot
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
b1e0fcbbb6
commit
2e64d989f6
+29
-4
@@ -543,7 +543,7 @@ def build_prompt(task: str, instruction: str, n_pictures: int, n_videos: int, ro
|
||||
def h3_render(model, clip, vae, audio_vae, prompt, refs, width, height, length, steps, sampler_name, scheduler,
|
||||
seed, panel=None, guide=None, guide_frame_idx=0, position="left", size=1.0, fit="contain", gap=0,
|
||||
panel_noise=0.0, hold="all frames", ref_image_size="match", decode_canvas=False,
|
||||
rope_mode="canvas", rope_gap=0.0, first_frame=None):
|
||||
rope_mode="canvas", rope_gap=0.0, first_frame=None, history=None, history_src=None):
|
||||
"""Reference to Video -> optional pinned panel -> optional aligned guide -> sample -> crop -> decode.
|
||||
|
||||
`refs` holds the Reference to Video inputs ({"ref_images": {...}, "ref_videos": {...}, ...}). Without a
|
||||
@@ -580,6 +580,28 @@ def h3_render(model, clip, vae, audio_vae, prompt, refs, width, height, length,
|
||||
positive = MiniMaxH3AddGuide.execute(positive=positive, latent=latent, frame_idx=guide_frame_idx,
|
||||
vae=vae, image=guide).args[0]
|
||||
|
||||
if history is not None: # continuity 'history': the previous result's end just before frame 0 (added last)
|
||||
if info is not None and rope_mode == "shifted":
|
||||
print("[BFS H3 Duet] history is not used with rope_mode 'shifted' (use canvas)", flush=True)
|
||||
else:
|
||||
hist = history[..., :3].float().cpu()
|
||||
if tuple(hist.shape[1:3]) != (height, width):
|
||||
hist = torch.nn.functional.interpolate(hist.movedim(-1, 1), size=(height, width), mode="bilinear",
|
||||
align_corners=False).movedim(1, -1)
|
||||
if info is not None: # the canvas: the source frames under the history go into the panel strip
|
||||
under = history_src if history_src is not None and history_src.shape[0] >= hist.shape[0] else hist
|
||||
strip = strip_frames(under[-hist.shape[0]:], list(range(hist.shape[0])), width, height, position,
|
||||
size, gap * PATCH_PX, fit)
|
||||
hist = compose(hist, strip, position)
|
||||
del strip
|
||||
try:
|
||||
from .bfs_h3_history import add_history
|
||||
except ImportError:
|
||||
from bfs_h3_history import add_history
|
||||
positive, n = add_history(positive, vae, hist)
|
||||
del hist
|
||||
if n:
|
||||
print(f"[BFS H3 Duet] history: the previous {n} frames go before frame 0 on H3's timeline", flush=True)
|
||||
if info is not None and rope_mode == "shifted":
|
||||
model = patch_model_rope(model, info, rope_gap)
|
||||
guider = Guider_Basic(model)
|
||||
@@ -971,20 +993,23 @@ class BFSShotH3Duet:
|
||||
imgs[f"ref_image_{len(imgs) + 1}"] = shot["ref2"]
|
||||
duet, guided = mode != self.MODES[1], mode != self.MODES[0]
|
||||
try:
|
||||
from .bfs_shot_loop import chain_image, remember_result
|
||||
from .bfs_shot_loop import chain_image, remember_result, shot_history
|
||||
except ImportError:
|
||||
chain_image = remember_result = None
|
||||
chain_image = remember_result = shot_history = None
|
||||
prev = chain_image(shot) if chain_image else None
|
||||
if prev is not None and shot.get("chain") == "reference":
|
||||
imgs[f"ref_image_{len(imgs) + 1}"] = prev
|
||||
text = _resolve_prompt(shot.get("prompt", ""), task, instruction, len(imgs), 0)
|
||||
hist, hsrc = shot_history(shot) if shot_history else (None, None)
|
||||
out = h3_render(model, clip, vae, audio_vae, text, {"ref_images": imgs}, shot["width"], shot["height"],
|
||||
shot["gen_length"], steps, sampler_name, scheduler, seed + int(shot.get("index", 0)),
|
||||
panel=shot["frames"] if duet else None, guide=shot["frames"] if guided else None,
|
||||
position=position, size=size, fit=fit, gap=gap, panel_noise=panel_noise,
|
||||
ref_image_size=ref_image_size, decode_canvas=decode_canvas,
|
||||
rope_mode=rope_mode, rope_gap=rope_gap,
|
||||
first_frame=prev if prev is not None and shot.get("chain") == "first frame" else None)
|
||||
first_frame=prev if prev is not None and shot.get("chain") == "first frame" else None,
|
||||
history=hist, history_src=hsrc)
|
||||
del hist, hsrc
|
||||
if remember_result:
|
||||
remember_result(shot, out[0]) # the next shot can continue from it (auto loop)
|
||||
return out[:4] + (out[5],)
|
||||
|
||||
+18
-10
@@ -2025,6 +2025,23 @@ def history_for(path, analysis, src, seg, i, prev_len, queue, rid, W, H, crop, f
|
||||
return _to_u8(hf), _to_u8(hs)
|
||||
|
||||
|
||||
def shot_history(shot: dict):
|
||||
"""(history, source under it) of a shot with continuity 'history', as float frames, taken OUT of the shot so nothing
|
||||
stays allocated afterwards; auto loop: from the previous render kept in memory. (None, None) when there is none."""
|
||||
if shot.get("chain") != "history" or int(shot.get("index", 0)) == 0:
|
||||
return None, None
|
||||
hist, hsrc = _from_u8(shot.pop("history", None)), _from_u8(shot.pop("history_src", None))
|
||||
if hist is None and not shot.get("crop") and _LAST_RESULT.get("index") == int(shot["index"]) - 1 \
|
||||
and _LAST_RESULT.get("count") == int(shot.get("count", 0)):
|
||||
prev_len = int(shot.get("prev_length") or _LAST_RESULT["frames"].shape[0])
|
||||
f = _LAST_RESULT["frames"][:prev_len]
|
||||
n = (min(int(shot.get("chain_history") or 17), f.shape[0]) // 17) * 17
|
||||
if n >= 17:
|
||||
hist = f[f.shape[0] - n:].float()
|
||||
hist = hist / 255.0 if hist.max() > 1.5 else hist
|
||||
return hist, hsrc
|
||||
|
||||
|
||||
def _to_u8(x: torch.Tensor) -> torch.Tensor:
|
||||
return (x.clamp(0, 1) * 255).round().to(torch.uint8).cpu()
|
||||
|
||||
@@ -2956,16 +2973,7 @@ class BFSShotH3Conditioning:
|
||||
def _history(shot, positive, vae, tag, on_canvas, duet, panel_position, panel_size):
|
||||
"""Continuity 'history': the end of the previous result as clean frames just before this shot on H3's
|
||||
timeline (added last, so the duet panel and the other guides leave it alone)."""
|
||||
if shot.get("chain") != "history" or int(shot.get("index", 0)) == 0:
|
||||
return positive
|
||||
# taken out of the shot: nothing stays allocated after this shot is conditioned
|
||||
hist, hsrc = _from_u8(shot.pop("history", None)), _from_u8(shot.pop("history_src", None))
|
||||
if hist is None and not shot.get("crop") and _LAST_RESULT.get("index") == int(shot["index"]) - 1:
|
||||
prev_len = int(shot.get("prev_length") or _LAST_RESULT["frames"].shape[0])
|
||||
f = _LAST_RESULT["frames"][:prev_len].float()
|
||||
f = f / 255.0 if f.max() > 1.5 else f
|
||||
n = (min(int(shot.get("chain_history") or 17), f.shape[0]) // 17) * 17
|
||||
hist = f[f.shape[0] - n:] if n >= 17 else None
|
||||
hist, hsrc = shot_history(shot)
|
||||
if hist is None:
|
||||
return positive
|
||||
if duet == "shifted RoPE":
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "bfsnodes"
|
||||
description = "Custom ComfyUI nodes for reference-guided video: BFS Sampler (guide-driven IC-LoRAs, auto crop, mask inpainting, temporal chunking), LTX Multiple Controls / Identity Transfer / Edit Anything, Head-Swap and Anime2Real conditioning, MiniMax-H3 latent guides, RoPE layouts and loaders, LoRA Surgery, and Shot Planner / Shot Loop (tabbed panel): split long videos into shots at camera cuts, face-tracked cast, SAM 3 mask per shot with four generation modes (full frame, mask only, crop, crop + mask), optional Qwen3-VL to suggest settings, describe references ({details}) and the person to replace ({target}), continuity between shots, H3 conditioning with an optional duet and prompt writer, join with the original timing, audio and a side-by-side comparison. H3 Duet: pin a source clip beside the video, training-free."
|
||||
version = "1.88.0"
|
||||
version = "1.89.0"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = [
|
||||
"torch",
|
||||
|
||||
@@ -837,5 +837,21 @@ class DuetPromptTest(unittest.TestCase):
|
||||
SL.vlm_generate, SL.vlm_describe = old
|
||||
|
||||
|
||||
class ShotHistoryTest(unittest.TestCase):
|
||||
def test_history_is_taken_out_of_the_shot(self):
|
||||
h = torch.full((17, 8, 8, 3), 255, dtype=torch.uint8)
|
||||
shot = {"chain": "history", "index": 1, "count": 2, "history": h, "history_src": h}
|
||||
hist, src = SL.shot_history(shot)
|
||||
self.assertEqual((tuple(hist.shape), float(hist.max()), "history" in shot), ((17, 8, 8, 3), 1.0, False))
|
||||
self.assertEqual(SL.shot_history({"chain": "off", "index": 1}), (None, None))
|
||||
|
||||
def test_auto_loop_takes_the_previous_render(self):
|
||||
SL._LAST_RESULT.clear()
|
||||
SL._LAST_RESULT.update(index=0, count=2, frames=torch.rand(40, 8, 8, 3))
|
||||
hist, _ = SL.shot_history({"chain": "history", "index": 1, "count": 2, "prev_length": 30, "chain_history": 34})
|
||||
self.assertEqual(hist.shape[0], 17) # 30 real frames -> one whole group of 17
|
||||
SL._LAST_RESULT.clear()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -314,6 +314,7 @@ that moves the video, its audio and the other guides past the history; ComfyUI f
|
||||
condition outside it), *Crop* (the history is cut with this shot's box) and the **duet canvas** (the source frames
|
||||
under the history go into the panel strip). Not with duet *shifted RoPE*. The aligned guide still drives this shot's
|
||||
own frames, so sync stays frame-exact.
|
||||
- **BFS Shot H3 Conditioning and BFS Shot H3 Duet** both use it (duet canvas or aligned guide).
|
||||
- **Queue loop:** shots run in order and each reads the stored result of the one before. Auto loop: taken from the
|
||||
previous render when it matches.
|
||||
- Video only for now (the previous audio is not carried). Costs one more 17/34/51-frame condition per shot.
|
||||
|
||||
Reference in New Issue
Block a user