diff --git a/README.md b/README.md index 852baca..3f968d0 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,24 @@ The offset is applied only to appearance/reference inputs. Guide videos, masks, and target video positions remain unchanged. In **LTX Multiple Controls**, the field is named `identity_temporal_offset_latents` and affects only `identity_image`. +## Shot Planner / Shot Loop (1.48.0) + +Split a long video into shots a video model can follow (at camera cuts via PySceneDetect, fixed +length, or by hand on a timeline), set a reference image and prompt per shot, run the rest of the +workflow once per shot, and join the results back with the original timing and soundtrack. + +- **BFS Shot Planner** — interactive timeline (thumbnails, cut detector curve, draggable + boundaries, per-shot reference gallery and prompt). Outputs the shots as a ComfyUI list, so + every node downstream runs once per shot. +- **BFS Shot Unpack** — guide frames, references, prompt and length of one shot, for any model. +- **BFS Shot H3 Conditioning** — MiniMax H3 conditioning for one shot with the native nodes + (aligned guide and/or native reference video). +- **BFS Shot Join** — concatenates, trims to the true lengths, cross-fades soft joins, returns audio. + +Two run modes: *auto loop* (all shots in one run) and *queue loop* (one shot per run, stored on +disk, re-queued automatically; nodes after the join only run on the last shot). See +[`web/docs/BFSShotPlanner.md`](web/docs/BFSShotPlanner.md). + ## Requirements - ComfyUI diff --git a/__init__.py b/__init__.py index 8778f46..33520bc 100644 --- a/__init__.py +++ b/__init__.py @@ -74,6 +74,12 @@ except Exception as _e: # noqa # gain came from the projector + LoRA, not the CAN). Kept the file but not registered. CAN_NODE_CLASS_MAPPINGS, CAN_NODE_DISPLAY_NAME_MAPPINGS = {}, {} +try: + from .bfs_shot_loop import NODE_CLASS_MAPPINGS as SHOT_NODE_CLASS_MAPPINGS + from .bfs_shot_loop import NODE_DISPLAY_NAME_MAPPINGS as SHOT_NODE_DISPLAY_NAME_MAPPINGS +except Exception as _e: # noqa + print(f"[BFSNodes] Shot loop nodes not loaded: {_e!r}") + SHOT_NODE_CLASS_MAPPINGS, SHOT_NODE_DISPLAY_NAME_MAPPINGS = {}, {} try: from .minimax_h3_downscaled_guide import NODE_CLASS_MAPPINGS as H3GUIDE_NODE_CLASS_MAPPINGS from .minimax_h3_downscaled_guide import NODE_DISPLAY_NAME_MAPPINGS as H3GUIDE_NODE_DISPLAY_NAME_MAPPINGS @@ -97,6 +103,7 @@ except Exception as _e: # noqa NODE_CLASS_MAPPINGS = { **H3GUIDE_NODE_CLASS_MAPPINGS, + **SHOT_NODE_CLASS_MAPPINGS, **GV_NODE_CLASS_MAPPINGS, **CAN_NODE_CLASS_MAPPINGS, **BFS_NODE_CLASS_MAPPINGS, @@ -122,6 +129,7 @@ NODE_CLASS_MAPPINGS = { NODE_DISPLAY_NAME_MAPPINGS = { **H3GUIDE_NODE_DISPLAY_NAME_MAPPINGS, + **SHOT_NODE_DISPLAY_NAME_MAPPINGS, **GV_NODE_DISPLAY_NAME_MAPPINGS, **CAN_NODE_DISPLAY_NAME_MAPPINGS, **BFS_NODE_DISPLAY_NAME_MAPPINGS, diff --git a/bfs_shot_loop.py b/bfs_shot_loop.py new file mode 100644 index 0000000..e19c99d --- /dev/null +++ b/bfs_shot_loop.py @@ -0,0 +1,783 @@ +"""BFS Shot Loop: plan a long video into model-sized shots, run any workflow once per shot, join. + +A video model only follows a guide reliably inside the clip length it was trained on (an H3 LoRA +trained on 107 frames loses the scene at 243). This splits the source into shots that fit, at +camera cuts when there are any, and hands them to the rest of the graph as a ComfyUI *list*. +ComfyUI then runs every node that receives a list once per item, pairing the lists by index, so +shot ``i`` always gets guide ``i``, reference ``i`` and prompt ``i``. ``BFS Shot Join`` takes the +decoded list back, trims each shot to its true length and concatenates them in order. + +Every shot is generated at a length the model accepts (for example 17n+5 frames for MiniMax H3). +When a shot is shorter than that, the extra guide frames are taken from the video that follows it, +so the guide never repeats or stretches; the join then cuts the result back to the shot's own +frames, which keeps the timing identical to the source. Those extra frames also give the join a +real overlap to cross-fade across boundaries that are not camera cuts. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import math +import os +import subprocess +from typing import Any + +import numpy as np +import torch + +import folder_paths + +# ---------------------------------------------------------------------------- frame grids + +GRIDS: dict[str, tuple[int, int]] = { + "H3 (17n+5)": (17, 5), + "LTX / Wan (8n+1)": (8, 1), + "Wan (4n+1)": (4, 1), + "any": (1, 0), +} +DEFAULT_GRID = "H3 (17n+5)" +VIDEO_EXTS = (".mp4", ".mov", ".webm", ".mkv", ".avi", ".m4v", ".gif") +IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".bmp") + + +def snap_up(n: int, grid: str) -> int: + """Smallest valid clip length >= n.""" + step, off = GRIDS.get(grid, GRIDS[DEFAULT_GRID]) + n = max(1, int(n)) + if step == 1: + return n + if n <= off: + return off + return off + step * math.ceil((n - off) / step) + + +def snap_down(n: int, grid: str) -> int: + """Largest valid clip length <= n (never below the grid's smallest length).""" + step, off = GRIDS.get(grid, GRIDS[DEFAULT_GRID]) + n = max(1, int(n)) + if step == 1: + return n + if n <= off: + return off + return off + step * ((n - off) // step) + + +# ---------------------------------------------------------------------------- video io + +def _input_path(name: str) -> str: + path = folder_paths.get_annotated_filepath(name) if name else "" + if not path or not os.path.isfile(path): + raise FileNotFoundError(f"BFS Shot Loop: file not found in the input folder: {name!r}") + return path + + +def _probe(path: str) -> dict: + import cv2 + cap = cv2.VideoCapture(path) + if not cap.isOpened(): + raise ValueError(f"BFS Shot Loop: cannot open video {path}") + fps = cap.get(cv2.CAP_PROP_FPS) or 0.0 + n = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0) + w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + cap.release() + if fps <= 0 or n <= 0: + raise ValueError(f"BFS Shot Loop: could not read fps/frame count from {path}") + return {"fps_src": float(fps), "n_src": n, "width": w, "height": h, "duration": n / fps} + + +def _timeline(n_src: int, fps_src: float, fps: float) -> np.ndarray: + """Source frame index for each frame of the resampled timeline (nearest frame, real time).""" + n = max(1, int(math.floor(n_src / fps_src * fps + 1e-6))) + idx = np.round(np.arange(n) * fps_src / fps).astype(np.int64) + return np.clip(idx, 0, n_src - 1) + + +def _read_frames(path: str, src_indices: np.ndarray, size: tuple[int, int] | None) -> list[np.ndarray]: + """Decode the requested source frames (sorted, may repeat) as RGB uint8, sequentially.""" + import cv2 + want = sorted(set(int(i) for i in src_indices)) + got: dict[int, np.ndarray] = {} + cap = cv2.VideoCapture(path) + i, k = 0, 0 + last = None + while k < len(want): + ok = cap.grab() + if not ok: + break + if i == want[k]: + ok, fr = cap.retrieve() + if ok: + fr = cv2.cvtColor(fr, cv2.COLOR_BGR2RGB) + if size is not None: + fr = _fit(fr, size) + last = fr + got[i] = fr + while k < len(want) and want[k] == i: + k += 1 + i += 1 + cap.release() + if not got: + raise ValueError(f"BFS Shot Loop: no frames decoded from {path}") + out = [] + for s in src_indices: + s = int(s) + out.append(got.get(s, last if s > max(got) else got[min(got, key=lambda x: abs(x - s))])) + return out + + +def _fit(img: np.ndarray, size: tuple[int, int]) -> np.ndarray: + """Cover-resize then center-crop to (w, h).""" + import cv2 + w, h = size + ih, iw = img.shape[:2] + s = max(w / iw, h / ih) + r = cv2.resize(img, (max(w, round(iw * s)), max(h, round(ih * s))), + interpolation=cv2.INTER_AREA if s < 1 else cv2.INTER_CUBIC) + y, x = (r.shape[0] - h) // 2, (r.shape[1] - w) // 2 + return r[y:y + h, x:x + w] + + +def generation_size(src_w: int, src_h: int, megapixels: float, multiple: int) -> tuple[int, int]: + """Aspect-preserving size with the given pixel area, both sides on the multiple grid.""" + multiple = max(8, int(multiple)) + area = max(0.05, float(megapixels)) * 1e6 + ar = src_w / max(1, src_h) + h = math.sqrt(area / ar) + w = h * ar + w = max(multiple, int(round(w / multiple)) * multiple) + h = max(multiple, int(round(h / multiple)) * multiple) + return w, h + + +def _read_audio(path: str, sample_rate: int = 44100) -> dict | None: + """Decode the whole soundtrack as float32 [1, C, T] with ffmpeg; None when silent.""" + try: + probe = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "a:0", "-show_entries", + "stream=channels", "-of", "csv=p=0", path], + capture_output=True, text=True, timeout=60) + ch = int((probe.stdout.strip() or "0").split(",")[0] or 0) + if ch <= 0: + return None + ch = min(ch, 2) + raw = subprocess.run(["ffmpeg", "-v", "error", "-i", path, "-vn", "-ac", str(ch), "-ar", str(sample_rate), + "-f", "f32le", "-"], capture_output=True, timeout=600).stdout + if not raw: + return None + a = np.frombuffer(raw, dtype=np.float32).reshape(-1, ch).T.copy() + return {"waveform": torch.from_numpy(a)[None], "sample_rate": sample_rate} + except Exception: # noqa: BLE001 - audio is optional + return None + + +def _slice_audio(audio: dict | None, start_s: float, dur_s: float) -> dict | None: + if audio is None: + return None + sr = audio["sample_rate"] + wf = audio["waveform"] + a, n = int(round(start_s * sr)), int(round(dur_s * sr)) + seg = wf[..., a:a + n] + if seg.shape[-1] < n: + seg = torch.nn.functional.pad(seg, (0, n - seg.shape[-1])) + return {"waveform": seg.contiguous(), "sample_rate": sr} + + +# ---------------------------------------------------------------------------- analysis + +_ANALYSIS_CACHE: dict[tuple, dict] = {} + + +def analyze(path: str, fps: float, thumbs: int = 120, thumb_h: int = 72) -> dict: + """Probe, per-frame cut score on the resampled timeline, and a strip of thumbnails.""" + import cv2 + key = (path, os.path.getmtime(path), round(float(fps), 4), thumbs, thumb_h) + if key in _ANALYSIS_CACHE: + return _ANALYSIS_CACHE[key] + info = _probe(path) + src = _timeline(info["n_src"], info["fps_src"], fps) + n = len(src) + small = [] + for fr in _read_frames(path, src, None): + g = cv2.resize(fr, (64, 36), interpolation=cv2.INTER_AREA) + small.append(g) + hsv = [cv2.calcHist([cv2.cvtColor(s, cv2.COLOR_RGB2HSV)], [0, 1], None, [16, 8], [0, 180, 0, 256]) for s in small] + hsv = [cv2.normalize(h, h).flatten() for h in hsv] + raw = np.zeros(n, np.float32) + for i in range(1, n): + pix = np.abs(small[i].astype(np.float32) - small[i - 1].astype(np.float32)).mean() / 255.0 + hist = cv2.compareHist(hsv[i - 1], hsv[i], cv2.HISTCMP_BHATTACHARYYA) + raw[i] = 0.5 * pix + 0.5 * float(hist) + # local normalisation: a cut stands out from the motion around it + score = np.zeros(n, np.float32) + w = max(4, int(round(fps))) + for i in range(1, n): + lo, hi = max(1, i - w), min(n, i + w + 1) + neigh = np.concatenate([raw[lo:i], raw[i + 1:hi]]) + base = float(np.median(neigh)) if len(neigh) else 0.0 + score[i] = raw[i] / (base + 0.02) + k = max(1, n // max(1, thumbs)) + th = [] + tw = max(16, int(round(thumb_h * info["width"] / max(1, info["height"])))) + frames_for_thumbs = _read_frames(path, src[::k], (tw, thumb_h)) + for j, fr in enumerate(frames_for_thumbs): + ok, buf = cv2.imencode(".jpg", cv2.cvtColor(fr, cv2.COLOR_RGB2BGR), [cv2.IMWRITE_JPEG_QUALITY, 70]) + th.append({"f": int(j * k), "src": "data:image/jpeg;base64," + base64.b64encode(buf.tobytes()).decode()}) + out = dict(info, fps=float(fps), n=n, thumbs=th, thumb_w=tw, thumb_h=thumb_h, + raw=[round(float(x), 4) for x in raw], score=[round(float(x), 3) for x in score]) + _ANALYSIS_CACHE[key] = out + return out + + +_PSD_CACHE: dict[tuple, list[float]] = {} + + +def scenedetect_cut_times(path: str, detector: str, sensitivity: float) -> list[float] | None: + """Cut times in seconds from PySceneDetect, or None when the package is missing.""" + try: + from scenedetect import SceneManager, open_video + from scenedetect.detectors import AdaptiveDetector, ContentDetector + except ImportError: + return None + sensitivity = float(min(1.0, max(0.0, sensitivity))) + key = (path, os.path.getmtime(path), detector, round(sensitivity, 3)) + if key in _PSD_CACHE: + return _PSD_CACHE[key] + video = open_video(path) + min_len = max(3, int(round(video.frame_rate * 0.25))) + sm = SceneManager() + if detector == "content": + sm.add_detector(ContentDetector(threshold=45.0 - 33.0 * sensitivity, min_scene_len=min_len)) + else: + sm.add_detector(AdaptiveDetector(adaptive_threshold=5.0 - 3.5 * sensitivity, min_scene_len=min_len)) + sm.detect_scenes(video, show_progress=False) + times = [float(a.get_seconds()) for a, _ in sm.get_scene_list()[1:]] + _PSD_CACHE[key] = times + return times + + +def find_cuts(path: str, analysis: dict, detector: str, sensitivity: float) -> tuple[list[int], str]: + """Cut frames on the resampled timeline and which detector produced them.""" + fps = float(analysis["fps"]) + if detector in ("adaptive", "content"): + times = scenedetect_cut_times(path, detector, sensitivity) + if times is not None: + n = analysis["n"] + return sorted({int(round(t * fps)) for t in times if 0 < round(t * fps) < n}), f"PySceneDetect {detector}" + return detect_cuts(analysis["score"], analysis["raw"], sensitivity, fps), "builtin" + + +def detect_cuts(score: list[float], raw: list[float], sensitivity: float, fps: float) -> list[int]: + """Frame indices that start a new shot (built-in detector). Higher sensitivity finds more cuts.""" + sensitivity = float(min(1.0, max(0.0, sensitivity))) + thr = 12.0 - 10.0 * sensitivity # local ratio threshold: 12 (strict) .. 2 (loose) + abs_min = 0.12 - 0.09 * sensitivity # ignore tiny global changes + gap = max(3, int(round(fps * 0.25))) # no two cuts closer than a quarter second + cand = [i for i in range(1, len(score)) if score[i] >= thr and raw[i] >= abs_min] + cuts: list[int] = [] + for i in sorted(cand, key=lambda j: -score[j]): + if all(abs(i - c) >= gap for c in cuts): + cuts.append(i) + return sorted(cuts) + + +# ---------------------------------------------------------------------------- planning + +def plan_segments(n: int, cuts: list[int], mode: str, max_len: int, min_len: int, + max_parts: int = 0, max_total: int = 0, manual: list[int] | None = None) -> list[dict]: + """Split [0, n) into shots. Returns [{start, end, cut_before}]. Lengths are in frames.""" + if max_total and max_total > 0: + n = min(n, int(max_total)) + max_len = max(1, int(max_len)) + min_len = max(1, min(int(min_len), max_len)) + cuts = sorted(c for c in set(cuts or []) if 0 < c < n) + cutset = set(cuts) + if mode == "manual" and manual: + bounds = sorted(set([0, n] + [b for b in manual if 0 < b < n])) + elif mode == "fixed": + k = max(1, math.ceil(n / max_len)) + bounds = sorted(set(round(n * j / k) for j in range(k + 1))) + else: # shots + bounds = [0] + cuts + [n] + # merge shots that are too short into a neighbour, as long as the merge still fits + changed = True + while changed: + changed = False + for j in range(len(bounds) - 1): + if bounds[j + 1] - bounds[j] >= min_len or len(bounds) <= 2: + continue + left = bounds[j + 1] - bounds[j - 1] if j > 0 else None + right = bounds[j + 2] - bounds[j] if j + 2 < len(bounds) else None + opts = [(v, side) for v, side in ((left, "l"), (right, "r")) if v is not None and v <= max_len] + if not opts: + continue + side = min(opts)[1] + bounds.pop(j if side == "l" else j + 1) + changed = True + break + # split shots that are too long into equal parts + out = [0] + for a, b in zip(bounds[:-1], bounds[1:]): + k = max(1, math.ceil((b - a) / max_len)) + out += [a + round((b - a) * j / k) for j in range(1, k + 1)] + bounds = sorted(set(out)) + segs = [{"start": a, "end": b, "cut_before": a in cutset} for a, b in zip(bounds[:-1], bounds[1:]) if b > a] + if max_parts and max_parts > 0: + segs = segs[:int(max_parts)] + return segs + + +DEFAULT_PLAN = { + "video": "", "fps": 24.0, "grid": DEFAULT_GRID, "mode": "shots", "max_s": 4.5, "min_s": 1.0, + "sensitivity": 0.5, "max_parts": 0, "max_total_s": 0.0, "bounds": [], "segs": [], + "global_ref": "", "global_ref2": "", "global_prompt": "", "megapixels": 0.15, "multiple": 32, + "detector": "adaptive", "run": "auto", +} + + +def _load_plan(plan_json: str) -> dict: + try: + p = json.loads(plan_json or "{}") + except json.JSONDecodeError as exc: + raise ValueError(f"BFS Shot Planner: the plan is not valid JSON ({exc})") from exc + out = dict(DEFAULT_PLAN) + out.update({k: v for k, v in p.items() if v is not None}) + return out + + +def resolve_plan(plan: dict, analysis: dict, path: str | None = None) -> list[dict]: + """Shots for this plan: the boundaries edited in the panel, or an automatic split.""" + fps = float(analysis["fps"]) + grid = plan["grid"] + max_len = snap_down(int(round(float(plan["max_s"]) * fps)), grid) + min_len = max(1, int(round(float(plan["min_s"]) * fps))) + max_total = int(round(float(plan.get("max_total_s") or 0) * fps)) + if path: + cuts, _ = find_cuts(path, analysis, plan.get("detector", "adaptive"), float(plan["sensitivity"])) + else: + cuts = detect_cuts(analysis["score"], analysis["raw"], float(plan["sensitivity"]), fps) + manual = plan.get("bounds") or None + mode = "manual" if manual else plan["mode"] + segs = plan_segments(analysis["n"], cuts, mode, max_len, min_len, + int(plan.get("max_parts") or 0), max_total, manual) + meta = plan.get("segs") or [] + for i, s in enumerate(segs): + m = meta[i] if i < len(meta) and isinstance(meta[i], dict) else {} + s["enabled"] = bool(m.get("enabled", True)) + s["ref"] = m.get("ref") or "" + s["ref2"] = m.get("ref2") or "" + s["prompt"] = m.get("prompt") or "" + s["cut_before"] = bool(s.get("cut_before")) or (s["start"] in cuts) + s["gen_len"] = snap_up(s["end"] - s["start"], grid) + return segs + + +# ---------------------------------------------------------------------------- queue loop state + +def run_id_for(plan_json: str, path: str) -> str: + h = hashlib.sha1((plan_json + str(os.path.getmtime(path))).encode()).hexdigest()[:16] + return h + + +def run_dir(run_id: str) -> str: + d = os.path.join(folder_paths.get_temp_directory(), "bfs_shotloop", run_id) + os.makedirs(d, exist_ok=True) + return d + + +def run_state(run_id: str) -> dict: + f = os.path.join(run_dir(run_id), "state.json") + if os.path.isfile(f): + with open(f) as fh: + return json.load(fh) + return {"done": [], "count": 0} + + +def save_state(run_id: str, st: dict) -> None: + with open(os.path.join(run_dir(run_id), "state.json"), "w") as fh: + json.dump(st, fh) + + +def _notify(event: str, data: dict) -> None: + try: + from server import PromptServer + PromptServer.instance.send_sync(event, data) + except Exception: # noqa: BLE001 - headless runs have no UI to notify + pass + + +# ---------------------------------------------------------------------------- images + +def _load_image(name: str) -> torch.Tensor | None: + if not name: + return None + from PIL import Image, ImageOps + img = ImageOps.exif_transpose(Image.open(_input_path(name))).convert("RGB") + return torch.from_numpy(np.asarray(img).astype(np.float32) / 255.0)[None] + + +def _grey(h: int = 64, w: int = 64) -> torch.Tensor: + return torch.full((1, h, w, 3), 0.5) + + +# ---------------------------------------------------------------------------- nodes + +class BFSShotPlanner: + """Split a video into shots that fit the model, with a reference and prompt per shot.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "plan": ("STRING", {"default": json.dumps(DEFAULT_PLAN), "multiline": True, + "tooltip": "The panel writes this. JSON with the video, split settings, " + "boundaries and the per-shot reference and prompt."}), + }, + "optional": { + "ref_image": ("IMAGE", {"tooltip": "Default reference for every shot that has none of its own " + "(overrides the panel's global reference)."}), + "ref_image_2": ("IMAGE", {"tooltip": "Default second reference (e.g. a full-body photo)."}), + "prompt": ("STRING", {"forceInput": True, + "tooltip": "Default prompt for shots without their own " + "(overrides the panel's global prompt)."}), + }, + } + + RETURN_TYPES = ("BFS_SHOT", "INT", "FLOAT", "INT", "INT", "AUDIO", "STRING") + RETURN_NAMES = ("shots", "count", "fps", "width", "height", "audio", "summary") + OUTPUT_IS_LIST = (True, False, False, False, False, False, False) + OUTPUT_TOOLTIPS = ( + "One item per shot. Every node that receives this list runs once per shot; connect it to " + "BFS Shot Unpack or BFS Shot H3 Conditioning, sample, decode, then BFS Shot Join.", + "Number of shots that will run.", "Timeline frame rate.", "Generation width.", + "Generation height.", "The whole soundtrack, trimmed to the planned duration.", + "Human-readable plan.") + FUNCTION = "plan_shots" + CATEGORY = "BFS/shot loop" + DESCRIPTION = ("Split a long video into model-sized shots (at camera cuts, fixed length, or by hand), " + "give every shot its own reference image and prompt, and run the rest of the graph once " + "per shot. Join the decoded shots with BFS Shot Join.") + + @classmethod + def IS_CHANGED(cls, plan, **kwargs): + try: + if _load_plan(plan).get("run") == "queue": + return float("nan") # the next shot depends on the loop state on disk + except ValueError: + pass + return plan + + def plan_shots(self, plan, ref_image=None, ref_image_2=None, prompt=None): + p = _load_plan(plan) + path = _input_path(p["video"]) + fps = float(p["fps"]) + a = analyze(path, fps) + segs = resolve_plan(p, a, path) + segs = [s for s in segs if s["enabled"]] + if not segs: + raise ValueError("BFS Shot Planner: no shots are enabled.") + W, H = generation_size(a["width"], a["height"], float(p["megapixels"]), int(p["multiple"])) + src = _timeline(a["n_src"], a["fps_src"], fps) + audio = _read_audio(path) + g_ref = ref_image if ref_image is not None else _load_image(p.get("global_ref", "")) + g_ref2 = ref_image_2 if ref_image_2 is not None else _load_image(p.get("global_ref2", "")) + g_prompt = prompt if prompt is not None else p.get("global_prompt", "") + queue = p.get("run") == "queue" + rid = run_id_for(plan, path) if queue else "" + todo = list(range(len(segs))) + if queue: + st = run_state(rid) + st["count"] = len(segs) + save_state(rid, st) + pending = [i for i in todo if i not in st["done"]] + todo = pending[:1] if pending else [len(segs) - 1] + shots = [] + for i, s in enumerate(segs): + if i not in todo: + continue + idx = np.arange(s["start"], s["start"] + s["gen_len"]) + idx = np.clip(idx, 0, a["n"] - 1) # past the end: hold the last frame + frames = _read_frames(path, src[idx], (W, H)) + ft = torch.from_numpy(np.stack(frames).astype(np.float32) / 255.0) + ref = _load_image(s["ref"]) if s["ref"] else g_ref + ref2 = _load_image(s["ref2"]) if s["ref2"] else g_ref2 + shots.append({ + "index": i, "count": len(segs), "start": s["start"], "end": s["end"], + "length": s["end"] - s["start"], "gen_length": s["gen_len"], "fps": fps, + "cut_before": s["cut_before"], "width": W, "height": H, "frames": ft, + "ref": ref, "ref2": ref2, "prompt": s["prompt"] or g_prompt or "", + "audio": _slice_audio(audio, s["start"] / fps, s["gen_len"] / fps), + "run_id": rid, "queue": queue, + }) + total = sum(s["end"] - s["start"] for s in segs) + full_audio = _slice_audio(audio, segs[0]["start"] / fps, total / fps) if audio else None + lines = [f"{len(segs)} shots, {total} frames ({total / fps:.2f}s) at {fps:g} fps, {W}x{H}" + + (f" | queue loop: running shot {shots[0]['index'] + 1}/{len(segs)}" if queue else "")] + for s in shots: + lines.append(f"#{s['index'] + 1}: frames {s['start']}-{s['end'] - 1} ({s['length']} -> generate " + f"{s['gen_length']}){' cut' if s['cut_before'] else ''}" + f"{' ref' if s['ref'] is not None else ''}{' ref2' if s['ref2'] is not None else ''}") + return (shots, len(segs), fps, W, H, full_audio, "\n".join(lines)) + + +class BFSShotUnpack: + """Open one shot into plain values for any workflow (runs once per shot).""" + + @classmethod + def INPUT_TYPES(cls): + return {"required": {"shot": ("BFS_SHOT",)}} + + RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE", "STRING", "INT", "IMAGE", "AUDIO", "INT", "INT", "INT") + RETURN_NAMES = ("guide_frames", "ref_image", "ref_image_2", "prompt", "length", "first_frame", + "audio", "width", "height", "index") + OUTPUT_TOOLTIPS = ( + "The shot's guide frames, already at a length the model accepts.", + "This shot's reference (a grey placeholder if none was set).", + "This shot's second reference (a grey placeholder if none was set).", + "This shot's prompt.", "Frames to generate (the grid-valid length).", + "First guide frame of the shot.", "Soundtrack of the shot (silence if the video has none).", + "Generation width.", "Generation height.", "Shot index (0-based).") + FUNCTION = "unpack" + CATEGORY = "BFS/shot loop" + DESCRIPTION = "Split one shot into its guide frames, references, prompt and length." + + def unpack(self, shot): + audio = shot["audio"] or {"waveform": torch.zeros(1, 1, int(44100 * shot["gen_length"] / shot["fps"])), + "sample_rate": 44100} + return (shot["frames"], shot["ref"] if shot["ref"] is not None else _grey(), + shot["ref2"] if shot["ref2"] is not None else _grey(), shot["prompt"], shot["gen_length"], + shot["frames"][:1], audio, shot["width"], shot["height"], shot["index"]) + + +class BFSShotH3Conditioning: + """Native MiniMax H3 conditioning for one shot: references, prompt and the shot as a guide.""" + + GUIDE_MODES = ["aligned guide (Add Guide)", "native reference video (