diff --git a/bfs_h3_side_panel.py b/bfs_h3_side_panel.py index bb69b4f..dc7932c 100644 --- a/bfs_h3_side_panel.py +++ b/bfs_h3_side_panel.py @@ -657,6 +657,11 @@ if _io is not None: _io.Image.Input("setting", optional=True, tooltip= "The place picture (BFS Setting Picture: a frame of the clip with the person in static). It becomes " "the last and the prompt says it shows the setting (write {setting} to place its tag)."), + _io.Clip.Input("vlm", optional=True, tooltip= + "Optional VLM (CLIPLoader with a Qwen3-VL text encoder) for the prompt. Empty prompt: it writes one " + "from the task + instruction (TSC's rules). Prompt with {...} fields (TSC's template): it fills them " + "from the clip and the pictures. Prompt without fields: used as written."), + _io.Int.Input("vlm_max_tokens", default=1024, min=128, max=4096, optional=True, advanced=True), ], outputs=[_io.Image.Output(display_name="images"), _io.Audio.Output(display_name="audio"), _io.Image.Output(display_name="canvas"), _io.String.Output(display_name="layout_text"), @@ -669,7 +674,8 @@ if _io is not None: def execute(cls, model, clip, vae, task, instruction, prompt, width, height, length, position, size, fit, gap, panel_noise, hold, rope_mode, rope_gap, ref_image_size, steps, sampler_name, scheduler, seed, decode_canvas, audio_vae=None, panel=None, guide=None, guide_frame_idx=0, ref_images=None, - ref_videos=None, ref_video_audios=None, ref_audios=None, panel_static="off", setting=None): + ref_videos=None, ref_video_audios=None, ref_audios=None, panel_static="off", setting=None, vlm=None, + vlm_max_tokens=1024): if panel is None and guide is None: raise ValueError("BFS H3 Duet needs a panel, a guide, or both") raw_panel = panel @@ -679,7 +685,9 @@ if _io is not None: length = src.shape[0] if src.shape[0] >= 5 else 124 ref_images = {k: v for k, v in (ref_images or {}).items() if v is not None} ref_videos = {k: v for k, v in (ref_videos or {}).items() if v is not None} - text = _resolve_prompt(prompt, task, instruction, len(ref_images), len(ref_videos)) + text = _shot_loop().duet_prompt(vlm, prompt, guide if guide is not None else raw_panel, list(ref_images.values()), + task, instruction, int(vlm_max_tokens), rope_mode, setting) + text = _resolve_prompt(text, task, instruction, len(ref_images), len(ref_videos)) if setting is not None: # the place picture is the last ref_images = {f"ref_image_{i}": v for i, v in enumerate([*ref_images.values(), setting])} text = with_setting(text, len(ref_images), task == "character swap" or "" in text) @@ -804,7 +812,9 @@ if _io is not None: _io.Float.Input("rope_gap", default=0.0, min=0.0, max=256.0, step=1.0, advanced=True), _io.Combo.Input("ref_image_size", options=["match", "max"], default="match", advanced=True), _io.Int.Input("vlm_max_tokens", default=1024, min=128, max=4096, advanced=True), - _io.Clip.Input("vlm", optional=True, tooltip="Optional VLM (CLIPLoader with a Qwen3-VL text encoder) that writes the prompt."), + _io.Clip.Input("vlm", optional=True, tooltip="Optional VLM (CLIPLoader with a Qwen3-VL text encoder). Empty prompt: " + "it writes one from the task + instruction. Prompt with {...} fields: it fills them from the " + "clip and the pictures. Prompt without fields: used as written."), _io.Model.Input("model", optional=True, tooltip="Needed for rope_mode = shifted."), _io.Vae.Input("audio_vae", optional=True), _io.Image.Input("guide", optional=True, tooltip="Optional aligned latent guide in the video area."), @@ -835,19 +845,12 @@ if _io is not None: setting=None): from comfy_extras.nodes_minimax_h3 import MiniMaxH3ReferenceToVideo refs = [v for v in (ref_images or {}).values() if v is not None] + raw_panel = panel panel = static_over_panel(panel, panel_static) if length <= 0: length = panel.shape[0] if panel.shape[0] >= 5 else 124 - text = prompt if prompt and prompt.strip() else "" - if not text and vlm is not None: - try: - from .bfs_shot_loop import write_duet_prompt - except ImportError: - write_duet_prompt = None - if write_duet_prompt: - idx = sorted(set(int(round(x)) for x in torch.linspace(0, panel.shape[0] - 1, min(4, panel.shape[0])).tolist())) - text = write_duet_prompt(vlm, [panel[i:i + 1] for i in idx], refs, task, instruction, int(vlm_max_tokens), - rope_mode) + text = _shot_loop().duet_prompt(vlm, prompt, raw_panel, refs, task, instruction, int(vlm_max_tokens), + rope_mode, setting) if not text: text = build_prompt(task if task != "custom" else "appearance", instruction, len(refs), 0, rope_mode) if setting is not None: # the place picture is the last diff --git a/bfs_shot_loop.py b/bfs_shot_loop.py index 6dd6f61..c0e37a3 100644 --- a/bfs_shot_loop.py +++ b/bfs_shot_loop.py @@ -19,6 +19,7 @@ from __future__ import annotations import base64 import hashlib import json +import re import math import os import subprocess @@ -1707,15 +1708,27 @@ def vlm_write_prompt(clip, path: str, analysis: dict, start: int, end: int, refs return write_duet_prompt(clip, fr, refs, task, change, max_tokens) +PLACE_Q = ("This picture shows a place; the patch of black-and-white noise only hides a person, ignore it. Describe the " + "place in ONE short phrase of concrete seen words, at most 25 words: what kind of place, then two or three " + "details (furniture, walls, light). Start with 'the'. The phrase only.") + + +def describe_place(clip, setting: torch.Tensor | None, max_tokens: int = 256) -> str: + if setting is None: + return "" + t = vlm_describe(clip, [setting], PLACE_Q, max_tokens).strip().strip('"').rstrip(".") + return t if not t or t.lower().startswith("the ") else "the " + t + + def write_duet_prompt(clip, frames: list, refs: list, task: str, change: str, max_tokens: int = 1024, - mode: str = "canvas") -> str: + mode: str = "canvas", setting: torch.Tensor | None = None) -> str: """A duet prompt from frames of the kept footage ([1,H,W,3] each) and the reference pictures. The VLM only fills a few fields (look, medium, camera and actions, sounds); the six REF2VA sections are assembled here, so a small model cannot break the format.""" refs = [r for r in refs if r is not None] fr = [f for f in frames if f is not None] key = ("write2", tuple(_image_key(f) for f in fr), tuple(_image_key(r) for r in refs), task, change, id(clip), - int(max_tokens), mode) + int(max_tokens), mode, _image_key(setting)) if key in _DESCRIBE_CACHE: return _DESCRIBE_CACHE[key] _status("Writing the prompt with the VLM", force=True) @@ -1767,6 +1780,15 @@ def write_duet_prompt(clip, frames: list, refs: list, task: str, change: str, ma keep = f"The performance and camera: fully_preserved - every movement and expression in sync with the kept footage." restate = f" The same performance, {what}, in sync with the kept footage." style = (f"The target video is in {chg}." if task == "style" and chg else f"The target video is in a realistic style, as {medium}.") + place = describe_place(clip, setting) + if setting is not None: # TSC's : the place picture; {setting} becomes its + p = place or "the place" + if swap: + defs += f"\n is {p} from {{setting}}." + summary = summary.replace(", in the same place.", f", in {p} from {{setting}}.") + keep += f"\n (appears in [Shot 1]): fully_preserved - {p} is retained." + else: + defs += f" {{setting}} shows the place: {p}." out = "\n\n".join([ "subject_definitions:\n" + defs, "summary:\n" + summary, @@ -1779,6 +1801,91 @@ def write_duet_prompt(clip, frames: list, refs: list, task: str, change: str, ma return out +RESERVED_SLOTS = ("layout", "setting", "target", "details") +_SLOT = re.compile(r"\{([^{}\n]{2,200})\}") + + +def template_slots(text: str) -> list[str]: + """The {...} fields of a hand-written prompt template that a VLM should fill ({layout}, {setting}, {target} and + {details} are filled by the nodes).""" + return [m for m in _SLOT.findall(text or "") if m.strip().lower() not in RESERVED_SLOTS] + + +def fill_duet_template(clip, template: str, frames: list, refs: list, task: str, change: str, max_tokens: int = 1024, + setting: torch.Tensor | None = None) -> str: + """A hand-written duet prompt with {...} fields (TSC's template) filled by the VLM. The facts are gathered with + separate questions (references, place, clip), then a text-only pass fills the fields and keeps every other word.""" + refs = [r for r in refs if r is not None] + fr = [f for f in frames if f is not None] + key = ("fill1", template, tuple(_image_key(f) for f in fr), tuple(_image_key(r) for r in refs), task, change, + id(clip), int(max_tokens), _image_key(setting)) + if key in _DESCRIBE_CACHE: + return _DESCRIBE_CACHE[key] + slots = template_slots(template) + if not slots: + return template + _status("Filling the prompt template with the VLM", force=True) + facts = [] + if refs: + pics = " and ".join(f"" for k in range(len(refs))) + look = vlm_describe(clip, refs, "Describe the subject of these reference pictures (a person, creature, robot or " + "animal) in concrete seen words: what it is, apparent gender and age when it has them, skin, " + "face shape, eye shape and colour, brows, nose, lips, hair colour, length and cut, and the " + "clothing piece by piece with colours. Looks only: never the pose, camera or background.", + max_tokens).strip() + facts.append(f"The new character ({pics}): {look}") + place = describe_place(clip, setting) + if place: + facts.append(f"The place ({{setting}}): {place}") + q = (f"These {len(fr)} images are frames, in time order, of one video. Answer with one JSON object only, no code " + "fence:\n{" + '"medium": "what the video is and how it was captured, e.g. handheld vertical smartphone footage ' + 'under warm lamp light", "camera": "camera distance, angle and movement", "light": "where the light comes from ' + 'and what it falls on", "actions": "what the person does, in time order, one short sentence per real action, ' + 'a time (MM:SS.mmm) only for the big moments; call them the person and never describe their looks", ' + '"cuts": "the times of hard cuts as MM:SS.mmm, or none", "sounds": "the sounds of the scene and when"}') + d = _parse_json(vlm_generate(clip, q, _letterbox_batch(fr), max_tokens)) if fr else {} + for k in ("medium", "camera", "light", "actions", "cuts", "sounds"): + if d.get(k): + facts.append(f"The kept footage, {k}: {d[k]}") + if change.strip(): + facts.append(f"The requested change: {change.strip()}") + fields = "\n".join(f"- {{{x}}}" for x in dict.fromkeys(slots)) + ask = ("Fill a video prompt template. Replace every field written in curly braces with concrete seen words taken " + "from the facts below, matching what the field asks for, and keep every other character of the template " + "exactly as it is (section names, tags like , , {setting}, {layout}, line breaks). " + "Rules: never describe the performer of the kept footage (looks, clothes); the new character's looks come only " + "from the character facts; no negations, no quality words; write 'they' for a character with no clear gender. " + "If a field is an optional extra shot that the facts do not support, delete that [Shot n] paragraph. " + f"Task: {task}.\n\nFacts:\n" + "\n".join(facts) + f"\n\nFields to fill:\n{fields}\n\nTemplate:\n" + + template + "\n\nReply with the filled template only.") + out = _clean_written(vlm_generate(clip, ask, None, max(int(max_tokens), 1536))) + if "subject_definitions:" not in out or len(out) < len(template) * 0.5: + print("[BFS] the VLM did not return the template; the fields stay as written", flush=True) + out = template + _DESCRIBE_CACHE[key] = out + return out + + +def duet_prompt(vlm, prompt: str, frames: torch.Tensor | None, refs: list, task: str, instruction: str, + max_tokens: int = 1024, mode: str = "canvas", setting: torch.Tensor | None = None) -> str: + """The prompt a duet node uses. Without a VLM: the prompt as written ('' = the node's template). With one: + a prompt with {...} fields gets them filled from the clip and the pictures; an empty prompt is written from the + task and the instruction; a prompt without fields is used as written.""" + text = (prompt or "").strip() + if vlm is None or frames is None: + if text and template_slots(text): + print(f"[BFS H3 Duet] the prompt still has fields to fill ({', '.join(template_slots(text)[:3])}...): connect " + "a VLM to fill them, or write them in", flush=True) + return text + idx = sorted(set(int(round(x)) for x in torch.linspace(0, frames.shape[0] - 1, min(4, frames.shape[0])).tolist())) + fr = [frames[i:i + 1, ..., :3].float() for i in idx] + if text and template_slots(text): + return fill_duet_template(vlm, text, fr, refs, task, instruction, max_tokens, setting) + if not text: + return write_duet_prompt(vlm, fr, refs, task, instruction, max_tokens, mode, setting) + return text + + # ---------------------------------------------------------------------------- continuity between shots CHAIN_MODES = ("off", "reference", "first frame") diff --git a/pyproject.toml b/pyproject.toml index 947c23a..e152d4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "bfsnodes" description = "Custom ComfyUI nodes for reference-guided video: BFS Sampler (guide-driven IC-LoRAs, auto crop, mask inpainting, temporal chunking), LTX Multiple Controls / Identity Transfer / Edit Anything, Head-Swap and Anime2Real conditioning, MiniMax-H3 latent guides, RoPE layouts and loaders, LoRA Surgery, and Shot Planner / Shot Loop (tabbed panel): split long videos into shots at camera cuts, face-tracked cast, SAM 3 mask per shot with four generation modes (full frame, mask only, crop, crop + mask), optional Qwen3-VL to suggest settings, describe references ({details}) and the person to replace ({target}), continuity between shots, H3 conditioning with an optional duet and prompt writer, join with the original timing, audio and a side-by-side comparison. H3 Duet: pin a source clip beside the video, training-free." -version = "1.84.1" +version = "1.85.0" license = {file = "LICENSE"} dependencies = [ "torch", diff --git a/tests/test_bfs_shot_loop.py b/tests/test_bfs_shot_loop.py index 4cfdbb7..7a99b2b 100644 --- a/tests/test_bfs_shot_loop.py +++ b/tests/test_bfs_shot_loop.py @@ -807,5 +807,35 @@ class TrackModeTest(unittest.TestCase): SL.track_video = orig +class DuetPromptTest(unittest.TestCase): + def test_fields_are_found_and_the_vlm_fills_them(self): + tpl = ("subject_definitions:\n is the woman from : {skin, face}.\n is the {room} " + "from {setting}.\n\nsummary:\nx {layout}") + self.assertEqual(SL.template_slots(tpl), ["skin, face", "room"]) + asked = [] + + def gen(clip, q, images, max_tokens, temperature=0.0, seed=0): + asked.append(q) + if images is None: + return tpl.replace("{skin, face}", "pale skin, oval face").replace("{room}", "kitchen") + return '{"medium": "phone footage", "actions": "the person waves"}' + + old = SL.vlm_generate, SL.vlm_describe + SL.vlm_generate, SL.vlm_describe = gen, lambda clip, ims, q, mt=320, temperature=0.0, seed=0: "the kitchen with white cabinets" + try: + fr = torch.rand(9, 32, 32, 3) + out = SL.duet_prompt(object(), tpl, fr, [torch.rand(1, 32, 32, 3)], "character swap", "", 512, + setting=torch.rand(1, 32, 32, 3)) + self.assertIn("pale skin, oval face", out); self.assertIn("{setting}", out); self.assertIn("{layout}", out) + self.assertIn("the person waves", asked[-1]) + self.assertEqual(SL.duet_prompt(None, tpl, fr, [], "character swap", ""), tpl.strip()) + self.assertEqual(SL.duet_prompt(object(), "a full prompt", fr, [], "character swap", ""), "a full prompt") + w = SL.duet_prompt(object(), "", fr, [torch.rand(1, 32, 32, 3)], "character swap", "", 512, + setting=torch.rand(1, 32, 32, 3)) + self.assertIn(" is the kitchen with white cabinets from {setting}.", w) + finally: + SL.vlm_generate, SL.vlm_describe = old + + if __name__ == "__main__": unittest.main() diff --git a/web/js/docs/BFSH3Duet.md b/web/js/docs/BFSH3Duet.md index 2c12c55..69bdb9d 100644 --- a/web/js/docs/BFSH3Duet.md +++ b/web/js/docs/BFSH3Duet.md @@ -110,6 +110,20 @@ TSC's two fixes for a swap in the same room with the same clothes, without a VLM in full detail; the noise patch in it is where stands."* (or write `{setting}` to place its tag yourself). This stops the background and framing from being taken from the character picture. +## Prompt from a VLM (optional, 1.85.0) + +Plug a Qwen3-VL CLIP (the same CLIPLoader as `clip` works) into `vlm`: + +- **prompt empty**: the VLM writes it from the task + instruction (TSC's six sections; with a `setting` picture the + place becomes ` ... from `). +- **prompt with `{...}` fields** (TSC's template): the VLM fills only the fields, from the clip's frames, the + reference pictures and the setting picture; every other word stays. `{layout}`, `{setting}`, `{target}` and + `{details}` are filled by the node, not the VLM. +- **prompt without fields**: used as written (the VLM is not called). + +Without `vlm`, a prompt that still has `{...}` fields logs a warning. The prompt actually used is the `prompt` output +and is printed under the comparison. + ## Settings | setting | effect |