Duet: optional VLM prompt - writes it from the task, or fills the {...} fields of a hand-written template (TSC's); setting picture described as <Subject 2>

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

bfs_h3_side_panel.py

@@ -657,6 +657,11 @@ if _io is not None:
                     _io.Image.Input("setting", optional=True, tooltip=
                         "The place picture (BFS Setting Picture: a frame of the clip with the person in static). It becomes "
                         "the last <Picture n> and the prompt says it shows the setting (write {setting} to place its tag)."),
+                    _io.Clip.Input("vlm", optional=True, tooltip=
+                        "Optional VLM (CLIPLoader with a Qwen3-VL text encoder) for the prompt. Empty prompt: it writes one "
+                        "from the task + instruction (TSC's rules). Prompt with {...} fields (TSC's template): it fills them "
+                        "from the clip and the pictures. Prompt without fields: used as written."),
+                    _io.Int.Input("vlm_max_tokens", default=1024, min=128, max=4096, optional=True, advanced=True),
                 ],
                 outputs=[_io.Image.Output(display_name="images"), _io.Audio.Output(display_name="audio"),
                          _io.Image.Output(display_name="canvas"), _io.String.Output(display_name="layout_text"),
@@ -669,7 +674,8 @@ if _io is not None:
         def execute(cls, model, clip, vae, task, instruction, prompt, width, height, length, position, size, fit, gap,
                     panel_noise, hold, rope_mode, rope_gap, ref_image_size, steps, sampler_name, scheduler, seed,
                     decode_canvas, audio_vae=None, panel=None, guide=None, guide_frame_idx=0, ref_images=None,
-                    ref_videos=None, ref_video_audios=None, ref_audios=None, panel_static="off", setting=None):
+                    ref_videos=None, ref_video_audios=None, ref_audios=None, panel_static="off", setting=None, vlm=None,
+                    vlm_max_tokens=1024):
             if panel is None and guide is None:
                 raise ValueError("BFS H3 Duet needs a panel, a guide, or both")
             raw_panel = panel
@@ -679,7 +685,9 @@ if _io is not None:
                 length = src.shape[0] if src.shape[0] >= 5 else 124
             ref_images = {k: v for k, v in (ref_images or {}).items() if v is not None}
             ref_videos = {k: v for k, v in (ref_videos or {}).items() if v is not None}
-            text = _resolve_prompt(prompt, task, instruction, len(ref_images), len(ref_videos))
+            text = _shot_loop().duet_prompt(vlm, prompt, guide if guide is not None else raw_panel, list(ref_images.values()),
+                                            task, instruction, int(vlm_max_tokens), rope_mode, setting)
+            text = _resolve_prompt(text, task, instruction, len(ref_images), len(ref_videos))
             if setting is not None:          # the place picture is the last <Picture n>
                 ref_images = {f"ref_image_{i}": v for i, v in enumerate([*ref_images.values(), setting])}
                 text = with_setting(text, len(ref_images), task == "character swap" or "<Subject 1>" in text)
@@ -804,7 +812,9 @@ if _io is not None:
                     _io.Float.Input("rope_gap", default=0.0, min=0.0, max=256.0, step=1.0, advanced=True),
                     _io.Combo.Input("ref_image_size", options=["match", "max"], default="match", advanced=True),
                     _io.Int.Input("vlm_max_tokens", default=1024, min=128, max=4096, advanced=True),
-                    _io.Clip.Input("vlm", optional=True, tooltip="Optional VLM (CLIPLoader with a Qwen3-VL text encoder) that writes the prompt."),
+                    _io.Clip.Input("vlm", optional=True, tooltip="Optional VLM (CLIPLoader with a Qwen3-VL text encoder). Empty prompt: "
+                                   "it writes one from the task + instruction. Prompt with {...} fields: it fills them from the "
+                                   "clip and the pictures. Prompt without fields: used as written."),
                     _io.Model.Input("model", optional=True, tooltip="Needed for rope_mode = shifted."),
                     _io.Vae.Input("audio_vae", optional=True),
                     _io.Image.Input("guide", optional=True, tooltip="Optional aligned latent guide in the video area."),
@@ -835,19 +845,12 @@ if _io is not None:
                     setting=None):
             from comfy_extras.nodes_minimax_h3 import MiniMaxH3ReferenceToVideo
             refs = [v for v in (ref_images or {}).values() if v is not None]
+            raw_panel = panel
             panel = static_over_panel(panel, panel_static)
             if length <= 0:
                 length = panel.shape[0] if panel.shape[0] >= 5 else 124
-            text = prompt if prompt and prompt.strip() else ""
-            if not text and vlm is not None:
-                try:
-                    from .bfs_shot_loop import write_duet_prompt
-                except ImportError:
-                    write_duet_prompt = None
-                if write_duet_prompt:
-                    idx = sorted(set(int(round(x)) for x in torch.linspace(0, panel.shape[0] - 1, min(4, panel.shape[0])).tolist()))
-                    text = write_duet_prompt(vlm, [panel[i:i + 1] for i in idx], refs, task, instruction, int(vlm_max_tokens),
-                                             rope_mode)
+            text = _shot_loop().duet_prompt(vlm, prompt, raw_panel, refs, task, instruction, int(vlm_max_tokens),
+                                            rope_mode, setting)
             if not text:
                 text = build_prompt(task if task != "custom" else "appearance", instruction, len(refs), 0, rope_mode)
             if setting is not None:          # the place picture is the last <Picture n>

bfs_shot_loop.py

@@ -19,6 +19,7 @@ from __future__ import annotations
 import base64
 import hashlib
 import json
+import re
 import math
 import os
 import subprocess
@@ -1707,15 +1708,27 @@ def vlm_write_prompt(clip, path: str, analysis: dict, start: int, end: int, refs
     return write_duet_prompt(clip, fr, refs, task, change, max_tokens)
 
 
+PLACE_Q = ("This picture shows a place; the patch of black-and-white noise only hides a person, ignore it. Describe the "
+           "place in ONE short phrase of concrete seen words, at most 25 words: what kind of place, then two or three "
+           "details (furniture, walls, light). Start with 'the'. The phrase only.")
+
+
+def describe_place(clip, setting: torch.Tensor | None, max_tokens: int = 256) -> str:
+    if setting is None:
+        return ""
+    t = vlm_describe(clip, [setting], PLACE_Q, max_tokens).strip().strip('"').rstrip(".")
+    return t if not t or t.lower().startswith("the ") else "the " + t
+
+
 def write_duet_prompt(clip, frames: list, refs: list, task: str, change: str, max_tokens: int = 1024,
-                      mode: str = "canvas") -> str:
+                      mode: str = "canvas", setting: torch.Tensor | None = None) -> str:
     """A duet prompt from frames of the kept footage ([1,H,W,3] each) and the reference pictures. The VLM only fills
     a few fields (look, medium, camera and actions, sounds); the six REF2VA sections are assembled here, so a small
     model cannot break the format."""
     refs = [r for r in refs if r is not None]
     fr = [f for f in frames if f is not None]
     key = ("write2", tuple(_image_key(f) for f in fr), tuple(_image_key(r) for r in refs), task, change, id(clip),
-           int(max_tokens), mode)
+           int(max_tokens), mode, _image_key(setting))
     if key in _DESCRIBE_CACHE:
         return _DESCRIBE_CACHE[key]
     _status("Writing the prompt with the VLM", force=True)
@@ -1767,6 +1780,15 @@ def write_duet_prompt(clip, frames: list, refs: list, task: str, change: str, ma
         keep = f"The performance and camera: fully_preserved - every movement and expression in sync with the kept footage."
         restate = f" The same performance, {what}, in sync with the kept footage."
         style = (f"The target video is in {chg}." if task == "style" and chg else f"The target video is in a realistic style, as {medium}.")
+    place = describe_place(clip, setting)
+    if setting is not None:          # TSC's <Subject 2>: the place picture; {setting} becomes its <Picture n>
+        p = place or "the place"
+        if swap:
+            defs += f"\n<Subject 2> is {p} from {{setting}}."
+            summary = summary.replace(", in the same place.", f", in {p} from {{setting}}.")
+            keep += f"\n<Subject 2> (appears in [Shot 1]): fully_preserved - {p} is retained."
+        else:
+            defs += f" {{setting}} shows the place: {p}."
     out = "\n\n".join([
         "subject_definitions:\n" + defs,
         "summary:\n" + summary,
@@ -1779,6 +1801,91 @@ def write_duet_prompt(clip, frames: list, refs: list, task: str, change: str, ma
     return out
 
 
+RESERVED_SLOTS = ("layout", "setting", "target", "details")
+_SLOT = re.compile(r"\{([^{}\n]{2,200})\}")
+
+
+def template_slots(text: str) -> list[str]:
+    """The {...} fields of a hand-written prompt template that a VLM should fill ({layout}, {setting}, {target} and
+    {details} are filled by the nodes)."""
+    return [m for m in _SLOT.findall(text or "") if m.strip().lower() not in RESERVED_SLOTS]
+
+
+def fill_duet_template(clip, template: str, frames: list, refs: list, task: str, change: str, max_tokens: int = 1024,
+                       setting: torch.Tensor | None = None) -> str:
+    """A hand-written duet prompt with {...} fields (TSC's template) filled by the VLM. The facts are gathered with
+    separate questions (references, place, clip), then a text-only pass fills the fields and keeps every other word."""
+    refs = [r for r in refs if r is not None]
+    fr = [f for f in frames if f is not None]
+    key = ("fill1", template, tuple(_image_key(f) for f in fr), tuple(_image_key(r) for r in refs), task, change,
+           id(clip), int(max_tokens), _image_key(setting))
+    if key in _DESCRIBE_CACHE:
+        return _DESCRIBE_CACHE[key]
+    slots = template_slots(template)
+    if not slots:
+        return template
+    _status("Filling the prompt template with the VLM", force=True)
+    facts = []
+    if refs:
+        pics = " and ".join(f"<Picture {k + 1}>" for k in range(len(refs)))
+        look = vlm_describe(clip, refs, "Describe the subject of these reference pictures (a person, creature, robot or "
+                            "animal) in concrete seen words: what it is, apparent gender and age when it has them, skin, "
+                            "face shape, eye shape and colour, brows, nose, lips, hair colour, length and cut, and the "
+                            "clothing piece by piece with colours. Looks only: never the pose, camera or background.",
+                            max_tokens).strip()
+        facts.append(f"The new character ({pics}): {look}")
+    place = describe_place(clip, setting)
+    if place:
+        facts.append(f"The place ({{setting}}): {place}")
+    q = (f"These {len(fr)} images are frames, in time order, of one video. Answer with one JSON object only, no code "
+         "fence:\n{" + '"medium": "what the video is and how it was captured, e.g. handheld vertical smartphone footage '
+         'under warm lamp light", "camera": "camera distance, angle and movement", "light": "where the light comes from '
+         'and what it falls on", "actions": "what the person does, in time order, one short sentence per real action, '
+         'a time (MM:SS.mmm) only for the big moments; call them the person and never describe their looks", '
+         '"cuts": "the times of hard cuts as MM:SS.mmm, or none", "sounds": "the sounds of the scene and when"}')
+    d = _parse_json(vlm_generate(clip, q, _letterbox_batch(fr), max_tokens)) if fr else {}
+    for k in ("medium", "camera", "light", "actions", "cuts", "sounds"):
+        if d.get(k):
+            facts.append(f"The kept footage, {k}: {d[k]}")
+    if change.strip():
+        facts.append(f"The requested change: {change.strip()}")
+    fields = "\n".join(f"- {{{x}}}" for x in dict.fromkeys(slots))
+    ask = ("Fill a video prompt template. Replace every field written in curly braces with concrete seen words taken "
+           "from the facts below, matching what the field asks for, and keep every other character of the template "
+           "exactly as it is (section names, tags like <Subject 1>, <Picture 1>, {setting}, {layout}, line breaks). "
+           "Rules: never describe the performer of the kept footage (looks, clothes); the new character's looks come only "
+           "from the character facts; no negations, no quality words; write 'they' for a character with no clear gender. "
+           "If a field is an optional extra shot that the facts do not support, delete that [Shot n] paragraph. "
+           f"Task: {task}.\n\nFacts:\n" + "\n".join(facts) + f"\n\nFields to fill:\n{fields}\n\nTemplate:\n"
+           + template + "\n\nReply with the filled template only.")
+    out = _clean_written(vlm_generate(clip, ask, None, max(int(max_tokens), 1536)))
+    if "subject_definitions:" not in out or len(out) < len(template) * 0.5:
+        print("[BFS] the VLM did not return the template; the fields stay as written", flush=True)
+        out = template
+    _DESCRIBE_CACHE[key] = out
+    return out
+
+
+def duet_prompt(vlm, prompt: str, frames: torch.Tensor | None, refs: list, task: str, instruction: str,
+                max_tokens: int = 1024, mode: str = "canvas", setting: torch.Tensor | None = None) -> str:
+    """The prompt a duet node uses. Without a VLM: the prompt as written ('' = the node's template). With one:
+    a prompt with {...} fields gets them filled from the clip and the pictures; an empty prompt is written from the
+    task and the instruction; a prompt without fields is used as written."""
+    text = (prompt or "").strip()
+    if vlm is None or frames is None:
+        if text and template_slots(text):
+            print(f"[BFS H3 Duet] the prompt still has fields to fill ({', '.join(template_slots(text)[:3])}...): connect "
+                  "a VLM to fill them, or write them in", flush=True)
+        return text
+    idx = sorted(set(int(round(x)) for x in torch.linspace(0, frames.shape[0] - 1, min(4, frames.shape[0])).tolist()))
+    fr = [frames[i:i + 1, ..., :3].float() for i in idx]
+    if text and template_slots(text):
+        return fill_duet_template(vlm, text, fr, refs, task, instruction, max_tokens, setting)
+    if not text:
+        return write_duet_prompt(vlm, fr, refs, task, instruction, max_tokens, mode, setting)
+    return text
+
+
 # ---------------------------------------------------------------------------- continuity between shots
 
 CHAIN_MODES = ("off", "reference", "first frame")

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "bfsnodes"
 description = "Custom ComfyUI nodes for reference-guided video: BFS Sampler (guide-driven IC-LoRAs, auto crop, mask inpainting, temporal chunking), LTX Multiple Controls / Identity Transfer / Edit Anything, Head-Swap and Anime2Real conditioning, MiniMax-H3 latent guides, RoPE layouts and loaders, LoRA Surgery, and Shot Planner / Shot Loop (tabbed panel): split long videos into shots at camera cuts, face-tracked cast, SAM 3 mask per shot with four generation modes (full frame, mask only, crop, crop + mask), optional Qwen3-VL to suggest settings, describe references ({details}) and the person to replace ({target}), continuity between shots, H3 conditioning with an optional duet and prompt writer, join with the original timing, audio and a side-by-side comparison. H3 Duet: pin a source clip beside the video, training-free."
-version = "1.84.1"
+version = "1.85.0"
 license = {file = "LICENSE"}
 dependencies = [
     "torch",

tests/test_bfs_shot_loop.py

@@ -807,5 +807,35 @@ class TrackModeTest(unittest.TestCase):
             SL.track_video = orig
 
 
+class DuetPromptTest(unittest.TestCase):
+    def test_fields_are_found_and_the_vlm_fills_them(self):
+        tpl = ("subject_definitions:\n<Subject 1> is the woman from <Picture 1>: {skin, face}.\n<Subject 2> is the {room} "
+               "from {setting}.\n\nsummary:\nx {layout}")
+        self.assertEqual(SL.template_slots(tpl), ["skin, face", "room"])
+        asked = []
+
+        def gen(clip, q, images, max_tokens, temperature=0.0, seed=0):
+            asked.append(q)
+            if images is None:
+                return tpl.replace("{skin, face}", "pale skin, oval face").replace("{room}", "kitchen")
+            return '{"medium": "phone footage", "actions": "the person waves"}'
+
+        old = SL.vlm_generate, SL.vlm_describe
+        SL.vlm_generate, SL.vlm_describe = gen, lambda clip, ims, q, mt=320, temperature=0.0, seed=0: "the kitchen with white cabinets"
+        try:
+            fr = torch.rand(9, 32, 32, 3)
+            out = SL.duet_prompt(object(), tpl, fr, [torch.rand(1, 32, 32, 3)], "character swap", "", 512,
+                                 setting=torch.rand(1, 32, 32, 3))
+            self.assertIn("pale skin, oval face", out); self.assertIn("{setting}", out); self.assertIn("{layout}", out)
+            self.assertIn("the person waves", asked[-1])
+            self.assertEqual(SL.duet_prompt(None, tpl, fr, [], "character swap", ""), tpl.strip())
+            self.assertEqual(SL.duet_prompt(object(), "a full prompt", fr, [], "character swap", ""), "a full prompt")
+            w = SL.duet_prompt(object(), "", fr, [torch.rand(1, 32, 32, 3)], "character swap", "", 512,
+                               setting=torch.rand(1, 32, 32, 3))
+            self.assertIn("<Subject 2> is the kitchen with white cabinets from {setting}.", w)
+        finally:
+            SL.vlm_generate, SL.vlm_describe = old
+
+
 if __name__ == "__main__":
     unittest.main()

web/js/docs/BFSH3Duet.md

@@ -110,6 +110,20 @@ TSC's two fixes for a swap in the same room with the same clothes, without a VLM
   in full detail; the noise patch in it is where <Subject 1> stands."* (or write `{setting}` to place its tag yourself).
   This stops the background and framing from being taken from the character picture.
 
+## Prompt from a VLM (optional, 1.85.0)
+
+Plug a Qwen3-VL CLIP (the same CLIPLoader as `clip` works) into `vlm`:
+
+- **prompt empty**: the VLM writes it from the task + instruction (TSC's six sections; with a `setting` picture the
+  place becomes `<Subject 2> ... from <Picture n>`).
+- **prompt with `{...}` fields** (TSC's template): the VLM fills only the fields, from the clip's frames, the
+  reference pictures and the setting picture; every other word stays. `{layout}`, `{setting}`, `{target}` and
+  `{details}` are filled by the node, not the VLM.
+- **prompt without fields**: used as written (the VLM is not called).
+
+Without `vlm`, a prompt that still has `{...}` fields logs a warning. The prompt actually used is the `prompt` output
+and is printed under the comparison.
+
 ## Settings
 
 | setting | effect |