feat: rule-based camera moves for [Shot N] blocks, Editor toggle

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## [Unreleased]
 
 ### Added
+- Camera moves for `[Shot N]` prompts (Engine settings, H3, off by default): each shot without a camera move gets one — close-up, focus, zoom in, move around — aimed at the most specific thing in it, never a bare `<Subject N>`. No language model; uses spaCy when installed (`spacy` in requirements).
 - H3 rating picker: "Disliked — bad image" and "Disliked — bad composition". Everything else reads them as Disliked; shot memory, decisiveness, late-branch and STAS learn only from composition blame, and DynaShift discards the latent when composition was the fault.
 - Late-branch guidance now measures how hard it actually pushed; a rating counts up to 2x (down to 0.25x) when that run was unusually strong (weak) for the key, after 3 runs of history.
 

conditioning.py

@@ -9275,6 +9275,29 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
             return head[:cut + 1].strip(), True
         return head.rsplit(" ", 1)[0].strip(), True
 
+    @staticmethod
+    def _v2_camera_moves(text, where):
+        """`text` with a camera move added to each [Shot N] block that lacks one."""
+        try:
+            try:
+                from . import shot_camera as _sc
+            except ImportError:
+                import shot_camera as _sc
+            out, report = _sc.add_camera_moves(text)
+        except Exception as e:  # noqa: BLE001
+            print(f"[FunPackVideoRefinerV2] camera moves: failed | {where}: {e}; prompt left as written")
+            return text
+        if not report:
+            return text
+        added = [r for r in report if r["move"]]
+        skipped = "; ".join(f"shot {r['shot']} {r['why']}" for r in report if not r["move"])
+        print(f"[FunPackVideoRefinerV2] camera moves: "
+              + (f"Active | {where}: {len(added)} of {len(report)} shot(s) — "
+                 + ", ".join(f"shot {r['shot']} {r['move'].rstrip('.')}" for r in added)
+                 if added else f"Inactive | {where}: no shot got one")
+              + (f" ({skipped})" if skipped else ""))
+        return out
+
     def _v2_note_enhanced(self, item):
         """Record one rewrite for the node's ui output AND publish it live, so the Editor
         can show it while the video is still rendering (and the user can stop a bad one)."""
@@ -10142,7 +10165,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
                   split_by_transitions=False, split_transition_placement="start", reference_injection=False,
                   value_guidance=True, latent=None, seed_output_connected=False,
                   steer_mode="relative", absolute_strength=0.6,
-                  h3_phrase_emphasis=False, h3_phrase_variability=0.0,
+                  h3_phrase_emphasis=False, h3_phrase_variability=0.0, camera_moves=False,
                   prompt_enhance=False, prompt_enhance_system="", prompt_enhance_reference_intro="",
                   prompt_enhance_temperature=0.7, prompt_enhance_top_p=0.92,
                   prompt_enhance_max_length=400, prompt_enhance_thinking=False,
@@ -10816,6 +10839,11 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
         if _prompt_variables:
             prompt_to_encode = _resolve_variables(prompt_to_encode, _prompt_variables)[0]
 
+        # Camera moves for MiniMax's own `[Shot N]` blocks (not timeline scenes): after
+        # shortcuts and $variables, before the enhancer, so the enhancer sees them.
+        if camera_moves and not split_by_transitions:
+            prompt_to_encode = self._v2_camera_moves(prompt_to_encode, "prompt")
+
         # ── prompt enhancer ────────────────────────────────────────────────────────────
         # Runs BEFORE the prompt becomes conditioning, so what ComfyUI generates from is the
         # enhanced text and nothing downstream has to know this happened. After shortcuts and
@@ -11126,6 +11154,9 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
                         split_scene_texts = [
                             _resolve_variables(t, _prompt_variables)[0] for t in split_scene_texts
                         ]
+                    if camera_moves and split_scene_texts:
+                        split_scene_texts = [self._v2_camera_moves(t, f"scene {_i + 1}")
+                                             for _i, t in enumerate(split_scene_texts)]
                     # Enhanced PER SCENE, not as one blob. The editor's scene list is
                     # authoritative on count, so rewriting the scenes as a single paragraph
                     # and re-splitting could return a different number of scenes and desync
@@ -14136,6 +14167,7 @@ class FunPackStudio:
             refinement_key_input=refinement_key_input,
             positive_conditioning=positive_conditioning,
             h3_phrase_emphasis=bool(rf.get("h3_phrase_emphasis", False)),
+            camera_moves=bool(rf.get("camera_moves", False)),
             h3_phrase_variability=float(rf.get("h3_phrase_variability", 0.0) or 0.0),
             prompt_enhance=bool(rf.get("prompt_enhance", False)),
             prompt_enhance_output=bool(rf.get("prompt_enhance_output", False)),

movie_editor/backend/settings_card.py

@@ -338,6 +338,7 @@ _EDITOR_DEFAULTS = {
     "h3_block_repeat_video_only": False,
     "h3_block_repeat_last_steps": 0,
     "h3_phrase_emphasis": False,
+    "camera_moves": False,
     "h3_phrase_variability": 0.0,
     "h3_repr_steering": False,
     "h3_repr_steering_block": "25",

movie_editor/frontend/engine_settings.js

@@ -119,6 +119,8 @@
       hint: "Puts each word back to its original strength after the change, so the result reads as 'less of that thing' rather than 'quieter prompt'. Off is the raw result." },
     { name: "h3_phrase_emphasis", label: "Rating-driven phrase emphasis (H3)", kind: "bool", default: false,
       hint: "EXPERIMENTAL, unvalidated. Boosts the attention paid to phrases the rating said were MISSING, by biasing their attention logits in H3's packed stream. Needs a rated run first. Turn it off if generations drift away from the prompt — and note it forces SLA to run dense." },
+    { name: "camera_moves", label: "Camera moves for [Shot N] (H3)", kind: "bool", default: false,
+      hint: "Adds a camera move (close-up, focus, zoom in...) to each [Shot N] block that has none, aimed at the most specific thing in it — a part of the subject, never the subject itself. No language model; instant. Blocks that already name a camera move, and music text, are left alone." },
     { name: "h3_phrase_variability", label: "Phrase emphasis variability", kind: "float", default: 0.0, min: 0.0, max: 1.0, step: 0.05,
       dependsOn: "h3_phrase_emphasis", dependsVals: [true],
       hint: "0 = only what the rating has cemented gets extra attention, nothing else. 1 = that emphasis is switched off entirely, so untrained phrasing and actions compete on equal footing and can bleed into the scene — including ones you have rated against before. Same bias channel as the toggle above, just scaled down." },

requirements.txt

@@ -14,3 +14,6 @@ gguf
 Pillow
 insightface
 opencv-python-headless
+# Picks the target of a camera move in [Shot N] prompts. Optional: without it (or without
+# its model, `python -m spacy download en_core_web_sm`) simpler word rules are used.
+spacy

shot_camera.py

@@ -0,0 +1,178 @@
+"""Add a camera move to each `[Shot N]` block of an H3 prompt that has none, aimed at the
+most specific thing in that shot. No language model: a small spaCy tagger (rules-only
+fallback when it is not installed) picks the target, a few words pick the move.
+
+`[Shot N]` here is MiniMax's own in-prompt shot, never a timeline scene. `<Subject N>` tags
+are the subjects every shot is about, so they are never a target on their own; a part or
+belonging of one ("<Subject 1>'s hand") is. A thing named in most shots (or in the text before
+the first shot) is a constant, not a target. Sound text (music, score...) is left alone.
+"""
+
+import math
+import re
+
+SHOT = re.compile(r"\[\s*Shot\s+(\d+)\s*\]", re.I)
+SUBJECT = re.compile(r"<\s*Subject\s*(\d+)\s*>", re.I)
+# A `<Subject N>` tag confuses the tagger ("Subject1 bites" reads as a noun); a real-looking
+# rare name parses like a person. Swapped in before tagging, swapped back after.
+NAMES = ("Mariel", "Tobias", "Odette", "Rurik", "Sabine", "Teodor", "Ilsa", "Corwin", "Yvette")
+STAND_IN = re.compile(r"\b(" + "|".join(NAMES) + r")\b")
+AUDIO = re.compile(r"\b(music|soundtrack|score|melody|instrumental|beat|bassline|synth|"
+                   r"song|tempo|audio|ambient sound|sound of)\b", re.I)
+CAMERA = re.compile(r"\b(camera|zoom(?:s|ing)?|close-?up|pan(?:s|ning)?|dolly|tracking|"
+                    r"rack focus|focus(?:es|ing)? (?:on|at)|tilt(?:s|ing)?|rotate|orbit|"
+                    r"push[- ]in|pull[- ]back|wide shot|handheld|crane)\b", re.I)
+GENERIC = {"camera", "video", "scene", "shot", "frame", "background", "foreground", "view",
+           "image", "screen", "moment", "time", "way", "side", "front", "middle", "one",
+           "other", "something", "everything", "thing", "music", "sound", "style", "lighting",
+           "light", "atmosphere", "mood", "setting"}
+PARTS = {"lip", "mouth", "tongue", "tooth", "teeth", "eye", "eyes", "face", "cheek", "hand",
+         "finger", "hair", "neck", "shoulder", "waist", "hip", "leg", "foot", "feet", "arm",
+         "skin", "expression", "smile", "gaze", "back", "chin", "brow"}
+PLACES = {"room", "street", "kitchen", "bedroom", "bed", "table", "floor", "stage", "hall",
+          "garden", "beach", "forest", "city", "car", "office", "bathroom", "field"}
+MOVES_DETAIL = ("Close-up on {x}.", "Focus at {x}.", "Zoom in at {x}.")
+MOVES_OTHER = ("Zoom in at {x}.", "Move around {x}.", "Rotate the camera to {x}.")
+DETERMINERS = {"the", "a", "an", "this", "that", "these", "those"}
+POSSESSIVE_PRONOUNS = {"his", "her", "their", "its", "my", "your", "our"}
+
+_nlp = None
+_tried = False
+
+
+def _spacy():
+    """The small English tagger, or None (said once) when it is not installed."""
+    global _nlp, _tried
+    if not _tried:
+        _tried = True
+        try:
+            import spacy
+            _nlp = spacy.load("en_core_web_sm")
+        except Exception as e:  # noqa: BLE001
+            try:
+                import funpack_log as _log
+            except ImportError:
+                from . import funpack_log as _log
+            _log.failed("FunPackCameraMoves", "spaCy tagger", e,
+                        "using the simple word rules; run `pip install spacy` and "
+                        "`python -m spacy download en_core_web_sm` for better targets")
+    return _nlp
+
+
+def _hide(text):
+    return SUBJECT.sub(lambda m: NAMES[(int(m.group(1)) - 1) % len(NAMES)], text)
+
+
+def _show(text):
+    return STAND_IN.sub(lambda m: f"<Subject {NAMES.index(m.group(1)) + 1}>", text)
+
+
+def _is_subject(text):
+    return bool(STAND_IN.fullmatch(re.sub(r"^(?:the\s+)", "", text.strip(), flags=re.I)))
+
+
+def _split_sound(body):
+    """-> (picture text, sound text): everything from the first sentence about sound on."""
+    for m in re.finditer(r"[^.!?]+[.!?]?\s*", body):
+        if AUDIO.search(m.group(0)):
+            return body[:m.start()], body[m.start():]
+    return body, ""
+
+
+def _phrase(tok):
+    """A chunk head with the words that name it ("Subject1's hand", "red lips")."""
+    keep = {tok}
+    for c in tok.children:
+        if c.dep_ in ("poss", "compound", "amod") and c.i < tok.i:
+            keep.add(c)
+            if c.dep_ == "poss":
+                keep.update(g for g in c.children if g.dep_ == "case")
+    toks = sorted(keep, key=lambda t: t.i)
+    text = " ".join(t.text for t in toks).strip().replace(" 's", "'s").replace(" ’s", "’s")
+    return text, any(c.dep_ == "poss" for c in tok.children)
+
+
+def candidates(picture):
+    """[(lemma, phrase, owned, position, dep)] for the nouns of one shot's picture text."""
+    hidden = _hide(picture)
+    nlp = _spacy()
+    out = []
+    if nlp is not None:
+        doc = nlp(hidden)
+        for chunk in doc.noun_chunks:
+            root = chunk.root
+            prev = doc[root.i - 1] if root.i else None
+            if prev is not None and STAND_IN.fullmatch(prev.text) and prev.dep_ != "poss":
+                continue                      # "<Subject 1> laughs": the tagger misread a verb
+            if root.pos_ in ("PRON", "NUM") or _is_subject(chunk.text):
+                continue
+            lemma = root.lemma_.lower()
+            if root.dep_ == "pobj" and root.head.lower_ == "of":
+                continue                      # "a glass of wine": the glass is the thing
+            text, owned = _phrase(root)
+            if _is_subject(text) or lemma in GENERIC or STAND_IN.fullmatch(root.text):
+                continue
+            out.append((lemma, text, owned, chunk.start, root.dep_))
+        return out
+    for m in re.finditer(rf"(?:({'|'.join(NAMES)})'s|\b(his|her|their|the|a|an))\s+((?:\w+\s+)?\w+)",
+                         hidden, re.I):
+        words = m.group(3).split()
+        lemma = words[-1].lower().rstrip("s") if words[-1].lower() not in PARTS else words[-1].lower()
+        if lemma in GENERIC:
+            continue
+        owner = m.group(1) or (m.group(2) if (m.group(2) or "").lower() in POSSESSIVE_PRONOUNS else "")
+        text = (f"{owner}'s " if m.group(1) else f"{owner} " if owner else "") + " ".join(words)
+        out.append((lemma, text.strip(), bool(owner), m.start(), "obj"))
+    return out
+
+
+def _score(c):
+    lemma, _text, owned, pos, dep = c
+    return ((2 if owned or lemma in PARTS else 0) + (1 if dep in ("dobj", "pobj") else 0)
+            - pos * 1e-4)
+
+
+def add_camera_moves(text):
+    """-> (new prompt, [per-shot report dicts]). Unchanged when there is no `[Shot N]`."""
+    marks = list(SHOT.finditer(text or ""))
+    if not marks:
+        return text, []
+    header = text[:marks[0].start()]
+    bodies = [text[m.end():(marks[i + 1].start() if i + 1 < len(marks) else len(text))]
+              for i, m in enumerate(marks)]
+    parts = [_split_sound(b) for b in bodies]
+    cands = [candidates(p[0]) for p in parts]
+    header_lemmas = {c[0] for c in candidates(header)}
+    need = max(2, math.ceil(0.6 * len(bodies))) if len(bodies) > 1 else 10 ** 9
+    seen = {}
+    for lst in cands:
+        for lemma in {c[0] for c in lst}:
+            seen[lemma] = seen.get(lemma, 0) + 1
+    constant = {l for l, n in seen.items() if n >= need} | header_lemmas
+    report, out, last = [], [header], None
+    for i, m in enumerate(marks):
+        picture, sound = parts[i]
+        entry = {"shot": int(m.group(1)), "move": None, "target": None, "why": ""}
+        pool = [c for c in cands[i] if c[0] not in constant]
+        if CAMERA.search(picture):
+            entry["why"] = "already has a camera move"
+        elif not pool:
+            entry["why"] = "nothing specific to aim at"
+        else:
+            best = max(pool, key=_score)
+            detail = best[2] or best[0] in PARTS
+            moves = MOVES_DETAIL if detail else MOVES_OTHER
+            move = next((mv for mv in moves if mv != last), moves[0])
+            target = _show(best[1])
+            if not re.match(r"(?:the|a|an|<Subject|his|her|their)\b", target, re.I) \
+                    and "'s " not in target:
+                target = "the " + target
+            entry.update(move=move.format(x=target), target=target)
+            last = move
+            core = picture.rstrip()
+            gap = picture[len(core):] or (" " if sound else "")
+            picture = core + ("" if core.endswith((".", "!", "?")) else ".") + " " \
+                + entry["move"] + gap
+        out.append(m.group(0) + picture + sound)
+        report.append(entry)
+    return "".join(out), report

tests/test_shot_camera.py

@@ -0,0 +1,102 @@
+"""Camera moves for [Shot N] blocks. Fixtures are neutral stand-ins with the real shape:
+<Subject N> tags everywhere, one shot per block, a music sentence at the end."""
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import shot_camera as sc  # noqa: E402
+
+HEAD = ("Two friends in a sunlit loft, <Subject 1> is a tall woman with red hair, "
+        "<Subject 2> is a man in a grey coat. Cinematic 35mm look. ")
+S1 = "[Shot 1] <Subject 1> is waving <Subject 1>'s hand, greeting <Subject 2>. "
+S2 = "[Shot 2] <Subject 1> bites <Subject 1>'s lower lip while <Subject 2> smiles at <Subject 1>. "
+S3 = "[Shot 3] <Subject 2> hands <Subject 1> a glass of wine across the table. <Subject 1> laughs. "
+MUSIC = "Upbeat jazz music with a soft bassline plays throughout."
+PROMPT = HEAD + S1 + S2 + S3 + MUSIC
+
+
+def test_no_shot_markers_leaves_the_prompt_alone():
+    assert sc.add_camera_moves("A woman walks. Music plays.") == ("A woman walks. Music plays.", [])
+
+
+def test_move_aims_at_a_part_of_the_subject_never_the_subject():
+    pytest.importorskip("spacy")
+    out, rep = sc.add_camera_moves(PROMPT)
+    assert rep[0]["target"] == "<Subject 1>'s hand"
+    assert rep[1]["target"] == "<Subject 1>'s lower lip"
+    for r in rep:
+        assert r["move"] and "<Subject 1>." not in r["move"].replace("'s", "")
+    assert "Close-up on <Subject 1>'s hand." in out
+
+
+def test_moves_sit_inside_their_shot_and_before_the_music():
+    pytest.importorskip("spacy")
+    out, rep = sc.add_camera_moves(PROMPT)
+    assert out.startswith(HEAD + "[Shot 1]")                 # intro untouched
+    assert out.endswith(" " + MUSIC)                         # music untouched, last
+    assert out.index(rep[2]["move"]) < out.index(MUSIC)
+    assert out.index("[Shot 2]") < out.index(rep[1]["move"]) < out.index("[Shot 3]")
+
+
+def test_a_shot_with_a_move_already_is_left_alone():
+    pytest.importorskip("spacy")
+    p = HEAD + "[Shot 1] The camera zooms in on <Subject 1>'s hand as it waves. " + S2
+    out, rep = sc.add_camera_moves(p)
+    assert rep[0]["move"] is None and "already" in rep[0]["why"]
+    assert "[Shot 1] The camera zooms in on <Subject 1>'s hand as it waves. [Shot 2]" in out
+
+
+def test_a_thing_in_most_shots_is_not_a_target():
+    pytest.importorskip("spacy")
+    p = (HEAD + "[Shot 1] <Subject 1> holds a rose. [Shot 2] <Subject 1> smells the rose. "
+         "[Shot 3] <Subject 2> hugs <Subject 1>'s shoulder near the rose.")
+    _out, rep = sc.add_camera_moves(p)
+    assert all(r["target"] != "the rose" for r in rep)
+    assert rep[2]["target"] == "<Subject 1>'s shoulder"
+
+
+def test_a_shot_with_only_subjects_gets_nothing():
+    pytest.importorskip("spacy")
+    _out, rep = sc.add_camera_moves(HEAD + "[Shot 1] <Subject 1> laughs with <Subject 2>. "
+                                    "[Shot 2] <Subject 2> nods at <Subject 1>.")
+    assert [r["move"] for r in rep] == [None, None]
+
+
+def test_the_same_move_does_not_repeat_between_shots():
+    pytest.importorskip("spacy")
+    p = HEAD + "[Shot 1] <Subject 1> raises <Subject 1>'s hand. [Shot 2] <Subject 2> lifts <Subject 2>'s chin."
+    _out, rep = sc.add_camera_moves(p)
+    assert rep[0]["move"].split(" ")[0] != rep[1]["move"].split(" ")[0]
+
+
+def test_without_spacy_the_rules_still_find_owned_parts(monkeypatch):
+    monkeypatch.setattr(sc, "_nlp", None)
+    monkeypatch.setattr(sc, "_tried", True)
+    _out, rep = sc.add_camera_moves(PROMPT)
+    assert rep[0]["target"] == "<Subject 1>'s hand"
+
+
+def test_tags_survive_the_round_trip():
+    pytest.importorskip("spacy")
+    out, _rep = sc.add_camera_moves(PROMPT)
+    assert out.count("<Subject 1>") >= PROMPT.count("<Subject 1>")
+    assert not any(n in out for n in sc.NAMES)
+
+
+def test_refiner_wrapper_returns_the_prompt_and_says_what_it_did(capsys):
+    pytest.importorskip("spacy")
+    import conditioning
+    out = conditioning.FunPackVideoRefinerV2._v2_camera_moves(PROMPT, "prompt")
+    assert "Close-up on <Subject 1>'s hand." in out
+    assert "camera moves: Active" in capsys.readouterr().out
+    assert conditioning.FunPackVideoRefinerV2._v2_camera_moves("no shots here", "prompt") == "no shots here"
+
+
+def test_refiner_wrapper_never_breaks_a_run(monkeypatch, capsys):
+    import conditioning
+    monkeypatch.setattr(sc, "add_camera_moves", lambda t: 1 / 0)
+    assert conditioning.FunPackVideoRefinerV2._v2_camera_moves(PROMPT, "prompt") == PROMPT
+    assert "camera moves: failed" in capsys.readouterr().out

web/funpack_studio.js

@@ -1138,6 +1138,11 @@ function openPanel(node) {
       settings.refiner.h3_phrase_emphasis = phraseEmph.checked;
     });
     body.append(row("Rating-driven phrase emphasis (H3)", phraseEmph));
+
+    const camMoves = el("input"); camMoves.type = "checkbox";
+    camMoves.checked = !!settings.refiner.camera_moves;
+    camMoves.addEventListener("change", () => { settings.refiner.camera_moves = camMoves.checked; });
+    body.append(row("Camera moves for [Shot N] (H3)", camMoves));
     body.append(el("div", "funpack-studio-hint",
       "EXPERIMENTAL, unvalidated. Boosts the attention paid to phrases the rating said were "
       + "MISSING, by biasing their attention logits in H3's packed stream. Needs a rated run "