diff --git a/CHANGELOG.md b/CHANGELOG.md index d9cfa0eb..4bb8705e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] ### Added +- Camera moves for `[Shot N]` prompts (Engine settings, H3, off by default): each shot without a camera move gets one — close-up, focus, zoom in, move around — aimed at the most specific thing in it, never a bare ``. No language model; uses spaCy when installed (`spacy` in requirements). - H3 rating picker: "Disliked — bad image" and "Disliked — bad composition". Everything else reads them as Disliked; shot memory, decisiveness, late-branch and STAS learn only from composition blame, and DynaShift discards the latent when composition was the fault. - Late-branch guidance now measures how hard it actually pushed; a rating counts up to 2x (down to 0.25x) when that run was unusually strong (weak) for the key, after 3 runs of history. diff --git a/conditioning.py b/conditioning.py index 687e2c3c..48d97eef 100644 --- a/conditioning.py +++ b/conditioning.py @@ -9275,6 +9275,29 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): return head[:cut + 1].strip(), True return head.rsplit(" ", 1)[0].strip(), True + @staticmethod + def _v2_camera_moves(text, where): + """`text` with a camera move added to each [Shot N] block that lacks one.""" + try: + try: + from . import shot_camera as _sc + except ImportError: + import shot_camera as _sc + out, report = _sc.add_camera_moves(text) + except Exception as e: # noqa: BLE001 + print(f"[FunPackVideoRefinerV2] camera moves: failed | {where}: {e}; prompt left as written") + return text + if not report: + return text + added = [r for r in report if r["move"]] + skipped = "; ".join(f"shot {r['shot']} {r['why']}" for r in report if not r["move"]) + print(f"[FunPackVideoRefinerV2] camera moves: " + + (f"Active | {where}: {len(added)} of {len(report)} shot(s) — " + + ", ".join(f"shot {r['shot']} {r['move'].rstrip('.')}" for r in added) + if added else f"Inactive | {where}: no shot got one") + + (f" ({skipped})" if skipped else "")) + return out + def _v2_note_enhanced(self, item): """Record one rewrite for the node's ui output AND publish it live, so the Editor can show it while the video is still rendering (and the user can stop a bad one).""" @@ -10142,7 +10165,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): split_by_transitions=False, split_transition_placement="start", reference_injection=False, value_guidance=True, latent=None, seed_output_connected=False, steer_mode="relative", absolute_strength=0.6, - h3_phrase_emphasis=False, h3_phrase_variability=0.0, + h3_phrase_emphasis=False, h3_phrase_variability=0.0, camera_moves=False, prompt_enhance=False, prompt_enhance_system="", prompt_enhance_reference_intro="", prompt_enhance_temperature=0.7, prompt_enhance_top_p=0.92, prompt_enhance_max_length=400, prompt_enhance_thinking=False, @@ -10816,6 +10839,11 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): if _prompt_variables: prompt_to_encode = _resolve_variables(prompt_to_encode, _prompt_variables)[0] + # Camera moves for MiniMax's own `[Shot N]` blocks (not timeline scenes): after + # shortcuts and $variables, before the enhancer, so the enhancer sees them. + if camera_moves and not split_by_transitions: + prompt_to_encode = self._v2_camera_moves(prompt_to_encode, "prompt") + # ── prompt enhancer ──────────────────────────────────────────────────────────── # Runs BEFORE the prompt becomes conditioning, so what ComfyUI generates from is the # enhanced text and nothing downstream has to know this happened. After shortcuts and @@ -11126,6 +11154,9 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): split_scene_texts = [ _resolve_variables(t, _prompt_variables)[0] for t in split_scene_texts ] + if camera_moves and split_scene_texts: + split_scene_texts = [self._v2_camera_moves(t, f"scene {_i + 1}") + for _i, t in enumerate(split_scene_texts)] # Enhanced PER SCENE, not as one blob. The editor's scene list is # authoritative on count, so rewriting the scenes as a single paragraph # and re-splitting could return a different number of scenes and desync @@ -14136,6 +14167,7 @@ class FunPackStudio: refinement_key_input=refinement_key_input, positive_conditioning=positive_conditioning, h3_phrase_emphasis=bool(rf.get("h3_phrase_emphasis", False)), + camera_moves=bool(rf.get("camera_moves", False)), h3_phrase_variability=float(rf.get("h3_phrase_variability", 0.0) or 0.0), prompt_enhance=bool(rf.get("prompt_enhance", False)), prompt_enhance_output=bool(rf.get("prompt_enhance_output", False)), diff --git a/movie_editor/backend/settings_card.py b/movie_editor/backend/settings_card.py index 5901cff5..5ea2fa11 100644 --- a/movie_editor/backend/settings_card.py +++ b/movie_editor/backend/settings_card.py @@ -338,6 +338,7 @@ _EDITOR_DEFAULTS = { "h3_block_repeat_video_only": False, "h3_block_repeat_last_steps": 0, "h3_phrase_emphasis": False, + "camera_moves": False, "h3_phrase_variability": 0.0, "h3_repr_steering": False, "h3_repr_steering_block": "25", diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index fb016a6f..f1ffe779 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -119,6 +119,8 @@ hint: "Puts each word back to its original strength after the change, so the result reads as 'less of that thing' rather than 'quieter prompt'. Off is the raw result." }, { name: "h3_phrase_emphasis", label: "Rating-driven phrase emphasis (H3)", kind: "bool", default: false, hint: "EXPERIMENTAL, unvalidated. Boosts the attention paid to phrases the rating said were MISSING, by biasing their attention logits in H3's packed stream. Needs a rated run first. Turn it off if generations drift away from the prompt — and note it forces SLA to run dense." }, + { name: "camera_moves", label: "Camera moves for [Shot N] (H3)", kind: "bool", default: false, + hint: "Adds a camera move (close-up, focus, zoom in...) to each [Shot N] block that has none, aimed at the most specific thing in it — a part of the subject, never the subject itself. No language model; instant. Blocks that already name a camera move, and music text, are left alone." }, { name: "h3_phrase_variability", label: "Phrase emphasis variability", kind: "float", default: 0.0, min: 0.0, max: 1.0, step: 0.05, dependsOn: "h3_phrase_emphasis", dependsVals: [true], hint: "0 = only what the rating has cemented gets extra attention, nothing else. 1 = that emphasis is switched off entirely, so untrained phrasing and actions compete on equal footing and can bleed into the scene — including ones you have rated against before. Same bias channel as the toggle above, just scaled down." }, diff --git a/requirements.txt b/requirements.txt index c3377240..434bb5a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,6 @@ gguf Pillow insightface opencv-python-headless +# Picks the target of a camera move in [Shot N] prompts. Optional: without it (or without +# its model, `python -m spacy download en_core_web_sm`) simpler word rules are used. +spacy diff --git a/shot_camera.py b/shot_camera.py new file mode 100644 index 00000000..594bb582 --- /dev/null +++ b/shot_camera.py @@ -0,0 +1,178 @@ +"""Add a camera move to each `[Shot N]` block of an H3 prompt that has none, aimed at the +most specific thing in that shot. No language model: a small spaCy tagger (rules-only +fallback when it is not installed) picks the target, a few words pick the move. + +`[Shot N]` here is MiniMax's own in-prompt shot, never a timeline scene. `` tags +are the subjects every shot is about, so they are never a target on their own; a part or +belonging of one ("'s hand") is. A thing named in most shots (or in the text before +the first shot) is a constant, not a target. Sound text (music, score...) is left alone. +""" + +import math +import re + +SHOT = re.compile(r"\[\s*Shot\s+(\d+)\s*\]", re.I) +SUBJECT = re.compile(r"<\s*Subject\s*(\d+)\s*>", re.I) +# A `` tag confuses the tagger ("Subject1 bites" reads as a noun); a real-looking +# rare name parses like a person. Swapped in before tagging, swapped back after. +NAMES = ("Mariel", "Tobias", "Odette", "Rurik", "Sabine", "Teodor", "Ilsa", "Corwin", "Yvette") +STAND_IN = re.compile(r"\b(" + "|".join(NAMES) + r")\b") +AUDIO = re.compile(r"\b(music|soundtrack|score|melody|instrumental|beat|bassline|synth|" + r"song|tempo|audio|ambient sound|sound of)\b", re.I) +CAMERA = re.compile(r"\b(camera|zoom(?:s|ing)?|close-?up|pan(?:s|ning)?|dolly|tracking|" + r"rack focus|focus(?:es|ing)? (?:on|at)|tilt(?:s|ing)?|rotate|orbit|" + r"push[- ]in|pull[- ]back|wide shot|handheld|crane)\b", re.I) +GENERIC = {"camera", "video", "scene", "shot", "frame", "background", "foreground", "view", + "image", "screen", "moment", "time", "way", "side", "front", "middle", "one", + "other", "something", "everything", "thing", "music", "sound", "style", "lighting", + "light", "atmosphere", "mood", "setting"} +PARTS = {"lip", "mouth", "tongue", "tooth", "teeth", "eye", "eyes", "face", "cheek", "hand", + "finger", "hair", "neck", "shoulder", "waist", "hip", "leg", "foot", "feet", "arm", + "skin", "expression", "smile", "gaze", "back", "chin", "brow"} +PLACES = {"room", "street", "kitchen", "bedroom", "bed", "table", "floor", "stage", "hall", + "garden", "beach", "forest", "city", "car", "office", "bathroom", "field"} +MOVES_DETAIL = ("Close-up on {x}.", "Focus at {x}.", "Zoom in at {x}.") +MOVES_OTHER = ("Zoom in at {x}.", "Move around {x}.", "Rotate the camera to {x}.") +DETERMINERS = {"the", "a", "an", "this", "that", "these", "those"} +POSSESSIVE_PRONOUNS = {"his", "her", "their", "its", "my", "your", "our"} + +_nlp = None +_tried = False + + +def _spacy(): + """The small English tagger, or None (said once) when it is not installed.""" + global _nlp, _tried + if not _tried: + _tried = True + try: + import spacy + _nlp = spacy.load("en_core_web_sm") + except Exception as e: # noqa: BLE001 + try: + import funpack_log as _log + except ImportError: + from . import funpack_log as _log + _log.failed("FunPackCameraMoves", "spaCy tagger", e, + "using the simple word rules; run `pip install spacy` and " + "`python -m spacy download en_core_web_sm` for better targets") + return _nlp + + +def _hide(text): + return SUBJECT.sub(lambda m: NAMES[(int(m.group(1)) - 1) % len(NAMES)], text) + + +def _show(text): + return STAND_IN.sub(lambda m: f"", text) + + +def _is_subject(text): + return bool(STAND_IN.fullmatch(re.sub(r"^(?:the\s+)", "", text.strip(), flags=re.I))) + + +def _split_sound(body): + """-> (picture text, sound text): everything from the first sentence about sound on.""" + for m in re.finditer(r"[^.!?]+[.!?]?\s*", body): + if AUDIO.search(m.group(0)): + return body[:m.start()], body[m.start():] + return body, "" + + +def _phrase(tok): + """A chunk head with the words that name it ("Subject1's hand", "red lips").""" + keep = {tok} + for c in tok.children: + if c.dep_ in ("poss", "compound", "amod") and c.i < tok.i: + keep.add(c) + if c.dep_ == "poss": + keep.update(g for g in c.children if g.dep_ == "case") + toks = sorted(keep, key=lambda t: t.i) + text = " ".join(t.text for t in toks).strip().replace(" 's", "'s").replace(" ’s", "’s") + return text, any(c.dep_ == "poss" for c in tok.children) + + +def candidates(picture): + """[(lemma, phrase, owned, position, dep)] for the nouns of one shot's picture text.""" + hidden = _hide(picture) + nlp = _spacy() + out = [] + if nlp is not None: + doc = nlp(hidden) + for chunk in doc.noun_chunks: + root = chunk.root + prev = doc[root.i - 1] if root.i else None + if prev is not None and STAND_IN.fullmatch(prev.text) and prev.dep_ != "poss": + continue # " laughs": the tagger misread a verb + if root.pos_ in ("PRON", "NUM") or _is_subject(chunk.text): + continue + lemma = root.lemma_.lower() + if root.dep_ == "pobj" and root.head.lower_ == "of": + continue # "a glass of wine": the glass is the thing + text, owned = _phrase(root) + if _is_subject(text) or lemma in GENERIC or STAND_IN.fullmatch(root.text): + continue + out.append((lemma, text, owned, chunk.start, root.dep_)) + return out + for m in re.finditer(rf"(?:({'|'.join(NAMES)})'s|\b(his|her|their|the|a|an))\s+((?:\w+\s+)?\w+)", + hidden, re.I): + words = m.group(3).split() + lemma = words[-1].lower().rstrip("s") if words[-1].lower() not in PARTS else words[-1].lower() + if lemma in GENERIC: + continue + owner = m.group(1) or (m.group(2) if (m.group(2) or "").lower() in POSSESSIVE_PRONOUNS else "") + text = (f"{owner}'s " if m.group(1) else f"{owner} " if owner else "") + " ".join(words) + out.append((lemma, text.strip(), bool(owner), m.start(), "obj")) + return out + + +def _score(c): + lemma, _text, owned, pos, dep = c + return ((2 if owned or lemma in PARTS else 0) + (1 if dep in ("dobj", "pobj") else 0) + - pos * 1e-4) + + +def add_camera_moves(text): + """-> (new prompt, [per-shot report dicts]). Unchanged when there is no `[Shot N]`.""" + marks = list(SHOT.finditer(text or "")) + if not marks: + return text, [] + header = text[:marks[0].start()] + bodies = [text[m.end():(marks[i + 1].start() if i + 1 < len(marks) else len(text))] + for i, m in enumerate(marks)] + parts = [_split_sound(b) for b in bodies] + cands = [candidates(p[0]) for p in parts] + header_lemmas = {c[0] for c in candidates(header)} + need = max(2, math.ceil(0.6 * len(bodies))) if len(bodies) > 1 else 10 ** 9 + seen = {} + for lst in cands: + for lemma in {c[0] for c in lst}: + seen[lemma] = seen.get(lemma, 0) + 1 + constant = {l for l, n in seen.items() if n >= need} | header_lemmas + report, out, last = [], [header], None + for i, m in enumerate(marks): + picture, sound = parts[i] + entry = {"shot": int(m.group(1)), "move": None, "target": None, "why": ""} + pool = [c for c in cands[i] if c[0] not in constant] + if CAMERA.search(picture): + entry["why"] = "already has a camera move" + elif not pool: + entry["why"] = "nothing specific to aim at" + else: + best = max(pool, key=_score) + detail = best[2] or best[0] in PARTS + moves = MOVES_DETAIL if detail else MOVES_OTHER + move = next((mv for mv in moves if mv != last), moves[0]) + target = _show(best[1]) + if not re.match(r"(?:the|a|an| tags everywhere, one shot per block, a music sentence at the end.""" +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import shot_camera as sc # noqa: E402 + +HEAD = ("Two friends in a sunlit loft, is a tall woman with red hair, " + " is a man in a grey coat. Cinematic 35mm look. ") +S1 = "[Shot 1] is waving 's hand, greeting . " +S2 = "[Shot 2] bites 's lower lip while smiles at . " +S3 = "[Shot 3] hands a glass of wine across the table. laughs. " +MUSIC = "Upbeat jazz music with a soft bassline plays throughout." +PROMPT = HEAD + S1 + S2 + S3 + MUSIC + + +def test_no_shot_markers_leaves_the_prompt_alone(): + assert sc.add_camera_moves("A woman walks. Music plays.") == ("A woman walks. Music plays.", []) + + +def test_move_aims_at_a_part_of_the_subject_never_the_subject(): + pytest.importorskip("spacy") + out, rep = sc.add_camera_moves(PROMPT) + assert rep[0]["target"] == "'s hand" + assert rep[1]["target"] == "'s lower lip" + for r in rep: + assert r["move"] and "." not in r["move"].replace("'s", "") + assert "Close-up on 's hand." in out + + +def test_moves_sit_inside_their_shot_and_before_the_music(): + pytest.importorskip("spacy") + out, rep = sc.add_camera_moves(PROMPT) + assert out.startswith(HEAD + "[Shot 1]") # intro untouched + assert out.endswith(" " + MUSIC) # music untouched, last + assert out.index(rep[2]["move"]) < out.index(MUSIC) + assert out.index("[Shot 2]") < out.index(rep[1]["move"]) < out.index("[Shot 3]") + + +def test_a_shot_with_a_move_already_is_left_alone(): + pytest.importorskip("spacy") + p = HEAD + "[Shot 1] The camera zooms in on 's hand as it waves. " + S2 + out, rep = sc.add_camera_moves(p) + assert rep[0]["move"] is None and "already" in rep[0]["why"] + assert "[Shot 1] The camera zooms in on 's hand as it waves. [Shot 2]" in out + + +def test_a_thing_in_most_shots_is_not_a_target(): + pytest.importorskip("spacy") + p = (HEAD + "[Shot 1] holds a rose. [Shot 2] smells the rose. " + "[Shot 3] hugs 's shoulder near the rose.") + _out, rep = sc.add_camera_moves(p) + assert all(r["target"] != "the rose" for r in rep) + assert rep[2]["target"] == "'s shoulder" + + +def test_a_shot_with_only_subjects_gets_nothing(): + pytest.importorskip("spacy") + _out, rep = sc.add_camera_moves(HEAD + "[Shot 1] laughs with . " + "[Shot 2] nods at .") + assert [r["move"] for r in rep] == [None, None] + + +def test_the_same_move_does_not_repeat_between_shots(): + pytest.importorskip("spacy") + p = HEAD + "[Shot 1] raises 's hand. [Shot 2] lifts 's chin." + _out, rep = sc.add_camera_moves(p) + assert rep[0]["move"].split(" ")[0] != rep[1]["move"].split(" ")[0] + + +def test_without_spacy_the_rules_still_find_owned_parts(monkeypatch): + monkeypatch.setattr(sc, "_nlp", None) + monkeypatch.setattr(sc, "_tried", True) + _out, rep = sc.add_camera_moves(PROMPT) + assert rep[0]["target"] == "'s hand" + + +def test_tags_survive_the_round_trip(): + pytest.importorskip("spacy") + out, _rep = sc.add_camera_moves(PROMPT) + assert out.count("") >= PROMPT.count("") + assert not any(n in out for n in sc.NAMES) + + +def test_refiner_wrapper_returns_the_prompt_and_says_what_it_did(capsys): + pytest.importorskip("spacy") + import conditioning + out = conditioning.FunPackVideoRefinerV2._v2_camera_moves(PROMPT, "prompt") + assert "Close-up on 's hand." in out + assert "camera moves: Active" in capsys.readouterr().out + assert conditioning.FunPackVideoRefinerV2._v2_camera_moves("no shots here", "prompt") == "no shots here" + + +def test_refiner_wrapper_never_breaks_a_run(monkeypatch, capsys): + import conditioning + monkeypatch.setattr(sc, "add_camera_moves", lambda t: 1 / 0) + assert conditioning.FunPackVideoRefinerV2._v2_camera_moves(PROMPT, "prompt") == PROMPT + assert "camera moves: failed" in capsys.readouterr().out diff --git a/web/funpack_studio.js b/web/funpack_studio.js index 4fbc2c53..0cba7836 100644 --- a/web/funpack_studio.js +++ b/web/funpack_studio.js @@ -1138,6 +1138,11 @@ function openPanel(node) { settings.refiner.h3_phrase_emphasis = phraseEmph.checked; }); body.append(row("Rating-driven phrase emphasis (H3)", phraseEmph)); + + const camMoves = el("input"); camMoves.type = "checkbox"; + camMoves.checked = !!settings.refiner.camera_moves; + camMoves.addEventListener("change", () => { settings.refiner.camera_moves = camMoves.checked; }); + body.append(row("Camera moves for [Shot N] (H3)", camMoves)); body.append(el("div", "funpack-studio-hint", "EXPERIMENTAL, unvalidated. Boosts the attention paid to phrases the rating said were " + "MISSING, by biasing their attention logits in H3's packed stream. Needs a rated run "