diff --git a/conditioning.py b/conditioning.py index 48d97eef..083366ef 100644 --- a/conditioning.py +++ b/conditioning.py @@ -9276,14 +9276,14 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): return head.rsplit(" ", 1)[0].strip(), True @staticmethod - def _v2_camera_moves(text, where): + def _v2_camera_moves(text, where, seed=None, chance=1.0): """`text` with a camera move added to each [Shot N] block that lacks one.""" try: try: from . import shot_camera as _sc except ImportError: import shot_camera as _sc - out, report = _sc.add_camera_moves(text) + out, report = _sc.add_camera_moves(text, seed=seed, chance=chance) except Exception as e: # noqa: BLE001 print(f"[FunPackVideoRefinerV2] camera moves: failed | {where}: {e}; prompt left as written") return text @@ -10165,7 +10165,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): split_by_transitions=False, split_transition_placement="start", reference_injection=False, value_guidance=True, latent=None, seed_output_connected=False, steer_mode="relative", absolute_strength=0.6, - h3_phrase_emphasis=False, h3_phrase_variability=0.0, camera_moves=False, + h3_phrase_emphasis=False, h3_phrase_variability=0.0, camera_moves=False, camera_moves_chance=0.7, prompt_enhance=False, prompt_enhance_system="", prompt_enhance_reference_intro="", prompt_enhance_temperature=0.7, prompt_enhance_top_p=0.92, prompt_enhance_max_length=400, prompt_enhance_thinking=False, @@ -10842,7 +10842,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): # Camera moves for MiniMax's own `[Shot N]` blocks (not timeline scenes): after # shortcuts and $variables, before the enhancer, so the enhancer sees them. if camera_moves and not split_by_transitions: - prompt_to_encode = self._v2_camera_moves(prompt_to_encode, "prompt") + prompt_to_encode = self._v2_camera_moves(prompt_to_encode, "prompt", seed, camera_moves_chance) # ── prompt enhancer ──────────────────────────────────────────────────────────── # Runs BEFORE the prompt becomes conditioning, so what ComfyUI generates from is the @@ -11155,7 +11155,8 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): _resolve_variables(t, _prompt_variables)[0] for t in split_scene_texts ] if camera_moves and split_scene_texts: - split_scene_texts = [self._v2_camera_moves(t, f"scene {_i + 1}") + split_scene_texts = [self._v2_camera_moves(t, f"scene {_i + 1}", f"{seed}:{_i}", + camera_moves_chance) for _i, t in enumerate(split_scene_texts)] # Enhanced PER SCENE, not as one blob. The editor's scene list is # authoritative on count, so rewriting the scenes as a single paragraph @@ -14168,6 +14169,7 @@ class FunPackStudio: positive_conditioning=positive_conditioning, h3_phrase_emphasis=bool(rf.get("h3_phrase_emphasis", False)), camera_moves=bool(rf.get("camera_moves", False)), + camera_moves_chance=float(rf.get("camera_moves_chance", 0.7)), h3_phrase_variability=float(rf.get("h3_phrase_variability", 0.0) or 0.0), prompt_enhance=bool(rf.get("prompt_enhance", False)), prompt_enhance_output=bool(rf.get("prompt_enhance_output", False)), diff --git a/movie_editor/backend/settings_card.py b/movie_editor/backend/settings_card.py index 5ea2fa11..7b1fd408 100644 --- a/movie_editor/backend/settings_card.py +++ b/movie_editor/backend/settings_card.py @@ -253,6 +253,7 @@ _OWNED_BY = { "negative_erase_strength": ("negative_erase", ('true',)), "output_guidance_strength": ("output_guidance", None), "h3_phrase_variability": ("h3_phrase_emphasis", ('true',)), + "camera_moves_chance": ("camera_moves", ('true',)), "trajectory_guidance_strength": ("trajectory_guidance", None), "phase_scale": ("identity_transfer_enabled", None), "score_slider_strength": ("score_slider", None), @@ -339,6 +340,7 @@ _EDITOR_DEFAULTS = { "h3_block_repeat_last_steps": 0, "h3_phrase_emphasis": False, "camera_moves": False, + "camera_moves_chance": 0.7, "h3_phrase_variability": 0.0, "h3_repr_steering": False, "h3_repr_steering_block": "25", diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index f1ffe779..ea019169 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -121,6 +121,9 @@ hint: "EXPERIMENTAL, unvalidated. Boosts the attention paid to phrases the rating said were MISSING, by biasing their attention logits in H3's packed stream. Needs a rated run first. Turn it off if generations drift away from the prompt — and note it forces SLA to run dense." }, { name: "camera_moves", label: "Camera moves for [Shot N] (H3)", kind: "bool", default: false, hint: "Adds a camera move (close-up, focus, zoom in...) to each [Shot N] block that has none, aimed at the most specific thing in it — a part of the subject, never the subject itself. No language model; instant. Blocks that already name a camera move, and music text, are left alone." }, + { name: "camera_moves_chance", label: "Camera move chance", kind: "float", default: 0.7, min: 0.0, max: 1.0, step: 0.05, + dependsOn: "camera_moves", dependsVals: [true], + hint: "How often a [Shot N] block gets camera moves at all. Each one that does gets one to three (mostly one), sometimes travelling from the first thing to the last, sometimes ending with a pull-back — drawn per shot from the seed, so no pattern repeats in every prompt. 1 = every eligible shot." }, { name: "h3_phrase_variability", label: "Phrase emphasis variability", kind: "float", default: 0.0, min: 0.0, max: 1.0, step: 0.05, dependsOn: "h3_phrase_emphasis", dependsVals: [true], hint: "0 = only what the rating has cemented gets extra attention, nothing else. 1 = that emphasis is switched off entirely, so untrained phrasing and actions compete on equal footing and can bleed into the scene — including ones you have rated against before. Same bias channel as the toggle above, just scaled down." }, diff --git a/shot_camera.py b/shot_camera.py index 37314d08..42cf3485 100644 --- a/shot_camera.py +++ b/shot_camera.py @@ -9,6 +9,7 @@ the first shot) is a constant, not a target. Sound text (music, score...) is lef """ import math +import random import re SHOT = re.compile(r"\[\s*Shot\s+(\d+)\s*\]", re.I) @@ -31,10 +32,19 @@ PARTS = {"lip", "mouth", "tongue", "tooth", "teeth", "eye", "eyes", "face", "che "skin", "expression", "smile", "gaze", "back", "chin", "brow"} PLACES = {"room", "street", "kitchen", "bedroom", "bed", "table", "floor", "stage", "hall", "garden", "beach", "forest", "city", "car", "office", "bathroom", "field"} -MOVES_DETAIL = ("Close-up on {x}.", "Focus at {x}.", "Zoom in at {x}.") -MOVES_OTHER = ("Zoom in at {x}.", "Move around {x}.", "Rotate the camera to {x}.") -MOVES_TRAVEL = ("Camera moves from {x} to {y}.", "Pan from {x} to {y}.", - "Focus shifts from {x} to {y}.") +MOVES_DETAIL = ("Camera moves into a close-up on {x}.", "Camera focuses at {x}.", + "Camera zooms in at {x}.") +MOVES_OTHER = ("Camera zooms in at {x}.", "Camera moves around {x}.", + "Camera rotates to {x}.") +MOVES_TRAVEL = ("Camera moves from {x} to {y}.", "Camera pans from {x} to {y}.", + "Camera shifts focus from {x} to {y}.") +# Chained moves (varied mode): a follow-up to a first move, and a closing pull-back. +MOVES_THEN = ("Then the camera moves to {y}.", "Then the camera pans to {y}.", + "Then the camera focuses at {y}.") +MOVES_FINISH = ("Then the camera zooms out.", "Then the camera pulls back.", + "Then the camera pulls back to a wider view.") +COUNT_WEIGHTS = (6, 3, 1) # one move / two / three, in a shot that gets any +TRAVEL_SHARE = 0.5 # of one-move shots with two topics, how many travel X -> Y DETERMINERS = {"the", "a", "an", "this", "that", "these", "those"} POSSESSIVE_PRONOUNS = {"his", "her", "their", "its", "my", "your", "our"} @@ -153,8 +163,51 @@ def _topics(pool): return out -def add_camera_moves(text): - """-> (new prompt, [per-shot report dicts]). Unchanged when there is no `[Shot N]`.""" +def _pick(options, last, rng): + """One of `options`, never the one the previous shot opened with when others exist.""" + pool = [o for o in options if o != last] or list(options) + return rng.choice(pool) if rng else pool[0] + + +def _plan(pool, last, rng): + """-> (moves text, target description, opening template). Fixed and minimal without + `rng` (one move, X -> Y when the shot has two topics); with it, how many moves, whether + they travel and which words are all drawn, so no shot pattern repeats by construction.""" + topics = _topics(pool) + best = max(pool, key=_score) + if rng is None: + k, travel = 1, len(topics) >= 2 + else: + k = rng.choices((1, 2, 3), COUNT_WEIGHTS)[0] + travel = k == 1 and len(topics) >= 2 and rng.random() < TRAVEL_SHARE + if travel: + x, y = _name(topics[0]), _name(topics[-1]) + move = _pick(MOVES_TRAVEL, last, rng) + return move.format(x=x, y=y), f"{x} -> {y}", move + first = topics[0] if (k > 1 and topics) else best + moves = MOVES_DETAIL if (first[2] or first[0] in PARTS) else MOVES_OTHER + opening = _pick(moves, last, rng) + target = _name(first) + sentences = [opening.format(x=target)] + desc = target + if k > 1: + if len(topics) >= 2: + y = _name(topics[-1]) + sentences.append(_pick(MOVES_THEN, None, rng).format(y=y)) + desc += f" -> {y}" + else: + sentences.append(_pick(MOVES_FINISH, None, rng)) + if k > 2 and len(topics) >= 2: + sentences.append(_pick(MOVES_FINISH, None, rng)) + return " ".join(sentences), desc, opening + + +def add_camera_moves(text, seed=None, chance=1.0): + """-> (new prompt, [per-shot report dicts]). Unchanged when there is no `[Shot N]`. + + `seed=None`: one plain move per shot (deterministic). With a `seed`, each shot is left + alone with probability 1 - `chance`, and otherwise gets one to three moves, drawn per + shot from the seed, so the same prompt and seed always give the same text.""" marks = list(SHOT.finditer(text or "")) if not marks: return text, [] @@ -174,28 +227,22 @@ def add_camera_moves(text): for i, m in enumerate(marks): picture, sound = parts[i] entry = {"shot": int(m.group(1)), "move": None, "target": None, "why": ""} + rng = random.Random(f"{seed}:{i}") if seed is not None else None pool = [c for c in cands[i] if c[0] not in constant] if CAMERA.search(picture): entry["why"] = "already has a camera move" elif not pool: entry["why"] = "nothing specific to aim at" + elif rng is not None and rng.random() >= chance: + entry["why"] = "left as written by chance" else: - topics = _topics(pool) - if len(topics) >= 2: - x, y = _name(topics[0]), _name(topics[-1]) - move = next((mv for mv in MOVES_TRAVEL if mv != last), MOVES_TRAVEL[0]) - entry.update(move=move.format(x=x, y=y), target=f"{x} -> {y}") - else: - best = max(pool, key=_score) - moves = MOVES_DETAIL if (best[2] or best[0] in PARTS) else MOVES_OTHER - move = next((mv for mv in moves if mv != last), moves[0]) - target = _name(best) - entry.update(move=move.format(x=target), target=target) - last = move + moves, desc, opening = _plan(pool, last, rng) + entry.update(move=moves, target=desc) + last = opening core = picture.rstrip() gap = picture[len(core):] or (" " if sound else "") picture = core + ("" if core.endswith((".", "!", "?")) else ".") + " " \ - + entry["move"] + gap + + moves + gap out.append(m.group(0) + picture + sound) report.append(entry) return "".join(out), report diff --git a/tests/test_shot_camera.py b/tests/test_shot_camera.py index 27d28a8f..6f045645 100644 --- a/tests/test_shot_camera.py +++ b/tests/test_shot_camera.py @@ -29,7 +29,7 @@ def test_move_aims_at_a_part_of_the_subject_never_the_subject(): assert rep[1]["target"] == "'s lower lip" for r in rep: assert r["move"] and "." not in r["move"].replace("'s", "") - assert "Close-up on 's hand." in out + assert "Camera moves into a close-up on 's hand." in out def test_moves_sit_inside_their_shot_and_before_the_music(): @@ -69,7 +69,7 @@ def test_the_same_move_does_not_repeat_between_shots(): pytest.importorskip("spacy") p = HEAD + "[Shot 1] raises 's hand. [Shot 2] lifts 's chin." _out, rep = sc.add_camera_moves(p) - assert rep[0]["move"].split(" ")[0] != rep[1]["move"].split(" ")[0] + assert rep[0]["move"].split(" ")[1:3] != rep[1]["move"].split(" ")[1:3] def test_without_spacy_the_rules_still_find_owned_parts(monkeypatch): @@ -90,7 +90,7 @@ def test_refiner_wrapper_returns_the_prompt_and_says_what_it_did(capsys): pytest.importorskip("spacy") import conditioning out = conditioning.FunPackVideoRefinerV2._v2_camera_moves(PROMPT, "prompt") - assert "Close-up on 's hand." in out + assert "Camera moves into a close-up on 's hand." in out assert "camera moves: Active" in capsys.readouterr().out assert conditioning.FunPackVideoRefinerV2._v2_camera_moves("no shots here", "prompt") == "no shots here" @@ -117,3 +117,38 @@ def test_three_topics_go_from_the_first_to_the_last(): "then opens the window.") _out, rep = sc.add_camera_moves(p) assert "chin" in rep[0]["move"] and "window" in rep[0]["move"] and "lamp" not in rep[0]["move"] + + +TWO = HEAD + "[Shot 1] touches 's chin, then reaches for the lamp." +ONE = HEAD + "[Shot 1] touches 's chin." + + +def test_varied_moves_are_reproducible_from_the_seed(): + pytest.importorskip("spacy") + assert sc.add_camera_moves(PROMPT, seed=7) == sc.add_camera_moves(PROMPT, seed=7) + assert len({sc.add_camera_moves(PROMPT, seed=s)[0] for s in range(20)}) > 5 + + +def test_chance_zero_leaves_every_shot_alone_and_says_why(): + pytest.importorskip("spacy") + out, rep = sc.add_camera_moves(PROMPT, seed=1, chance=0.0) + assert out == PROMPT and all("chance" in r["why"] for r in rep) + + +def test_no_single_pattern_is_every_prompt(): + """Over many seeds a shot with two topics gets one, two and three moves, some travelling + X -> Y and some not: 'zoom in, move to Y, zoom out' is never the rule.""" + pytest.importorskip("spacy") + sentences, travel = set(), set() + for seed in range(200): + _out, rep = sc.add_camera_moves(TWO, seed=seed) + sentences.add(rep[0]["move"].count(". ") + 1) + travel.add(" from " in rep[0]["move"]) + assert sentences == {1, 2, 3} and travel == {True, False} + + +def test_a_single_topic_shot_can_still_chain_with_a_pull_back(): + pytest.importorskip("spacy") + moves = {sc.add_camera_moves(ONE, seed=s)[1][0]["move"] for s in range(60)} + assert any("Then the camera" in m for m in moves) and any("Then" not in m for m in moves) + assert all(m.startswith("Camera") for m in moves)