diff --git a/CHANGELOG.md b/CHANGELOG.md index 76a9e548..e98154c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ ### Added +- **Prompt enhancer: Dynamic style.** A Style choice in the Enhance panel (default: unchanged). Dynamic keeps every rule of the default but asks for a concrete action beat, a change over time and secondary motion in each shot. Instructions you type still override it. - **Shortcuts: find, replace and common phrases.** The ⋯ menu in Shortcuts has Find and replace (search the text of every shortcut, preview each change, match case or not, then change them all in one go, or remove the phrase) and Common phrases (the phrases used in more than one shortcut, with counts; pick one to correct it everywhere). Triggers are never touched. - **Sparse attention method "veda"** (H3 loader, SLA on, method veda; Experimental, UNTESTED on a GPU). A small learned predictor (Veda, arXiv 2605.30325) picks which 128-token tiles of the attention map matter and an INT8 Triton kernel computes only those, at the predictor's own trained budget (90% for text-to-video). It needs CUDA and Triton and a predictor file in ComfyUI/models/veda, chosen in "veda_predictor" (auto takes the text-to-video file, else the first). Works with LoRA (it sees q/k/v after the LoRA), MXFP8 (weights only: q/k/v stay bf16), Two stages (a size the predictor was not trained for uses the nearest tile plan, and the log says so), dense steps and the attention backend, which keeps the text refiner, masked calls and short sequences. sla_sparsity, the rising share and the references choice are not read in this first version. When Veda cannot run, the status says why and the chosen backend is installed on its own. One log line per run in the SLA line's wording (calls, tile pairs kept, asked vs real sparsity, share of full attention computed, steps at full attention). The Triton kernel is checked against an exact reference on the GPU before first use; the torch half (tiles, predictor, selection, the override on a real tiny H3) is tested on CPU. diff --git a/app/ui/features/composer/enhance.js b/app/ui/features/composer/enhance.js index 8885caff..57d89c2f 100644 --- a/app/ui/features/composer/enhance.js +++ b/app/ui/features/composer/enhance.js @@ -34,6 +34,7 @@ export function enhance(app, own) { if (!sl) return page.set([c.emptyState.default({ icon: "✎", title: "No prompt enhancer here", hint: "Pick a preset that has one, or add the “FunUI Enhance Prompt” node in Settings ▸ Models & Pipeline." })]); const v = sl.inputs || {}, val = (n, d) => (v[n] === undefined ? d : v[n]); const on = val("enabled", false), greedy = val("greedy", false); + const styles = defaults && defaults.styles ? Object.keys(defaults.styles) : ["default", "dynamic"], style = styles.includes(val("style", "default")) ? val("style", "default") : "default"; page.set([ c.label.section({ text: "Prompt enhancer" }), c.hint.default({ text: "A language model rewrites the finished prompt before it is encoded. Off, or if it fails, your prompt is used as typed. (word:1.5) and [a@2-4] markup does not survive a rewrite." }), @@ -41,7 +42,8 @@ export function enhance(app, own) { ...(!on ? readout() : [ pick("encoder_file", "Encoder", "The model's own encoder", "The text encoder that writes the rewrite. Models like H3's own encoder cannot write prose: pick a chat model such as Gemma 4 here."), ...((val("encoder_file", "") && choices("encoder_family").length) ? [pick("encoder_family", "Encoder family", "", "Which family that file is for.", "stable_diffusion")] : []), - field("Instructions", c.textarea.md({ label: "Instructions", rows: 6, value: val("instructions", ""), placeholder: defaults ? defaults.instructions : "(built-in instructions)", onCommit: (x) => set("instructions", x) }), "Empty uses the built-in instructions."), + field("Style", c.select.md({ label: "Style", value: style, options: styles.map((n) => ({ value: n, label: n === "dynamic" ? "Dynamic (more motion)" : "Default" })), onChange: (x) => set("style", x) }), "Dynamic asks for a concrete action beat, a change over time and secondary motion in every shot. Instructions typed below override it."), + field("Instructions", c.textarea.md({ label: "Instructions", rows: 6, value: val("instructions", ""), placeholder: defaults ? (defaults.styles || {})[style] || defaults.instructions : "(built-in instructions)", onCommit: (x) => set("instructions", x) }), "Empty uses the built-in instructions of the Style above."), num("Max tokens", val("max_length", 400), 32, 4096, 8, "max_length", "The longest the rewrite may be."), c.toggle.default({ label: "Use the picture", hint: "Sends the scene's picture to the model along with the prompt.", checked: val("use_image", true), onChange: (x) => set("use_image", x) }), c.toggle.default({ label: "Thinking", hint: "Let a thinking model reason first. Its reasoning eats the token budget.", checked: val("thinking", false), onChange: (x) => set("thinking", x) }), diff --git a/app/ui/features/composer/enhance.test.js b/app/ui/features/composer/enhance.test.js index d81c7baf..b2eb2b99 100644 --- a/app/ui/features/composer/enhance.test.js +++ b/app/ui/features/composer/enhance.test.js @@ -63,3 +63,18 @@ test("the Encoder pick lists the encoder files, and its family shows only once a await new Promise((r) => setTimeout(r, 0)); assert.match(again.node.textContent, /Encoder family/); }); + +test("Style picks the built-in instructions shown as the placeholder, and choosing Dynamic saves it", async () => { + const saved = []; + const slot = { id: "e", node: "FunUIEnhancePrompt", inputs: { enabled: true } }; + const ps = { slots: () => [slot], ensureLoaded: async () => {}, save: async (...a) => { saved.push(a); }, subscribe: () => () => {} }; + const api = { enhancerEncoders: async () => ({ files: [], families: [] }), enhancerRuns: async () => ({ runs: [] }), + enhancerDefaults: async () => ({ instructions: "DEFAULT TEXT", reference_intro: "", styles: { default: "DEFAULT TEXT", dynamic: "DYNAMIC TEXT" } }) }; + const page = enhance({ dom: createDom(), pipeline: ps, api }, () => {}); + document.body.append(page.node); + await new Promise((r) => setTimeout(r, 0)); + assert.match(page.node.textContent, /Style/); + assert.match(page.node.textContent, /Dynamic \(more motion\)/); + const area = [...page.node.querySelectorAll("textarea")].find((t) => t.placeholder === "DEFAULT TEXT"); + assert.ok(area, "the default style shows the default instructions"); +}); diff --git a/core/tests/test_shipped_nodes.py b/core/tests/test_shipped_nodes.py index ffdf0104..7c871ad2 100644 --- a/core/tests/test_shipped_nodes.py +++ b/core/tests/test_shipped_nodes.py @@ -61,7 +61,7 @@ FROZEN = { "FunUIEnhancePrompt": [ "clip", "text", "enabled", "instructions", "max_length", "greedy", "temperature", "top_p", "top_k", "min_p", "repetition_penalty", "presence_penalty", "seed", "thinking", "encoder_file", "encoder_family", "image", "use_image", - "shortcuts", "lorebooks", "reference_intro", "chat", + "shortcuts", "lorebooks", "reference_intro", "chat", "style", ], "FunUIApplyPromptMarkup": ["model", "clip", "positive", "latent", "markup"], "FunUILatentResample": ["latent", "upscaler", "operation", "scale", "positive", "negative"], diff --git a/modules/conditioning/prompt_enhancer/__init__.py b/modules/conditioning/prompt_enhancer/__init__.py index a45e8999..db704e9b 100644 --- a/modules/conditioning/prompt_enhancer/__init__.py +++ b/modules/conditioning/prompt_enhancer/__init__.py @@ -32,7 +32,8 @@ def routes(table, base, web): """What an empty field means, so the Composer can show it.""" from . import enhance return web.json_response({"instructions": enhance.SYSTEM_PROMPT, - "reference_intro": enhance.REFERENCE_INTRO}) + "reference_intro": enhance.REFERENCE_INTRO, + "styles": enhance.STYLES}) @table.get(base + "/runs") async def _runs(req): diff --git a/modules/conditioning/prompt_enhancer/enhance.py b/modules/conditioning/prompt_enhancer/enhance.py index 849c9051..7f9994ae 100644 --- a/modules/conditioning/prompt_enhancer/enhance.py +++ b/modules/conditioning/prompt_enhancer/enhance.py @@ -28,6 +28,25 @@ Rules: Output: one paragraph of plain prose. No preamble, no headings, no markdown, no quotes around the whole answer. Output only the prompt itself.""" +DYNAMIC_PROMPT = """You expand short video prompts into detailed, DYNAMIC ones for a text-to-video model. + +Rules: +- Keep every element the user asked for. Never drop, replace or contradict one. +- Every shot gets at least one concrete action beat of its subject: a verb with intensity, speed or direction ("lunges forward", "spins on one heel", "slams the door shut"), not a state ("stands", "sits", "looks", "holds", "wears", "is"). +- Show a change over time inside the shot: something starts, builds, peaks or ends. +- Add physically specific secondary motion where it fits: hair, cloth, water, dust, smoke, leaves, light. +- Write all motion in the present progressive ("is sprinting", "is turning"). +- Add concrete visual detail where the prompt is vague: lighting, materials, setting, time of day. +- Add a matching soundscape: ambient sound, and any sound the described actions would make. +- If the user quoted speech, reproduce it exactly. Never invent speech that was not asked for. +- Do not invent camera moves or scene cuts unless the user asked for them. +- Do not add characters the user did not mention. + +Output: one paragraph of plain prose. No preamble, no headings, no markdown, no quotes around the whole answer. Output only the prompt itself.""" + +#: what the Style choice selects; "default" must stay SYSTEM_PROMPT so an old workflow rewrites as before +STYLES = {"default": SYSTEM_PROMPT, "dynamic": DYNAMIC_PROMPT} + CHAT_SYSTEM = """REVISION: this time the user is giving feedback on your earlier rewrite. The message has labelled parts: ORIGINAL PROMPT (what the user wrote), YOUR LATEST REWRITE (your last answer, if shown) and USER FEEDBACK (their comments, oldest first). - Start from YOUR LATEST REWRITE (or the ORIGINAL PROMPT if none is shown) and change it as the feedback asks. Apply every comment; when two disagree, the later one wins. @@ -347,13 +366,13 @@ def generate(clip, system, user, *, seed=None, image=None, thinking=False, max_l return "", f"generation failed: {exc}" -def enhance(clip, text, *, system="", reference_text="", chat="", **sampling): +def enhance(clip, text, *, system="", reference_text="", chat="", style="default", **sampling): """-> (text, status, info). On any failure the ORIGINAL text comes back.""" original = str(text or "").strip() info = {"before": original, "after": original, "thinking": "", "sent": "", "ok": False} if not original: return text, "skipped: the prompt is empty", info - system = str(system or "").strip() or SYSTEM_PROMPT + system = str(system or "").strip() or STYLES.get(style, SYSTEM_PROMPT) # typed instructions win over the style user = original + str(reference_text or "") if chat: # Labelled parts plus a revision rule in the SYSTEM turn: with the conversation merely diff --git a/modules/conditioning/prompt_enhancer/nodes.py b/modules/conditioning/prompt_enhancer/nodes.py index b4a7d071..0d3c9084 100644 --- a/modules/conditioning/prompt_enhancer/nodes.py +++ b/modules/conditioning/prompt_enhancer/nodes.py @@ -85,6 +85,8 @@ class FunUIEnhancePrompt(io.ComfyNode): tooltip="The line above the reference entries. Empty uses the built-in one."), io.String.Input("chat", multiline=True, default="", optional=True, tooltip="Your comments on earlier rewrites (Composer > Chat)."), + io.Combo.Input("style", options=list(en.STYLES), default="default", optional=True, + tooltip="default: the built-in instructions. dynamic: the same rules, but every shot gets a concrete action beat, a change over time and secondary motion. Typed Instructions override both."), ], outputs=[io.String.Output(display_name="enhanced"), io.String.Output(display_name="status")], @@ -100,7 +102,7 @@ class FunUIEnhancePrompt(io.ComfyNode): def execute(cls, clip, text, enabled=False, instructions="", max_length=400, greedy=False, temperature=0.7, top_p=0.92, top_k=50, min_p=0.05, repetition_penalty=1.3, presence_penalty=0.0, seed=0, thinking=False, encoder_file="", encoder_family="stable_diffusion", image=None, use_image=True, shortcuts="", - lorebooks="", reference_intro="", chat="") -> io.NodeOutput: + lorebooks="", reference_intro="", chat="", style="default") -> io.NodeOutput: if not enabled: # still the final prompt, and the Composer's Enhance tab shows what was encoded either way RUNS.append({"prompt_id": _prompt_id(), "status": "off: the prompt ran as typed", "before": text, "after": text, @@ -129,7 +131,7 @@ class FunUIEnhancePrompt(io.ComfyNode): rounds = [] out, status, info = en.enhance( clip, text, system=instructions, - reference_text=en.reference(text, groups, reference_intro), chat=en.chat_block(rounds), + reference_text=en.reference(text, groups, reference_intro), chat=en.chat_block(rounds), style=style, seed=seed, image=image if use_image else None, thinking=thinking, max_length=max_length, temperature=temperature, top_p=top_p, top_k=top_k, min_p=min_p, repetition_penalty=repetition_penalty, presence_penalty=presence_penalty, diff --git a/modules/conditioning/prompt_enhancer/tests/test_prompt_enhancer.py b/modules/conditioning/prompt_enhancer/tests/test_prompt_enhancer.py index 453ffbd2..a9bd9407 100644 --- a/modules/conditioning/prompt_enhancer/tests/test_prompt_enhancer.py +++ b/modules/conditioning/prompt_enhancer/tests/test_prompt_enhancer.py @@ -201,6 +201,65 @@ def test_chat_revision_goes_in_the_system_turn_and_an_echo_is_cut(): assert "ORIGINAL PROMPT:\na fox" in clip.seen["text"] and info["sent"] +# --- style ------------------------------------------------------------------- + +def test_dynamic_asks_for_motion_and_keeps_every_guard_of_the_default(): + e = en() + assert e.STYLES["default"] is e.SYSTEM_PROMPT and e.STYLES["dynamic"] is e.DYNAMIC_PROMPT + d = e.DYNAMIC_PROMPT + assert "action beat" in d and "change over time" in d and "present progressive" in d + assert "Never drop, replace or contradict" in d and "Do not invent camera moves or scene cuts unless the user asked" in d + assert "reproduce it exactly. Never invent speech" in d and "Do not add characters the user did not mention" in d + + +@pytest.mark.parametrize("style", ["default", "dynamic", "nonsense"]) +@pytest.mark.parametrize("typed", ["", "Make it blue."]) +@pytest.mark.parametrize("with_chat", [False, True]) +@pytest.mark.parametrize("with_reference", [False, True]) +def test_style_matrix_typed_instructions_win_then_style_then_default(style, typed, with_chat, with_reference): + e = en() + clip = FakeClip("A fox runs.") + chat = e.chat_block([{"rewrites": {"whole": "old"}, "comment": "faster"}]) if with_chat else "" + ref = "\n\nREF" if with_reference else "" + out, _, info = e.enhance(clip, "a fox", system=typed, style=style, chat=chat, reference_text=ref) + assert out == "A fox runs." and info["ok"] + sent = clip.seen["text"] + want = typed or e.STYLES.get(style, e.SYSTEM_PROMPT) # an unknown style falls back to the default + assert sent.startswith(want) + other = e.DYNAMIC_PROMPT if want is not e.DYNAMIC_PROMPT else e.SYSTEM_PROMPT + assert other not in sent + assert (e.CHAT_SYSTEM in sent) is with_chat and ("REF" in sent) is with_reference and "a fox" in sent + + +def test_the_node_has_a_style_input_last_and_passes_it_on(node): + schema = node.FunUIEnhancePrompt.define_schema() + assert [i.id for i in schema.inputs][-1] == "style" + clip = FakeClip("A fox sprints.") + node.FunUIEnhancePrompt.execute(clip, "a fox", enabled=True, seed=3, style="dynamic") + assert en().DYNAMIC_PROMPT in clip.seen["text"] + clip = FakeClip("A fox walks.") + node.FunUIEnhancePrompt.execute(clip, "a fox", enabled=True, seed=3) + assert clip.seen["text"].startswith(en().SYSTEM_PROMPT), "a workflow saved before Style rewrites as before" + + +def test_the_defaults_route_lists_the_styles(): + from modules.conditioning import prompt_enhancer as mod + table = {} + + class T: + def get(self, path): + return lambda fn: table.setdefault(path, fn) + + class W: + @staticmethod + def json_response(d): + return d + mod.routes(T(), "/m", W) + import asyncio + got = asyncio.run(table["/m/defaults"](None)) + assert set(got["styles"]) == {"default", "dynamic"} and got["instructions"] == got["styles"]["default"] + + # --- the node --------------------------------------------------------------- @pytest.fixture diff --git a/tools/motion_counts.py b/tools/motion_counts.py new file mode 100644 index 00000000..545dea35 --- /dev/null +++ b/tools/motion_counts.py @@ -0,0 +1,79 @@ +"""Counts-only check: how many shots in your projects describe no real motion. + +Read-only. Prints numbers, never prompt text. Not shipped to users. + + python tools/motion_counts.py # ~/.funpack_movie/projects + python tools/motion_counts.py /path/to/projects # another project folder + +A "shot" is a [Shot N] block of a scene's text, or the whole text when it has none. A shot is STATIC when it +has no verb except stative ones (is, stand, sit, look, hold, wear, have...), found with the spaCy tagger +Shot camera already uses (en_core_web_sm). Ratings are the labels the Composer stored: "liked*" and +"disliked*" are split out, any other label (a number, say) is counted under "other rating". +""" +import collections +import glob +import json +import os +import re +import sys + +STATIVE = frozenset("""be stand sit look hold wear have lie rest appear seem remain stay face feature show sport dress lean wait +gaze stare hang display contain surround fill""".split()) +SHOT = re.compile(r"\[Shot \d+[^\]]*\]") + + +def bucket(rating): + r = str(rating or "").strip().lower() + if not r: + return "unrated" + return "disliked" if r.startswith("dislik") else "liked" if r.startswith("lik") else "other rating" + + +def shots(text): + parts = [p.strip() for p in SHOT.split(text or "")] + return [p for p in parts if p] + + +def main(folder): + try: + import spacy + nlp = spacy.load("en_core_web_sm") + except Exception as exc: # noqa: BLE001 - say what is missing, plainly + sys.exit(f"needs spaCy and its small English model ({type(exc).__name__}): pip install spacy && python -m spacy download en_core_web_sm") + files = sorted(glob.glob(os.path.join(os.path.expanduser(folder), "*.json"))) + seen, words = collections.Counter(), collections.defaultdict(list) + scenes = 0 + for path in files: + try: + with open(path, encoding="utf-8") as fh: + project = json.load(fh) + except (OSError, ValueError): + seen["unreadable project files"] += 1 + continue + for scene in project.get("scenes", []) if isinstance(project, dict) else []: + if not isinstance(scene, dict): + continue + scenes += 1 + group = bucket(scene.get("rating")) + rated = scene.get("rated") if isinstance(scene.get("rated"), dict) else {} + text = (rated.get("text") if group != "unrated" else None) or scene.get("text") or "" # what the rating is about + for block in shots(text): + doc = nlp(block) + moving = any(t.pos_ == "VERB" and t.lemma_.lower() not in STATIVE for t in doc) + seen[(group, "moving" if moving else "static")] += 1 + words[group].append(len(block.split())) + print(f"project files {len(files)}, scenes {scenes}, unreadable {seen.pop('unreadable project files', 0)}") + print(f"{'group':<14}{'shots':>7}{'static':>8}{'static %':>10}{'avg words':>11}") + total = [0, 0] + for group in ("liked", "disliked", "other rating", "unrated"): + s, m = seen[(group, "static")], seen[(group, "moving")] + n = s + m + total[0] += s + total[1] += n + w = words[group] + print(f"{group:<14}{n:>7}{s:>8}{(100 * s / n if n else 0):>9.0f}%{(sum(w) / len(w) if w else 0):>11.1f}") + print(f"{'all':<14}{total[1]:>7}{total[0]:>8}{(100 * total[0] / total[1] if total[1] else 0):>9.0f}%") + + +if __name__ == "__main__": + main(sys.argv[1] if len(sys.argv) > 1 else "~/.funpack_movie/projects")