Prompt enhancer: Dynamic style for more motion; counts-only motion check script
This commit is contained in:
digital-garbage
@@ -32,7 +32,8 @@ def routes(table, base, web):
|
||||
"""What an empty field means, so the Composer can show it."""
|
||||
from . import enhance
|
||||
return web.json_response({"instructions": enhance.SYSTEM_PROMPT,
|
||||
"reference_intro": enhance.REFERENCE_INTRO})
|
||||
"reference_intro": enhance.REFERENCE_INTRO,
|
||||
"styles": enhance.STYLES})
|
||||
|
||||
@table.get(base + "/runs")
|
||||
async def _runs(req):
|
||||
|
||||
@@ -28,6 +28,25 @@ Rules:
|
||||
|
||||
Output: one paragraph of plain prose. No preamble, no headings, no markdown, no quotes around the whole answer. Output only the prompt itself."""
|
||||
|
||||
DYNAMIC_PROMPT = """You expand short video prompts into detailed, DYNAMIC ones for a text-to-video model.
|
||||
|
||||
Rules:
|
||||
- Keep every element the user asked for. Never drop, replace or contradict one.
|
||||
- Every shot gets at least one concrete action beat of its subject: a verb with intensity, speed or direction ("lunges forward", "spins on one heel", "slams the door shut"), not a state ("stands", "sits", "looks", "holds", "wears", "is").
|
||||
- Show a change over time inside the shot: something starts, builds, peaks or ends.
|
||||
- Add physically specific secondary motion where it fits: hair, cloth, water, dust, smoke, leaves, light.
|
||||
- Write all motion in the present progressive ("is sprinting", "is turning").
|
||||
- Add concrete visual detail where the prompt is vague: lighting, materials, setting, time of day.
|
||||
- Add a matching soundscape: ambient sound, and any sound the described actions would make.
|
||||
- If the user quoted speech, reproduce it exactly. Never invent speech that was not asked for.
|
||||
- Do not invent camera moves or scene cuts unless the user asked for them.
|
||||
- Do not add characters the user did not mention.
|
||||
|
||||
Output: one paragraph of plain prose. No preamble, no headings, no markdown, no quotes around the whole answer. Output only the prompt itself."""
|
||||
|
||||
#: what the Style choice selects; "default" must stay SYSTEM_PROMPT so an old workflow rewrites as before
|
||||
STYLES = {"default": SYSTEM_PROMPT, "dynamic": DYNAMIC_PROMPT}
|
||||
|
||||
CHAT_SYSTEM = """REVISION: this time the user is giving feedback on your earlier rewrite.
|
||||
The message has labelled parts: ORIGINAL PROMPT (what the user wrote), YOUR LATEST REWRITE (your last answer, if shown) and USER FEEDBACK (their comments, oldest first).
|
||||
- Start from YOUR LATEST REWRITE (or the ORIGINAL PROMPT if none is shown) and change it as the feedback asks. Apply every comment; when two disagree, the later one wins.
|
||||
@@ -347,13 +366,13 @@ def generate(clip, system, user, *, seed=None, image=None, thinking=False, max_l
|
||||
return "", f"generation failed: {exc}"
|
||||
|
||||
|
||||
def enhance(clip, text, *, system="", reference_text="", chat="", **sampling):
|
||||
def enhance(clip, text, *, system="", reference_text="", chat="", style="default", **sampling):
|
||||
"""-> (text, status, info). On any failure the ORIGINAL text comes back."""
|
||||
original = str(text or "").strip()
|
||||
info = {"before": original, "after": original, "thinking": "", "sent": "", "ok": False}
|
||||
if not original:
|
||||
return text, "skipped: the prompt is empty", info
|
||||
system = str(system or "").strip() or SYSTEM_PROMPT
|
||||
system = str(system or "").strip() or STYLES.get(style, SYSTEM_PROMPT) # typed instructions win over the style
|
||||
user = original + str(reference_text or "")
|
||||
if chat:
|
||||
# Labelled parts plus a revision rule in the SYSTEM turn: with the conversation merely
|
||||
|
||||
@@ -85,6 +85,8 @@ class FunUIEnhancePrompt(io.ComfyNode):
|
||||
tooltip="The line above the reference entries. Empty uses the built-in one."),
|
||||
io.String.Input("chat", multiline=True, default="", optional=True,
|
||||
tooltip="Your comments on earlier rewrites (Composer > Chat)."),
|
||||
io.Combo.Input("style", options=list(en.STYLES), default="default", optional=True,
|
||||
tooltip="default: the built-in instructions. dynamic: the same rules, but every shot gets a concrete action beat, a change over time and secondary motion. Typed Instructions override both."),
|
||||
],
|
||||
outputs=[io.String.Output(display_name="enhanced"),
|
||||
io.String.Output(display_name="status")],
|
||||
@@ -100,7 +102,7 @@ class FunUIEnhancePrompt(io.ComfyNode):
|
||||
def execute(cls, clip, text, enabled=False, instructions="", max_length=400, greedy=False,
|
||||
temperature=0.7, top_p=0.92, top_k=50, min_p=0.05, repetition_penalty=1.3,
|
||||
presence_penalty=0.0, seed=0, thinking=False, encoder_file="", encoder_family="stable_diffusion", image=None, use_image=True, shortcuts="",
|
||||
lorebooks="", reference_intro="", chat="") -> io.NodeOutput:
|
||||
lorebooks="", reference_intro="", chat="", style="default") -> io.NodeOutput:
|
||||
if not enabled:
|
||||
# still the final prompt, and the Composer's Enhance tab shows what was encoded either way
|
||||
RUNS.append({"prompt_id": _prompt_id(), "status": "off: the prompt ran as typed", "before": text, "after": text,
|
||||
@@ -129,7 +131,7 @@ class FunUIEnhancePrompt(io.ComfyNode):
|
||||
rounds = []
|
||||
out, status, info = en.enhance(
|
||||
clip, text, system=instructions,
|
||||
reference_text=en.reference(text, groups, reference_intro), chat=en.chat_block(rounds),
|
||||
reference_text=en.reference(text, groups, reference_intro), chat=en.chat_block(rounds), style=style,
|
||||
seed=seed, image=image if use_image else None, thinking=thinking, max_length=max_length,
|
||||
temperature=temperature, top_p=top_p, top_k=top_k, min_p=min_p,
|
||||
repetition_penalty=repetition_penalty, presence_penalty=presence_penalty,
|
||||
|
||||
@@ -201,6 +201,65 @@ def test_chat_revision_goes_in_the_system_turn_and_an_echo_is_cut():
|
||||
assert "ORIGINAL PROMPT:\na fox" in clip.seen["text"] and info["sent"]
|
||||
|
||||
|
||||
# --- style -------------------------------------------------------------------
|
||||
|
||||
def test_dynamic_asks_for_motion_and_keeps_every_guard_of_the_default():
|
||||
e = en()
|
||||
assert e.STYLES["default"] is e.SYSTEM_PROMPT and e.STYLES["dynamic"] is e.DYNAMIC_PROMPT
|
||||
d = e.DYNAMIC_PROMPT
|
||||
assert "action beat" in d and "change over time" in d and "present progressive" in d
|
||||
assert "Never drop, replace or contradict" in d and "Do not invent camera moves or scene cuts unless the user asked" in d
|
||||
assert "reproduce it exactly. Never invent speech" in d and "Do not add characters the user did not mention" in d
|
||||
|
||||
|
||||
@pytest.mark.parametrize("style", ["default", "dynamic", "nonsense"])
|
||||
@pytest.mark.parametrize("typed", ["", "Make it blue."])
|
||||
@pytest.mark.parametrize("with_chat", [False, True])
|
||||
@pytest.mark.parametrize("with_reference", [False, True])
|
||||
def test_style_matrix_typed_instructions_win_then_style_then_default(style, typed, with_chat, with_reference):
|
||||
e = en()
|
||||
clip = FakeClip("A fox runs.")
|
||||
chat = e.chat_block([{"rewrites": {"whole": "old"}, "comment": "faster"}]) if with_chat else ""
|
||||
ref = "\n\nREF" if with_reference else ""
|
||||
out, _, info = e.enhance(clip, "a fox", system=typed, style=style, chat=chat, reference_text=ref)
|
||||
assert out == "A fox runs." and info["ok"]
|
||||
sent = clip.seen["text"]
|
||||
want = typed or e.STYLES.get(style, e.SYSTEM_PROMPT) # an unknown style falls back to the default
|
||||
assert sent.startswith(want)
|
||||
other = e.DYNAMIC_PROMPT if want is not e.DYNAMIC_PROMPT else e.SYSTEM_PROMPT
|
||||
assert other not in sent
|
||||
assert (e.CHAT_SYSTEM in sent) is with_chat and ("REF" in sent) is with_reference and "a fox" in sent
|
||||
|
||||
|
||||
def test_the_node_has_a_style_input_last_and_passes_it_on(node):
|
||||
schema = node.FunUIEnhancePrompt.define_schema()
|
||||
assert [i.id for i in schema.inputs][-1] == "style"
|
||||
clip = FakeClip("A fox sprints.")
|
||||
node.FunUIEnhancePrompt.execute(clip, "a fox", enabled=True, seed=3, style="dynamic")
|
||||
assert en().DYNAMIC_PROMPT in clip.seen["text"]
|
||||
clip = FakeClip("A fox walks.")
|
||||
node.FunUIEnhancePrompt.execute(clip, "a fox", enabled=True, seed=3)
|
||||
assert clip.seen["text"].startswith(en().SYSTEM_PROMPT), "a workflow saved before Style rewrites as before"
|
||||
|
||||
|
||||
def test_the_defaults_route_lists_the_styles():
|
||||
from modules.conditioning import prompt_enhancer as mod
|
||||
table = {}
|
||||
|
||||
class T:
|
||||
def get(self, path):
|
||||
return lambda fn: table.setdefault(path, fn)
|
||||
|
||||
class W:
|
||||
@staticmethod
|
||||
def json_response(d):
|
||||
return d
|
||||
mod.routes(T(), "/m", W)
|
||||
import asyncio
|
||||
got = asyncio.run(table["/m/defaults"](None))
|
||||
assert set(got["styles"]) == {"default", "dynamic"} and got["instructions"] == got["styles"]["default"]
|
||||
|
||||
|
||||
# --- the node ---------------------------------------------------------------
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
Reference in New Issue
Block a user