feat: prompt enhancer moves to Composer ▸ Enhance

Own tab with its full sampling settings (temperature, top-p, top-k, min-p,
repetition/presence penalty, seed, greedy) and a last-run before/after view.
Studio's source_image now reaches the enhancer on the per-scene path too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
digital-garbage
2026-09-23 22:11:42 +03:00
co-authored by Claude Opus 5.5
parent bb59062b3e
commit 0bb682b67c
10 changed files with 434 additions and 56 deletions
+53 -15
View File
@@ -8936,7 +8936,8 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
def _v2_generate_advisor_text(self, clip, system_prompt, user_prompt, seed=None, image=None,
thinking=True, max_length=800, temperature=0.7, top_p=0.92,
top_k=50, repetition_penalty=1.3, min_floor=128,
label="Advisor"):
label="Advisor", min_p=0.05, presence_penalty=0.0,
do_sample=True):
"""Generate text through a CLIP that exposes ComfyUI's generate/decode pair.
The sampling parameters default to exactly what the repair advisor has always used, so
@@ -8985,15 +8986,15 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
except TypeError:
tokens = clip.tokenize(advisor_prompt)
generate_kwargs = dict(
do_sample=True,
do_sample=bool(do_sample),
max_length=max_length,
temperature=float(temperature),
top_k=int(top_k),
top_p=float(top_p),
min_p=0.05,
min_p=float(min_p),
repetition_penalty=float(repetition_penalty),
no_repeat_ngram_size=5,
presence_penalty=0.0,
presence_penalty=float(presence_penalty),
seed=seed if seed else None,
)
try:
@@ -9017,7 +9018,8 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
#: repeats an anchor line across scenes, and each call is a full LLM generation
def _v2_enhance_prompt(self, clip, text, system_prompt, cache=None, seed=None,
temperature=0.7, top_p=0.92, max_length=400, thinking=False,
image=None):
image=None, top_k=50, min_p=0.05, repetition_penalty=1.3,
presence_penalty=0.0, do_sample=True):
"""Rewrite one prompt through the advisor LLM, BEFORE it is encoded.
This is not the repair advisor. That one is triggered by a rating and fixes a named
@@ -9037,7 +9039,9 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
raw, status = self._v2_generate_advisor_text(
clip, system_prompt, original, seed=seed, image=image, thinking=bool(thinking),
max_length=max_length, temperature=temperature, top_p=top_p,
min_floor=32, label="Prompt enhancer",
min_floor=32, label="Prompt enhancer", top_k=top_k, min_p=min_p,
repetition_penalty=repetition_penalty, presence_penalty=presence_penalty,
do_sample=do_sample,
)
enhanced, overran = self._v2_trim_runaway_prompt(
self._v2_clean_enhanced_prompt(raw), max_length=max_length)
@@ -9962,6 +9966,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
prompt_enhance=False, prompt_enhance_system="",
prompt_enhance_temperature=0.7, prompt_enhance_top_p=0.92,
prompt_enhance_max_length=400, prompt_enhance_thinking=False,
prompt_enhance_image=None, prompt_enhance_sampling=None,
_seed=None, _seed_source="fresh seed", _scene_seeds=None, _velocity_keys=None,
batch_variants=1, guess_mode=False, guess_direction="up", guess_range=1.0,
guess_freeze_seed=True, movie_editor_scene_ratings=None, scene_segments=None,
@@ -10604,23 +10609,34 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
# `advisor_clip` if one is wired, the generation CLIP otherwise.
_enhance_cache = {}
_enhance_status = ""
# What the enhancer wrote, per encoded prompt — Studio hands it to the Editor as a
# node ui output so the Composer can show before/after. A readout, never read back.
self._v2_enhanced_prompts = []
_enhance_clip = advisor_clip # already falls back to `clip` at the top
_enhance_system = str(prompt_enhance_system or "").strip() or V2_PROMPT_ENHANCER_SYSTEM_PROMPT
# When the run splits into scenes, the per-scene conditionings REPLACE this base entry
# — so enhancing the base too would be a second full generation whose result is then
# thrown away. One generation per prompt that is actually encoded, never two.
_enhance_base = prompt_enhance and not split_by_transitions
# top_k / min_p / penalties / greedy; `seed` 0 = follow the generation seed.
_enhance_sampling = dict(prompt_enhance_sampling or {})
_enhance_seed = int(_enhance_sampling.pop("seed", 0) or 0) or seed
if prompt_enhance and not _enhance_base:
print("[FunPackVideoRefinerV2] Prompt enhancer: base prompt skipped — this run "
"splits into scenes and each scene is enhanced instead.")
if _enhance_base:
_before = prompt_to_encode
prompt_to_encode, _enhance_status = self._v2_enhance_prompt(
_enhance_clip, prompt_to_encode, _enhance_system, cache=_enhance_cache,
seed=seed, temperature=prompt_enhance_temperature,
seed=_enhance_seed, temperature=prompt_enhance_temperature,
top_p=prompt_enhance_top_p, max_length=prompt_enhance_max_length,
thinking=prompt_enhance_thinking, image=source_image,
thinking=prompt_enhance_thinking, image=prompt_enhance_image,
**_enhance_sampling,
)
print(f"[FunPackVideoRefinerV2] {_enhance_status}")
self._v2_enhanced_prompts.append({
"scene": None, "before": _before, "after": prompt_to_encode,
"status": _enhance_status, "image": prompt_enhance_image is not None})
cond, meta, encode_status, conditioning_owner = self._v2_conditioning_source(
clip,
@@ -10856,15 +10872,21 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
# and re-splitting could return a different number of scenes and desync
# every clip from its anchor. One call per distinct text, cached.
if prompt_enhance and split_scene_texts:
split_scene_texts = [
self._v2_enhance_prompt(
_enhanced_texts = []
for _i, t in enumerate(split_scene_texts):
_after, _st = self._v2_enhance_prompt(
_enhance_clip, t, _enhance_system, cache=_enhance_cache,
seed=seed, temperature=prompt_enhance_temperature,
seed=_enhance_seed, temperature=prompt_enhance_temperature,
top_p=prompt_enhance_top_p,
max_length=prompt_enhance_max_length,
thinking=prompt_enhance_thinking)[0]
for t in split_scene_texts
]
thinking=prompt_enhance_thinking, image=prompt_enhance_image,
**_enhance_sampling)
print(f"[FunPackVideoRefinerV2] Scene {_i + 1}: {_st}")
self._v2_enhanced_prompts.append({
"scene": _i, "before": t, "after": _after, "status": _st,
"image": prompt_enhance_image is not None})
_enhanced_texts.append(_after)
split_scene_texts = _enhanced_texts
scene_refinement_keys = [set(s.get("keys") or set()) for s in canon_scenes]
current_scene_seeds = self._v2_scene_seed_values(seed, len(split_scene_texts), _scene_seeds)
current_scene_seed_source = (
@@ -13853,6 +13875,17 @@ class FunPackStudio:
prompt_enhance_top_p=float(rf.get("prompt_enhance_top_p", 0.92)),
prompt_enhance_max_length=int(rf.get("prompt_enhance_max_length", 400)),
prompt_enhance_thinking=bool(rf.get("prompt_enhance_thinking", False)),
# Its own switch, not vision_conditioning's: showing the enhancer the picture is
# a different decision from writing the picture into the conditioning.
prompt_enhance_image=source_image if rf.get("prompt_enhance_use_image", True) else None,
prompt_enhance_sampling={
"top_k": int(rf.get("prompt_enhance_top_k", 50)),
"min_p": float(rf.get("prompt_enhance_min_p", 0.05)),
"repetition_penalty": float(rf.get("prompt_enhance_repetition_penalty", 1.3)),
"presence_penalty": float(rf.get("prompt_enhance_presence_penalty", 0.0)),
"do_sample": not bool(rf.get("prompt_enhance_greedy", False)),
"seed": int(rf.get("prompt_enhance_seed", 0) or 0),
},
clip_vision_output=clip_vision_output,
source_image=source_image if vision_conditioning else None,
model=model,
@@ -13942,7 +13975,12 @@ class FunPackStudio:
cond = _h3_reconcile_token_tags(cond, "positive")
out_negative = _h3_reconcile_token_tags(out_negative, "negative")
return (out_model, cond, out_negative, seed, high_sampler, high_sigmas, low_sampler, low_sigmas, loss_graph, status, training_info, encoded_prompts, video_latent)
result = (out_model, cond, out_negative, seed, high_sampler, high_sigmas, low_sampler, low_sigmas, loss_graph, status, training_info, encoded_prompts, video_latent)
enhanced = getattr(refiner, "_v2_enhanced_prompts", None)
if enhanced:
# History carries every node's ui output; the Editor's /status reads this key.
return {"ui": {"funpack_enhanced": enhanced}, "result": result}
return result
@staticmethod
def _parse_sigmas(text):
+20 -5
View File
@@ -177,6 +177,19 @@ _OWNED_EXTRA = {
"detail_threshold": ("segmented_detailing", None),
"detail_max_area": ("segmented_detailing", None),
"detail_mode": ("segmented_detailing", None),
# The prompt enhancer's settings live in the Composer (composer.js), not engine_settings.js.
"prompt_enhance_max_length": ("prompt_enhance", ('true',)),
"prompt_enhance_system": ("prompt_enhance", ('true',)),
"prompt_enhance_temperature": ("prompt_enhance", ('true',)),
"prompt_enhance_thinking": ("prompt_enhance", ('true',)),
"prompt_enhance_top_p": ("prompt_enhance", ('true',)),
"prompt_enhance_use_image": ("prompt_enhance", ('true',)),
"prompt_enhance_top_k": ("prompt_enhance", ('true',)),
"prompt_enhance_min_p": ("prompt_enhance", ('true',)),
"prompt_enhance_repetition_penalty": ("prompt_enhance", ('true',)),
"prompt_enhance_presence_penalty": ("prompt_enhance", ('true',)),
"prompt_enhance_seed": ("prompt_enhance", ('true',)),
"prompt_enhance_greedy": ("prompt_enhance", ('true',)),
}
_OWNED_BY = {
@@ -226,11 +239,6 @@ _OWNED_BY = {
"h3_phrase_variability": ("h3_phrase_emphasis", ('true',)),
"trajectory_guidance_strength": ("trajectory_guidance", None),
"phase_scale": ("identity_transfer_enabled", None),
"prompt_enhance_max_length": ("prompt_enhance", ('true',)),
"prompt_enhance_system": ("prompt_enhance", ('true',)),
"prompt_enhance_temperature": ("prompt_enhance", ('true',)),
"prompt_enhance_thinking": ("prompt_enhance", ('true',)),
"prompt_enhance_top_p": ("prompt_enhance", ('true',)),
"score_slider_strength": ("score_slider", None),
"source_id": ("identity_transfer_enabled", None),
"v2a_grad_scale": ("joyai_audio_memory", None),
@@ -328,6 +336,13 @@ _EDITOR_DEFAULTS = {
"prompt_enhance_temperature": 0.7,
"prompt_enhance_thinking": False,
"prompt_enhance_top_p": 0.92,
"prompt_enhance_use_image": True,
"prompt_enhance_top_k": 50,
"prompt_enhance_min_p": 0.05,
"prompt_enhance_repetition_penalty": 1.3,
"prompt_enhance_presence_penalty": 0.0,
"prompt_enhance_seed": 0,
"prompt_enhance_greedy": False,
"reference_injection": False,
"score_slider": False,
"score_slider_strength": 1.0,
+207 -5
View File
@@ -929,8 +929,180 @@
return wrap;
}
// ── enhance (LLM prompt enhancer) ───────────────────────────────────────────────
// Studio rewrites each prompt through a language model right before it is encoded — after
// shortcuts and $variables, so the model reads exactly what would otherwise be used. The
// settings are Studio's refiner settings; the last run's rewrite comes back from the run's
// history (Store.enhanced). Also mounted in Engine settings for Simple mode (no Composer).
const ENH_DEFAULTS = {
prompt_enhance: false, prompt_enhance_system: "", prompt_enhance_max_length: 400,
prompt_enhance_temperature: 0.7, prompt_enhance_top_p: 0.92, prompt_enhance_top_k: 50,
prompt_enhance_min_p: 0.05, prompt_enhance_repetition_penalty: 1.3,
prompt_enhance_presence_penalty: 0, prompt_enhance_seed: 0, prompt_enhance_greedy: false,
prompt_enhance_thinking: false, prompt_enhance_use_image: true,
};
// `sampled`: only read while sampling. Greedy returns the single likeliest word before
// ComfyUI looks at any of them, so they disappear rather than sit there doing nothing.
const ENH_GROUPS = [
{ title: "Behaviour", knobs: [
{ name: "prompt_enhance_use_image", label: "Show it Studio's image", kind: "bool",
hint: "Hands it the picture connected to Studio's source_image, if there is one. Only helps a model that can see." },
{ name: "prompt_enhance_thinking", label: "Think first", kind: "bool",
hint: "Reasons before answering, if the model can. Slower; the reasoning is removed." },
{ name: "prompt_enhance_max_length", label: "Max tokens", kind: "int", min: 32, max: 4096, step: 32,
hint: "Most it may write. Longer is richer and slower." },
] },
{ title: "Sampling", knobs: [
{ name: "prompt_enhance_greedy", label: "Same wording every time", kind: "bool",
hint: "Greedy: always picks the likeliest next word. Repeatable, but plainer." },
{ name: "prompt_enhance_temperature", label: "Temperature", kind: "range", min: 0.01, max: 2, step: 0.01, sampled: true,
hint: "Low stays close to your wording, high invents more." },
{ name: "prompt_enhance_top_p", label: "Top-p", kind: "range", min: 0, max: 1, step: 0.01, sampled: true,
hint: "Keeps only the likeliest words that add up to this share. Lower is safer." },
{ name: "prompt_enhance_top_k", label: "Top-k", kind: "int", min: 1, max: 500, step: 1, sampled: true,
hint: "Picks from at most this many candidate words. Lower is safer." },
{ name: "prompt_enhance_min_p", label: "Min-p", kind: "range", min: 0, max: 1, step: 0.01, sampled: true,
hint: "Drops words less than this fraction as likely as the best one." },
{ name: "prompt_enhance_repetition_penalty", label: "Repetition penalty", kind: "range", min: 1, max: 2, step: 0.01, sampled: true,
hint: "Discourages reusing words already written. 1 is off." },
{ name: "prompt_enhance_presence_penalty", label: "Presence penalty", kind: "range", min: 0, max: 2, step: 0.01, sampled: true,
hint: "A flat push away from any word already used. 0 is off." },
{ name: "prompt_enhance_seed", label: "Seed", kind: "int", min: 0, max: 2147483647, step: 1, sampled: true,
hint: "0 follows the video's seed. Any other number keeps the rewrite fixed while the video re-rolls." },
] },
];
function enhanceKnobRow(f, value, patch) {
const row = el("div", "sw-row enh-row");
const main = el("div", "sw-row-main");
main.append(el("div", "sw-row-title", f.label), el("div", "sw-row-hint", f.hint));
row.append(main);
const ctrl = el("input");
ctrl.dataset.k = "rf-" + f.name;
if (f.kind === "bool") {
ctrl.type = "checkbox"; ctrl.className = "enh-switch"; ctrl.checked = !!value;
ctrl.onchange = () => patch({ [f.name]: ctrl.checked }, true);
row.append(ctrl);
return row;
}
ctrl.type = f.kind === "range" ? "range" : "number";
ctrl.className = f.kind === "range" ? "enh-range" : "lib-in enh-num";
ctrl.min = String(f.min); ctrl.max = String(f.max); ctrl.step = String(f.step);
ctrl.value = String(value);
const out = el("span", "enh-val", f.kind === "range" ? Number(value).toFixed(2) : "");
ctrl.oninput = () => {
const n = parseFloat(ctrl.value);
if (!Number.isFinite(n)) return;
if (f.kind === "range") out.textContent = n.toFixed(2);
patch({ [f.name]: f.kind === "int" ? Math.round(n) : n }, false);
};
row.append(ctrl);
if (f.kind === "range") row.append(out);
return row;
}
function enhanceResultCard(it) {
const card = el("div", "enh-card");
const before = String(it.before || "").trim();
const after = String(it.after || "").trim();
const changed = !!after && after !== before;
const head = el("div", "enh-card-head");
head.append(el("span", "enh-card-title", it.scene == null ? "Prompt" : `Scene ${it.scene + 1}`));
if (it.image) head.append(el("span", "enh-chip", "image sent"));
if (!changed) head.append(el("span", "enh-chip enh-chip-warn", "unchanged"));
const copy = el("button", "btn ghost tiny enh-copy", "Copy");
copy.type = "button";
copy.onclick = async () => {
try { await navigator.clipboard.writeText(after); copy.textContent = "Copied"; }
catch (_) { copy.textContent = "Copy failed"; }
setTimeout(() => { copy.textContent = "Copy"; }, 1400);
};
head.append(copy);
card.append(head, el("div", "enh-after", after || before));
if (changed) {
const det = el("details", "enh-before");
det.append(el("summary", null, "What went in"), el("div", "enh-before-text", before));
card.append(det);
}
if (it.status) card.append(el("div", "enh-status", String(it.status)));
return card;
}
function enhanceTab(st) {
const wrap = el("div", "bin enh-bin");
const SS = window.StudioSettings;
if (!st.project || !SS) {
wrap.append(el("div", "insp-hint", "Open a project to set up the prompt enhancer."));
return wrap;
}
const { rf } = SS.read(st.project);
const val = (k) => (rf[k] != null ? rf[k] : ENH_DEFAULTS[k]);
const on = !!val("prompt_enhance");
const hero = el("div", "enh-hero" + (on ? " on" : ""));
const top = el("label", "enh-hero-top");
const titles = el("div", "enh-hero-titles");
titles.append(el("div", "enh-hero-title", "Prompt enhancer"),
el("div", "enh-hero-sub", "A language model rewrites your prompt into a richer one right before the video is made."));
const sw = el("input", "enh-switch"); sw.type = "checkbox"; sw.checked = on;
sw.dataset.k = "rf-prompt_enhance";
sw.onchange = () => SS.patchRefiner({ prompt_enhance: sw.checked }, true);
top.append(titles, sw);
const flow = el("div", "enh-flow");
["Your prompt", "Shortcuts", "$variables", "Enhancer", "Video"].forEach((step, i) => {
if (i) flow.append(el("span", "enh-flow-arrow", "›"));
flow.append(el("span", "enh-flow-step" + (step === "Enhancer" ? " is-enh" : ""), step));
});
hero.append(top, flow);
if (on) {
hero.append(el("div", "enh-hero-note",
"Needs a model that can write text, connected to Studio's advisor_clip (or its clip). Each scene is rewritten on its own."));
}
wrap.append(hero);
if (on) {
const instHead = el("div", "enh-section-head");
instHead.append(el("div", "sw-rows-label", "Instructions"));
if (String(val("prompt_enhance_system") || "").trim()) {
const reset = el("button", "btn ghost tiny", "Use built-in");
reset.type = "button";
reset.title = "Clear your instructions and go back to FunPack's own";
reset.onclick = () => { SS.patchRefiner({ prompt_enhance_system: "" }, true); repaintAll(); };
instHead.append(reset);
}
wrap.append(instHead);
const ta = el("textarea", "lib-in enh-system"); ta.rows = 6;
ta.dataset.k = "rf-prompt_enhance_system";
ta.value = String(val("prompt_enhance_system") || "");
ta.placeholder = "Empty uses FunPack's built-in instructions: add detail and a soundscape, never invent characters, camera moves or speech.";
ta.oninput = () => SS.patchRefiner({ prompt_enhance_system: ta.value }, false);
ta.onchange = () => SS.patchRefiner({ prompt_enhance_system: ta.value }, true);
wrap.append(ta);
const greedy = !!val("prompt_enhance_greedy");
ENH_GROUPS.forEach((g) => {
wrap.append(el("div", "sw-rows-label", g.title));
const rows = el("div", "sw-rows");
g.knobs.filter((f) => !(greedy && f.sampled))
.forEach((f) => rows.append(enhanceKnobRow(f, val(f.name), SS.patchRefiner)));
wrap.append(rows);
});
}
wrap.append(el("div", "sw-rows-label", "Last run"));
const items = (st.enhanced && st.enhanced.items) || [];
if (!items.length) {
wrap.append(el("div", "enh-empty", on
? "Nothing yet. Generate, and the rewritten prompt shows up here next to what went in."
: "Turn the enhancer on and generate. Its rewrite shows up here."));
} else {
items.forEach((it) => wrap.append(enhanceResultCard(it)));
}
return wrap;
}
// ── window + tabs ────────────────────────────────────────────────────────────────
const TABS = ["Compose", "Shortcuts", "Splits", "Files"];
const TABS = ["Compose", "Enhance", "Shortcuts", "Splits", "Files"];
function render() {
if (!win) return;
const st = S.get();
@@ -942,6 +1114,7 @@
const b = el("button", "bin-tab" + (tab === name ? " active" : ""), name);
b.title = name === "Splits" ? "Split markers (generation prompt)"
: name === "Compose" ? "Global prompt — the whole montage"
: name === "Enhance" ? "Let a language model rewrite the prompt before generating"
: name === "Files" ? "FunPack files on disk — audit & purge" : name;
b.onclick = () => {
if (tab === name) return;
@@ -955,6 +1128,7 @@
const scroll = el("div", "composer-scroll");
scroll.append(
tab === "Compose" ? composeTab(st)
: tab === "Enhance" ? enhanceTab(st)
: tab === "Shortcuts" ? shortcutsTab(st)
: tab === "Splits" ? splitMarkersTab(st)
: filesTab(),
@@ -986,7 +1160,7 @@
// The setup wizard shows Shortcuts and Splits as full screens of its own, so the flow
// stays one continuous surface instead of spawning a floating Composer over it. Same
// builders, same editors, same store — only the container differs.
const PANE_BUILDERS = { shortcuts: shortcutsTab, splits: splitMarkersTab };
const PANE_BUILDERS = { shortcuts: shortcutsTab, splits: splitMarkersTab, enhance: enhanceTab };
const _panes = new Set(); // { kind, host }
function mountPane(kind, host) {
@@ -999,10 +1173,32 @@
}
function paintPane(entry) {
// A host whose container re-rendered without unmounting (Engine settings rebuilds its
// whole pane) is gone for good — drop it rather than paint into a detached node.
if (!entry.host.isConnected) { _panes.delete(entry); return; }
clear(entry.host);
entry.host.append(PANE_BUILDERS[entry.kind](S.get()));
}
// A repaint replaces every field, so never do it under the caret: wait until focus leaves.
function typingIn(root) {
const a = document.activeElement;
if (!root || !a || !root.contains(a)) return false;
if (a.tagName === "TEXTAREA") return true;
return a.tagName === "INPUT" && !/^(checkbox|radio|button)$/i.test(a.type || "text");
}
let _repaintDeferred = false;
function repaintAll() {
const hosts = [isOpen() ? win.body : null, ...[..._panes].map((e) => e.host)];
if (hosts.some(typingIn)) { _repaintDeferred = true; return; }
_repaintDeferred = false;
if (isOpen()) render();
_panes.forEach(paintPane);
}
document.addEventListener("focusout", () => {
if (_repaintDeferred) setTimeout(() => { if (_repaintDeferred) repaintAll(); }, 0);
});
let lastFp = null;
S.subscribe((st) => {
if (!isOpen() && !_panes.size) return;
@@ -1015,11 +1211,17 @@
// fingerprinted — typing goes through quiet patches and the field itself is
// the source of those edits.
sd: !!(st.project?.sampler_inputs?.segmented_detailing),
// Enhance tab: the switches that show/hide rows, and which run's rewrite is shown.
// Typed values stay out: a repaint per keystroke would fight the caret.
en: !!(st.project && window.StudioSettings?.read(st.project).rf.prompt_enhance),
eg: !!(st.project && window.StudioSettings?.read(st.project).rf.prompt_enhance_greedy),
eh: st.enhanced?.promptId || null,
});
if (fp === lastFp) return;
// A deferred repaint also flushes on the next notify: focusout never fires when focus
// moves while the window itself is in the background.
if (fp === lastFp) { if (_repaintDeferred) repaintAll(); return; }
lastFp = fp;
if (isOpen()) render();
_panes.forEach(paintPane);
repaintAll();
});
// Keep the Compose textarea in sync with timeline-driven global-prompt changes,
+11 -30
View File
@@ -119,25 +119,6 @@
{ name: "h3_phrase_variability", label: "Phrase emphasis variability", kind: "float", default: 0.0, min: 0.0, max: 1.0, step: 0.05,
dependsOn: "h3_phrase_emphasis", dependsVals: [true],
hint: "0 = only what the rating has cemented gets extra attention, nothing else. 1 = that emphasis is switched off entirely, so untrained phrasing and actions compete on equal footing and can bleed into the scene — including ones you have rated against before. Same bias channel as the toggle above, just scaled down." },
{ name: "prompt_enhance", label: "Enhance the prompt with an LLM", kind: "bool", default: false,
hint: "Before anything is generated, the fully expanded prompt is rewritten by a language model and the result is what the video is made from. Needs a text encoder that can GENERATE text wired into the Studio node's advisor_clip (or a FunPack Advisor LLM node); without one it says so and leaves the prompt alone.",
detail: "Runs after shortcuts and $variables resolve, before encoding — so the model is handed exactly the text that would otherwise have been used, and nothing downstream needs to know it happened. On a multi-scene timeline each scene is enhanced separately: the scene list is authoritative on count, and rewriting them as one paragraph could come back with a different number of scenes. One generation per distinct scene text, so a long timeline costs real time." },
{ name: "prompt_enhance_system", label: "Enhancer instructions", kind: "textarea", rows: 10, default: "",
dependsOn: "prompt_enhance", dependsVals: [true],
placeholder: "Leave empty to use the built-in instructions.",
hint: "The system prompt the enhancer runs under. Empty uses FunPack's built-in one, which expands detail and adds a soundscape without inventing characters, camera moves or speech." },
{ name: "prompt_enhance_max_length", label: "Enhancer length limit", kind: "int", default: 400, min: 32, max: 4096, step: 32,
dependsOn: "prompt_enhance", dependsVals: [true],
hint: "Maximum tokens the enhancer may write. Higher allows a richer prompt and costs more time." },
{ name: "prompt_enhance_temperature", label: "Enhancer temperature", kind: "float", default: 0.7, min: 0.01, max: 2.0, step: 0.05,
dependsOn: "prompt_enhance", dependsVals: [true],
hint: "How freely the enhancer writes. Lower sticks closer to your wording, higher invents more detail." },
{ name: "prompt_enhance_top_p", label: "Enhancer top-p", kind: "float", default: 0.92, min: 0.0, max: 1.0, step: 0.01,
dependsOn: "prompt_enhance", dependsVals: [true],
hint: "Nucleus sampling cutoff for the enhancer. Lower is more predictable wording." },
{ name: "prompt_enhance_thinking", label: "Enhancer thinking mode", kind: "bool", default: false,
dependsOn: "prompt_enhance", dependsVals: [true],
hint: "Let the model reason before answering, if it supports it. Slower, and the reasoning is stripped from the result." },
];
function parseStudioSettings(p) {
@@ -214,17 +195,6 @@
ctrl = el("select"); ctrl.dataset.k = "rf-" + f.name;
(f.choices || []).forEach((c) => { const o = new Option(c, c); if (c === val) o.selected = true; ctrl.append(o); });
ctrl.onchange = () => persistStudioRefiner({ [f.name]: ctrl.value }, true);
} else if (f.kind === "textarea") {
// A system prompt is paragraphs, not a value. Quiet while typing (no repaint under the
// caret, same rule the sampler's text knobs follow), committed on blur.
ctrl = el("textarea"); ctrl.dataset.k = "rf-" + f.name;
ctrl.rows = f.rows || 8;
ctrl.value = val != null ? String(val) : "";
if (f.placeholder) ctrl.placeholder = f.placeholder;
ctrl.style.width = "100%";
ctrl.style.resize = "vertical";
ctrl.style.fontFamily = "inherit";
ctrl.onchange = () => persistStudioRefiner({ [f.name]: ctrl.value }, true);
} else {
ctrl = el("input"); ctrl.type = "number";
if (f.step != null) ctrl.step = String(f.step);
@@ -681,6 +651,14 @@
STUDIO_REFINER_ADVANCED.filter((f) => !EASY() || !RATING_GATED_STUDIO.has(f.name))
.forEach((f) => renderStudioRefinerField(gAdv, rf, f));
// The prompt enhancer lives in the Composer. Simple mode has no Composer, so the same
// pane is mounted here instead; it unmounts itself once this render replaces it.
if (EASY() && window.Composer) {
const host = el("div", "eng-enhance-host");
pane.append(host);
window.Composer.mountPane("enhance", host);
}
if (EASY()) {
pane.append(hintEl(
"Studio runs in Prompt-only mode here: it shapes and splits the prompt, nothing more. "
@@ -1167,6 +1145,9 @@
},
});
// Studio's refiner settings, for the Composer's Enhance tab — one reader, one writer.
window.StudioSettings = { read: parseStudioSettings, patchRefiner: persistStudioRefiner };
window.EngineSettingsModal = {
open: () => window.SettingsWindow.open("engine"),
close: () => window.SettingsWindow.close(),
+8
View File
@@ -33,6 +33,9 @@
shortcutCategories: [], // managed grouping list: [{name, sub_categories:[]}]
imageTargets: [], // where an image asset can be wired [{value,label}]
ratingLabels: [], // FunPack Studio V2 rating options
// What the prompt enhancer wrote on the last run: {promptId, items:[{scene,before,after,
// status,image}]} or null. A readout for the Composer's Enhance tab; never read back.
enhanced: null,
};
// ── editor settings ─────────────────────────────────────────────────────────
@@ -821,6 +824,7 @@
state.selectedAudioTrackId = null;
_selectionAnchorId = first;
state.gen = { state: "idle", promptId: null, media: [], msg: "" };
state.enhanced = null;
state.sceneRenders = JSON.parse(JSON.stringify(state.project.scene_renders || {}));
state.sceneGhosts = JSON.parse(JSON.stringify(state.project.scene_ghosts || []));
_ensureTimelineOrder();
@@ -3497,6 +3501,10 @@
try {
const s = await API.status(state.project.id, promptId);
transientStreak = 0;
// Studio runs before sampling, so a run that fails later still carries this.
if (s.enhanced && state.enhanced?.promptId !== promptId) {
set({ enhanced: { promptId, items: s.enhanced } });
}
if (s.state === "error") {
_clearGenTimers();
const msg = s.error ? `ComfyUI error: ${s.error}` : "Generation failed inside ComfyUI — check the ComfyUI terminal for details.";
+54
View File
@@ -2064,6 +2064,60 @@ body.tour-mode { overflow: hidden; }
.lib-cat-tag { margin-left: 8px; padding: 1px 7px; border-radius: 10px; background: var(--teal-soft); color: var(--teal);
font-family: var(--font-ui); font-size: 9px; letter-spacing: .5px; text-transform: uppercase; vertical-align: middle; }
/* Composer ▸ Enhance (LLM prompt enhancer) */
.enh-bin { display: flex; flex-direction: column; gap: 6px; }
.enh-hero { border: 1px solid var(--line); border-radius: 12px; padding: 12px 14px; background: var(--ink-2);
transition: border-color .2s ease, box-shadow .2s ease; }
.enh-hero.on { border-color: var(--teal); box-shadow: 0 0 0 1px var(--teal-soft) inset; }
.enh-hero-top { display: flex; align-items: flex-start; gap: 12px; cursor: pointer; }
.enh-hero-titles { flex: 1; min-width: 0; }
.enh-hero-title { font-size: 14px; font-weight: 600; color: var(--text); }
.enh-hero-sub { margin-top: 2px; font-size: 12px; line-height: 1.45; color: var(--muted); }
.enh-hero-note { margin-top: 10px; font-size: 11px; line-height: 1.45; color: var(--faint); }
.enh-flow { display: flex; flex-wrap: wrap; align-items: center; gap: 4px; margin-top: 10px; }
.enh-flow-step { padding: 2px 8px; border-radius: 10px; border: 1px solid var(--line); font-size: 10px;
color: var(--muted); white-space: nowrap; }
.enh-flow-step.is-enh { border-color: var(--teal); color: var(--teal); background: var(--teal-soft); }
.enh-flow-arrow { font-size: 11px; color: var(--faint); }
.enh-hero:not(.on) .enh-flow-step.is-enh { border-style: dashed; background: transparent; opacity: .7; }
/* iOS-style switch, used for every on/off in this tab */
/* min-height/padding/border reset: the global input rule (30px tall, padded) would stretch it */
.enh-switch { appearance: none; -webkit-appearance: none; flex: 0 0 auto; width: 30px; height: 18px; margin: 0;
min-height: 0; padding: 0; border: 0;
border-radius: 9px; background: color-mix(in srgb, var(--faint) 55%, transparent); position: relative; cursor: pointer;
transition: background .15s ease; }
.enh-switch::after { content: ""; position: absolute; top: 2px; left: 2px; width: 14px; height: 14px; border-radius: 50%;
background: var(--text); transition: transform .15s ease; }
.enh-switch:checked { background: var(--teal); }
.enh-switch:checked::after { transform: translateX(12px); background: var(--on-teal); }
.enh-switch:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
.enh-section-head { display: flex; align-items: flex-end; justify-content: space-between; gap: 8px; }
.enh-section-head .btn { margin-bottom: 3px; }
.enh-system { width: 100%; min-height: 96px; resize: vertical; font-family: var(--font-ui); font-size: 12px; line-height: 1.5; }
.enh-row .sw-row-main { min-width: 0; }
.enh-range { flex: 0 1 120px; min-width: 70px; accent-color: var(--teal); }
.enh-val { flex: 0 0 34px; text-align: right; font-size: 11px; color: var(--muted); font-variant-numeric: tabular-nums; }
.enh-num { flex: 0 0 76px; width: 76px; }
.enh-empty { padding: 16px 12px; border: 1px dashed var(--line); border-radius: 10px; text-align: center;
font-size: 12px; line-height: 1.5; color: var(--faint); }
.enh-card { border: 1px solid var(--line); border-radius: 10px; background: var(--ink-2); padding: 10px 12px; }
.enh-card + .enh-card { margin-top: 2px; }
.enh-card-head { display: flex; align-items: center; gap: 6px; margin-bottom: 6px; }
.enh-card-title { font-size: 11px; font-weight: 700; letter-spacing: .5px; text-transform: uppercase; color: var(--teal); }
.enh-chip { padding: 1px 7px; border-radius: 10px; font-size: 10px; color: var(--muted); border: 1px solid var(--line); }
.enh-chip-warn { color: var(--warn); border-color: var(--warn); }
.enh-copy { margin-left: auto; }
.enh-after { font-size: 12.5px; line-height: 1.55; color: var(--text); white-space: pre-wrap; overflow-wrap: anywhere; user-select: text; }
.enh-before { margin-top: 8px; }
.enh-before summary { font-size: 11px; color: var(--muted); cursor: pointer; }
.enh-before-text { margin-top: 4px; padding-left: 8px; border-left: 2px solid var(--line); font-size: 12px; line-height: 1.5;
color: var(--muted); white-space: pre-wrap; overflow-wrap: anywhere; user-select: text; }
.enh-status { margin-top: 8px; font-size: 10.5px; line-height: 1.4; color: var(--faint); }
.eng-enhance-host { margin-top: 16px; }
/* + Add ▾ split-button dropdown (Shortcuts / Splits toolbars) */
.composer-add { position: relative; flex-shrink: 0; }
.composer-add-menu { position: absolute; top: calc(100% + 4px); left: 0; z-index: 10; min-width: 150px;
+11
View File
@@ -89,6 +89,14 @@ def _media_from_history(hist_entry: dict) -> list[dict]:
return out
def _enhanced_from_history(hist_entry: dict) -> list:
"""What Studio's prompt enhancer wrote this run (its `funpack_enhanced` ui output)."""
out: list = []
for node_out in (hist_entry.get("outputs") or {}).values():
out.extend(x for x in (node_out.get("funpack_enhanced") or []) if isinstance(x, dict))
return out
def _scene_layout_from_history(hist_entry: dict, fps: float) -> Optional[list]:
from .backend.chain_layout import layout_from_history_entry
return layout_from_history_entry(hist_entry, fps)
@@ -2053,6 +2061,9 @@ if web is not None and PromptServer is not None:
}
if scene_layout:
payload["scene_layout"] = scene_layout
enhanced = _enhanced_from_history(entry)
if enhanced:
payload["enhanced"] = enhanced
return web.json_response(payload)
try:
running = await bridge.is_running(prompt_id)
+9
View File
@@ -687,3 +687,12 @@ def test_the_ports_endpoint_answers_for_the_project_not_the_global_default(monke
assert pipeline_wiring.family_of(nodes.load_models()) == "ltxav"
# No pid — an app with no project open — still answers with the global default.
assert pipeline_wiring.family_of(_project_models(None)) == "ltxav"
def test_enhanced_prompts_are_read_from_any_node_in_history():
entry = {"outputs": {
"7": {"funpack_enhanced": [{"scene": 0, "before": "a", "after": "b"}, "junk"]},
"9": {"gifs": [{"filename": "x.mp4"}]},
}}
assert srv._enhanced_from_history(entry) == [{"scene": 0, "before": "a", "after": "b"}]
assert srv._enhanced_from_history({}) == []
+60
View File
@@ -378,3 +378,63 @@ def test_the_enhancer_never_inherits_the_advisors_thinking_default(studio):
clip = FakeClip()
studio._v2_enhance_prompt(clip, "a cat", "sys")
assert clip.calls[0][1].get("thinking") is False
# ── the Studio image reaches the enhancer on BOTH paths ─────────────────────
#
# The Editor forces the per-scene path, which used to call the enhancer with no image at all —
# so only a single-prompt run could ever show it the picture.
def test_the_image_reaches_the_enhancer():
clip = FakeClip()
image = object()
studio = C.FunPackVideoRefinerV2.__new__(C.FunPackVideoRefinerV2)
studio._v2_enhance_prompt(clip, "a cat", "sys", image=image)
assert clip.calls[0][1].get("image") is image
def test_both_paths_hand_over_the_enhancer_image():
src = _refine_source()
assert src.count("image=prompt_enhance_image") == 2
assert "thinking=prompt_enhance_thinking, image=source_image" not in src
def test_studio_gates_the_image_on_its_own_switch():
"""Not on vision_conditioning: showing the enhancer the picture is a separate choice."""
import inspect
src = inspect.getsource(C.FunPackStudio.run)
assert ('prompt_enhance_image=source_image if rf.get("prompt_enhance_use_image", True) '
'else None') in src
def test_every_rewrite_is_reported_for_the_composer():
src = _refine_source()
assert src.count("self._v2_enhanced_prompts.append(") == 2
import inspect
assert '{"ui": {"funpack_enhanced": enhanced}, "result": result}' in inspect.getsource(C.FunPackStudio.run)
# ── every sampling knob in the Composer reaches clip.generate ───────────────
def test_the_sampling_settings_reach_the_generator(studio):
clip = FakeClip()
studio._v2_enhance_prompt(clip, "a cat", "sys", seed=7, temperature=0.4, top_p=0.8,
top_k=12, min_p=0.1, repetition_penalty=1.1,
presence_penalty=0.5, do_sample=False)
g = clip.last_generate
assert (g["temperature"], g["top_p"], g["top_k"], g["min_p"]) == (0.4, 0.8, 12, 0.1)
assert (g["repetition_penalty"], g["presence_penalty"], g["do_sample"], g["seed"]) == (1.1, 0.5, False, 7)
def test_the_repair_advisor_keeps_its_old_sampling(studio):
"""New knobs are the enhancer's; the advisor's defaults must not move."""
clip = FakeClip()
studio._v2_generate_advisor_text(clip, "sys", "a cat")
g = clip.last_generate
assert (g["min_p"], g["presence_penalty"], g["do_sample"], g["top_k"]) == (0.05, 0.0, True, 50)
def test_a_fixed_enhancer_seed_overrides_the_run_seed():
src = _refine_source()
assert '_enhance_seed = int(_enhance_sampling.pop("seed", 0) or 0) or seed' in src
assert src.count("seed=_enhance_seed") == 2
+1 -1
View File
@@ -85,7 +85,7 @@ function defaultSettings() {
return {
refinement_key: "",
overrides: { refinement_key: false, feedback_prompt: false, user_intent_prompt: false, negative_prompt: false },
refiner: { mode: "Refine", advisor_mode: "Off", advisor_thinking: true, im_feeling_lucky: false, reset_session: false, feedback_prompt: "", user_intent_prompt_override: "", negative_prompt: "", temporal_style: "natural", split_by_transitions: false, split_transition_placement: "start", reference_injection: false, vision_conditioning: true, value_guidance: true, steer_mode: "relative", absolute_strength: 0.6, negative_erase: false, negative_erase_strength: 0.5, negative_erase_mode: "project", negative_erase_renorm: true, h3_phrase_emphasis: false, h3_render_gains: true, prompt_enhance: false, prompt_enhance_system: "", prompt_enhance_temperature: 0.7, prompt_enhance_top_p: 0.92, prompt_enhance_max_length: 400, prompt_enhance_thinking: false },
refiner: { mode: "Refine", advisor_mode: "Off", advisor_thinking: true, im_feeling_lucky: false, reset_session: false, feedback_prompt: "", user_intent_prompt_override: "", negative_prompt: "", temporal_style: "natural", split_by_transitions: false, split_transition_placement: "start", reference_injection: false, vision_conditioning: true, value_guidance: true, steer_mode: "relative", absolute_strength: 0.6, negative_erase: false, negative_erase_strength: 0.5, negative_erase_mode: "project", negative_erase_renorm: true, h3_phrase_emphasis: false, h3_render_gains: true, prompt_enhance: false, prompt_enhance_system: "", prompt_enhance_temperature: 0.7, prompt_enhance_top_p: 0.92, prompt_enhance_max_length: 400, prompt_enhance_thinking: false, prompt_enhance_use_image: true, prompt_enhance_top_k: 50, prompt_enhance_min_p: 0.05, prompt_enhance_repetition_penalty: 1.3, prompt_enhance_presence_penalty: 0, prompt_enhance_seed: 0, prompt_enhance_greedy: false },
advisor_llm: { enabled: false, model_path: "huihui-ai/Huihui-Qwen3-8B-abliterated-v2", dtype: "bfloat16" },
loras: [],
loras_config: { mode: "ltx2", per_block: false },