diff --git a/conditioning.py b/conditioning.py index 7b2865f7..939ede25 100644 --- a/conditioning.py +++ b/conditioning.py @@ -8936,7 +8936,8 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): def _v2_generate_advisor_text(self, clip, system_prompt, user_prompt, seed=None, image=None, thinking=True, max_length=800, temperature=0.7, top_p=0.92, top_k=50, repetition_penalty=1.3, min_floor=128, - label="Advisor"): + label="Advisor", min_p=0.05, presence_penalty=0.0, + do_sample=True): """Generate text through a CLIP that exposes ComfyUI's generate/decode pair. The sampling parameters default to exactly what the repair advisor has always used, so @@ -8985,15 +8986,15 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): except TypeError: tokens = clip.tokenize(advisor_prompt) generate_kwargs = dict( - do_sample=True, + do_sample=bool(do_sample), max_length=max_length, temperature=float(temperature), top_k=int(top_k), top_p=float(top_p), - min_p=0.05, + min_p=float(min_p), repetition_penalty=float(repetition_penalty), no_repeat_ngram_size=5, - presence_penalty=0.0, + presence_penalty=float(presence_penalty), seed=seed if seed else None, ) try: @@ -9017,7 +9018,8 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): #: repeats an anchor line across scenes, and each call is a full LLM generation def _v2_enhance_prompt(self, clip, text, system_prompt, cache=None, seed=None, temperature=0.7, top_p=0.92, max_length=400, thinking=False, - image=None): + image=None, top_k=50, min_p=0.05, repetition_penalty=1.3, + presence_penalty=0.0, do_sample=True): """Rewrite one prompt through the advisor LLM, BEFORE it is encoded. This is not the repair advisor. That one is triggered by a rating and fixes a named @@ -9037,7 +9039,9 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): raw, status = self._v2_generate_advisor_text( clip, system_prompt, original, seed=seed, image=image, thinking=bool(thinking), max_length=max_length, temperature=temperature, top_p=top_p, - min_floor=32, label="Prompt enhancer", + min_floor=32, label="Prompt enhancer", top_k=top_k, min_p=min_p, + repetition_penalty=repetition_penalty, presence_penalty=presence_penalty, + do_sample=do_sample, ) enhanced, overran = self._v2_trim_runaway_prompt( self._v2_clean_enhanced_prompt(raw), max_length=max_length) @@ -9962,6 +9966,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): prompt_enhance=False, prompt_enhance_system="", prompt_enhance_temperature=0.7, prompt_enhance_top_p=0.92, prompt_enhance_max_length=400, prompt_enhance_thinking=False, + prompt_enhance_image=None, prompt_enhance_sampling=None, _seed=None, _seed_source="fresh seed", _scene_seeds=None, _velocity_keys=None, batch_variants=1, guess_mode=False, guess_direction="up", guess_range=1.0, guess_freeze_seed=True, movie_editor_scene_ratings=None, scene_segments=None, @@ -10604,23 +10609,34 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): # `advisor_clip` if one is wired, the generation CLIP otherwise. _enhance_cache = {} _enhance_status = "" + # What the enhancer wrote, per encoded prompt — Studio hands it to the Editor as a + # node ui output so the Composer can show before/after. A readout, never read back. + self._v2_enhanced_prompts = [] _enhance_clip = advisor_clip # already falls back to `clip` at the top _enhance_system = str(prompt_enhance_system or "").strip() or V2_PROMPT_ENHANCER_SYSTEM_PROMPT # When the run splits into scenes, the per-scene conditionings REPLACE this base entry # — so enhancing the base too would be a second full generation whose result is then # thrown away. One generation per prompt that is actually encoded, never two. _enhance_base = prompt_enhance and not split_by_transitions + # top_k / min_p / penalties / greedy; `seed` 0 = follow the generation seed. + _enhance_sampling = dict(prompt_enhance_sampling or {}) + _enhance_seed = int(_enhance_sampling.pop("seed", 0) or 0) or seed if prompt_enhance and not _enhance_base: print("[FunPackVideoRefinerV2] Prompt enhancer: base prompt skipped — this run " "splits into scenes and each scene is enhanced instead.") if _enhance_base: + _before = prompt_to_encode prompt_to_encode, _enhance_status = self._v2_enhance_prompt( _enhance_clip, prompt_to_encode, _enhance_system, cache=_enhance_cache, - seed=seed, temperature=prompt_enhance_temperature, + seed=_enhance_seed, temperature=prompt_enhance_temperature, top_p=prompt_enhance_top_p, max_length=prompt_enhance_max_length, - thinking=prompt_enhance_thinking, image=source_image, + thinking=prompt_enhance_thinking, image=prompt_enhance_image, + **_enhance_sampling, ) print(f"[FunPackVideoRefinerV2] {_enhance_status}") + self._v2_enhanced_prompts.append({ + "scene": None, "before": _before, "after": prompt_to_encode, + "status": _enhance_status, "image": prompt_enhance_image is not None}) cond, meta, encode_status, conditioning_owner = self._v2_conditioning_source( clip, @@ -10856,15 +10872,21 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): # and re-splitting could return a different number of scenes and desync # every clip from its anchor. One call per distinct text, cached. if prompt_enhance and split_scene_texts: - split_scene_texts = [ - self._v2_enhance_prompt( + _enhanced_texts = [] + for _i, t in enumerate(split_scene_texts): + _after, _st = self._v2_enhance_prompt( _enhance_clip, t, _enhance_system, cache=_enhance_cache, - seed=seed, temperature=prompt_enhance_temperature, + seed=_enhance_seed, temperature=prompt_enhance_temperature, top_p=prompt_enhance_top_p, max_length=prompt_enhance_max_length, - thinking=prompt_enhance_thinking)[0] - for t in split_scene_texts - ] + thinking=prompt_enhance_thinking, image=prompt_enhance_image, + **_enhance_sampling) + print(f"[FunPackVideoRefinerV2] Scene {_i + 1}: {_st}") + self._v2_enhanced_prompts.append({ + "scene": _i, "before": t, "after": _after, "status": _st, + "image": prompt_enhance_image is not None}) + _enhanced_texts.append(_after) + split_scene_texts = _enhanced_texts scene_refinement_keys = [set(s.get("keys") or set()) for s in canon_scenes] current_scene_seeds = self._v2_scene_seed_values(seed, len(split_scene_texts), _scene_seeds) current_scene_seed_source = ( @@ -13853,6 +13875,17 @@ class FunPackStudio: prompt_enhance_top_p=float(rf.get("prompt_enhance_top_p", 0.92)), prompt_enhance_max_length=int(rf.get("prompt_enhance_max_length", 400)), prompt_enhance_thinking=bool(rf.get("prompt_enhance_thinking", False)), + # Its own switch, not vision_conditioning's: showing the enhancer the picture is + # a different decision from writing the picture into the conditioning. + prompt_enhance_image=source_image if rf.get("prompt_enhance_use_image", True) else None, + prompt_enhance_sampling={ + "top_k": int(rf.get("prompt_enhance_top_k", 50)), + "min_p": float(rf.get("prompt_enhance_min_p", 0.05)), + "repetition_penalty": float(rf.get("prompt_enhance_repetition_penalty", 1.3)), + "presence_penalty": float(rf.get("prompt_enhance_presence_penalty", 0.0)), + "do_sample": not bool(rf.get("prompt_enhance_greedy", False)), + "seed": int(rf.get("prompt_enhance_seed", 0) or 0), + }, clip_vision_output=clip_vision_output, source_image=source_image if vision_conditioning else None, model=model, @@ -13942,7 +13975,12 @@ class FunPackStudio: cond = _h3_reconcile_token_tags(cond, "positive") out_negative = _h3_reconcile_token_tags(out_negative, "negative") - return (out_model, cond, out_negative, seed, high_sampler, high_sigmas, low_sampler, low_sigmas, loss_graph, status, training_info, encoded_prompts, video_latent) + result = (out_model, cond, out_negative, seed, high_sampler, high_sigmas, low_sampler, low_sigmas, loss_graph, status, training_info, encoded_prompts, video_latent) + enhanced = getattr(refiner, "_v2_enhanced_prompts", None) + if enhanced: + # History carries every node's ui output; the Editor's /status reads this key. + return {"ui": {"funpack_enhanced": enhanced}, "result": result} + return result @staticmethod def _parse_sigmas(text): diff --git a/movie_editor/backend/settings_card.py b/movie_editor/backend/settings_card.py index d12b968f..8f2473d2 100644 --- a/movie_editor/backend/settings_card.py +++ b/movie_editor/backend/settings_card.py @@ -177,6 +177,19 @@ _OWNED_EXTRA = { "detail_threshold": ("segmented_detailing", None), "detail_max_area": ("segmented_detailing", None), "detail_mode": ("segmented_detailing", None), + # The prompt enhancer's settings live in the Composer (composer.js), not engine_settings.js. + "prompt_enhance_max_length": ("prompt_enhance", ('true',)), + "prompt_enhance_system": ("prompt_enhance", ('true',)), + "prompt_enhance_temperature": ("prompt_enhance", ('true',)), + "prompt_enhance_thinking": ("prompt_enhance", ('true',)), + "prompt_enhance_top_p": ("prompt_enhance", ('true',)), + "prompt_enhance_use_image": ("prompt_enhance", ('true',)), + "prompt_enhance_top_k": ("prompt_enhance", ('true',)), + "prompt_enhance_min_p": ("prompt_enhance", ('true',)), + "prompt_enhance_repetition_penalty": ("prompt_enhance", ('true',)), + "prompt_enhance_presence_penalty": ("prompt_enhance", ('true',)), + "prompt_enhance_seed": ("prompt_enhance", ('true',)), + "prompt_enhance_greedy": ("prompt_enhance", ('true',)), } _OWNED_BY = { @@ -226,11 +239,6 @@ _OWNED_BY = { "h3_phrase_variability": ("h3_phrase_emphasis", ('true',)), "trajectory_guidance_strength": ("trajectory_guidance", None), "phase_scale": ("identity_transfer_enabled", None), - "prompt_enhance_max_length": ("prompt_enhance", ('true',)), - "prompt_enhance_system": ("prompt_enhance", ('true',)), - "prompt_enhance_temperature": ("prompt_enhance", ('true',)), - "prompt_enhance_thinking": ("prompt_enhance", ('true',)), - "prompt_enhance_top_p": ("prompt_enhance", ('true',)), "score_slider_strength": ("score_slider", None), "source_id": ("identity_transfer_enabled", None), "v2a_grad_scale": ("joyai_audio_memory", None), @@ -328,6 +336,13 @@ _EDITOR_DEFAULTS = { "prompt_enhance_temperature": 0.7, "prompt_enhance_thinking": False, "prompt_enhance_top_p": 0.92, + "prompt_enhance_use_image": True, + "prompt_enhance_top_k": 50, + "prompt_enhance_min_p": 0.05, + "prompt_enhance_repetition_penalty": 1.3, + "prompt_enhance_presence_penalty": 0.0, + "prompt_enhance_seed": 0, + "prompt_enhance_greedy": False, "reference_injection": False, "score_slider": False, "score_slider_strength": 1.0, diff --git a/movie_editor/frontend/composer.js b/movie_editor/frontend/composer.js index 5139d94a..d3e2f189 100644 --- a/movie_editor/frontend/composer.js +++ b/movie_editor/frontend/composer.js @@ -929,8 +929,180 @@ return wrap; } + // ── enhance (LLM prompt enhancer) ─────────────────────────────────────────────── + // Studio rewrites each prompt through a language model right before it is encoded — after + // shortcuts and $variables, so the model reads exactly what would otherwise be used. The + // settings are Studio's refiner settings; the last run's rewrite comes back from the run's + // history (Store.enhanced). Also mounted in Engine settings for Simple mode (no Composer). + const ENH_DEFAULTS = { + prompt_enhance: false, prompt_enhance_system: "", prompt_enhance_max_length: 400, + prompt_enhance_temperature: 0.7, prompt_enhance_top_p: 0.92, prompt_enhance_top_k: 50, + prompt_enhance_min_p: 0.05, prompt_enhance_repetition_penalty: 1.3, + prompt_enhance_presence_penalty: 0, prompt_enhance_seed: 0, prompt_enhance_greedy: false, + prompt_enhance_thinking: false, prompt_enhance_use_image: true, + }; + // `sampled`: only read while sampling. Greedy returns the single likeliest word before + // ComfyUI looks at any of them, so they disappear rather than sit there doing nothing. + const ENH_GROUPS = [ + { title: "Behaviour", knobs: [ + { name: "prompt_enhance_use_image", label: "Show it Studio's image", kind: "bool", + hint: "Hands it the picture connected to Studio's source_image, if there is one. Only helps a model that can see." }, + { name: "prompt_enhance_thinking", label: "Think first", kind: "bool", + hint: "Reasons before answering, if the model can. Slower; the reasoning is removed." }, + { name: "prompt_enhance_max_length", label: "Max tokens", kind: "int", min: 32, max: 4096, step: 32, + hint: "Most it may write. Longer is richer and slower." }, + ] }, + { title: "Sampling", knobs: [ + { name: "prompt_enhance_greedy", label: "Same wording every time", kind: "bool", + hint: "Greedy: always picks the likeliest next word. Repeatable, but plainer." }, + { name: "prompt_enhance_temperature", label: "Temperature", kind: "range", min: 0.01, max: 2, step: 0.01, sampled: true, + hint: "Low stays close to your wording, high invents more." }, + { name: "prompt_enhance_top_p", label: "Top-p", kind: "range", min: 0, max: 1, step: 0.01, sampled: true, + hint: "Keeps only the likeliest words that add up to this share. Lower is safer." }, + { name: "prompt_enhance_top_k", label: "Top-k", kind: "int", min: 1, max: 500, step: 1, sampled: true, + hint: "Picks from at most this many candidate words. Lower is safer." }, + { name: "prompt_enhance_min_p", label: "Min-p", kind: "range", min: 0, max: 1, step: 0.01, sampled: true, + hint: "Drops words less than this fraction as likely as the best one." }, + { name: "prompt_enhance_repetition_penalty", label: "Repetition penalty", kind: "range", min: 1, max: 2, step: 0.01, sampled: true, + hint: "Discourages reusing words already written. 1 is off." }, + { name: "prompt_enhance_presence_penalty", label: "Presence penalty", kind: "range", min: 0, max: 2, step: 0.01, sampled: true, + hint: "A flat push away from any word already used. 0 is off." }, + { name: "prompt_enhance_seed", label: "Seed", kind: "int", min: 0, max: 2147483647, step: 1, sampled: true, + hint: "0 follows the video's seed. Any other number keeps the rewrite fixed while the video re-rolls." }, + ] }, + ]; + + function enhanceKnobRow(f, value, patch) { + const row = el("div", "sw-row enh-row"); + const main = el("div", "sw-row-main"); + main.append(el("div", "sw-row-title", f.label), el("div", "sw-row-hint", f.hint)); + row.append(main); + const ctrl = el("input"); + ctrl.dataset.k = "rf-" + f.name; + if (f.kind === "bool") { + ctrl.type = "checkbox"; ctrl.className = "enh-switch"; ctrl.checked = !!value; + ctrl.onchange = () => patch({ [f.name]: ctrl.checked }, true); + row.append(ctrl); + return row; + } + ctrl.type = f.kind === "range" ? "range" : "number"; + ctrl.className = f.kind === "range" ? "enh-range" : "lib-in enh-num"; + ctrl.min = String(f.min); ctrl.max = String(f.max); ctrl.step = String(f.step); + ctrl.value = String(value); + const out = el("span", "enh-val", f.kind === "range" ? Number(value).toFixed(2) : ""); + ctrl.oninput = () => { + const n = parseFloat(ctrl.value); + if (!Number.isFinite(n)) return; + if (f.kind === "range") out.textContent = n.toFixed(2); + patch({ [f.name]: f.kind === "int" ? Math.round(n) : n }, false); + }; + row.append(ctrl); + if (f.kind === "range") row.append(out); + return row; + } + + function enhanceResultCard(it) { + const card = el("div", "enh-card"); + const before = String(it.before || "").trim(); + const after = String(it.after || "").trim(); + const changed = !!after && after !== before; + const head = el("div", "enh-card-head"); + head.append(el("span", "enh-card-title", it.scene == null ? "Prompt" : `Scene ${it.scene + 1}`)); + if (it.image) head.append(el("span", "enh-chip", "image sent")); + if (!changed) head.append(el("span", "enh-chip enh-chip-warn", "unchanged")); + const copy = el("button", "btn ghost tiny enh-copy", "Copy"); + copy.type = "button"; + copy.onclick = async () => { + try { await navigator.clipboard.writeText(after); copy.textContent = "Copied"; } + catch (_) { copy.textContent = "Copy failed"; } + setTimeout(() => { copy.textContent = "Copy"; }, 1400); + }; + head.append(copy); + card.append(head, el("div", "enh-after", after || before)); + if (changed) { + const det = el("details", "enh-before"); + det.append(el("summary", null, "What went in"), el("div", "enh-before-text", before)); + card.append(det); + } + if (it.status) card.append(el("div", "enh-status", String(it.status))); + return card; + } + + function enhanceTab(st) { + const wrap = el("div", "bin enh-bin"); + const SS = window.StudioSettings; + if (!st.project || !SS) { + wrap.append(el("div", "insp-hint", "Open a project to set up the prompt enhancer.")); + return wrap; + } + const { rf } = SS.read(st.project); + const val = (k) => (rf[k] != null ? rf[k] : ENH_DEFAULTS[k]); + const on = !!val("prompt_enhance"); + + const hero = el("div", "enh-hero" + (on ? " on" : "")); + const top = el("label", "enh-hero-top"); + const titles = el("div", "enh-hero-titles"); + titles.append(el("div", "enh-hero-title", "Prompt enhancer"), + el("div", "enh-hero-sub", "A language model rewrites your prompt into a richer one right before the video is made.")); + const sw = el("input", "enh-switch"); sw.type = "checkbox"; sw.checked = on; + sw.dataset.k = "rf-prompt_enhance"; + sw.onchange = () => SS.patchRefiner({ prompt_enhance: sw.checked }, true); + top.append(titles, sw); + const flow = el("div", "enh-flow"); + ["Your prompt", "Shortcuts", "$variables", "Enhancer", "Video"].forEach((step, i) => { + if (i) flow.append(el("span", "enh-flow-arrow", "›")); + flow.append(el("span", "enh-flow-step" + (step === "Enhancer" ? " is-enh" : ""), step)); + }); + hero.append(top, flow); + if (on) { + hero.append(el("div", "enh-hero-note", + "Needs a model that can write text, connected to Studio's advisor_clip (or its clip). Each scene is rewritten on its own.")); + } + wrap.append(hero); + + if (on) { + const instHead = el("div", "enh-section-head"); + instHead.append(el("div", "sw-rows-label", "Instructions")); + if (String(val("prompt_enhance_system") || "").trim()) { + const reset = el("button", "btn ghost tiny", "Use built-in"); + reset.type = "button"; + reset.title = "Clear your instructions and go back to FunPack's own"; + reset.onclick = () => { SS.patchRefiner({ prompt_enhance_system: "" }, true); repaintAll(); }; + instHead.append(reset); + } + wrap.append(instHead); + const ta = el("textarea", "lib-in enh-system"); ta.rows = 6; + ta.dataset.k = "rf-prompt_enhance_system"; + ta.value = String(val("prompt_enhance_system") || ""); + ta.placeholder = "Empty uses FunPack's built-in instructions: add detail and a soundscape, never invent characters, camera moves or speech."; + ta.oninput = () => SS.patchRefiner({ prompt_enhance_system: ta.value }, false); + ta.onchange = () => SS.patchRefiner({ prompt_enhance_system: ta.value }, true); + wrap.append(ta); + + const greedy = !!val("prompt_enhance_greedy"); + ENH_GROUPS.forEach((g) => { + wrap.append(el("div", "sw-rows-label", g.title)); + const rows = el("div", "sw-rows"); + g.knobs.filter((f) => !(greedy && f.sampled)) + .forEach((f) => rows.append(enhanceKnobRow(f, val(f.name), SS.patchRefiner))); + wrap.append(rows); + }); + } + + wrap.append(el("div", "sw-rows-label", "Last run")); + const items = (st.enhanced && st.enhanced.items) || []; + if (!items.length) { + wrap.append(el("div", "enh-empty", on + ? "Nothing yet. Generate, and the rewritten prompt shows up here next to what went in." + : "Turn the enhancer on and generate. Its rewrite shows up here.")); + } else { + items.forEach((it) => wrap.append(enhanceResultCard(it))); + } + return wrap; + } + // ── window + tabs ──────────────────────────────────────────────────────────────── - const TABS = ["Compose", "Shortcuts", "Splits", "Files"]; + const TABS = ["Compose", "Enhance", "Shortcuts", "Splits", "Files"]; function render() { if (!win) return; const st = S.get(); @@ -942,6 +1114,7 @@ const b = el("button", "bin-tab" + (tab === name ? " active" : ""), name); b.title = name === "Splits" ? "Split markers (generation prompt)" : name === "Compose" ? "Global prompt — the whole montage" + : name === "Enhance" ? "Let a language model rewrite the prompt before generating" : name === "Files" ? "FunPack files on disk — audit & purge" : name; b.onclick = () => { if (tab === name) return; @@ -955,6 +1128,7 @@ const scroll = el("div", "composer-scroll"); scroll.append( tab === "Compose" ? composeTab(st) + : tab === "Enhance" ? enhanceTab(st) : tab === "Shortcuts" ? shortcutsTab(st) : tab === "Splits" ? splitMarkersTab(st) : filesTab(), @@ -986,7 +1160,7 @@ // The setup wizard shows Shortcuts and Splits as full screens of its own, so the flow // stays one continuous surface instead of spawning a floating Composer over it. Same // builders, same editors, same store — only the container differs. - const PANE_BUILDERS = { shortcuts: shortcutsTab, splits: splitMarkersTab }; + const PANE_BUILDERS = { shortcuts: shortcutsTab, splits: splitMarkersTab, enhance: enhanceTab }; const _panes = new Set(); // { kind, host } function mountPane(kind, host) { @@ -999,10 +1173,32 @@ } function paintPane(entry) { + // A host whose container re-rendered without unmounting (Engine settings rebuilds its + // whole pane) is gone for good — drop it rather than paint into a detached node. + if (!entry.host.isConnected) { _panes.delete(entry); return; } clear(entry.host); entry.host.append(PANE_BUILDERS[entry.kind](S.get())); } + // A repaint replaces every field, so never do it under the caret: wait until focus leaves. + function typingIn(root) { + const a = document.activeElement; + if (!root || !a || !root.contains(a)) return false; + if (a.tagName === "TEXTAREA") return true; + return a.tagName === "INPUT" && !/^(checkbox|radio|button)$/i.test(a.type || "text"); + } + let _repaintDeferred = false; + function repaintAll() { + const hosts = [isOpen() ? win.body : null, ...[..._panes].map((e) => e.host)]; + if (hosts.some(typingIn)) { _repaintDeferred = true; return; } + _repaintDeferred = false; + if (isOpen()) render(); + _panes.forEach(paintPane); + } + document.addEventListener("focusout", () => { + if (_repaintDeferred) setTimeout(() => { if (_repaintDeferred) repaintAll(); }, 0); + }); + let lastFp = null; S.subscribe((st) => { if (!isOpen() && !_panes.size) return; @@ -1015,11 +1211,17 @@ // fingerprinted — typing goes through quiet patches and the field itself is // the source of those edits. sd: !!(st.project?.sampler_inputs?.segmented_detailing), + // Enhance tab: the switches that show/hide rows, and which run's rewrite is shown. + // Typed values stay out: a repaint per keystroke would fight the caret. + en: !!(st.project && window.StudioSettings?.read(st.project).rf.prompt_enhance), + eg: !!(st.project && window.StudioSettings?.read(st.project).rf.prompt_enhance_greedy), + eh: st.enhanced?.promptId || null, }); - if (fp === lastFp) return; + // A deferred repaint also flushes on the next notify: focusout never fires when focus + // moves while the window itself is in the background. + if (fp === lastFp) { if (_repaintDeferred) repaintAll(); return; } lastFp = fp; - if (isOpen()) render(); - _panes.forEach(paintPane); + repaintAll(); }); // Keep the Compose textarea in sync with timeline-driven global-prompt changes, diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index fa9be3e3..41d4f038 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -119,25 +119,6 @@ { name: "h3_phrase_variability", label: "Phrase emphasis variability", kind: "float", default: 0.0, min: 0.0, max: 1.0, step: 0.05, dependsOn: "h3_phrase_emphasis", dependsVals: [true], hint: "0 = only what the rating has cemented gets extra attention, nothing else. 1 = that emphasis is switched off entirely, so untrained phrasing and actions compete on equal footing and can bleed into the scene — including ones you have rated against before. Same bias channel as the toggle above, just scaled down." }, - { name: "prompt_enhance", label: "Enhance the prompt with an LLM", kind: "bool", default: false, - hint: "Before anything is generated, the fully expanded prompt is rewritten by a language model and the result is what the video is made from. Needs a text encoder that can GENERATE text wired into the Studio node's advisor_clip (or a FunPack Advisor LLM node); without one it says so and leaves the prompt alone.", - detail: "Runs after shortcuts and $variables resolve, before encoding — so the model is handed exactly the text that would otherwise have been used, and nothing downstream needs to know it happened. On a multi-scene timeline each scene is enhanced separately: the scene list is authoritative on count, and rewriting them as one paragraph could come back with a different number of scenes. One generation per distinct scene text, so a long timeline costs real time." }, - { name: "prompt_enhance_system", label: "Enhancer instructions", kind: "textarea", rows: 10, default: "", - dependsOn: "prompt_enhance", dependsVals: [true], - placeholder: "Leave empty to use the built-in instructions.", - hint: "The system prompt the enhancer runs under. Empty uses FunPack's built-in one, which expands detail and adds a soundscape without inventing characters, camera moves or speech." }, - { name: "prompt_enhance_max_length", label: "Enhancer length limit", kind: "int", default: 400, min: 32, max: 4096, step: 32, - dependsOn: "prompt_enhance", dependsVals: [true], - hint: "Maximum tokens the enhancer may write. Higher allows a richer prompt and costs more time." }, - { name: "prompt_enhance_temperature", label: "Enhancer temperature", kind: "float", default: 0.7, min: 0.01, max: 2.0, step: 0.05, - dependsOn: "prompt_enhance", dependsVals: [true], - hint: "How freely the enhancer writes. Lower sticks closer to your wording, higher invents more detail." }, - { name: "prompt_enhance_top_p", label: "Enhancer top-p", kind: "float", default: 0.92, min: 0.0, max: 1.0, step: 0.01, - dependsOn: "prompt_enhance", dependsVals: [true], - hint: "Nucleus sampling cutoff for the enhancer. Lower is more predictable wording." }, - { name: "prompt_enhance_thinking", label: "Enhancer thinking mode", kind: "bool", default: false, - dependsOn: "prompt_enhance", dependsVals: [true], - hint: "Let the model reason before answering, if it supports it. Slower, and the reasoning is stripped from the result." }, ]; function parseStudioSettings(p) { @@ -214,17 +195,6 @@ ctrl = el("select"); ctrl.dataset.k = "rf-" + f.name; (f.choices || []).forEach((c) => { const o = new Option(c, c); if (c === val) o.selected = true; ctrl.append(o); }); ctrl.onchange = () => persistStudioRefiner({ [f.name]: ctrl.value }, true); - } else if (f.kind === "textarea") { - // A system prompt is paragraphs, not a value. Quiet while typing (no repaint under the - // caret, same rule the sampler's text knobs follow), committed on blur. - ctrl = el("textarea"); ctrl.dataset.k = "rf-" + f.name; - ctrl.rows = f.rows || 8; - ctrl.value = val != null ? String(val) : ""; - if (f.placeholder) ctrl.placeholder = f.placeholder; - ctrl.style.width = "100%"; - ctrl.style.resize = "vertical"; - ctrl.style.fontFamily = "inherit"; - ctrl.onchange = () => persistStudioRefiner({ [f.name]: ctrl.value }, true); } else { ctrl = el("input"); ctrl.type = "number"; if (f.step != null) ctrl.step = String(f.step); @@ -681,6 +651,14 @@ STUDIO_REFINER_ADVANCED.filter((f) => !EASY() || !RATING_GATED_STUDIO.has(f.name)) .forEach((f) => renderStudioRefinerField(gAdv, rf, f)); + // The prompt enhancer lives in the Composer. Simple mode has no Composer, so the same + // pane is mounted here instead; it unmounts itself once this render replaces it. + if (EASY() && window.Composer) { + const host = el("div", "eng-enhance-host"); + pane.append(host); + window.Composer.mountPane("enhance", host); + } + if (EASY()) { pane.append(hintEl( "Studio runs in Prompt-only mode here: it shapes and splits the prompt, nothing more. " @@ -1167,6 +1145,9 @@ }, }); + // Studio's refiner settings, for the Composer's Enhance tab — one reader, one writer. + window.StudioSettings = { read: parseStudioSettings, patchRefiner: persistStudioRefiner }; + window.EngineSettingsModal = { open: () => window.SettingsWindow.open("engine"), close: () => window.SettingsWindow.close(), diff --git a/movie_editor/frontend/store.js b/movie_editor/frontend/store.js index 9c48ad30..edf8b431 100644 --- a/movie_editor/frontend/store.js +++ b/movie_editor/frontend/store.js @@ -33,6 +33,9 @@ shortcutCategories: [], // managed grouping list: [{name, sub_categories:[]}] imageTargets: [], // where an image asset can be wired [{value,label}] ratingLabels: [], // FunPack Studio V2 rating options + // What the prompt enhancer wrote on the last run: {promptId, items:[{scene,before,after, + // status,image}]} or null. A readout for the Composer's Enhance tab; never read back. + enhanced: null, }; // ── editor settings ───────────────────────────────────────────────────────── @@ -821,6 +824,7 @@ state.selectedAudioTrackId = null; _selectionAnchorId = first; state.gen = { state: "idle", promptId: null, media: [], msg: "" }; + state.enhanced = null; state.sceneRenders = JSON.parse(JSON.stringify(state.project.scene_renders || {})); state.sceneGhosts = JSON.parse(JSON.stringify(state.project.scene_ghosts || [])); _ensureTimelineOrder(); @@ -3497,6 +3501,10 @@ try { const s = await API.status(state.project.id, promptId); transientStreak = 0; + // Studio runs before sampling, so a run that fails later still carries this. + if (s.enhanced && state.enhanced?.promptId !== promptId) { + set({ enhanced: { promptId, items: s.enhanced } }); + } if (s.state === "error") { _clearGenTimers(); const msg = s.error ? `ComfyUI error: ${s.error}` : "Generation failed inside ComfyUI — check the ComfyUI terminal for details."; diff --git a/movie_editor/frontend/styles.css b/movie_editor/frontend/styles.css index ec5c388d..85a1e9c6 100644 --- a/movie_editor/frontend/styles.css +++ b/movie_editor/frontend/styles.css @@ -2064,6 +2064,60 @@ body.tour-mode { overflow: hidden; } .lib-cat-tag { margin-left: 8px; padding: 1px 7px; border-radius: 10px; background: var(--teal-soft); color: var(--teal); font-family: var(--font-ui); font-size: 9px; letter-spacing: .5px; text-transform: uppercase; vertical-align: middle; } +/* Composer ▸ Enhance (LLM prompt enhancer) */ +.enh-bin { display: flex; flex-direction: column; gap: 6px; } +.enh-hero { border: 1px solid var(--line); border-radius: 12px; padding: 12px 14px; background: var(--ink-2); + transition: border-color .2s ease, box-shadow .2s ease; } +.enh-hero.on { border-color: var(--teal); box-shadow: 0 0 0 1px var(--teal-soft) inset; } +.enh-hero-top { display: flex; align-items: flex-start; gap: 12px; cursor: pointer; } +.enh-hero-titles { flex: 1; min-width: 0; } +.enh-hero-title { font-size: 14px; font-weight: 600; color: var(--text); } +.enh-hero-sub { margin-top: 2px; font-size: 12px; line-height: 1.45; color: var(--muted); } +.enh-hero-note { margin-top: 10px; font-size: 11px; line-height: 1.45; color: var(--faint); } +.enh-flow { display: flex; flex-wrap: wrap; align-items: center; gap: 4px; margin-top: 10px; } +.enh-flow-step { padding: 2px 8px; border-radius: 10px; border: 1px solid var(--line); font-size: 10px; + color: var(--muted); white-space: nowrap; } +.enh-flow-step.is-enh { border-color: var(--teal); color: var(--teal); background: var(--teal-soft); } +.enh-flow-arrow { font-size: 11px; color: var(--faint); } +.enh-hero:not(.on) .enh-flow-step.is-enh { border-style: dashed; background: transparent; opacity: .7; } + +/* iOS-style switch, used for every on/off in this tab */ +/* min-height/padding/border reset: the global input rule (30px tall, padded) would stretch it */ +.enh-switch { appearance: none; -webkit-appearance: none; flex: 0 0 auto; width: 30px; height: 18px; margin: 0; + min-height: 0; padding: 0; border: 0; + border-radius: 9px; background: color-mix(in srgb, var(--faint) 55%, transparent); position: relative; cursor: pointer; + transition: background .15s ease; } +.enh-switch::after { content: ""; position: absolute; top: 2px; left: 2px; width: 14px; height: 14px; border-radius: 50%; + background: var(--text); transition: transform .15s ease; } +.enh-switch:checked { background: var(--teal); } +.enh-switch:checked::after { transform: translateX(12px); background: var(--on-teal); } +.enh-switch:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } + +.enh-section-head { display: flex; align-items: flex-end; justify-content: space-between; gap: 8px; } +.enh-section-head .btn { margin-bottom: 3px; } +.enh-system { width: 100%; min-height: 96px; resize: vertical; font-family: var(--font-ui); font-size: 12px; line-height: 1.5; } +.enh-row .sw-row-main { min-width: 0; } +.enh-range { flex: 0 1 120px; min-width: 70px; accent-color: var(--teal); } +.enh-val { flex: 0 0 34px; text-align: right; font-size: 11px; color: var(--muted); font-variant-numeric: tabular-nums; } +.enh-num { flex: 0 0 76px; width: 76px; } + +.enh-empty { padding: 16px 12px; border: 1px dashed var(--line); border-radius: 10px; text-align: center; + font-size: 12px; line-height: 1.5; color: var(--faint); } +.enh-card { border: 1px solid var(--line); border-radius: 10px; background: var(--ink-2); padding: 10px 12px; } +.enh-card + .enh-card { margin-top: 2px; } +.enh-card-head { display: flex; align-items: center; gap: 6px; margin-bottom: 6px; } +.enh-card-title { font-size: 11px; font-weight: 700; letter-spacing: .5px; text-transform: uppercase; color: var(--teal); } +.enh-chip { padding: 1px 7px; border-radius: 10px; font-size: 10px; color: var(--muted); border: 1px solid var(--line); } +.enh-chip-warn { color: var(--warn); border-color: var(--warn); } +.enh-copy { margin-left: auto; } +.enh-after { font-size: 12.5px; line-height: 1.55; color: var(--text); white-space: pre-wrap; overflow-wrap: anywhere; user-select: text; } +.enh-before { margin-top: 8px; } +.enh-before summary { font-size: 11px; color: var(--muted); cursor: pointer; } +.enh-before-text { margin-top: 4px; padding-left: 8px; border-left: 2px solid var(--line); font-size: 12px; line-height: 1.5; + color: var(--muted); white-space: pre-wrap; overflow-wrap: anywhere; user-select: text; } +.enh-status { margin-top: 8px; font-size: 10.5px; line-height: 1.4; color: var(--faint); } +.eng-enhance-host { margin-top: 16px; } + /* ＋ Add ▾ split-button dropdown (Shortcuts / Splits toolbars) */ .composer-add { position: relative; flex-shrink: 0; } .composer-add-menu { position: absolute; top: calc(100% + 4px); left: 0; z-index: 10; min-width: 150px; diff --git a/movie_editor/server.py b/movie_editor/server.py index dd8493e9..bc1bc390 100644 --- a/movie_editor/server.py +++ b/movie_editor/server.py @@ -89,6 +89,14 @@ def _media_from_history(hist_entry: dict) -> list[dict]: return out +def _enhanced_from_history(hist_entry: dict) -> list: + """What Studio's prompt enhancer wrote this run (its `funpack_enhanced` ui output).""" + out: list = [] + for node_out in (hist_entry.get("outputs") or {}).values(): + out.extend(x for x in (node_out.get("funpack_enhanced") or []) if isinstance(x, dict)) + return out + + def _scene_layout_from_history(hist_entry: dict, fps: float) -> Optional[list]: from .backend.chain_layout import layout_from_history_entry return layout_from_history_entry(hist_entry, fps) @@ -2053,6 +2061,9 @@ if web is not None and PromptServer is not None: } if scene_layout: payload["scene_layout"] = scene_layout + enhanced = _enhanced_from_history(entry) + if enhanced: + payload["enhanced"] = enhanced return web.json_response(payload) try: running = await bridge.is_running(prompt_id) diff --git a/movie_editor/tests/test_server.py b/movie_editor/tests/test_server.py index 97b39e2d..e14723c4 100644 --- a/movie_editor/tests/test_server.py +++ b/movie_editor/tests/test_server.py @@ -687,3 +687,12 @@ def test_the_ports_endpoint_answers_for_the_project_not_the_global_default(monke assert pipeline_wiring.family_of(nodes.load_models()) == "ltxav" # No pid — an app with no project open — still answers with the global default. assert pipeline_wiring.family_of(_project_models(None)) == "ltxav" + + +def test_enhanced_prompts_are_read_from_any_node_in_history(): + entry = {"outputs": { + "7": {"funpack_enhanced": [{"scene": 0, "before": "a", "after": "b"}, "junk"]}, + "9": {"gifs": [{"filename": "x.mp4"}]}, + }} + assert srv._enhanced_from_history(entry) == [{"scene": 0, "before": "a", "after": "b"}] + assert srv._enhanced_from_history({}) == [] diff --git a/tests/test_prompt_enhancer.py b/tests/test_prompt_enhancer.py index 0d49145f..1607707b 100644 --- a/tests/test_prompt_enhancer.py +++ b/tests/test_prompt_enhancer.py @@ -378,3 +378,63 @@ def test_the_enhancer_never_inherits_the_advisors_thinking_default(studio): clip = FakeClip() studio._v2_enhance_prompt(clip, "a cat", "sys") assert clip.calls[0][1].get("thinking") is False + + +# ── the Studio image reaches the enhancer on BOTH paths ───────────────────── +# +# The Editor forces the per-scene path, which used to call the enhancer with no image at all — +# so only a single-prompt run could ever show it the picture. + +def test_the_image_reaches_the_enhancer(): + clip = FakeClip() + image = object() + studio = C.FunPackVideoRefinerV2.__new__(C.FunPackVideoRefinerV2) + studio._v2_enhance_prompt(clip, "a cat", "sys", image=image) + assert clip.calls[0][1].get("image") is image + + +def test_both_paths_hand_over_the_enhancer_image(): + src = _refine_source() + assert src.count("image=prompt_enhance_image") == 2 + assert "thinking=prompt_enhance_thinking, image=source_image" not in src + + +def test_studio_gates_the_image_on_its_own_switch(): + """Not on vision_conditioning: showing the enhancer the picture is a separate choice.""" + import inspect + src = inspect.getsource(C.FunPackStudio.run) + assert ('prompt_enhance_image=source_image if rf.get("prompt_enhance_use_image", True) ' + 'else None') in src + + +def test_every_rewrite_is_reported_for_the_composer(): + src = _refine_source() + assert src.count("self._v2_enhanced_prompts.append(") == 2 + import inspect + assert '{"ui": {"funpack_enhanced": enhanced}, "result": result}' in inspect.getsource(C.FunPackStudio.run) + + +# ── every sampling knob in the Composer reaches clip.generate ─────────────── + +def test_the_sampling_settings_reach_the_generator(studio): + clip = FakeClip() + studio._v2_enhance_prompt(clip, "a cat", "sys", seed=7, temperature=0.4, top_p=0.8, + top_k=12, min_p=0.1, repetition_penalty=1.1, + presence_penalty=0.5, do_sample=False) + g = clip.last_generate + assert (g["temperature"], g["top_p"], g["top_k"], g["min_p"]) == (0.4, 0.8, 12, 0.1) + assert (g["repetition_penalty"], g["presence_penalty"], g["do_sample"], g["seed"]) == (1.1, 0.5, False, 7) + + +def test_the_repair_advisor_keeps_its_old_sampling(studio): + """New knobs are the enhancer's; the advisor's defaults must not move.""" + clip = FakeClip() + studio._v2_generate_advisor_text(clip, "sys", "a cat") + g = clip.last_generate + assert (g["min_p"], g["presence_penalty"], g["do_sample"], g["top_k"]) == (0.05, 0.0, True, 50) + + +def test_a_fixed_enhancer_seed_overrides_the_run_seed(): + src = _refine_source() + assert '_enhance_seed = int(_enhance_sampling.pop("seed", 0) or 0) or seed' in src + assert src.count("seed=_enhance_seed") == 2 diff --git a/web/funpack_studio.js b/web/funpack_studio.js index 4a309c98..91760b5f 100644 --- a/web/funpack_studio.js +++ b/web/funpack_studio.js @@ -85,7 +85,7 @@ function defaultSettings() { return { refinement_key: "", overrides: { refinement_key: false, feedback_prompt: false, user_intent_prompt: false, negative_prompt: false }, - refiner: { mode: "Refine", advisor_mode: "Off", advisor_thinking: true, im_feeling_lucky: false, reset_session: false, feedback_prompt: "", user_intent_prompt_override: "", negative_prompt: "", temporal_style: "natural", split_by_transitions: false, split_transition_placement: "start", reference_injection: false, vision_conditioning: true, value_guidance: true, steer_mode: "relative", absolute_strength: 0.6, negative_erase: false, negative_erase_strength: 0.5, negative_erase_mode: "project", negative_erase_renorm: true, h3_phrase_emphasis: false, h3_render_gains: true, prompt_enhance: false, prompt_enhance_system: "", prompt_enhance_temperature: 0.7, prompt_enhance_top_p: 0.92, prompt_enhance_max_length: 400, prompt_enhance_thinking: false }, + refiner: { mode: "Refine", advisor_mode: "Off", advisor_thinking: true, im_feeling_lucky: false, reset_session: false, feedback_prompt: "", user_intent_prompt_override: "", negative_prompt: "", temporal_style: "natural", split_by_transitions: false, split_transition_placement: "start", reference_injection: false, vision_conditioning: true, value_guidance: true, steer_mode: "relative", absolute_strength: 0.6, negative_erase: false, negative_erase_strength: 0.5, negative_erase_mode: "project", negative_erase_renorm: true, h3_phrase_emphasis: false, h3_render_gains: true, prompt_enhance: false, prompt_enhance_system: "", prompt_enhance_temperature: 0.7, prompt_enhance_top_p: 0.92, prompt_enhance_max_length: 400, prompt_enhance_thinking: false, prompt_enhance_use_image: true, prompt_enhance_top_k: 50, prompt_enhance_min_p: 0.05, prompt_enhance_repetition_penalty: 1.3, prompt_enhance_presence_penalty: 0, prompt_enhance_seed: 0, prompt_enhance_greedy: false }, advisor_llm: { enabled: false, model_path: "huihui-ai/Huihui-Qwen3-8B-abliterated-v2", dtype: "bfloat16" }, loras: [], loras_config: { mode: "ltx2", per_block: false },