From 648ea94c4a02ab70f2e13fec42eb6833c7eefd6d Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:59:48 +0300 Subject: [PATCH] fix: text-only runs do not hand the grey stand-in to the enhancer; the encoder pick is text so a vanished file cannot stop a run with Enhance off; truer retain_first, montage and upscaler hints (attacker round) Co-Authored-By: Claude Sonnet 5.5 --- app/ui/features/composer/enhance.js | 12 ++++++------ app/ui/features/composer/enhance.test.js | 8 +++----- app/ui/features/generate/inputs.js | 2 ++ app/ui/features/generate/inputs.test.js | 9 ++++++--- app/ui/features/montage/build.test.js | 2 +- app/ui/features/montage/index.js | 2 +- app/ui/shell/api.js | 1 + modules/conditioning/prompt_enhancer/__init__.py | 6 ++++++ modules/conditioning/prompt_enhancer/nodes.py | 9 +++++---- .../prompt_enhancer/tests/test_prompt_enhancer.py | 10 +++++++++- modules/latent/second_pass/nodes.py | 2 +- modules/models/minimax_h3/pipeline.py | 3 ++- modules/sampling/context_windows/__init__.py | 2 +- 13 files changed, 44 insertions(+), 24 deletions(-) diff --git a/app/ui/features/composer/enhance.js b/app/ui/features/composer/enhance.js index 9c4be5c9..298e9c71 100644 --- a/app/ui/features/composer/enhance.js +++ b/app/ui/features/composer/enhance.js @@ -9,7 +9,7 @@ const DIALS = [["temperature", 0.7, 0, 2, 0.05, "Higher gives more varied wordin export function enhance(app, own) { const ps = app.pipeline, page = c.region.stack({ gap: "sm" }); - let defaults = null, last = null, alive = true, spec = null; + let defaults = null, last = null, alive = true, encoders = { files: [], families: [] }; own(() => { alive = false; }); const slot = () => (ps.slots() || []).find((s) => s.node === NODE) || null; const set = async (name, value) => { const sl = slot(); if (sl) { await ps.save({ inputs: { [sl.id]: { [name]: value } } }); draw(); } }; @@ -22,9 +22,9 @@ export function enhance(app, own) { return [c.label.section({ text: "Last run" }), c.hint.default({ text: last.status || "" }), part("Before", last.before), c.field.default({ label: "After", control: c.text.sm({ text: last.after || "" }) }), part("Model's thinking", last.thinking), part("Sent to the model (chat run)", last.sent)].filter(Boolean); } - const choices = (name) => (((spec && spec.widgets) || []).find((w) => w.name === name) || {}).choices || []; - const pick = (name, label, first, hint) => { - const cur = (slot().inputs || {})[name] || "", list = choices(name); + const choices = (name) => (name === "encoder_file" ? encoders.files : encoders.families); + const pick = (name, label, first, hint, dflt = "") => { + const cur = (slot().inputs || {})[name] || dflt, list = choices(name); const opts = [...(first ? [{ value: "", label: first }] : []), ...[...list, ...(cur && !list.includes(cur) ? [cur] : [])].map((v) => ({ value: v, label: String(v) }))]; return field(label, c.select.md({ label, value: cur, options: opts, onChange: (v) => set(name, v) }), hint); }; @@ -40,7 +40,7 @@ export function enhance(app, own) { c.toggle.default({ label: "Enhance prompt", checked: on, onChange: (x) => set("enabled", x) }), ...(!on ? readout() : [ pick("encoder_file", "Encoder", "The model's own encoder", "The text encoder that writes the rewrite. Models like H3's own encoder cannot write prose: pick a chat model such as Gemma 4 here."), - ...((val("encoder_file", "") && choices("encoder_family").length) ? [pick("encoder_family", "Encoder family", "", "Which family that file is for.")] : []), + ...((val("encoder_file", "") && choices("encoder_family").length) ? [pick("encoder_family", "Encoder family", "", "Which family that file is for.", "stable_diffusion")] : []), field("Instructions", c.textarea.md({ label: "Instructions", rows: 6, value: val("instructions", ""), placeholder: defaults ? defaults.instructions : "(built-in instructions)", onCommit: (x) => set("instructions", x) }), "Empty uses the built-in instructions."), num("Max tokens", val("max_length", 400), 32, 4096, 8, "max_length", "The longest the rewrite may be."), c.toggle.default({ label: "Use the picture", hint: "Sends the scene's picture to the model along with the prompt.", checked: val("use_image", true), onChange: (x) => set("use_image", x) }), @@ -57,7 +57,7 @@ export function enhance(app, own) { } draw(); ps.ensureLoaded().then(draw); - app.api.describeNodes([NODE]).then((r) => { spec = (r.nodes || {})[NODE] || null; draw(); }).catch(() => {}); + app.api.enhancerEncoders().then((r) => { encoders = { files: r.files || [], families: r.families || [] }; draw(); }).catch(() => {}); // Another project's pipeline, or an edit made elsewhere: redraw, but never under a field being typed in. const typing = () => page.node.contains(document.activeElement); own(ps.subscribe(() => { if (!typing()) draw(); })); diff --git a/app/ui/features/composer/enhance.test.js b/app/ui/features/composer/enhance.test.js index 033aa093..5d0a82b2 100644 --- a/app/ui/features/composer/enhance.test.js +++ b/app/ui/features/composer/enhance.test.js @@ -9,7 +9,7 @@ test.after(() => teardownDom()); test("the Enhance tab follows the open project's pipeline: no enhancer there, no toggle shown", async () => { let live = [{ id: "e", node: "FunPackEnhancePrompt", inputs: { enabled: true } }], heard; const ps = { slots: () => live, ensureLoaded: async () => {}, save: async () => {}, subscribe: (fn) => { heard = fn; return () => {}; } }; - const api = { describeNodes: async () => ({ nodes: {} }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) }; + const api = { enhancerEncoders: async () => ({ files: [], families: [] }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) }; const page = enhance({ pipeline: ps, api }, () => {}); document.body.append(page.node); assert.match(page.node.textContent, /Enhance prompt/); @@ -21,7 +21,7 @@ test("the Enhance tab follows the open project's pipeline: no enhancer there, no test("a run that ends while Enhance is open shows its final prompt without reopening the tab", async () => { const ps = { slots: () => [{ id: "e", node: "FunPackEnhancePrompt", inputs: {} }], ensureLoaded: async () => {}, save: async () => {}, subscribe: () => () => {} }; let runs = [], tell; - const api = { describeNodes: async () => ({ nodes: {} }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs }) }; + const api = { enhancerEncoders: async () => ({ files: [], families: [] }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs }) }; const generate = { subscribe: (fn) => { tell = fn; fn({ phase: "idle" }); return () => {}; } }; const page = enhance({ pipeline: ps, api, generate }, () => {}); await new Promise((r) => setTimeout(r, 0)); @@ -48,8 +48,7 @@ test("Chat on a pipeline with no enhancer says so instead of taking comments tha test("the Encoder pick lists the encoder files, and its family shows only once a file is picked", async () => { let inputs = { enabled: true }, saved; const ps = { slots: () => [{ id: "e", node: "FunPackEnhancePrompt", inputs }], ensureLoaded: async () => {}, save: async (b) => { saved = b; inputs = { ...inputs, ...b.inputs.e }; }, subscribe: () => () => {} }; - const widgets = [{ name: "encoder_file", type: "COMBO", choices: ["", "gemma4.safetensors"] }, { name: "encoder_family", type: "COMBO", choices: ["stable_diffusion", "ltxv"] }]; - const api = { describeNodes: async () => ({ nodes: { FunPackEnhancePrompt: { widgets } } }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) }; + const api = { enhancerEncoders: async () => ({ files: ["gemma4.safetensors"], families: ["stable_diffusion", "ltxv"] }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) }; const page = enhance({ pipeline: ps, api }, () => {}); document.body.append(page.node); await new Promise((r) => setTimeout(r, 0)); @@ -57,7 +56,6 @@ test("the Encoder pick lists the encoder files, and its family shows only once a assert.match(page.node.textContent, /gemma4\.safetensors/); assert.doesNotMatch(page.node.textContent, /Encoder family/); inputs = { enabled: true, encoder_file: "gemma4.safetensors" }; - page.node.dispatchEvent(new Event("noop")); const ps2 = { ...ps, slots: () => [{ id: "e", node: "FunPackEnhancePrompt", inputs }] }; const again = enhance({ pipeline: ps2, api }, () => {}); document.body.append(again.node); diff --git a/app/ui/features/generate/inputs.js b/app/ui/features/generate/inputs.js index 17d4f9c0..b59fcc3e 100644 --- a/app/ui/features/generate/inputs.js +++ b/app/ui/features/generate/inputs.js @@ -24,6 +24,8 @@ export async function buildInputs({ project, scene, slots, expand, frames, hooks const size = (name) => (raw[target.id] || {})[name] ?? (target.inputs || {})[name]; put(source.slot, "blank_width", Number(size(like.width)) || 0); put(source.slot, "blank_height", Number(size(like.height)) || 0); + const off = like.unsend && (slots || []).find((s) => s.id === like.unsend.slot); // a stand-in is not a picture to show anyone + if (off) put(off, like.unsend.input, false); } const plan = sceneReferences(scene, project.fixed_references); const { overrides, unwired } = wireReferences(plan.ids, slots, source ? [source.slot.id] : [], plan.pins); diff --git a/app/ui/features/generate/inputs.test.js b/app/ui/features/generate/inputs.test.js index 054728de..de8265a9 100644 --- a/app/ui/features/generate/inputs.test.js +++ b/app/ui/features/generate/inputs.test.js @@ -84,11 +84,14 @@ test("the shot camera node is told which project this is, whatever roles the sav }); test("text-only: a slot that can stand in for the picture is told the size of the slot that shapes it, and a picked picture is sent as is", async () => { - const like = { slot: "t", width: "target_width", height: "target_height" }; - const s = [{ id: "src", roles: [{ at: "assets.source_image", input: "media_id", blank_like: like }] }, + const like = { slot: "t", width: "target_width", height: "target_height", unsend: { slot: "e", input: "use_image" } }; + const s = [{ id: "src", roles: [{ at: "assets.source_image", input: "media_id", blank_like: like }] }, { id: "e", inputs: { use_image: true } }, { id: "t", inputs: { target_width: 1344, target_height: 768 }, roles: [{ at: "project.video", input: "target_width" }] }]; const run = (project, scene) => buildInputs({ project, scene, slots: s, expand: async () => null }); - assert.deepEqual((await run({ generation_mode: "t2v", video: { target_width: 640 } }, { source_image: "m1" })).inputs.src, { media_id: "", blank_width: 640, blank_height: 768 }); + const t2v = await run({ generation_mode: "t2v", video: { target_width: 640 } }, { source_image: "m1" }); + assert.deepEqual(t2v.inputs.src, { media_id: "", blank_width: 640, blank_height: 768 }); + assert.deepEqual(t2v.inputs.e, { use_image: false }); // the enhancer is not handed the grey stand-in + assert.equal((await run({ generation_mode: "i2v" }, { source_image: "m1" })).inputs.e, undefined); assert.deepEqual((await run({ generation_mode: "i2v" }, { source_image: "m1" })).inputs.src, { media_id: "m1" }); assert.deepEqual((await run({ generation_mode: "i2v" }, {})).inputs.src, { media_id: "", blank_width: 1344, blank_height: 768 }); }); diff --git a/app/ui/features/montage/build.test.js b/app/ui/features/montage/build.test.js index 3b98241f..160317b4 100644 --- a/app/ui/features/montage/build.test.js +++ b/app/ui/features/montage/build.test.js @@ -9,7 +9,7 @@ const project = () => ({ num_frames_per_scene: 97, frame_rate: 25, scenes: [{ id: "L", text: "lead" }, { id: "A", text: "a" }, { id: "B", text: "b" }, { id: "N", text: "no render" }], scene_renders: { L: { media: media("L") }, A: { media: media("A"), inSec: 1 }, B: { media: media("B") } } }); -test("chop: segments shrink by the decay, never below 9 frames, and cover the whole length", () => { +test("chop: segments shrink by the decay, never below 9 frames, and stop within the length (the last few frames may be left over)", () => { const segs = chop(97, 40, 0.5); assert.ok(segs.every((s) => s.frames >= 9)); assert.ok(segs.every((s, i) => i === 0 || s.start === segs[i - 1].start + segs[i - 1].frames), "pieces butt up exactly: no overlap, no gap"); diff --git a/app/ui/features/montage/index.js b/app/ui/features/montage/index.js index dd8ea545..48d77ad0 100644 --- a/app/ui/features/montage/index.js +++ b/app/ui/features/montage/index.js @@ -30,7 +30,7 @@ export default { c.field.default({ label: "Lead clip (plays in order)", control: c.select.md({ label: "Lead clip", value: lead, options: list.map((s) => ({ value: s.id, label: label(s) })), onChange: (v) => { lead = v; pool.delete(v); draw(); } }) }), c.field.default({ label: "Cutaway pool (random draw, in order within each clip)", control: c.checklist.default({ label: "Cutaway pool", values: [...pool], items: list.filter((s) => s.id !== lead).map((s) => ({ value: s.id, label: label(s) })), onChange: (v) => { pool.clear(); v.forEach((x) => pool.add(x)); } }) }), - c.field.default({ label: "Segment length (frames)", hint: "How long the lead clip's first piece and each cutaway piece are, in frames. Rounded down to a length the model can make (9, 17, 25 ...).", control: c.number.md({ label: "Segment length", value: seg, min: 9, step: 1, precision: 0, onChange: (v) => { seg = v; } }) }), + c.field.default({ label: "Segment length (frames)", hint: "How long the lead clip's first piece and each cutaway piece are, in frames. Rounded down to 9, 17, 25 ... frames.", control: c.number.md({ label: "Segment length", value: seg, min: 9, step: 1, precision: 0, onChange: (v) => { seg = v; } }) }), c.field.default({ label: "Acceleration", hint: "1.0 = equal segments; lower = the lead clip's segments get shorter toward the end", control: c.slider.readout({ label: "Acceleration", value: decay, min: 0.5, max: 1, step: 0.05, onCommit: (v) => { decay = v; } }) }), ]); draw(); diff --git a/app/ui/shell/api.js b/app/ui/shell/api.js index 359746aa..c87e03bb 100644 --- a/app/ui/shell/api.js +++ b/app/ui/shell/api.js @@ -50,6 +50,7 @@ export const api = { revolver: () => call("GET", "/api/shortcuts/revolver"), setRevolver: (enabled, random) => call("POST", "/api/shortcuts/revolver", { enabled, random }), enhancerDefaults: () => call("GET", "/api/m/conditioning_prompt_enhancer/defaults"), + enhancerEncoders: () => call("GET", "/api/m/conditioning_prompt_enhancer/encoders"), enhancerRuns: () => call("GET", "/api/m/conditioning_prompt_enhancer/runs"), rateTaste: (promptId, rating, axis) => call("POST", "/api/m/taste/rate", { prompt_id: promptId, rating, axis: axis || null }), newTasteGeneration: (promptId) => call("POST", "/api/m/taste/generation", promptId ? { prompt_id: promptId } : {}), diff --git a/modules/conditioning/prompt_enhancer/__init__.py b/modules/conditioning/prompt_enhancer/__init__.py index f22d32b7..fcd02596 100644 --- a/modules/conditioning/prompt_enhancer/__init__.py +++ b/modules/conditioning/prompt_enhancer/__init__.py @@ -41,5 +41,11 @@ def routes(table, base, web): runs = [r for r in RUNS if not want or r["prompt_id"] == want] return web.json_response({"runs": runs}) + @table.get(base + "/encoders") + async def _encoders(_req): + """Encoder files and families the Enhance panel can offer (read fresh: files come and go).""" + from ...loaders.clip import nodes as clip_nodes + return web.json_response({"files": clip_nodes.encoder_files(), "families": clip_nodes.clip_types()}) + PROVIDES = {"routes": routes} diff --git a/modules/conditioning/prompt_enhancer/nodes.py b/modules/conditioning/prompt_enhancer/nodes.py index 3a1e0aad..01ce508a 100644 --- a/modules/conditioning/prompt_enhancer/nodes.py +++ b/modules/conditioning/prompt_enhancer/nodes.py @@ -69,10 +69,11 @@ class FunPackEnhancePrompt(io.ComfyNode): tooltip="0 = a fresh rewrite every run. Any other number repeats the same rewrite."), io.Boolean.Input("thinking", default=False, optional=True, tooltip="Let a thinking model reason first. Its reasoning eats the token budget."), - io.Combo.Input("encoder_file", options=[""] + clip_nodes.encoder_files(), default="", optional=True, - tooltip="A text encoder file just for rewriting (Gemma, Qwen...). Empty uses the model's own encoder."), - io.Combo.Input("encoder_family", options=clip_nodes.clip_types(), default="stable_diffusion", optional=True, - tooltip="Which family that file is for. Only used with an encoder file."), + # Text, not a list: a file that has gone must not stop a run where Enhance is off. The Enhance panel offers the files. + io.String.Input("encoder_file", default="", optional=True, + tooltip="A text encoder file just for rewriting (Gemma, Qwen...). Empty uses the model's own encoder."), + io.String.Input("encoder_family", default="stable_diffusion", optional=True, + tooltip="Which family that file is for. Only used with an encoder file."), io.Image.Input("image", optional=True, tooltip="Shown to the model alongside the prompt."), io.Boolean.Input("use_image", default=True, optional=True, tooltip="Send the picture (when one is connected) to the model."), diff --git a/modules/conditioning/prompt_enhancer/tests/test_prompt_enhancer.py b/modules/conditioning/prompt_enhancer/tests/test_prompt_enhancer.py index 3cd88ba7..9272de7e 100644 --- a/modules/conditioning/prompt_enhancer/tests/test_prompt_enhancer.py +++ b/modules/conditioning/prompt_enhancer/tests/test_prompt_enhancer.py @@ -247,7 +247,7 @@ def test_the_readout_routes_are_provided_and_answer(): def json_response(d): return d mod.routes(T(), "/m", W) - assert set(table) == {"/m/defaults", "/m/runs"} + assert set(table) == {"/m/defaults", "/m/runs", "/m/encoders"} def test_a_tokenizer_that_reads_the_picture_out_of_kwargs_gets_it(): @@ -270,3 +270,11 @@ def test_a_picked_encoder_that_will_not_load_keeps_the_prompt_and_says_why(monke out = nodes.FunPackEnhancePrompt.execute(clip=object(), text="a fox", enabled=True, encoder_file="x.gguf") assert out.args[0] == "a fox" and out.args[1] == "encoder failed" assert "x.gguf" in nodes.RUNS[-1]["status"] and not nodes.RUNS[-1]["ok"] + + +def test_a_gone_encoder_file_is_text_the_graph_does_not_refuse(comfyui): + from modules.conditioning.prompt_enhancer import nodes + schema = nodes.FunPackEnhancePrompt.define_schema() + for name in ("encoder_file", "encoder_family"): + inp = next(i for i in schema.inputs if i.id == name) + assert inp.get_io_type() == "STRING" # a list would refuse a vanished file even with Enhance off diff --git a/modules/latent/second_pass/nodes.py b/modules/latent/second_pass/nodes.py index bf0c2005..fee2895b 100644 --- a/modules/latent/second_pass/nodes.py +++ b/modules/latent/second_pass/nodes.py @@ -41,7 +41,7 @@ class FunPackLatentUpscalerLoader(io.ComfyNode): log.info("FunPack Latent Upscaler", f"{upscaler_name}: {spec.title} upscaler") return io.NodeOutput(model) newer = ("It looks like the newer attention-based H3 upscaler, which FunPack cannot build yet: " - "use the older convolution one (h3_clean_latent_upscaler_v1). ") if any(str(k).startswith("base.") for k in sd) else "" + "use an older convolution-based H3 upscaler. ") if any(str(k).startswith("base.") for k in sd) else "" raise RuntimeError( f"{upscaler_name} is not an upscaler any installed model module can load. {newer}" f"If it is Lightricks' or Hunyuan's, ComfyUI's own Load Latent Upscale Model " diff --git a/modules/models/minimax_h3/pipeline.py b/modules/models/minimax_h3/pipeline.py index 058a4cfa..af34458f 100644 --- a/modules/models/minimax_h3/pipeline.py +++ b/modules/models/minimax_h3/pipeline.py @@ -64,7 +64,8 @@ def h3_reference_to_video(): # `blank_like`: with no picture (text-only), buildInputs gives this node the size of that # slot's target_width/height, so a plain grey stand-in carries the project's own shape. "roles": [{"at": "assets.source_image", "input": "media_id", - "blank_like": {"slot": "image_transform", "width": "target_width", "height": "target_height"}}], + "blank_like": {"slot": "image_transform", "width": "target_width", "height": "target_height", + "unsend": {"slot": "enhance", "input": "use_image"}}}], "inputs": {"media_id": "", "blank_width": 0, "blank_height": 0}}, {"id": "image_transform", "group": "Reference media", "node": "ImageTransformKJ", # target_width/height are the PIXEL BUDGET, set in Project settings diff --git a/modules/sampling/context_windows/__init__.py b/modules/sampling/context_windows/__init__.py index 8953e9a6..a9c43c1f 100644 --- a/modules/sampling/context_windows/__init__.py +++ b/modules/sampling/context_windows/__init__.py @@ -51,7 +51,7 @@ SETTINGS = { "freenoise": {"type": "bool", "default": True, "label": "FreeNoise blending", "when": {"enabled": True}, "hint": "Shuffle the starting noise so neighbouring windows share it."}, "retain_first": {"type": "bool", "default": False, "label": "Pin anchor in every window", - "hint": "Keeps the first frame in every window so the whole clip stays tied to your starting image.", + "hint": "Keeps the first frame in every window so the whole clip stays tied to its first frame (your starting image, when there is one).", "when": {"enabled": True}}, }