fix: text-only runs do not hand the grey stand-in to the enhancer; the encoder pick is text so a vanished file cannot stop a run with Enhance off; truer retain_first, montage and upscaler hints (attacker round)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
digital-garbage
2026-10-15 16:59:48 +03:00
co-authored by Claude Sonnet 5.5
parent 630bd63bed
commit 648ea94c4a
13 changed files with 44 additions and 24 deletions
+6 -6
View File
@@ -9,7 +9,7 @@ const DIALS = [["temperature", 0.7, 0, 2, 0.05, "Higher gives more varied wordin
export function enhance(app, own) {
const ps = app.pipeline, page = c.region.stack({ gap: "sm" });
let defaults = null, last = null, alive = true, spec = null;
let defaults = null, last = null, alive = true, encoders = { files: [], families: [] };
own(() => { alive = false; });
const slot = () => (ps.slots() || []).find((s) => s.node === NODE) || null;
const set = async (name, value) => { const sl = slot(); if (sl) { await ps.save({ inputs: { [sl.id]: { [name]: value } } }); draw(); } };
@@ -22,9 +22,9 @@ export function enhance(app, own) {
return [c.label.section({ text: "Last run" }), c.hint.default({ text: last.status || "" }), part("Before", last.before),
c.field.default({ label: "After", control: c.text.sm({ text: last.after || "" }) }), part("Model's thinking", last.thinking), part("Sent to the model (chat run)", last.sent)].filter(Boolean);
}
const choices = (name) => (((spec && spec.widgets) || []).find((w) => w.name === name) || {}).choices || [];
const pick = (name, label, first, hint) => {
const cur = (slot().inputs || {})[name] || "", list = choices(name);
const choices = (name) => (name === "encoder_file" ? encoders.files : encoders.families);
const pick = (name, label, first, hint, dflt = "") => {
const cur = (slot().inputs || {})[name] || dflt, list = choices(name);
const opts = [...(first ? [{ value: "", label: first }] : []), ...[...list, ...(cur && !list.includes(cur) ? [cur] : [])].map((v) => ({ value: v, label: String(v) }))];
return field(label, c.select.md({ label, value: cur, options: opts, onChange: (v) => set(name, v) }), hint);
};
@@ -40,7 +40,7 @@ export function enhance(app, own) {
c.toggle.default({ label: "Enhance prompt", checked: on, onChange: (x) => set("enabled", x) }),
...(!on ? readout() : [
pick("encoder_file", "Encoder", "The model's own encoder", "The text encoder that writes the rewrite. Models like H3's own encoder cannot write prose: pick a chat model such as Gemma 4 here."),
...((val("encoder_file", "") && choices("encoder_family").length) ? [pick("encoder_family", "Encoder family", "", "Which family that file is for.")] : []),
...((val("encoder_file", "") && choices("encoder_family").length) ? [pick("encoder_family", "Encoder family", "", "Which family that file is for.", "stable_diffusion")] : []),
field("Instructions", c.textarea.md({ label: "Instructions", rows: 6, value: val("instructions", ""), placeholder: defaults ? defaults.instructions : "(built-in instructions)", onCommit: (x) => set("instructions", x) }), "Empty uses the built-in instructions."),
num("Max tokens", val("max_length", 400), 32, 4096, 8, "max_length", "The longest the rewrite may be."),
c.toggle.default({ label: "Use the picture", hint: "Sends the scene's picture to the model along with the prompt.", checked: val("use_image", true), onChange: (x) => set("use_image", x) }),
@@ -57,7 +57,7 @@ export function enhance(app, own) {
}
draw();
ps.ensureLoaded().then(draw);
app.api.describeNodes([NODE]).then((r) => { spec = (r.nodes || {})[NODE] || null; draw(); }).catch(() => {});
app.api.enhancerEncoders().then((r) => { encoders = { files: r.files || [], families: r.families || [] }; draw(); }).catch(() => {});
// Another project's pipeline, or an edit made elsewhere: redraw, but never under a field being typed in.
const typing = () => page.node.contains(document.activeElement);
own(ps.subscribe(() => { if (!typing()) draw(); }));
+3 -5
View File
@@ -9,7 +9,7 @@ test.after(() => teardownDom());
test("the Enhance tab follows the open project's pipeline: no enhancer there, no toggle shown", async () => {
let live = [{ id: "e", node: "FunPackEnhancePrompt", inputs: { enabled: true } }], heard;
const ps = { slots: () => live, ensureLoaded: async () => {}, save: async () => {}, subscribe: (fn) => { heard = fn; return () => {}; } };
const api = { describeNodes: async () => ({ nodes: {} }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) };
const api = { enhancerEncoders: async () => ({ files: [], families: [] }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) };
const page = enhance({ pipeline: ps, api }, () => {});
document.body.append(page.node);
assert.match(page.node.textContent, /Enhance prompt/);
@@ -21,7 +21,7 @@ test("the Enhance tab follows the open project's pipeline: no enhancer there, no
test("a run that ends while Enhance is open shows its final prompt without reopening the tab", async () => {
const ps = { slots: () => [{ id: "e", node: "FunPackEnhancePrompt", inputs: {} }], ensureLoaded: async () => {}, save: async () => {}, subscribe: () => () => {} };
let runs = [], tell;
const api = { describeNodes: async () => ({ nodes: {} }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs }) };
const api = { enhancerEncoders: async () => ({ files: [], families: [] }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs }) };
const generate = { subscribe: (fn) => { tell = fn; fn({ phase: "idle" }); return () => {}; } };
const page = enhance({ pipeline: ps, api, generate }, () => {});
await new Promise((r) => setTimeout(r, 0));
@@ -48,8 +48,7 @@ test("Chat on a pipeline with no enhancer says so instead of taking comments tha
test("the Encoder pick lists the encoder files, and its family shows only once a file is picked", async () => {
let inputs = { enabled: true }, saved;
const ps = { slots: () => [{ id: "e", node: "FunPackEnhancePrompt", inputs }], ensureLoaded: async () => {}, save: async (b) => { saved = b; inputs = { ...inputs, ...b.inputs.e }; }, subscribe: () => () => {} };
const widgets = [{ name: "encoder_file", type: "COMBO", choices: ["", "gemma4.safetensors"] }, { name: "encoder_family", type: "COMBO", choices: ["stable_diffusion", "ltxv"] }];
const api = { describeNodes: async () => ({ nodes: { FunPackEnhancePrompt: { widgets } } }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) };
const api = { enhancerEncoders: async () => ({ files: ["gemma4.safetensors"], families: ["stable_diffusion", "ltxv"] }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) };
const page = enhance({ pipeline: ps, api }, () => {});
document.body.append(page.node);
await new Promise((r) => setTimeout(r, 0));
@@ -57,7 +56,6 @@ test("the Encoder pick lists the encoder files, and its family shows only once a
assert.match(page.node.textContent, /gemma4\.safetensors/);
assert.doesNotMatch(page.node.textContent, /Encoder family/);
inputs = { enabled: true, encoder_file: "gemma4.safetensors" };
page.node.dispatchEvent(new Event("noop"));
const ps2 = { ...ps, slots: () => [{ id: "e", node: "FunPackEnhancePrompt", inputs }] };
const again = enhance({ pipeline: ps2, api }, () => {});
document.body.append(again.node);
+2
View File
@@ -24,6 +24,8 @@ export async function buildInputs({ project, scene, slots, expand, frames, hooks
const size = (name) => (raw[target.id] || {})[name] ?? (target.inputs || {})[name];
put(source.slot, "blank_width", Number(size(like.width)) || 0);
put(source.slot, "blank_height", Number(size(like.height)) || 0);
const off = like.unsend && (slots || []).find((s) => s.id === like.unsend.slot); // a stand-in is not a picture to show anyone
if (off) put(off, like.unsend.input, false);
}
const plan = sceneReferences(scene, project.fixed_references);
const { overrides, unwired } = wireReferences(plan.ids, slots, source ? [source.slot.id] : [], plan.pins);
+6 -3
View File
@@ -84,11 +84,14 @@ test("the shot camera node is told which project this is, whatever roles the sav
});
test("text-only: a slot that can stand in for the picture is told the size of the slot that shapes it, and a picked picture is sent as is", async () => {
const like = { slot: "t", width: "target_width", height: "target_height" };
const s = [{ id: "src", roles: [{ at: "assets.source_image", input: "media_id", blank_like: like }] },
const like = { slot: "t", width: "target_width", height: "target_height", unsend: { slot: "e", input: "use_image" } };
const s = [{ id: "src", roles: [{ at: "assets.source_image", input: "media_id", blank_like: like }] }, { id: "e", inputs: { use_image: true } },
{ id: "t", inputs: { target_width: 1344, target_height: 768 }, roles: [{ at: "project.video", input: "target_width" }] }];
const run = (project, scene) => buildInputs({ project, scene, slots: s, expand: async () => null });
assert.deepEqual((await run({ generation_mode: "t2v", video: { target_width: 640 } }, { source_image: "m1" })).inputs.src, { media_id: "", blank_width: 640, blank_height: 768 });
const t2v = await run({ generation_mode: "t2v", video: { target_width: 640 } }, { source_image: "m1" });
assert.deepEqual(t2v.inputs.src, { media_id: "", blank_width: 640, blank_height: 768 });
assert.deepEqual(t2v.inputs.e, { use_image: false }); // the enhancer is not handed the grey stand-in
assert.equal((await run({ generation_mode: "i2v" }, { source_image: "m1" })).inputs.e, undefined);
assert.deepEqual((await run({ generation_mode: "i2v" }, { source_image: "m1" })).inputs.src, { media_id: "m1" });
assert.deepEqual((await run({ generation_mode: "i2v" }, {})).inputs.src, { media_id: "", blank_width: 1344, blank_height: 768 });
});
+1 -1
View File
@@ -9,7 +9,7 @@ const project = () => ({ num_frames_per_scene: 97, frame_rate: 25,
scenes: [{ id: "L", text: "lead" }, { id: "A", text: "a" }, { id: "B", text: "b" }, { id: "N", text: "no render" }],
scene_renders: { L: { media: media("L") }, A: { media: media("A"), inSec: 1 }, B: { media: media("B") } } });
test("chop: segments shrink by the decay, never below 9 frames, and cover the whole length", () => {
test("chop: segments shrink by the decay, never below 9 frames, and stop within the length (the last few frames may be left over)", () => {
const segs = chop(97, 40, 0.5);
assert.ok(segs.every((s) => s.frames >= 9));
assert.ok(segs.every((s, i) => i === 0 || s.start === segs[i - 1].start + segs[i - 1].frames), "pieces butt up exactly: no overlap, no gap");
+1 -1
View File
@@ -30,7 +30,7 @@ export default {
c.field.default({ label: "Lead clip (plays in order)", control: c.select.md({ label: "Lead clip", value: lead, options: list.map((s) => ({ value: s.id, label: label(s) })), onChange: (v) => { lead = v; pool.delete(v); draw(); } }) }),
c.field.default({ label: "Cutaway pool (random draw, in order within each clip)", control: c.checklist.default({ label: "Cutaway pool", values: [...pool],
items: list.filter((s) => s.id !== lead).map((s) => ({ value: s.id, label: label(s) })), onChange: (v) => { pool.clear(); v.forEach((x) => pool.add(x)); } }) }),
c.field.default({ label: "Segment length (frames)", hint: "How long the lead clip's first piece and each cutaway piece are, in frames. Rounded down to a length the model can make (9, 17, 25 ...).", control: c.number.md({ label: "Segment length", value: seg, min: 9, step: 1, precision: 0, onChange: (v) => { seg = v; } }) }),
c.field.default({ label: "Segment length (frames)", hint: "How long the lead clip's first piece and each cutaway piece are, in frames. Rounded down to 9, 17, 25 ... frames.", control: c.number.md({ label: "Segment length", value: seg, min: 9, step: 1, precision: 0, onChange: (v) => { seg = v; } }) }),
c.field.default({ label: "Acceleration", hint: "1.0 = equal segments; lower = the lead clip's segments get shorter toward the end", control: c.slider.readout({ label: "Acceleration", value: decay, min: 0.5, max: 1, step: 0.05, onCommit: (v) => { decay = v; } }) }),
]);
draw();
+1
View File
@@ -50,6 +50,7 @@ export const api = {
revolver: () => call("GET", "/api/shortcuts/revolver"),
setRevolver: (enabled, random) => call("POST", "/api/shortcuts/revolver", { enabled, random }),
enhancerDefaults: () => call("GET", "/api/m/conditioning_prompt_enhancer/defaults"),
enhancerEncoders: () => call("GET", "/api/m/conditioning_prompt_enhancer/encoders"),
enhancerRuns: () => call("GET", "/api/m/conditioning_prompt_enhancer/runs"),
rateTaste: (promptId, rating, axis) => call("POST", "/api/m/taste/rate", { prompt_id: promptId, rating, axis: axis || null }),
newTasteGeneration: (promptId) => call("POST", "/api/m/taste/generation", promptId ? { prompt_id: promptId } : {}),
@@ -41,5 +41,11 @@ def routes(table, base, web):
runs = [r for r in RUNS if not want or r["prompt_id"] == want]
return web.json_response({"runs": runs})
@table.get(base + "/encoders")
async def _encoders(_req):
"""Encoder files and families the Enhance panel can offer (read fresh: files come and go)."""
from ...loaders.clip import nodes as clip_nodes
return web.json_response({"files": clip_nodes.encoder_files(), "families": clip_nodes.clip_types()})
PROVIDES = {"routes": routes}
@@ -69,10 +69,11 @@ class FunPackEnhancePrompt(io.ComfyNode):
tooltip="0 = a fresh rewrite every run. Any other number repeats the same rewrite."),
io.Boolean.Input("thinking", default=False, optional=True,
tooltip="Let a thinking model reason first. Its reasoning eats the token budget."),
io.Combo.Input("encoder_file", options=[""] + clip_nodes.encoder_files(), default="", optional=True,
tooltip="A text encoder file just for rewriting (Gemma, Qwen...). Empty uses the model's own encoder."),
io.Combo.Input("encoder_family", options=clip_nodes.clip_types(), default="stable_diffusion", optional=True,
tooltip="Which family that file is for. Only used with an encoder file."),
# Text, not a list: a file that has gone must not stop a run where Enhance is off. The Enhance panel offers the files.
io.String.Input("encoder_file", default="", optional=True,
tooltip="A text encoder file just for rewriting (Gemma, Qwen...). Empty uses the model's own encoder."),
io.String.Input("encoder_family", default="stable_diffusion", optional=True,
tooltip="Which family that file is for. Only used with an encoder file."),
io.Image.Input("image", optional=True, tooltip="Shown to the model alongside the prompt."),
io.Boolean.Input("use_image", default=True, optional=True,
tooltip="Send the picture (when one is connected) to the model."),
@@ -247,7 +247,7 @@ def test_the_readout_routes_are_provided_and_answer():
def json_response(d):
return d
mod.routes(T(), "/m", W)
assert set(table) == {"/m/defaults", "/m/runs"}
assert set(table) == {"/m/defaults", "/m/runs", "/m/encoders"}
def test_a_tokenizer_that_reads_the_picture_out_of_kwargs_gets_it():
@@ -270,3 +270,11 @@ def test_a_picked_encoder_that_will_not_load_keeps_the_prompt_and_says_why(monke
out = nodes.FunPackEnhancePrompt.execute(clip=object(), text="a fox", enabled=True, encoder_file="x.gguf")
assert out.args[0] == "a fox" and out.args[1] == "encoder failed"
assert "x.gguf" in nodes.RUNS[-1]["status"] and not nodes.RUNS[-1]["ok"]
def test_a_gone_encoder_file_is_text_the_graph_does_not_refuse(comfyui):
from modules.conditioning.prompt_enhancer import nodes
schema = nodes.FunPackEnhancePrompt.define_schema()
for name in ("encoder_file", "encoder_family"):
inp = next(i for i in schema.inputs if i.id == name)
assert inp.get_io_type() == "STRING" # a list would refuse a vanished file even with Enhance off
+1 -1
View File
@@ -41,7 +41,7 @@ class FunPackLatentUpscalerLoader(io.ComfyNode):
log.info("FunPack Latent Upscaler", f"{upscaler_name}: {spec.title} upscaler")
return io.NodeOutput(model)
newer = ("It looks like the newer attention-based H3 upscaler, which FunPack cannot build yet: "
"use the older convolution one (h3_clean_latent_upscaler_v1). ") if any(str(k).startswith("base.") for k in sd) else ""
"use an older convolution-based H3 upscaler. ") if any(str(k).startswith("base.") for k in sd) else ""
raise RuntimeError(
f"{upscaler_name} is not an upscaler any installed model module can load. {newer}"
f"If it is Lightricks' or Hunyuan's, ComfyUI's own Load Latent Upscale Model "
+2 -1
View File
@@ -64,7 +64,8 @@ def h3_reference_to_video():
# `blank_like`: with no picture (text-only), buildInputs gives this node the size of that
# slot's target_width/height, so a plain grey stand-in carries the project's own shape.
"roles": [{"at": "assets.source_image", "input": "media_id",
"blank_like": {"slot": "image_transform", "width": "target_width", "height": "target_height"}}],
"blank_like": {"slot": "image_transform", "width": "target_width", "height": "target_height",
"unsend": {"slot": "enhance", "input": "use_image"}}}],
"inputs": {"media_id": "", "blank_width": 0, "blank_height": 0}},
{"id": "image_transform", "group": "Reference media", "node": "ImageTransformKJ",
# target_width/height are the PIXEL BUDGET, set in Project settings
+1 -1
View File
@@ -51,7 +51,7 @@ SETTINGS = {
"freenoise": {"type": "bool", "default": True, "label": "FreeNoise blending", "when": {"enabled": True},
"hint": "Shuffle the starting noise so neighbouring windows share it."},
"retain_first": {"type": "bool", "default": False, "label": "Pin anchor in every window",
"hint": "Keeps the first frame in every window so the whole clip stays tied to your starting image.",
"hint": "Keeps the first frame in every window so the whole clip stays tied to its first frame (your starting image, when there is one).",
"when": {"enabled": True}},
}