fix: text-only runs do not hand the grey stand-in to the enhancer; the encoder pick is text so a vanished file cannot stop a run with Enhance off; truer retain_first, montage and upscaler hints (attacker round)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
parent
630bd63bed
commit
648ea94c4a
@@ -9,7 +9,7 @@ const DIALS = [["temperature", 0.7, 0, 2, 0.05, "Higher gives more varied wordin
|
||||
|
||||
export function enhance(app, own) {
|
||||
const ps = app.pipeline, page = c.region.stack({ gap: "sm" });
|
||||
let defaults = null, last = null, alive = true, spec = null;
|
||||
let defaults = null, last = null, alive = true, encoders = { files: [], families: [] };
|
||||
own(() => { alive = false; });
|
||||
const slot = () => (ps.slots() || []).find((s) => s.node === NODE) || null;
|
||||
const set = async (name, value) => { const sl = slot(); if (sl) { await ps.save({ inputs: { [sl.id]: { [name]: value } } }); draw(); } };
|
||||
@@ -22,9 +22,9 @@ export function enhance(app, own) {
|
||||
return [c.label.section({ text: "Last run" }), c.hint.default({ text: last.status || "" }), part("Before", last.before),
|
||||
c.field.default({ label: "After", control: c.text.sm({ text: last.after || "" }) }), part("Model's thinking", last.thinking), part("Sent to the model (chat run)", last.sent)].filter(Boolean);
|
||||
}
|
||||
const choices = (name) => (((spec && spec.widgets) || []).find((w) => w.name === name) || {}).choices || [];
|
||||
const pick = (name, label, first, hint) => {
|
||||
const cur = (slot().inputs || {})[name] || "", list = choices(name);
|
||||
const choices = (name) => (name === "encoder_file" ? encoders.files : encoders.families);
|
||||
const pick = (name, label, first, hint, dflt = "") => {
|
||||
const cur = (slot().inputs || {})[name] || dflt, list = choices(name);
|
||||
const opts = [...(first ? [{ value: "", label: first }] : []), ...[...list, ...(cur && !list.includes(cur) ? [cur] : [])].map((v) => ({ value: v, label: String(v) }))];
|
||||
return field(label, c.select.md({ label, value: cur, options: opts, onChange: (v) => set(name, v) }), hint);
|
||||
};
|
||||
@@ -40,7 +40,7 @@ export function enhance(app, own) {
|
||||
c.toggle.default({ label: "Enhance prompt", checked: on, onChange: (x) => set("enabled", x) }),
|
||||
...(!on ? readout() : [
|
||||
pick("encoder_file", "Encoder", "The model's own encoder", "The text encoder that writes the rewrite. Models like H3's own encoder cannot write prose: pick a chat model such as Gemma 4 here."),
|
||||
...((val("encoder_file", "") && choices("encoder_family").length) ? [pick("encoder_family", "Encoder family", "", "Which family that file is for.")] : []),
|
||||
...((val("encoder_file", "") && choices("encoder_family").length) ? [pick("encoder_family", "Encoder family", "", "Which family that file is for.", "stable_diffusion")] : []),
|
||||
field("Instructions", c.textarea.md({ label: "Instructions", rows: 6, value: val("instructions", ""), placeholder: defaults ? defaults.instructions : "(built-in instructions)", onCommit: (x) => set("instructions", x) }), "Empty uses the built-in instructions."),
|
||||
num("Max tokens", val("max_length", 400), 32, 4096, 8, "max_length", "The longest the rewrite may be."),
|
||||
c.toggle.default({ label: "Use the picture", hint: "Sends the scene's picture to the model along with the prompt.", checked: val("use_image", true), onChange: (x) => set("use_image", x) }),
|
||||
@@ -57,7 +57,7 @@ export function enhance(app, own) {
|
||||
}
|
||||
draw();
|
||||
ps.ensureLoaded().then(draw);
|
||||
app.api.describeNodes([NODE]).then((r) => { spec = (r.nodes || {})[NODE] || null; draw(); }).catch(() => {});
|
||||
app.api.enhancerEncoders().then((r) => { encoders = { files: r.files || [], families: r.families || [] }; draw(); }).catch(() => {});
|
||||
// Another project's pipeline, or an edit made elsewhere: redraw, but never under a field being typed in.
|
||||
const typing = () => page.node.contains(document.activeElement);
|
||||
own(ps.subscribe(() => { if (!typing()) draw(); }));
|
||||
|
||||
@@ -9,7 +9,7 @@ test.after(() => teardownDom());
|
||||
test("the Enhance tab follows the open project's pipeline: no enhancer there, no toggle shown", async () => {
|
||||
let live = [{ id: "e", node: "FunPackEnhancePrompt", inputs: { enabled: true } }], heard;
|
||||
const ps = { slots: () => live, ensureLoaded: async () => {}, save: async () => {}, subscribe: (fn) => { heard = fn; return () => {}; } };
|
||||
const api = { describeNodes: async () => ({ nodes: {} }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) };
|
||||
const api = { enhancerEncoders: async () => ({ files: [], families: [] }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) };
|
||||
const page = enhance({ pipeline: ps, api }, () => {});
|
||||
document.body.append(page.node);
|
||||
assert.match(page.node.textContent, /Enhance prompt/);
|
||||
@@ -21,7 +21,7 @@ test("the Enhance tab follows the open project's pipeline: no enhancer there, no
|
||||
test("a run that ends while Enhance is open shows its final prompt without reopening the tab", async () => {
|
||||
const ps = { slots: () => [{ id: "e", node: "FunPackEnhancePrompt", inputs: {} }], ensureLoaded: async () => {}, save: async () => {}, subscribe: () => () => {} };
|
||||
let runs = [], tell;
|
||||
const api = { describeNodes: async () => ({ nodes: {} }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs }) };
|
||||
const api = { enhancerEncoders: async () => ({ files: [], families: [] }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs }) };
|
||||
const generate = { subscribe: (fn) => { tell = fn; fn({ phase: "idle" }); return () => {}; } };
|
||||
const page = enhance({ pipeline: ps, api, generate }, () => {});
|
||||
await new Promise((r) => setTimeout(r, 0));
|
||||
@@ -48,8 +48,7 @@ test("Chat on a pipeline with no enhancer says so instead of taking comments tha
|
||||
test("the Encoder pick lists the encoder files, and its family shows only once a file is picked", async () => {
|
||||
let inputs = { enabled: true }, saved;
|
||||
const ps = { slots: () => [{ id: "e", node: "FunPackEnhancePrompt", inputs }], ensureLoaded: async () => {}, save: async (b) => { saved = b; inputs = { ...inputs, ...b.inputs.e }; }, subscribe: () => () => {} };
|
||||
const widgets = [{ name: "encoder_file", type: "COMBO", choices: ["", "gemma4.safetensors"] }, { name: "encoder_family", type: "COMBO", choices: ["stable_diffusion", "ltxv"] }];
|
||||
const api = { describeNodes: async () => ({ nodes: { FunPackEnhancePrompt: { widgets } } }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) };
|
||||
const api = { enhancerEncoders: async () => ({ files: ["gemma4.safetensors"], families: ["stable_diffusion", "ltxv"] }), enhancerDefaults: async () => ({}), enhancerRuns: async () => ({ runs: [] }) };
|
||||
const page = enhance({ pipeline: ps, api }, () => {});
|
||||
document.body.append(page.node);
|
||||
await new Promise((r) => setTimeout(r, 0));
|
||||
@@ -57,7 +56,6 @@ test("the Encoder pick lists the encoder files, and its family shows only once a
|
||||
assert.match(page.node.textContent, /gemma4\.safetensors/);
|
||||
assert.doesNotMatch(page.node.textContent, /Encoder family/);
|
||||
inputs = { enabled: true, encoder_file: "gemma4.safetensors" };
|
||||
page.node.dispatchEvent(new Event("noop"));
|
||||
const ps2 = { ...ps, slots: () => [{ id: "e", node: "FunPackEnhancePrompt", inputs }] };
|
||||
const again = enhance({ pipeline: ps2, api }, () => {});
|
||||
document.body.append(again.node);
|
||||
|
||||
@@ -24,6 +24,8 @@ export async function buildInputs({ project, scene, slots, expand, frames, hooks
|
||||
const size = (name) => (raw[target.id] || {})[name] ?? (target.inputs || {})[name];
|
||||
put(source.slot, "blank_width", Number(size(like.width)) || 0);
|
||||
put(source.slot, "blank_height", Number(size(like.height)) || 0);
|
||||
const off = like.unsend && (slots || []).find((s) => s.id === like.unsend.slot); // a stand-in is not a picture to show anyone
|
||||
if (off) put(off, like.unsend.input, false);
|
||||
}
|
||||
const plan = sceneReferences(scene, project.fixed_references);
|
||||
const { overrides, unwired } = wireReferences(plan.ids, slots, source ? [source.slot.id] : [], plan.pins);
|
||||
|
||||
@@ -84,11 +84,14 @@ test("the shot camera node is told which project this is, whatever roles the sav
|
||||
});
|
||||
|
||||
test("text-only: a slot that can stand in for the picture is told the size of the slot that shapes it, and a picked picture is sent as is", async () => {
|
||||
const like = { slot: "t", width: "target_width", height: "target_height" };
|
||||
const s = [{ id: "src", roles: [{ at: "assets.source_image", input: "media_id", blank_like: like }] },
|
||||
const like = { slot: "t", width: "target_width", height: "target_height", unsend: { slot: "e", input: "use_image" } };
|
||||
const s = [{ id: "src", roles: [{ at: "assets.source_image", input: "media_id", blank_like: like }] }, { id: "e", inputs: { use_image: true } },
|
||||
{ id: "t", inputs: { target_width: 1344, target_height: 768 }, roles: [{ at: "project.video", input: "target_width" }] }];
|
||||
const run = (project, scene) => buildInputs({ project, scene, slots: s, expand: async () => null });
|
||||
assert.deepEqual((await run({ generation_mode: "t2v", video: { target_width: 640 } }, { source_image: "m1" })).inputs.src, { media_id: "", blank_width: 640, blank_height: 768 });
|
||||
const t2v = await run({ generation_mode: "t2v", video: { target_width: 640 } }, { source_image: "m1" });
|
||||
assert.deepEqual(t2v.inputs.src, { media_id: "", blank_width: 640, blank_height: 768 });
|
||||
assert.deepEqual(t2v.inputs.e, { use_image: false }); // the enhancer is not handed the grey stand-in
|
||||
assert.equal((await run({ generation_mode: "i2v" }, { source_image: "m1" })).inputs.e, undefined);
|
||||
assert.deepEqual((await run({ generation_mode: "i2v" }, { source_image: "m1" })).inputs.src, { media_id: "m1" });
|
||||
assert.deepEqual((await run({ generation_mode: "i2v" }, {})).inputs.src, { media_id: "", blank_width: 1344, blank_height: 768 });
|
||||
});
|
||||
|
||||
@@ -9,7 +9,7 @@ const project = () => ({ num_frames_per_scene: 97, frame_rate: 25,
|
||||
scenes: [{ id: "L", text: "lead" }, { id: "A", text: "a" }, { id: "B", text: "b" }, { id: "N", text: "no render" }],
|
||||
scene_renders: { L: { media: media("L") }, A: { media: media("A"), inSec: 1 }, B: { media: media("B") } } });
|
||||
|
||||
test("chop: segments shrink by the decay, never below 9 frames, and cover the whole length", () => {
|
||||
test("chop: segments shrink by the decay, never below 9 frames, and stop within the length (the last few frames may be left over)", () => {
|
||||
const segs = chop(97, 40, 0.5);
|
||||
assert.ok(segs.every((s) => s.frames >= 9));
|
||||
assert.ok(segs.every((s, i) => i === 0 || s.start === segs[i - 1].start + segs[i - 1].frames), "pieces butt up exactly: no overlap, no gap");
|
||||
|
||||
@@ -30,7 +30,7 @@ export default {
|
||||
c.field.default({ label: "Lead clip (plays in order)", control: c.select.md({ label: "Lead clip", value: lead, options: list.map((s) => ({ value: s.id, label: label(s) })), onChange: (v) => { lead = v; pool.delete(v); draw(); } }) }),
|
||||
c.field.default({ label: "Cutaway pool (random draw, in order within each clip)", control: c.checklist.default({ label: "Cutaway pool", values: [...pool],
|
||||
items: list.filter((s) => s.id !== lead).map((s) => ({ value: s.id, label: label(s) })), onChange: (v) => { pool.clear(); v.forEach((x) => pool.add(x)); } }) }),
|
||||
c.field.default({ label: "Segment length (frames)", hint: "How long the lead clip's first piece and each cutaway piece are, in frames. Rounded down to a length the model can make (9, 17, 25 ...).", control: c.number.md({ label: "Segment length", value: seg, min: 9, step: 1, precision: 0, onChange: (v) => { seg = v; } }) }),
|
||||
c.field.default({ label: "Segment length (frames)", hint: "How long the lead clip's first piece and each cutaway piece are, in frames. Rounded down to 9, 17, 25 ... frames.", control: c.number.md({ label: "Segment length", value: seg, min: 9, step: 1, precision: 0, onChange: (v) => { seg = v; } }) }),
|
||||
c.field.default({ label: "Acceleration", hint: "1.0 = equal segments; lower = the lead clip's segments get shorter toward the end", control: c.slider.readout({ label: "Acceleration", value: decay, min: 0.5, max: 1, step: 0.05, onCommit: (v) => { decay = v; } }) }),
|
||||
]);
|
||||
draw();
|
||||
|
||||
@@ -50,6 +50,7 @@ export const api = {
|
||||
revolver: () => call("GET", "/api/shortcuts/revolver"),
|
||||
setRevolver: (enabled, random) => call("POST", "/api/shortcuts/revolver", { enabled, random }),
|
||||
enhancerDefaults: () => call("GET", "/api/m/conditioning_prompt_enhancer/defaults"),
|
||||
enhancerEncoders: () => call("GET", "/api/m/conditioning_prompt_enhancer/encoders"),
|
||||
enhancerRuns: () => call("GET", "/api/m/conditioning_prompt_enhancer/runs"),
|
||||
rateTaste: (promptId, rating, axis) => call("POST", "/api/m/taste/rate", { prompt_id: promptId, rating, axis: axis || null }),
|
||||
newTasteGeneration: (promptId) => call("POST", "/api/m/taste/generation", promptId ? { prompt_id: promptId } : {}),
|
||||
|
||||
@@ -41,5 +41,11 @@ def routes(table, base, web):
|
||||
runs = [r for r in RUNS if not want or r["prompt_id"] == want]
|
||||
return web.json_response({"runs": runs})
|
||||
|
||||
@table.get(base + "/encoders")
|
||||
async def _encoders(_req):
|
||||
"""Encoder files and families the Enhance panel can offer (read fresh: files come and go)."""
|
||||
from ...loaders.clip import nodes as clip_nodes
|
||||
return web.json_response({"files": clip_nodes.encoder_files(), "families": clip_nodes.clip_types()})
|
||||
|
||||
|
||||
PROVIDES = {"routes": routes}
|
||||
|
||||
@@ -69,10 +69,11 @@ class FunPackEnhancePrompt(io.ComfyNode):
|
||||
tooltip="0 = a fresh rewrite every run. Any other number repeats the same rewrite."),
|
||||
io.Boolean.Input("thinking", default=False, optional=True,
|
||||
tooltip="Let a thinking model reason first. Its reasoning eats the token budget."),
|
||||
io.Combo.Input("encoder_file", options=[""] + clip_nodes.encoder_files(), default="", optional=True,
|
||||
tooltip="A text encoder file just for rewriting (Gemma, Qwen...). Empty uses the model's own encoder."),
|
||||
io.Combo.Input("encoder_family", options=clip_nodes.clip_types(), default="stable_diffusion", optional=True,
|
||||
tooltip="Which family that file is for. Only used with an encoder file."),
|
||||
# Text, not a list: a file that has gone must not stop a run where Enhance is off. The Enhance panel offers the files.
|
||||
io.String.Input("encoder_file", default="", optional=True,
|
||||
tooltip="A text encoder file just for rewriting (Gemma, Qwen...). Empty uses the model's own encoder."),
|
||||
io.String.Input("encoder_family", default="stable_diffusion", optional=True,
|
||||
tooltip="Which family that file is for. Only used with an encoder file."),
|
||||
io.Image.Input("image", optional=True, tooltip="Shown to the model alongside the prompt."),
|
||||
io.Boolean.Input("use_image", default=True, optional=True,
|
||||
tooltip="Send the picture (when one is connected) to the model."),
|
||||
|
||||
@@ -247,7 +247,7 @@ def test_the_readout_routes_are_provided_and_answer():
|
||||
def json_response(d):
|
||||
return d
|
||||
mod.routes(T(), "/m", W)
|
||||
assert set(table) == {"/m/defaults", "/m/runs"}
|
||||
assert set(table) == {"/m/defaults", "/m/runs", "/m/encoders"}
|
||||
|
||||
|
||||
def test_a_tokenizer_that_reads_the_picture_out_of_kwargs_gets_it():
|
||||
@@ -270,3 +270,11 @@ def test_a_picked_encoder_that_will_not_load_keeps_the_prompt_and_says_why(monke
|
||||
out = nodes.FunPackEnhancePrompt.execute(clip=object(), text="a fox", enabled=True, encoder_file="x.gguf")
|
||||
assert out.args[0] == "a fox" and out.args[1] == "encoder failed"
|
||||
assert "x.gguf" in nodes.RUNS[-1]["status"] and not nodes.RUNS[-1]["ok"]
|
||||
|
||||
|
||||
def test_a_gone_encoder_file_is_text_the_graph_does_not_refuse(comfyui):
|
||||
from modules.conditioning.prompt_enhancer import nodes
|
||||
schema = nodes.FunPackEnhancePrompt.define_schema()
|
||||
for name in ("encoder_file", "encoder_family"):
|
||||
inp = next(i for i in schema.inputs if i.id == name)
|
||||
assert inp.get_io_type() == "STRING" # a list would refuse a vanished file even with Enhance off
|
||||
|
||||
@@ -41,7 +41,7 @@ class FunPackLatentUpscalerLoader(io.ComfyNode):
|
||||
log.info("FunPack Latent Upscaler", f"{upscaler_name}: {spec.title} upscaler")
|
||||
return io.NodeOutput(model)
|
||||
newer = ("It looks like the newer attention-based H3 upscaler, which FunPack cannot build yet: "
|
||||
"use the older convolution one (h3_clean_latent_upscaler_v1). ") if any(str(k).startswith("base.") for k in sd) else ""
|
||||
"use an older convolution-based H3 upscaler. ") if any(str(k).startswith("base.") for k in sd) else ""
|
||||
raise RuntimeError(
|
||||
f"{upscaler_name} is not an upscaler any installed model module can load. {newer}"
|
||||
f"If it is Lightricks' or Hunyuan's, ComfyUI's own Load Latent Upscale Model "
|
||||
|
||||
@@ -64,7 +64,8 @@ def h3_reference_to_video():
|
||||
# `blank_like`: with no picture (text-only), buildInputs gives this node the size of that
|
||||
# slot's target_width/height, so a plain grey stand-in carries the project's own shape.
|
||||
"roles": [{"at": "assets.source_image", "input": "media_id",
|
||||
"blank_like": {"slot": "image_transform", "width": "target_width", "height": "target_height"}}],
|
||||
"blank_like": {"slot": "image_transform", "width": "target_width", "height": "target_height",
|
||||
"unsend": {"slot": "enhance", "input": "use_image"}}}],
|
||||
"inputs": {"media_id": "", "blank_width": 0, "blank_height": 0}},
|
||||
{"id": "image_transform", "group": "Reference media", "node": "ImageTransformKJ",
|
||||
# target_width/height are the PIXEL BUDGET, set in Project settings
|
||||
|
||||
@@ -51,7 +51,7 @@ SETTINGS = {
|
||||
"freenoise": {"type": "bool", "default": True, "label": "FreeNoise blending", "when": {"enabled": True},
|
||||
"hint": "Shuffle the starting noise so neighbouring windows share it."},
|
||||
"retain_first": {"type": "bool", "default": False, "label": "Pin anchor in every window",
|
||||
"hint": "Keeps the first frame in every window so the whole clip stays tied to your starting image.",
|
||||
"hint": "Keeps the first frame in every window so the whole clip stays tied to its first frame (your starting image, when there is one).",
|
||||
"when": {"enabled": True}},
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user