feat(models): a new project starts as FunPack's loaders, already wired

Seeded server-side on first read, once, per family — plus the family's own plumbing, so
nothing required is left to wire by hand. A seed never reaches the global default, which
would replace configured loaders with empty ones. Wizard and node picker follow.
This commit is contained in:
digital-garbage
2026-08-20 19:01:59 +03:00
parent 8d4a199e1f
commit 41bba44b54
6 changed files with 395 additions and 6 deletions
+107
View File
@@ -389,6 +389,113 @@ def validate_models_wiring(models: Any) -> list[str]:
# ── the default pipeline ──────────────────────────────────────────────────────
# A fresh project starts with FunPack's own loaders, already wired to the core ports it
# needs, so setting up a model is choosing files and nothing else. Every one of these is an
# ordinary slot afterwards: rename it, swap the node class, rewire it, delete it.
#
# (role, node class, label, input sources). Outputs are read from the live schema and wired
# through DEFAULT_WIRES_BY_ROLE, the same table used when a node is added by hand.
FUNPACK_DEFAULT_LOADERS: list[tuple[str, str, str, dict]] = [
("unet", "FunPackDiffusionModelLoader", "Diffusion model", {}),
("clip", "FunPackCLIPLoader", "Text encoder", {}),
("video_vae", "FunPackVAELoader", "Video VAE", {}),
("audio_vae", "FunPackVAELoader", "Audio VAE", {}),
]
# The rest of what a family needs before it can generate. Not loaders and not FunPack's, but
# leaving them out would mean "the pipeline sets itself up" still ended in a required slot
# the user has to find, add and wire by hand — which is the thing this is here to remove.
# Sources name seeded slots by their fixed ids, and the project's own frames / fps
# primitives, so the node follows the project instead of holding a second copy of its length.
FAMILY_DEFAULT_EXTRAS: dict[str, list[tuple[str, str, str, dict]]] = {
"ltxav": [
("audio_encoder", "LTXVEmptyLatentAudio", "Audio latent",
{"audio_vae": "out:fp_audio_vae:VAE",
"frames_number": "core:frames:0",
"frame_rate": "core:fps:0"}),
],
"minimax_h3": [
# H3 emits both streams from one node, so this IS the latent the sampler starts from.
("empty_latent", "EmptyMiniMaxH3LatentAV", "AV latent", {"length": "core:frames:0"}),
],
}
def declared_widget_defaults(node_def: Optional[dict]) -> dict:
"""A node's DECLARED widget defaults — no first-choice fallback.
`nodes.widget_inputs` fills a combo's default with its first option, which is right when
someone adds a node by hand and wrong here: it would pre-select an arbitrary model file
and make an unconfigured loader look configured. Only what the node actually declares is
seeded, so the file pickers stay empty and say so.
"""
out = {}
inp = (node_def or {}).get("input") or {}
for group in ("required", "optional"):
for name, spec in (inp.get(group) or {}).items():
if not isinstance(spec, (list, tuple)) or not spec:
continue
opts = spec[1] if len(spec) > 1 and isinstance(spec[1], dict) else {}
if opts.get("forceInput") or "default" not in opts:
continue
out[name] = opts["default"]
return out
def default_pipeline_slots(family: str = DEFAULT_FAMILY,
object_info: Optional[dict] = None) -> list[dict]:
"""Slot recipes for a project that has none yet, wired for `family`.
A loader with nothing to feed in this family is skipped rather than seeded as a node
wired to nowhere.
"""
from .nodes import node_outputs # lazy: nodes imports this module's siblings
family = family if family in FAMILY_WIRING else DEFAULT_FAMILY
wires_by_role = _default_wires(family)
out = []
for role, cls, label, sources in (FUNPACK_DEFAULT_LOADERS
+ FAMILY_DEFAULT_EXTRAS.get(family, [])):
node_def = (object_info or {}).get(cls)
if node_def is None:
continue # this ComfyUI does not have the node
role_wires = wires_by_role.get(role) or {}
# By output NAME first, then by type — the same rule the add-a-node path uses, and
# necessary because the tables are keyed by type while a node names its own outputs.
wires = {}
for o in node_outputs(node_def):
target = role_wires.get(o["name"]) or role_wires.get(o["type"])
if target:
wires[o["name"]] = [target]
if not wires:
continue # nothing to feed in this family: not a node, just clutter
out.append({"id": f"fp_{role}", "role": role, "role_label": label,
"node_class": cls, "inputs": declared_widget_defaults(node_def),
"input_sources": dict(sources), "wires": wires})
return out
def seed_default_pipeline(models: dict, object_info: Optional[dict] = None) -> dict:
"""Give a pipeline config that has never been set up FunPack's own loaders.
Recorded with `defaults_seeded` so it happens exactly once: a pipeline someone emptied
on purpose stays empty, and so does one built around an imported workflow.
"""
if not isinstance(models, dict) or models.get("defaults_seeded"):
return models
if models.get("slots") or models.get("workflow_import") or models.get("disable_core"):
models["defaults_seeded"] = True # nothing to seed, and never seed it later
return models
if not isinstance(object_info, dict):
# No live schema means no way to tell whether FunPack's loaders are even installed,
# and no widget defaults to seed them with. Leave it unmarked and try again later.
return models
models["slots"] = default_pipeline_slots(family_of(models), object_info)
models["defaults_seeded"] = True
return models
def wiring_rules_payload(family: str = DEFAULT_FAMILY) -> dict[str, Any]:
"""Static rules for the Models UI, for one model family."""
family = family if family in FAMILY_WIRING else DEFAULT_FAMILY
+47 -2
View File
@@ -20,6 +20,7 @@
let coreProducers = []; // [{id,type,label}]
let requirements = []; // [{id,type,label,required,role_hint,hint}]
let wiringRules = {}; // guided wiring rules from /pipeline-ports
let defaultSlots = []; // what a project with no pipeline yet starts as
let container = null; // mounted content root inside the Settings window
let linkMode = false; // selecting inputs to bind into one shared control
let linkSel = []; // [{slotId, input, kind, choices, label}]
@@ -1200,6 +1201,13 @@
}
search.oninput = () => { listOpen = true; renderList(); };
// Opening "Add loader → Text encoder" on FunPack's own loader is the difference between
// choosing a file and choosing a node first. Any other node is still one search away.
function preferredClassFor(role) {
const recipe = defaultSlots.find((r) => r.role === role);
return recipe ? recipe.node_class : null;
}
async function pick(cls) {
curCand = specByClass[cls];
if (!curCand) { try { curCand = await loadSpec(cls); } catch (_) { return; } }
@@ -1298,7 +1306,12 @@
listBox.append(el("div", "ns-empty", "Loading nodes…"));
(roleKey === "__any__" ? ensureAllNodes() : candidates(roleKey))
.then((list) => { curList = list || []; renderList(); })
.then((list) => {
curList = list || [];
const preferred = preferredClassFor(roleFinal);
if (preferred && curList.some((c) => c.class === preferred)) { pick(preferred); return; }
renderList();
})
.catch((e) => {
clear(listBox);
listBox.append(el("div", "ns-empty", "Node list unavailable: " + (e && e.message ? e.message : e)));
@@ -2145,6 +2158,29 @@
return add;
}
// Recipes whose role nothing fills yet. A project made before FunPack had its own
// loaders — or one inherited from a global config — never gets seeded, so the offer has
// to be reachable by hand too. Only MISSING roles are ever added: an existing loader is
// someone's choice, and in guided mode a second producer on the same port would take it.
function missingDefaultLoaders() {
if (!pipelineLocked()) return [];
return defaultSlots.filter((r) => !(config.slots || []).some((s) => s.role === r.role));
}
async function addMissingDefaultLoaders() {
const missing = missingDefaultLoaders();
if (!missing.length) return;
for (const recipe of missing) {
const id = (config.slots || []).some((s) => s.id === recipe.id)
? recipe.id + "_" + Math.random().toString(36).slice(2, 7) : recipe.id;
config.slots.push({ ...recipe, id, inputs: { ...recipe.inputs } });
}
reconcileOpenPortWiring();
await persistNow();
await prewarmSpecs();
render();
}
function loaderSection(slots, v) {
const sec = el("div", "node-group node-group-pinned");
const head = el("div", "node-group-head");
@@ -2155,6 +2191,14 @@
(n, s) => n + ((v.perSlot[s.id] || []).some((m) => m.level === "error") ? 1 : 0), 0);
if (bad) title.append(el("span", "node-group-bad", `${bad} to fix`));
head.append(title);
const missing = missingDefaultLoaders();
if (missing.length && !searchQ) {
const use = el("button", "btn ghost tiny", `Use FunPack loaders (${missing.length})`);
use.title = "Add FunPack's own loaders for the roles nothing fills yet, already wired "
+ "to the pipeline. Loaders you have set up are left exactly as they are.";
use.onclick = addMissingDefaultLoaders;
head.append(use);
}
sec.append(head);
sec.append(el("div", "node-group-note", "Your model files. Pick them here — the rest of the pipeline wires itself."));
const grid = _grid(slots, v);
@@ -2318,7 +2362,8 @@
coreProducers = pp.core_producers || [];
requirements = pp.requirements || [];
wiringRules = pp.wiring || {};
} catch (_) { ports = []; coreProducers = []; requirements = []; wiringRules = {}; }
defaultSlots = pp.default_slots || [];
} catch (_) { ports = []; coreProducers = []; requirements = []; wiringRules = {}; defaultSlots = []; }
reconcileOpenPortWiring();
try { coreNodes = (await API.coreGraph(window.Store?.get().project?.id)).nodes || []; } catch (_) { coreNodes = []; }
await prewarmSpecs();
+19 -3
View File
@@ -466,10 +466,26 @@
render() {
const box = el("div", "oo-step");
const own = ctx.family === "__own__";
box.append(head("models", own ? "Wire your pipeline" : "Point it at your models",
box.append(head("models", own ? "Wire your pipeline" : "Choose your model files",
own
? "Nothing is generated until a final IMAGE reaches the global video output."
: "Pick the checkpoint, text encoder and VAE files this model uses."));
: "The loaders are already in place and wired. All that is left is picking files."));
if (!own) {
const panel = el("div", "oo-panel");
panel.append(el("div", "oo-list-label", "Already wired — pick a file for each"));
const list = el("ul", "oo-list");
[["Diffusion model", "the model itself, plus how it is quantized and which attention backend it uses"],
["Text encoder", "one file, or two for LTX-2.3's Gemma3 and its connector"],
["Video VAE", "decodes the picture"],
["Audio VAE", "decodes the sound"]].forEach(([name, why]) => {
const li = el("li", null, name);
li.append(el("span", "oo-list-sub", why));
list.append(li);
});
panel.append(list);
box.append(panel);
}
const p = el("p", "oo-body");
p.textContent = ctx.genType === "t2v"
@@ -478,7 +494,7 @@
box.append(p);
box.append(actions({
primary: "Set up models…",
primary: own ? "Set up models…" : "Choose files…",
onPrimary: () => openPane("models"),
secondary: "Continue",
onSecondary: next,
+37 -1
View File
@@ -2396,6 +2396,9 @@ if web is not None and PromptServer is not None:
"core_producers": nodes.core_producers(oi),
"requirements": nodes.pipeline_requirements(fam),
"wiring": pipeline_wiring.wiring_rules_payload(fam),
# What a project with no pipeline yet should start as: FunPack's own loaders,
# already wired. Sent with the rules because it is derived from the same family.
"default_slots": pipeline_wiring.default_pipeline_slots(fam, oi) if oi else [],
})
@routes.get(UI_PREFIX + "/api/image-targets")
@@ -2554,7 +2557,20 @@ if web is not None and PromptServer is not None:
@routes.get(UI_PREFIX + "/api/projects/{pid}/models")
async def _project_models_get(req):
return web.json_response(_project_models(_project_or_404(req.match_info["pid"])))
p = _project_or_404(req.match_info["pid"])
models = _project_models(p)
# First look at a pipeline that has never been set up: it becomes FunPack's own
# loaders, already wired. Seeded HERE rather than in the panel so it happens once,
# server-side, and never travels into the global default the way a save does.
if not models.get("defaults_seeded") and not models.get("slots"):
try:
oi = await bridge.object_info()
except Exception:
oi = None
if pipeline_wiring.seed_default_pipeline(models, oi).get("slots"):
p.models = models
projects.save(p)
return web.json_response(models)
@routes.put(UI_PREFIX + "/api/projects/{pid}/models")
async def _project_models_put(req):
@@ -2563,12 +2579,32 @@ if web is not None and PromptServer is not None:
if not isinstance(body, dict):
body = {"slots": []}
body.setdefault("slots", [])
# A pipeline that has never been set up starts as FunPack's own loaders, already
# wired — the wizard's family step reaches here before the Models panel is ever
# opened, so seeding only in the panel would leave "Continue" with no pipeline.
seeded_now = False
if not body.get("defaults_seeded"):
oi = None
if not body.get("slots"):
try:
oi = await bridge.object_info()
except Exception:
oi = None
before = len(body.get("slots") or [])
pipeline_wiring.seed_default_pipeline(body, oi)
seeded_now = len(body.get("slots") or []) > before
p.models = body
projects.save(p)
# Keep the global default in sync (it seeds new projects) — but never let a save that
# simply omits `model_family` erase the one already recorded there. A caller that
# means to change the family sends it; one that is only rearranging loaders does not,
# and silently clearing it is how every later project ends up with no family at all.
#
# A SEED is not a user edit, and must not reach the global at all: opening a new
# project would otherwise replace the loaders someone configured — the very thing
# the global exists to hand on — with four empty ones.
if seeded_now:
return web.json_response(body)
glob = dict(body)
if not glob.get("model_family"):
inherited = nodes.load_models().get("model_family")
+77
View File
@@ -920,3 +920,80 @@ def test_bypassing_a_connected_run_of_nodes_is_not_refused_by_its_own_members():
assert not any("bypass" in b for b in report["blocking"]), report["blocking"]
# both collapse away and Studio ends up on the loader itself
assert graph["studio"]["inputs"]["model"] == ["slot_u", 0]
# ── the default pipeline actually builds ──────────────────────────────────────
# Seeding loaders is only worth anything if the graph they produce is complete. This runs
# the real recipe through the real builder: what a user gets on a fresh project, minus the
# model files they still have to pick.
from movie_editor.backend import pipeline_wiring # noqa: E402
OI_DEFAULTS = dict(OI, **{
"FunPackDiffusionModelLoader": {
"input": {"required": {"model_name": [["m.safetensors"]],
"weight_dtype": [["default", "fp8_e4m3fn"], {"default": "default"}]}},
"output": ["MODEL", "STRING"], "output_name": ["MODEL", "status"]},
"FunPackCLIPLoader": {
"input": {"required": {"clip_list": ["STRING", {"default": "[]", "funpack_list": {}}],
"type": [["ltxv"], {"default": "ltxv"}]}},
"output": ["CLIP", "STRING"], "output_name": ["CLIP", "status"]},
"FunPackVAELoader": {
"input": {"required": {"vae_name": [["v.safetensors"]],
"dtype": [["default", "bf16"], {"default": "default"}]}},
"output": ["VAE", "STRING"], "output_name": ["VAE", "status"]},
"LTXVEmptyLatentAudio": {
"input": {"required": {"frames_number": ["INT", {"default": 97}],
"frame_rate": ["FLOAT,INT", {"default": 25.0, "widgetType": "FLOAT"}],
"audio_vae": ["VAE", {}]}},
"output": ["LATENT"], "output_name": ["Latent"]},
})
def _seeded_models(family="ltxav"):
models = {"slots": [], "model_family": family}
pipeline_wiring.seed_default_pipeline(models, OI_DEFAULTS)
for slot in models["slots"]: # the files the user picks; everything else is set
if slot["node_class"] == "FunPackDiffusionModelLoader":
slot["inputs"]["model_name"] = "m.safetensors"
if slot["node_class"] == "FunPackVAELoader":
slot["inputs"]["vae_name"] = "v.safetensors"
return models
def test_a_freshly_seeded_pipeline_builds_with_nothing_left_to_wire():
"""t2v: only the i2v anchor is unfed, and that comes from a scene's image, not a loader."""
_graph, report = builder.build(OI_DEFAULTS, _seeded_models(), PARAMS)
assert report["blocking"] == []
assert len(report["unsatisfied"]) == 1
assert "source_image" in report["unsatisfied"][0]
def test_a_freshly_seeded_pipeline_is_complete_for_i2v_too():
"""With a scene image the builder materialises it as a LoadImage, which is the IMAGE
producer Studio's anchor was waiting for — so nothing is left over at all."""
_graph, report = builder.build(OI_DEFAULTS, _seeded_models(), PARAMS,
media={"filename": "scene.png"})
assert report["blocking"] == []
assert report["unsatisfied"] == []
def test_the_seeded_loaders_reach_the_core_ports_they_were_wired_to():
graph, _ = builder.build(OI_DEFAULTS, _seeded_models(), PARAMS)
by_class = {n["class_type"]: nid for nid, n in graph.items()}
assert graph["studio"]["inputs"]["model"] == [by_class["FunPackDiffusionModelLoader"], 0]
assert graph["studio"]["inputs"]["clip"] == [by_class["FunPackCLIPLoader"], 0]
# two VAE loaders: the video one feeds the sampler, the audio one the audio decode
video_vae = graph["sampler"]["inputs"]["vae"]
audio_vae = graph["audiodec"]["inputs"]["audio_vae"]
assert graph[video_vae[0]]["class_type"] == "FunPackVAELoader"
assert graph[audio_vae[0]]["class_type"] == "FunPackVAELoader"
assert video_vae[0] != audio_vae[0]
def test_the_seeded_audio_latent_follows_the_project_not_its_own_widgets():
graph, _ = builder.build(OI_DEFAULTS, _seeded_models(), PARAMS)
audio = next(n for n in graph.values() if n["class_type"] == "LTXVEmptyLatentAudio")
assert audio["inputs"]["frames_number"] == ["frames", 0]
assert audio["inputs"]["frame_rate"] == ["fps", 0]
# and it takes the AUDIO vae, not whichever VAE auto-wire happened to reach first
assert audio["inputs"]["audio_vae"] == graph["audiodec"]["inputs"]["audio_vae"]
+108
View File
@@ -213,3 +213,111 @@ def test_the_panel_filters_exactly_as_the_builder_validates():
# and nothing internal leaks into it
for ports in payload["type_fallback_ports"].values():
assert not set(ports) & set(payload["guided_hidden_ports"])
# ── the default pipeline ──────────────────────────────────────────────────────
# Shipping FunPack's own loaders was pointless if a new project still started empty: the
# stated goal is that setting up a model is choosing files and nothing else.
OI_LOADERS = {
"FunPackDiffusionModelLoader": {"input": {"required": {
"model_name": [["a.safetensors", "b.safetensors"]],
"weight_dtype": [["default", "fp8_e4m3fn"], {"default": "default"}]}},
"output": ["MODEL"]},
"FunPackCLIPLoader": {"input": {"required": {
"clip_list": ["STRING", {"default": "[]", "funpack_list": {}}],
"type": [["ltxv", "minimax"], {"default": "ltxv"}]}},
"output": ["CLIP"]},
"FunPackVAELoader": {"input": {"required": {
"vae_name": [["v.safetensors"]],
"dtype": [["default", "bf16"], {"default": "default"}]}},
"output": ["VAE"]},
}
def test_a_fresh_pipeline_is_funpacks_own_loaders_already_wired():
slots = pipeline_wiring.default_pipeline_slots("ltxav", OI_LOADERS)
assert [s["role"] for s in slots] == ["unet", "clip", "video_vae", "audio_vae"]
wired = {s["role"]: list(s["wires"].values())[0][0] for s in slots}
assert wired["unet"] == "port:FunPackStudio.model"
assert wired["clip"] == "port:FunPackStudio.clip"
assert wired["video_vae"] == "port:FunPackLTXAVSceneChainSampler.vae"
assert wired["audio_vae"] == "port:LTXVAudioVAEDecode.audio_vae"
def test_the_default_pipeline_follows_the_family():
"""H3 decodes audio with core's node, so the audio VAE lands somewhere else entirely."""
slots = pipeline_wiring.default_pipeline_slots("minimax_h3", OI_LOADERS)
wired = {s["role"]: list(s["wires"].values())[0][0] for s in slots}
assert wired["audio_vae"] == "port:VAEDecodeAudio.vae"
def test_seeded_loaders_carry_declared_defaults_but_no_model_file():
"""Pre-selecting the first file would make an unconfigured loader look configured."""
slots = {s["role"]: s for s in pipeline_wiring.default_pipeline_slots("ltxav", OI_LOADERS)}
assert slots["unet"]["inputs"] == {"weight_dtype": "default"}
assert "model_name" not in slots["unet"]["inputs"]
assert slots["clip"]["inputs"] == {"clip_list": "[]", "type": "ltxv"}
def test_loaders_this_comfyui_does_not_have_are_not_seeded():
assert pipeline_wiring.default_pipeline_slots("ltxav", {}) == []
def test_seeding_happens_once_and_is_recorded():
models = {"slots": []}
pipeline_wiring.seed_default_pipeline(models, OI_LOADERS)
assert models["defaults_seeded"] is True and len(models["slots"]) == 4
models["slots"] = []
pipeline_wiring.seed_default_pipeline(models, OI_LOADERS)
assert models["slots"] == [] # emptied on purpose stays empty
def test_an_existing_pipeline_is_never_replaced():
models = {"slots": [{"id": "mine", "role": "unet", "node_class": "UNETLoader"}]}
pipeline_wiring.seed_default_pipeline(models, OI_LOADERS)
assert [s["id"] for s in models["slots"]] == ["mine"]
assert models["defaults_seeded"] is True
def test_an_imported_workflow_is_never_seeded_over():
models = {"slots": [], "workflow_import": {"name": "mine.json"}}
pipeline_wiring.seed_default_pipeline(models, OI_LOADERS)
assert models["slots"] == []
def test_seeding_is_deferred_when_the_node_schema_is_unavailable():
"""Marking it done without a schema would mean this project never gets loaders at all."""
models = {"slots": []}
pipeline_wiring.seed_default_pipeline(models, None)
assert "defaults_seeded" not in models
pipeline_wiring.seed_default_pipeline(models, OI_LOADERS)
assert len(models["slots"]) == 4
OI_WITH_PLUMBING = dict(OI_LOADERS, **{
"LTXVEmptyLatentAudio": {"input": {"required": {
"frames_number": ["INT", {"default": 97}],
"frame_rate": ["FLOAT,INT", {"default": 25.0, "widgetType": "FLOAT"}],
"audio_vae": ["VAE", {}]}},
"output": ["LATENT"], "output_name": ["Latent"]},
})
def test_the_default_pipeline_is_complete_not_just_the_loaders():
"""A required slot the user still has to find, add and wire by hand is the thing this
is here to remove — so the family's own plumbing is seeded with it."""
slots = {s["role"]: s for s in
pipeline_wiring.default_pipeline_slots("ltxav", OI_WITH_PLUMBING)}
assert "audio_encoder" in slots
audio = slots["audio_encoder"]
# wired by output NAME ("Latent"), while the rules table is keyed by type ("LATENT")
assert audio["wires"] == {"Latent": ["port:LTXVConcatAVLatent.audio_latent"]}
assert audio["input_sources"]["audio_vae"] == "out:fp_audio_vae:VAE"
# length and rate come from the project, not from a second copy inside the node
assert audio["input_sources"]["frames_number"] == "core:frames:0"
assert audio["input_sources"]["frame_rate"] == "core:fps:0"
def test_plumbing_for_another_family_is_not_seeded():
slots = pipeline_wiring.default_pipeline_slots("minimax_h3", OI_WITH_PLUMBING)
assert "audio_encoder" not in {s["role"] for s in slots}