diff --git a/movie_editor/backend/pipeline_deps.py b/movie_editor/backend/pipeline_deps.py index 2c41acd7..4e70b423 100644 --- a/movie_editor/backend/pipeline_deps.py +++ b/movie_editor/backend/pipeline_deps.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, Callable, Optional from . import config -from .builder import CORE +from .builder import CORE, family_core MANAGER_FOLDER = "ComfyUI-Manager" MANAGER_GIT_URL = "https://github.com/ComfyUI-Manager/ComfyUI-Manager" @@ -51,6 +51,107 @@ PIPELINE_PACKS: list[dict[str, Any]] = [ _PACK_BY_ID = {p["id"]: p for p in PIPELINE_PACKS} _ALL_MAPPED = frozenset().union(*(p["classes"] for p in PIPELINE_PACKS)) + +# ── per-family setup: what a project needs before it can generate ───────────── +# Declarative on purpose. MiniMax H3 is not released yet — its nodes are in an unmerged +# ComfyUI PR and no weights exist — so the entries below are PLACEHOLDERS: they describe +# what to look for and say plainly that it is not out. When it ships, the only thing that +# changes here is the data (flip `released`, fill in the real filename hints); the setup +# flow, the panel and the tests already work against it. +# +# `nodes` are slot node classes (not core graph nodes — those are covered by PIPELINE_PACKS). +# `models` are files the user has to download; `folder` is the ComfyUI models/ subdirectory. +FAMILY_SETUP: dict[str, dict[str, Any]] = { + "ltxav": { + "label": "LTX-2 / LTXAV", + "released": True, + "summary": "Lightricks LTX-2 / LTXAV. Gemma3 text encoder, separate video and audio latents.", + "nodes": [], + "models": [ + {"role": "unet", "label": "LTX-2 diffusion model", "folder": "diffusion_models"}, + {"role": "clip", "label": "Gemma3 text encoder", "folder": "text_encoders"}, + {"role": "video_vae", "label": "LTX-2 video VAE", "folder": "vae"}, + {"role": "audio_vae", "label": "LTX-2 audio VAE", "folder": "vae"}, + ], + }, + "minimax_h3": { + "label": "MiniMax H3 (Hailuo)", + "released": False, + "summary": "MiniMax H3. Qwen3-VL text encoder, one joint AV latent, 24 fps, " + "native reference conditioning (ref2va).", + "not_released_note": + "MiniMax H3 is not available yet. Its nodes are in an unmerged ComfyUI pull " + "request and no weights have been published. You can select it now to set the " + "project up — the pipeline is already wired for it, and it will start working " + "as soon as ComfyUI ships the nodes and the weights appear in your models " + "folders. Nothing here needs another FunPack update.", + "source_url": "https://github.com/Comfy-Org/ComfyUI/pull/15224", + "source_title": "ComfyUI PR #15224 — MiniMax H3 support", + "nodes": [ + {"class": "EmptyMiniMaxH3LatentAV", "label": "Empty MiniMax H3 AV Latent", + "role": "empty_latent", + "why": "Makes the joint video + audio latent that feeds the Chain Sampler."}, + {"class": "MiniMaxH3SigmaShift", "label": "MiniMax H3 Sigma Shift", + "role": None, "optional": True, + "why": "Optional — sets the video/audio flow shifts (defaults 12.0 / 3.0)."}, + ], + "models": [ + {"role": "unet", "label": "MiniMax H3 diffusion model", "folder": "diffusion_models", + "hint": "single-stream packed AV DiT"}, + {"role": "clip", "label": "Qwen3-VL-32B text encoder (50 layers)", "folder": "text_encoders", + "hint": "H3 consumes the unnormalized hidden state after layer 50"}, + {"role": "video_vae", "label": "MiniMax H3 video VAE", "folder": "vae", + "hint": "24-channel latent, 16x spatial"}, + {"role": "audio_vae", "label": "MiniMax H3 audio VAE", "folder": "vae", + "hint": "32 kHz stereo; also encodes audio references for ref2va"}, + ], + }, +} + +DEFAULT_FAMILY = "ltxav" + + +def family_setup(family: str | None) -> dict: + key = str(family or DEFAULT_FAMILY).strip().lower() + return FAMILY_SETUP.get(key) or FAMILY_SETUP[DEFAULT_FAMILY] + + +def families_payload() -> list[dict]: + """The setup picker's options, in display order.""" + return [ + {"key": key, "label": spec["label"], "released": bool(spec.get("released")), + "summary": spec.get("summary", ""), + "note": spec.get("not_released_note"), + "source_url": spec.get("source_url"), "source_title": spec.get("source_title")} + for key, spec in FAMILY_SETUP.items() + ] + + +def family_readiness(object_info: dict | None, family: str) -> dict: + """What this family still needs — nodes not installed, models not downloaded. + + Model files cannot be checked from object_info alone (a loader lists them as combo + choices only once the node exists), so those are reported as EXPECTED rather than + missing: the honest statement is "here is what you will need", not a false negative. + """ + oi = object_info or {} + spec = family_setup(family) + nodes = [] + for n in spec.get("nodes", []): + nodes.append({**n, "installed": n["class"] in oi}) + return { + "family": str(family or DEFAULT_FAMILY).lower(), + "label": spec["label"], + "released": bool(spec.get("released")), + "summary": spec.get("summary", ""), + "note": spec.get("not_released_note"), + "source_url": spec.get("source_url"), + "source_title": spec.get("source_title"), + "nodes": nodes, + "missing_nodes": [n for n in nodes if not n["installed"] and not n.get("optional")], + "models": spec.get("models", []), + } + _install_jobs: dict[str, dict] = {} @@ -75,17 +176,24 @@ def manager_dir_on_disk() -> bool: return d.is_dir() and ((d / ".git").exists() or (d / "__init__.py").exists()) -def required_core_classes() -> list[str]: - return sorted(set(CORE.values())) +def required_core_classes(family: str | None = None) -> list[str]: + return sorted(set(_core_for(family).values())) -def missing_core_classes(object_info: dict | None) -> list[str]: +def _core_for(family: str | None) -> dict: + """The fixed-core node classes for `family` (H3 drops three of LTXAV's).""" + if not family: + return CORE + return family_core(str(family).strip().lower())[0] + + +def missing_core_classes(object_info: dict | None, family: str | None = None) -> list[str]: oi = object_info or {} - return sorted(cls for cls in CORE.values() if cls not in oi) + return sorted(cls for cls in _core_for(family).values() if cls not in oi) -def missing_packs(object_info: dict | None) -> list[dict]: - missing = set(missing_core_classes(object_info)) +def missing_packs(object_info: dict | None, family: str | None = None) -> list[dict]: + missing = set(missing_core_classes(object_info, family)) out: list[dict] = [] for pack in PIPELINE_PACKS: hit = pack["classes"] & missing @@ -99,8 +207,8 @@ def missing_packs(object_info: dict | None) -> list[dict]: return out -def unmapped_missing_classes(object_info: dict | None) -> list[str]: - missing = set(missing_core_classes(object_info)) +def unmapped_missing_classes(object_info: dict | None, family: str | None = None) -> list[str]: + missing = set(missing_core_classes(object_info, family)) return sorted(missing - _ALL_MAPPED) @@ -109,18 +217,25 @@ def status_payload( *, manager_available: bool, manager_on_disk: bool | None = None, + family: str | None = None, ) -> dict: - packs = missing_packs(object_info) - unmapped = unmapped_missing_classes(object_info) + packs = missing_packs(object_info, family) + unmapped = unmapped_missing_classes(object_info, family) on_disk = manager_dir_on_disk() if manager_on_disk is None else manager_on_disk - needs_setup = bool(packs) - needs_manager = needs_setup and not manager_available + readiness = family_readiness(object_info, family or DEFAULT_FAMILY) + # A family whose nodes are not out yet is a setup problem too — the modal has to open + # and say so, rather than reporting a clean pipeline that cannot actually generate. + needs_setup = bool(packs) or bool(readiness["missing_nodes"]) + needs_manager = bool(packs) and not manager_available return { + "family": readiness["family"], + "families": families_payload(), + "readiness": readiness, "manager_available": manager_available, "manager_on_disk": on_disk, "needs_manager_install": needs_manager and not on_disk, "needs_manager_restart": needs_manager and on_disk, - "missing_classes": missing_core_classes(object_info), + "missing_classes": missing_core_classes(object_info, family), "missing_packs": packs, "unmapped_classes": unmapped, "needs_install": needs_setup, diff --git a/movie_editor/frontend/models.js b/movie_editor/frontend/models.js index e54f6db6..5ed6e9a2 100644 --- a/movie_editor/frontend/models.js +++ b/movie_editor/frontend/models.js @@ -1231,6 +1231,11 @@ }; row.append(b); }); + const setup = el("button", "btn ghost tiny", "Setup…"); + setup.title = "What this model family needs — nodes and model files, including anything " + + "not released yet."; + setup.onclick = () => window.PipelineSetup?.open(); + row.append(setup); sec.append(row); return sec; } diff --git a/movie_editor/frontend/pipeline_setup.js b/movie_editor/frontend/pipeline_setup.js index 616681a8..c56944e7 100644 --- a/movie_editor/frontend/pipeline_setup.js +++ b/movie_editor/frontend/pipeline_setup.js @@ -178,6 +178,73 @@ } } + // The family decides which nodes and which model files the project needs, so it is the + // FIRST question — asking it after an install would have installed the wrong thing. + async function chooseFamily(key) { + if (!S.get().project) return; + const models = JSON.parse(JSON.stringify(S.get().models || { slots: [] })); + models.model_family = key; + await API.saveModels(S.get().project.id, models); + await S.loadModels(); + let deps; + try { deps = await API.pipelineDeps(); } catch (_) { closeModal(); return; } + if (!deps?.needs_setup) { closeModal(); return; } + openModal(deps); + } + + function familyStep(deps) { + const box = el("div", "pipe-setup-family"); + box.append(el("div", "pipe-setup-manual-title", "Which model is this project for?")); + (deps.families || []).forEach((f) => { + const active = f.key === deps.family; + const b = el("button", "btn ghost pipe-setup-family-btn" + (active ? " active" : "")); + b.type = "button"; + const title = el("div", "pipe-setup-family-title", f.label); + if (!f.released) title.append(el("span", "pipe-setup-sub", "not released yet")); + b.append(title); + b.append(el("div", "pipe-setup-sub", f.summary || "")); + b.onclick = () => chooseFamily(f.key); + box.append(b); + }); + return box; + } + + // What the chosen family still needs. For a family that is not out yet this is the + // whole point of the panel: say what is missing and why, instead of offering an + // install button that cannot possibly succeed. + function readinessBlock(r) { + const box = el("div", "pipe-setup-readiness"); + if (r.note) box.append(el("div", "pipe-setup-hint", r.note)); + if (r.source_url) { + const a = el("a", "pipe-setup-link", r.source_title || r.source_url); + a.href = r.source_url; a.target = "_blank"; a.rel = "noopener"; + box.append(a); + } + const missing = r.missing_nodes || []; + if (missing.length) { + box.append(el("div", "pipe-setup-manual-title", "Nodes not in your ComfyUI yet:")); + const ul = el("ul", "pipe-setup-list"); + missing.forEach((n) => { + const li = el("li", null, n.label); + li.append(el("span", "pipe-setup-sub", n.class + (n.why ? " — " + n.why : ""))); + ul.append(li); + }); + box.append(ul); + } + if ((r.models || []).length) { + box.append(el("div", "pipe-setup-manual-title", + r.released ? "Model files this pipeline uses:" : "Model files to download when they are published:")); + const ul = el("ul", "pipe-setup-list"); + r.models.forEach((m) => { + const li = el("li", null, m.label); + li.append(el("span", "pipe-setup-sub", "models/" + m.folder + (m.hint ? " — " + m.hint : ""))); + ul.append(li); + }); + box.append(ul); + } + return box; + } + function openModal(deps) { closeModal(); overlay = el("div", "modal-overlay pipe-setup-overlay"); @@ -187,24 +254,34 @@ modal.append(head); const content = el("div", "modal-content"); - if (deps.needs_manager_install) { + content.append(familyStep(deps)); + const r = deps.readiness; + const unreleased = r && !r.released; + if (unreleased || (r && (r.missing_nodes || []).length)) { + content.append(readinessBlock(r)); + } + // Only talk about installing when there is something Manager can actually install. + // A family that is not released yet has nothing to fetch, and offering to fetch it + // would be a promise the button cannot keep. + const hasPacks = (deps.missing_packs || []).length > 0; + if (hasPacks && deps.needs_manager_install) { content.append(el("p", "pipe-setup-lead", "Nodes required for the built-in pipeline are missing. Install ComfyUI-Manager first, then we can fetch the rest.")); - } else if (deps.needs_manager_restart) { + } else if (hasPacks && deps.needs_manager_restart) { content.append(el("p", "pipe-setup-lead", "ComfyUI-Manager is on disk but not running yet. Restart ComfyUI to load it, then install the missing node packs.")); - } else { + } else if (hasPacks) { content.append(el("p", "pipe-setup-lead", - "Nodes required for the built-in pipeline are missing. Do you want to install them?")); + "Node packs required for the built-in pipeline are missing. Do you want to install them?")); } - if ((deps.missing_packs || []).length) { + if (hasPacks) { content.append(packList(deps)); } - if (deps.needs_manager_install) { + if (hasPacks && deps.needs_manager_install) { content.append(el("div", "pipe-setup-hint", "ComfyUI-Manager handles downloads and updates for custom node packs.")); content.append(manualBlock(deps)); - } else if (!deps.manager_available) { + } else if (hasPacks && !deps.manager_available) { content.append(manualBlock(deps)); } if ((deps.unmapped_classes || []).length) { @@ -214,17 +291,19 @@ modal.append(content); const foot = el("div", "pipe-setup-foot"); - if (deps.needs_manager_install) { + if (!hasPacks) { + // nothing installable — the only useful action left is to close and come back + } else if (deps.needs_manager_install) { const mgrBtn = el("button", "btn primary", "Install ComfyUI-Manager"); mgrBtn.type = "button"; mgrBtn.onclick = () => startManagerInstall(); foot.append(mgrBtn); - } else if (deps.needs_manager_restart) { + } else if (hasPacks && deps.needs_manager_restart) { const restartBtn = el("button", "btn primary", "Restart ComfyUI"); restartBtn.type = "button"; restartBtn.onclick = () => restartComfyOnly(); foot.append(restartBtn); - } else if (deps.manager_available) { + } else if (hasPacks && deps.manager_available) { const installBtn = el("button", "btn primary", "Install missing nodes"); installBtn.type = "button"; installBtn.onclick = () => startInstall(deps); @@ -267,5 +346,14 @@ openModal(deps); } - window.PipelineSetup = { maybePrompt, close: closeModal }; + // Opened on demand from Models → Model family, so the setup panel is reachable again + // after it has been dismissed once (the auto-prompt is one-shot per browser). + async function open() { + if (!S.get().project) return; + let deps; + try { deps = await API.pipelineDeps(); } catch (_) { return; } + openModal(deps); + } + + window.PipelineSetup = { maybePrompt, open, close: closeModal }; })(); diff --git a/movie_editor/frontend/styles.css b/movie_editor/frontend/styles.css index 9011899f..554c7cea 100644 --- a/movie_editor/frontend/styles.css +++ b/movie_editor/frontend/styles.css @@ -1335,6 +1335,20 @@ label.field.linked input, label.field.linked select { opacity: .6; } .pipe-setup-manual-title { font-size: 12px; color: var(--muted); margin-bottom: 6px; } .pipe-setup-foot { display: flex; flex-wrap: wrap; gap: 8px; padding: 12px 16px 16px; border-top: 1px solid var(--line); } .pipe-setup-hint { margin: 0 0 10px; font-size: 12px; color: var(--muted); line-height: 1.45; } +/* Model family picker — the first question in setup, since it decides which nodes and + which model files everything after it looks for. */ +.pipe-setup-family { display: flex; flex-direction: column; gap: 8px; margin-bottom: 14px; } +.pipe-setup-family-btn { display: block; width: 100%; text-align: left; padding: 10px 12px; + border-radius: 8px; line-height: 1.4; } +.pipe-setup-family-btn.active { border-color: var(--accent); background: var(--accent-soft, rgba(255,170,60,.10)); } +.pipe-setup-family-title { font-size: 13px; color: var(--text); display: flex; align-items: baseline; + gap: 8px; flex-wrap: wrap; } +.pipe-setup-family-title .pipe-setup-sub { display: inline; margin-top: 0; font-family: inherit; + text-transform: uppercase; letter-spacing: .04em; font-size: 10px; } +.pipe-setup-family-btn .pipe-setup-sub { font-family: inherit; font-size: 11px; color: var(--muted); } +.pipe-setup-readiness { margin: 0 0 14px; padding: 10px 12px; border-radius: 8px; + border: 1px solid var(--line); background: var(--panel-2, rgba(255,255,255,.02)); } +.pipe-setup-link { display: inline-block; margin-bottom: 10px; font-size: 12px; } .restart-card .restart-cancel { margin-top: 4px; } /* restart overlay */ diff --git a/movie_editor/server.py b/movie_editor/server.py index 2ec02246..f1f422e8 100644 --- a/movie_editor/server.py +++ b/movie_editor/server.py @@ -2208,6 +2208,7 @@ if web is not None and PromptServer is not None: oi, manager_available=mgr, manager_on_disk=pipeline_deps.manager_dir_on_disk(), + family=pipeline_wiring.family_of(nodes.load_models()), )) @routes.post(UI_PREFIX + "/api/pipeline-deps/install") diff --git a/movie_editor/tests/test_pipeline_deps.py b/movie_editor/tests/test_pipeline_deps.py index 9894f955..5f61a1ff 100644 --- a/movie_editor/tests/test_pipeline_deps.py +++ b/movie_editor/tests/test_pipeline_deps.py @@ -80,3 +80,69 @@ def test_custom_nodes_dir_uses_folder_paths(tmp_path, monkeypatch): fp = type("FP", (), {"get_folder_paths": staticmethod(lambda _: [str(cn)])})() monkeypatch.setitem(__import__("sys").modules, "folder_paths", fp) assert pipeline_deps.custom_nodes_dir() == cn + + +# ── per-family setup placeholders ───────────────────────────────────────────── +# MiniMax H3 is not released — no merged nodes, no published weights. The setup flow has +# to say that plainly rather than either hiding the family or offering an install that +# cannot succeed. These pin the shape the placeholders must keep, so shipping the real +# thing is a data edit (flip `released`, fill the hints) and not a code change. + +def test_both_families_are_offered_and_h3_is_marked_unreleased(): + from movie_editor.backend import pipeline_deps as pd + fams = {f["key"]: f for f in pd.families_payload()} + assert set(fams) == {"ltxav", "minimax_h3"} + assert fams["ltxav"]["released"] is True + assert fams["minimax_h3"]["released"] is False + # a user picking it must be told why nothing generates yet, and where it comes from + assert "not available yet" in fams["minimax_h3"]["note"] + assert "15224" in fams["minimax_h3"]["source_url"] + assert fams["ltxav"]["note"] is None + + +def test_h3_readiness_lists_the_nodes_and_model_files_it_is_waiting_on(): + from movie_editor.backend import pipeline_deps as pd + r = pd.family_readiness({}, "minimax_h3") + assert r["released"] is False + # the AV latent node is required; the sigma-shift node is optional and must not block + missing = {n["class"] for n in r["missing_nodes"]} + assert missing == {"EmptyMiniMaxH3LatentAV"} + assert {m["role"] for m in r["models"]} == {"unet", "clip", "video_vae", "audio_vae"} + assert all(m["folder"] for m in r["models"]) + + # once ComfyUI ships the node, it stops being listed as missing — no code change + r2 = pd.family_readiness({"EmptyMiniMaxH3LatentAV": {}}, "minimax_h3") + assert r2["missing_nodes"] == [] + + +def test_an_unreleased_family_still_counts_as_needing_setup(): + """Otherwise the modal never opens and the project looks ready when it cannot generate.""" + from movie_editor.backend import pipeline_deps as pd + full_oi = {cls: {} for cls in pd.required_core_classes("minimax_h3")} + s = pd.status_payload(full_oi, manager_available=True, family="minimax_h3") + assert s["missing_packs"] == [] # nothing for Manager to install + assert s["needs_setup"] is True # ... but setup is not finished + assert s["readiness"]["released"] is False + assert s["family"] == "minimax_h3" + + # with the H3 node present too, setup is genuinely done + s2 = pd.status_payload({**full_oi, "EmptyMiniMaxH3LatentAV": {}}, + manager_available=True, family="minimax_h3") + assert s2["needs_setup"] is False + + +def test_h3_does_not_ask_for_the_ltx_only_core_packs(): + """H3's core drops LTXVConditioning / Concat / LTXVAudioVAEDecode, so a user on H3 + must not be told to install a pack for nodes their graph never emits.""" + from movie_editor.backend import pipeline_deps as pd + oi = {cls: {} for cls in pd.required_core_classes("minimax_h3")} + assert pd.missing_core_classes(oi, "minimax_h3") == [] + # the same object_info is NOT enough for LTXAV + ltx_missing = pd.missing_core_classes(oi, "ltxav") + assert "LTXVConditioning" in ltx_missing and "LTXVConcatAVLatent" in ltx_missing + + +def test_family_lookup_falls_back_instead_of_raising(): + from movie_editor.backend import pipeline_deps as pd + assert pd.family_setup("hailuo-9000")["label"] == pd.family_setup("ltxav")["label"] + assert pd.family_setup(None)["label"] == pd.family_setup("ltxav")["label"]