diff --git a/CHANGELOG.md b/CHANGELOG.md index afd87599..7be2d2fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ What is new in each version, in plain words. Newest first. ## [Unreleased] -- **MiniMax H3-Image** (experimental): a new "Start from" pipeline, MiniMax H3-Image · Text to Image and Edit. Write a prompt for a picture, or add reference pictures and refer to them as ``, ``, ... in the prompt (the first is the picture to edit; up to 16). The picture is held for the clip's length like other image models. You need the model, its VAE and its text encoder files; the model's licence excludes the EU, the UK, South Korea and the United States. +- **MiniMax H3-Image** (experimental): a new "Start from" pipeline, MiniMax H3-Image · Text to Image and Edit. Write a prompt for a picture, or add reference pictures and refer to them as ``, ``, ... in the prompt (the first is the picture to edit; up to 16). The result is a PNG picture, not a clip, and Length and FPS are not offered for it (both are set to 1). You need the model, its VAE and its text encoder files; the model's licence excludes the EU, the UK, South Korea and the United States. - **Noise merging is now called Step merging**, because most of its modes merge the model's answers at a step, not noise. Settings are kept. - **Alternating prompts** (Settings > general): write `[dance|zoom]` in a prompt, or `DANCE|ZOOM` between two shortcuts, for two options, A and B. "Main prompt reads" sets what the prompt says by default: "A while B" (default), "A B", only A or only B. Switch on "Read A and B separately" (off by default) to also assign steps: "Steps that read only A" and "Steps that read only B" give each step you name its own option, with no extra model pass. A `|` next to anything that is not a shortcut is left as typed. Two options per bracket. - **Simultaneous actions** (Step merging, needs "Read A and B separately"): with `[A|B]` in the prompt, each merged step is answered once on A and once on B from the same picture, and the two answers are mixed at your strength. "Second pass new noise" sets how much of the noise B starts from is new (0 keeps the same noise for both). Strength 0 reads only A, and strength 1 with new noise 1 reads only B, each with a single pass. Steps named in Step merging win over the per-step lists. Costs one extra model pass per merged step. diff --git a/app/ui/features/inspector/scene.js b/app/ui/features/inspector/scene.js index 737e6390..b75a24d7 100644 --- a/app/ui/features/inspector/scene.js +++ b/app/ui/features/inspector/scene.js @@ -2,7 +2,7 @@ import { composer as c } from "../../composer/composer.js"; import { namesOf, pick } from "./mediapick.js"; import { effFrames, effFps, genUnitId, isSubclip, seconds, snapFrames } from "../../shell/scenes.js"; -import { gridOf } from "../../shell/frame_grid.js"; +import { gridOf, offersLength } from "../../shell/frame_grid.js"; import { restoreToPlan } from "../../shell/edits.js"; const MODES = [{ value: "project", label: "Project default" }, { value: "timeline", label: "Timeline trim" }, { value: "custom", label: "Custom" }]; @@ -37,7 +37,7 @@ function drift(sc, open) { export const card = (...children) => c.card.default({ children: children.flat().filter(Boolean) }); export function sceneRows(p, app) { - const sc = p.selected, open = p.project; + const sc = p.selected, open = p.project, slots = app.pipeline && (app.pipeline.view || app.pipeline.slots)(); if (!sc) return [c.emptyState.default({ icon: "▭", title: "No scene", hint: "Add one on the timeline." })]; const cuts = p.scenes.filter((s) => genUnitId(s) === genUnitId(sc)).length > 1; return [ @@ -49,13 +49,14 @@ export function sceneRows(p, app) { card(mediaField(app, p, sc, { label: "Aspect ratio source", hint: "The scene's picture. Its shape sets the aspect ratio; the project's Width and Height set the size.", key: "source_image", multiple: false, kinds: ["image", "video"], empty: "— choose aspect ratio source —" }), mediaField(app, p, sc, { label: "References", key: "references", multiple: true, kinds: ["image", "video", "audio"], empty: "+ Add reference" })), - c.label.section({ text: "Length" }), - card(c.field.row({ fields: [ - lengthField(p, sc, "Frames", "frames_mode", "frames", effFrames(sc, open), gridOf(app.pipeline && (app.pipeline.view || app.pipeline.slots)())), - lengthField(p, sc, "FPS", "fps_mode", "fps", effFps(sc, open)), - ] }), - drift(sc, open), - c.button.md({ label: "Generate this scene", tone: "primary", onClick: () => app.say("generate.scene") })), + ...(offersLength(slots) ? [c.label.section({ text: "Length" }), + card(c.field.row({ fields: [ + lengthField(p, sc, "Frames", "frames_mode", "frames", effFrames(sc, open), gridOf(slots)), + lengthField(p, sc, "FPS", "fps_mode", "fps", effFps(sc, open)), + ] }), + drift(sc, open), + c.button.md({ label: "Generate this scene", tone: "primary", onClick: () => app.say("generate.scene") }))] : [ + card(c.button.md({ label: "Generate this scene", tone: "primary", onClick: () => app.say("generate.scene") }))]), card(c.checkboxRow.default({ label: "Exclude from full generation", hint: "Generate leaves this scene out.", checked: Boolean(sc.excluded), onChange: (v) => p.setScene(sc.id, "excluded", v) })), c.banner.info({ text: "Made with the project's pipeline and Engine settings", action: { label: "Engine settings →", onClick: () => app.openSettings && app.openSettings("engine") } }), ]; diff --git a/app/ui/features/settings/models.js b/app/ui/features/settings/models.js index 3d8fbb04..f87e979c 100644 --- a/app/ui/features/settings/models.js +++ b/app/ui/features/settings/models.js @@ -228,7 +228,10 @@ export const models = (app) => function mount() { const pick = found.find((p) => p.id === id); if (!pick || !(await confirm("Replace pipeline", `Replace the whole pipeline with “${pick.title}”? Every file and value you set here is cleared. Revert puts it back until you leave this page.`, "danger"))) return; const r = await ps.restore({ slots: pick.slots, removed: [], unwired: {} }); - if (!r.refused.length) Object.entries(pick.prefs || {}).forEach(([k, v]) => app.project.setPref(k, v)); // project settings the preset goes with (render steps) + if (!r.refused.length) { // project settings the preset goes with (render steps, a picture model's one frame) + Object.entries(pick.prefs || {}).forEach(([k, v]) => app.project.setPref(k, v)); + Object.entries(pick.project || {}).forEach(([k, v]) => app.project.setField(k, v)); + } specs = {}; await describe(); say(r.refused.length ? `Not changed: ${r.refused[0]}` : ""); } async function importWorkflow() { diff --git a/app/ui/features/wizard/index.js b/app/ui/features/wizard/index.js index 0343fd09..2de0d857 100644 --- a/app/ui/features/wizard/index.js +++ b/app/ui/features/wizard/index.js @@ -28,7 +28,7 @@ export default { const pick = presets.find((p) => p.id === ctx.preset); try { await app.project.newProject((ctx.name || "").trim() || "Untitled montage"); - if (pick) { const r = await app.pipeline.restore({ slots: pick.slots, removed: [], unwired: {} }); if (r.refused.length) c.toast.warn({ text: `The model's pipeline was not loaded: ${r.refused[0]}` }); else Object.entries(pick.prefs || {}).forEach(([k, v]) => app.project.setPref(k, v)); } + if (pick) { const r = await app.pipeline.restore({ slots: pick.slots, removed: [], unwired: {} }); if (r.refused.length) c.toast.warn({ text: `The model's pipeline was not loaded: ${r.refused[0]}` }); else { Object.entries(pick.prefs || {}).forEach(([k, v]) => app.project.setPref(k, v)); Object.entries(pick.project || {}).forEach(([k, v]) => app.project.setField(k, v)); } } } catch (err) { busy = false; return c.toast.warn({ text: `Could not make the project: ${err.message}` }); } busy = false; close(); diff --git a/app/ui/shell/frame_grid.js b/app/ui/shell/frame_grid.js index 6bee9470..1e1c3b1b 100644 --- a/app/ui/shell/frame_grid.js +++ b/app/ui/shell/frame_grid.js @@ -21,3 +21,7 @@ export async function learnGrid(api, slots) { try { Object.assign(specs, (await api.describeNodes(missing)).nodes || {}); } catch { missing.forEach((n) => { specs[n] = null; }); } return true; } + +/** Whether the pipeline takes a length or a rate from the project at all. A model that makes a picture holds it for a fixed time, so + * there is nothing to set: the Length and FPS controls are not offered. A pipeline not known yet offers them. */ +export const offersLength = (slots) => !(slots && slots.length) || slots.some((s) => (s.roles || []).some((r) => r.drives === "frames" || r.drives === "fps")); diff --git a/app/ui/shell/frame_grid.test.js b/app/ui/shell/frame_grid.test.js index a35c5980..86469e1e 100644 --- a/app/ui/shell/frame_grid.test.js +++ b/app/ui/shell/frame_grid.test.js @@ -1,6 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { gridOf, learnGrid } from "./frame_grid.js"; +import { gridOf, learnGrid, offersLength } from "./frame_grid.js"; test("the frame grid is read from the model's own length input: H3 17k+5, LTX 8k+1, a plain count none", async () => { const slots = [{ id: "r2v", node: "H3Node", roles: [{ at: "project.video", input: "length", drives: "frames" }] }, @@ -14,3 +14,14 @@ test("the frame grid is read from the model's own length input: H3 17k+5, LTX 8k await learnGrid({ describeNodes: async () => ({ nodes: { LtxNode: { widgets: [{ name: "length", min: 1, step: 8 }] } } }) }, ltx); assert.deepEqual(gridOf(ltx), { step: 8, base: 1, fps: null }); }); + +test("Length and FPS are offered when a pipeline takes them, or is not known yet, and not for one that holds a picture for a fixed time", () => { + const length = { id: "r", roles: [{ at: "project.video", input: "length", drives: "frames" }] }; + const rate = { id: "s", roles: [{ at: "project.video", input: "fps", drives: "fps" }] }; + const size = { id: "a", roles: [{ at: "project.video", input: "width" }] }; + assert.equal(offersLength([size, length]), true); + assert.equal(offersLength([size, rate]), true); + assert.equal(offersLength([size, { id: "x" }]), false); + assert.equal(offersLength([]), true); + assert.equal(offersLength(undefined), true); +}); diff --git a/core/routes.py b/core/routes.py index 8f5b7e0f..56b169f6 100644 --- a/core/routes.py +++ b/core/routes.py @@ -328,7 +328,9 @@ def register(routes, prefix=None): if isinstance(preset, dict) and preset.get("id") and preset.get("slots"): found.append({"id": preset["id"], "title": preset.get("title") or preset["id"], "module": spec.id, "slots": preset["slots"], - "prefs": preset.get("prefs") if isinstance(preset.get("prefs"), dict) else {}}) + "prefs": preset.get("prefs") if isinstance(preset.get("prefs"), dict) else {}, + # project numbers the pipeline goes with (a picture model: 1 frame at 1 fps) + "project": {k: v for k, v in (preset.get("project") or {}).items() if k in ("num_frames_per_scene", "frame_rate") and isinstance(v, int)}}) return web.json_response({"presets": found}) @routes.post(P + "/api/pipeline/import") diff --git a/core/tests/test_routes_pipeline.py b/core/tests/test_routes_pipeline.py index 70f876d2..567b01a7 100644 --- a/core/tests/test_routes_pipeline.py +++ b/core/tests/test_routes_pipeline.py @@ -532,6 +532,13 @@ def test_a_module_provided_preset_is_offered(server, registered): assert preset["slots"] +def test_a_picture_model_preset_carries_the_project_numbers_it_goes_with(server, registered): + _status, body = _request(server, "GET", "/fun/api/pipeline/presets") + image = next(p for p in body["presets"] if p["id"] == "minimax_h3_image") + assert image["project"] == {"num_frames_per_scene": 1, "frame_rate": 1} + assert next(p for p in body["presets"] if p["id"] == "minimax_h3_reference_to_video")["project"] == {} # a video model changes none + + def test_a_comfyui_workflow_imports_into_slots_that_build_a_real_graph(server, registered): """Over HTTP, against ComfyUI's own nodes: the API export of a plain text-to-image graph.""" from core import graph diff --git a/docs/how-to-start.md b/docs/how-to-start.md index 3b021df9..af18b1a2 100644 --- a/docs/how-to-start.md +++ b/docs/how-to-start.md @@ -24,7 +24,7 @@ Start ComfyUI as usual and open `/fun/` on the same address, for example `http:/ |---|---| | MiniMax H3 · Reference to Video, MiniMax H3 · Fast & Simple R2V | The model FunUI is built and tested around. | | Stable Diffusion 1.5 · Text to Image (held as a clip) | Experimental. | -| MiniMax H3-Image · Text to Image and Edit (held as a clip) | Experimental. Add pictures as references: the first is the picture to edit (`` in the prompt), the others are references (``, ...), up to 16. Uses 6 steps at CFG 1; Width and Height set the size (about 4 megapixels is a good start). | +| MiniMax H3-Image · Text to Image and Edit | Experimental. Add pictures as references: the first is the picture to edit (`` in the prompt), the others are references (``, ...), up to 16. The result is a picture file (PNG), and Length and FPS are not offered (both are set to 1). Uses 6 steps at CFG 1; Width and Height set the size (about 4 megapixels is a good start). | | LTX 2.3 · Image to Video, from References, LTX 2.3 · Video to Video, direct | Tested on a real model, with LoRAs, Step merging and Attention explorer. Other enhancements are not tested on LTX yet; please report problems in [Issues](https://github.com/digital-garbage/FunUI/issues). | ## 4. Pick your files diff --git a/modules/models/minimax_h3_image/__init__.py b/modules/models/minimax_h3_image/__init__.py index e312eb8c..bca3c8a8 100644 --- a/modules/models/minimax_h3_image/__init__.py +++ b/modules/models/minimax_h3_image/__init__.py @@ -5,9 +5,8 @@ the 16x latent grid, has no sound stream and reads at most 16 pictures: the firs picture to edit (`` in the prompt), the others are references (``, ...). It samples in 6 steps at CFG 1 and works best around 4 to 16 megapixels. -Like every image model here, its picture is held for the clip's length, so the editor gets a clip -made the same way as any other. A held 16-megapixel frame is about 48 MB in memory: keep the -length short at large sizes. +The result is a picture file (PNG), not a clip. The pipeline offers no Length or FPS; the preset sets +the project's Length and FPS to 1, so a scene made with it counts as one second on the timeline. The checkpoint is recognised by its header: the packed-DiT keys of H3 with no audio projection. Everything the model needs at run time (the image latent, the picture labels, the sigma grid @@ -31,6 +30,9 @@ _ABSENT = "audio_patch_proj.weight" #: How many pictures the model reads (the one to edit and its references). MAX_REFERENCES = 16 +#: The project numbers the preset sets: one frame at one frame per second. +PROJECT = {"num_frames_per_scene": 1, "frame_rate": 1} + def detect(keys) -> bool: keyset = set(keys) @@ -52,7 +54,6 @@ def probe_traits(keys) -> list: def text_to_image(): - video = {"at": "project.video", "input": "fps", "label": "FPS", "drives": "fps"} slots = [ {"id": "model", "group": "Loaders", "node": "FunUIDiffusionModelLoader", "inputs": { "weight_dtype": "default", "compute_dtype": "default", "attention": "default"}}, @@ -94,11 +95,8 @@ def text_to_image(): {"id": "decode", "group": "Render", "node": "FunUIDecode", "inputs": {"samples": ["sampler", 0], "vae": ["vae", 0], "model": ["model", 0]}}, - {"id": "hold", "group": "Render", "node": "RepeatImageBatch", - "roles": [{"at": "project.video", "input": "amount", "label": "Length", "drives": "frames"}], - "inputs": {"image": ["decode", 0], "amount": 1}}, - {"id": "save", "group": "Render", "node": "FunUISaveVideo", "roles": [video], - "inputs": {"images": ["hold", 0], "fps": 24.0, "filename_prefix": "FunUI", "settings": ["settings", 0]}}, + {"id": "save", "group": "Render", "node": "SaveImage", + "inputs": {"images": ["decode", 0], "filename_prefix": "FunUI"}}, ] for n in range(1, MAX_REFERENCES + 1): # Not wired here: an unused picture input is simply absent. The app wires the ones a scene has, per run. @@ -111,8 +109,8 @@ def text_to_image(): def presets(): - return [{"id": "minimax_h3_image", "title": "MiniMax H3-Image · Text to Image and Edit (held as a clip)", - "slots": text_to_image()}] + return [{"id": "minimax_h3_image", "title": "MiniMax H3-Image · Text to Image and Edit", + "slots": text_to_image(), "project": PROJECT}] PROVIDES = {"detect": detect, "probe_traits": probe_traits, "pipeline_presets": presets} diff --git a/modules/models/minimax_h3_image/tests/test_h3_image.py b/modules/models/minimax_h3_image/tests/test_h3_image.py index 61ea8d4c..5054be5e 100644 --- a/modules/models/minimax_h3_image/tests/test_h3_image.py +++ b/modules/models/minimax_h3_image/tests/test_h3_image.py @@ -54,3 +54,17 @@ def test_the_preset_builds_a_real_graph_and_is_offered(registered): assert refs[0]["roles"][0]["wireTo"] == {"slot": "cond", "input": "images.image_1"} # image_1 is the picture to edit sampler = next(s for s in slots if s["id"] == "sampler")["inputs"] assert (sampler["steps"], sampler["cfg"], sampler["sampler_name"], sampler["scheduler"]) == (6, 1.0, "euler", "simple") + + +def test_it_saves_a_picture_and_offers_no_length_or_fps(registered): + from core import graph, routes + from modules.models.minimax_h3_image import PROJECT, presets, text_to_image + + slots = {s["id"]: s for s in text_to_image()} + assert not [r for s in slots.values() for r in s.get("roles") or [] if r.get("drives") in ("frames", "fps")] + assert slots["save"]["node"] == "SaveImage" and "hold" not in slots # a PNG, not a clip + assert PROJECT == {"num_frames_per_scene": 1, "frame_rate": 1} and presets()[0]["project"] == PROJECT + prompt, _problems = graph.build(list(slots.values())) + assert [n["class_type"] for n in prompt.values()].count("SaveImage") == 1 + served = [p for _s, make in routes.modules().providers("pipeline_presets") for p in make() if p["id"] == "minimax_h3_image"] + assert served and served[0]["project"] == PROJECT