diff --git a/app/ui/features/generate/index.js b/app/ui/features/generate/index.js new file mode 100644 index 00000000..f57ef2da --- /dev/null +++ b/app/ui/features/generate/index.js @@ -0,0 +1,62 @@ +// Generate / Selected: queue the plan's scenes one after another and put each result on its clip. +import { composer as c } from "../../composer/composer.js"; +import { isGenerative, genUnitId, effFrames, effFps } from "../../shell/scenes.js"; +import { buildInputs } from "./inputs.js"; + +const expand = (body) => fetch("/funpack/api/prompt/expand", { method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ...body, seed: Math.floor(Math.random() * 2 ** 31) || 1 }) }).then((r) => (r.ok ? r.json() : null)); + +export default { + id: "generate", + mount: "timeline.actions", + needs: ["project", "pipeline", "generate"], + setup({ host, app }) { + const p = app.project, g = app.generate; + let busy = false; + + const record = (pid, id, image) => { + if (!image || !p.project || p.project.id !== pid) return; + p.edit((pr) => { + const sc = pr.scenes.find((s) => s.id === id); + if (!sc) return false; + (pr.scene_renders ||= {})[id] = { media: { filename: image.filename, subfolder: image.subfolder || "", type: image.type || "output" }, + durationSec: effFrames(sc, pr) / effFps(sc, pr), inSec: 0 }; + return true; + }); + }; + + async function runAll(scenes) { + if (busy) return; + busy = true; draw(); + const pid = p.project.id, seen = new Set(); + try { + for (const sc of scenes) { + if (!isGenerative(sc) || sc.excluded || seen.has(genUnitId(sc))) continue; + seen.add(genUnitId(sc)); + const { inputs, unwired } = await buildInputs({ project: p.project, scene: sc, slots: app.pipeline.slots(), expand }); + if (unwired) c.toast.warn({ text: `${unwired} reference(s) did not fit this pipeline and are not used.` }); + const done = g.waitForTerminal(); // listening before the run starts, so a fast one is not missed + if (!(await g.generate({ sceneId: sc.id, projectId: pid, inputs }))) { done.cancel(); break; } + const end = await done; + if (end !== g.DONE) break; + const images = g.run.state.images; + record(pid, sc.id, images[images.length - 1]); + } + } finally { busy = false; draw(); } + } + + let nodes = []; + function draw() { + const open = p.project; + const go = c.button.sm({ label: busy ? "Generating…" : "▶ Generate", tone: "primary", disabled: busy || !open, + onClick: () => runAll(p.scenes) }); + const one = c.button.sm({ label: "Selected", disabled: busy || !p.selected, onClick: () => runAll([p.selected]) }); + const stop = c.button.sm({ label: "■ Stop", tone: "danger", disabled: !busy, onClick: () => g.cancel() }); + const next = [go, one, stop].map((b) => b.node); + if (nodes.length) nodes.forEach((n, i) => n.replaceWith(next[i])); else host.append(...next); // in place: keeps its spot in the row + nodes = next; + } + draw(); + return app.on(draw); + }, +}; diff --git a/app/ui/features/generate/inputs.js b/app/ui/features/generate/inputs.js new file mode 100644 index 00000000..1a9ea363 --- /dev/null +++ b/app/ui/features/generate/inputs.js @@ -0,0 +1,28 @@ +// What one run sends: the scene's prompt (expanded with anchor/postfix/$variables), the project's video +// settings, the scene's picked media -- each placed by the ROLE a pipeline slot declares, never by slot id. +import { wireReferences } from "../../shell/reference_wiring.js"; + +const rolesAt = (slots, at) => (slots || []).flatMap((s) => (s.roles || []).filter((r) => r.at === at).map((role) => ({ slot: s, role }))); + +export async function buildInputs({ project, scene, slots, expand }) { + const raw = {}; + const put = (slot, name, value) => { raw[slot.id] = { ...raw[slot.id], [name]: value }; }; + + for (const { slot, role } of (slots || []).flatMap((s) => (s.roles || []).filter((r) => (r.at || "").startsWith("project.") && r.input).map((role) => ({ slot: s, role })))) { + const v = role.at === "project.negative" ? project.negative : (project.video || {})[role.input]; + if (v !== undefined && v !== null && v !== "") put(slot, role.input, v); + } + const source = rolesAt(slots, "assets.source_image")[0]; + if (source) put(source.slot, "media_id", scene.source_image || ""); + const { overrides, unwired } = wireReferences(scene.references || [], slots, source ? [source.slot.id] : []); + for (const [id, fields] of Object.entries(overrides)) raw[id] = { ...raw[id], ...fields }; + + const prompt = rolesAt(slots, "generation.prompt")[0]; + if (prompt) { + const text = scene.text || ""; + const body = await expand({ text, anchor: project.anchor, postfix: project.postfix, postfix_enabled: project.postfix_enabled, variables: project.variables }) + .catch(() => null); // a failing prompt-craft feature must not block the run: send what was typed + put(prompt.slot, prompt.role.input, body && typeof body.text === "string" ? body.text : text); + } + return { inputs: raw, unwired }; +} diff --git a/app/ui/features/generate/inputs.test.js b/app/ui/features/generate/inputs.test.js new file mode 100644 index 00000000..7d5fe43c --- /dev/null +++ b/app/ui/features/generate/inputs.test.js @@ -0,0 +1,20 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { buildInputs } from "./inputs.js"; + +const slots = [ + { id: "p", roles: [{ at: "generation.prompt", input: "text" }] }, + { id: "s", roles: [{ at: "assets.source_image" }] }, + { id: "k", roles: [{ at: "project.video", input: "steps" }, { at: "project.negative", input: "neg" }] }, +]; + +test("each value lands on the slot whose role asks for it", async () => { + const { inputs } = await buildInputs({ project: { video: { steps: 8 }, negative: "blur" }, scene: { text: "a cat", source_image: "m1" }, slots, + expand: async ({ text }) => ({ text: `[${text}]` }) }); + assert.deepEqual(inputs, { p: { text: "[a cat]" }, s: { media_id: "m1" }, k: { steps: 8, neg: "blur" } }); +}); + +test("a failing expander sends the typed text", async () => { + const { inputs } = await buildInputs({ project: {}, scene: { text: "x" }, slots, expand: async () => { throw new Error("down"); } }); + assert.equal(inputs.p.text, "x"); +}); diff --git a/app/ui/features/placeholders/index.js b/app/ui/features/placeholders/index.js index 360ea3a1..846cb6ed 100644 --- a/app/ui/features/placeholders/index.js +++ b/app/ui/features/placeholders/index.js @@ -17,7 +17,7 @@ const items = { ], - "timeline.actions": () => [btn("▶ Generate", "primary"), btn("Selected"), btn("⧉ Render", "render"), btn("⚡ Auto Montage")], + "timeline.actions": () => [btn("⧉ Render", "render"), btn("⚡ Auto Montage")], "timeline.status": () => [btn("⏱ Sampler", "neutral"), c.text.sm({ text: "0 clips" }), dockTab("Assets"), dockTab("Preview"), dockTab("Properties"), btn("◆ Composer", "neutral")], timeline: () => [ diff --git a/app/ui/modules.js b/app/ui/modules.js index 896530f1..0dd3c8a3 100644 --- a/app/ui/modules.js +++ b/app/ui/modules.js @@ -4,6 +4,7 @@ export default [ "./features/projects/index.js", "./features/timeline/index.js", "./features/inspector/index.js", + "./features/generate/index.js", "./features/placeholders/index.js", "./features/temp/index.js", ].map((path) => new URL(path, import.meta.url).href);