diff --git a/app/ui/boot.js b/app/ui/boot.js index 41f50b98..fd866077 100644 --- a/app/ui/boot.js +++ b/app/ui/boot.js @@ -29,7 +29,7 @@ linkPipeline({ project, pipeline, onOpen: (fn) => on((what) => { if (what === "o const frame = build(document.getElementById("app")); const selection = createSelection({ project }); selection.on(() => say("select")); -const app = { project, pipeline, generate, api, on, say, title: frame.setZoneTitle, selection, playhead: createPlayhead(), theme: window.ComposerTheme, mode: createMode(), panel: frame.togglePanel, closePanels: frame.closePanels, dock: createDock(), maintenance: createMaintenance({ api, flush: async () => { await project.flush(); if (project.unsaved) throw new Error("Your latest edits could not be saved, so nothing was changed. Try again in a moment."); } }), keys: createKeys(), resetLayout: () => { frame.resetLayout(); app.dock.reset(); } }; +const app = { editorSettings: [], project, pipeline, generate, api, on, say, title: frame.setZoneTitle, selection, playhead: createPlayhead(), theme: window.ComposerTheme, mode: createMode(), panel: frame.togglePanel, closePanels: frame.closePanels, dock: createDock(), maintenance: createMaintenance({ api, flush: async () => { await project.flush(); if (project.unsaved) throw new Error("Your latest edits could not be saved, so nothing was changed. Try again in a moment."); } }), keys: createKeys(), resetLayout: () => { frame.resetLayout(); app.dock.reset(); } }; const settings = createSettings({ menubar: hostFor("menubar.menus") }); const result = await loadFeatures(paths, { app }); settings.addButton(); diff --git a/app/ui/features/settings/editor.js b/app/ui/features/settings/editor.js new file mode 100644 index 00000000..e3435908 --- /dev/null +++ b/app/ui/features/settings/editor.js @@ -0,0 +1,7 @@ +// Editor: how the editor behaves. Each feature that has a preference puts a section on app.editorSettings; nothing is named here. +import { composer as c } from "../../composer/composer.js"; + +export const editor = (app) => function mount() { + const parts = app.editorSettings.map((section) => section()); + return parts.length ? c.region.stack({ gap: "lg", children: parts }) : c.emptyState.default({ icon: "☰", title: "Nothing to set", hint: "No editor feature has a preference to show." }); +}; diff --git a/app/ui/features/settings/index.js b/app/ui/features/settings/index.js index 08b5b20c..e9aec560 100644 --- a/app/ui/features/settings/index.js +++ b/app/ui/features/settings/index.js @@ -8,10 +8,7 @@ import { modules } from "./modules.js"; import { system } from "./system.js"; import { packs } from "./packs.js"; import { taste } from "./taste.js"; - -const later = (what) => function mount() { - return c.emptyState.default({ icon: "◌", title: "Not built yet", hint: what }); -}; +import { editor } from "./editor.js"; export default { id: "settings-sections", @@ -22,7 +19,7 @@ export default { add("about", "About FunPack", "", "◎", "accent", about(app.api), "", "version commit branch cpu memory gpu disk python torch"); add("appearance", "Appearance", "Light, dark, or follow the system.", "◐", "neutral", appearance(app.theme), "", "theme colour light dark auto"); add("editor", "Editor", "How the editor behaves. The open project remembers these, so they follow it to another machine.", "☰", "neutral", - later("Upscale model, prompt autocomplete, shortcut ideas, i2v options."), "", "upscale autocomplete shortcuts ideas preferences"); + editor(app), "", "upscale autocomplete shortcuts ideas preferences"); add("engine", "Engine", "Settings every installed module has volunteered, grouped by what they're for.", "⚡", "warn", engine(app), "Generation", "engine module settings"); add("models", "Models & Pipeline", "Loaders and nodes wired into the live pipeline.", "⬡", "accent", diff --git a/app/ui/features/upscale/index.js b/app/ui/features/upscale/index.js new file mode 100644 index 00000000..4efdf08f --- /dev/null +++ b/app/ui/features/upscale/index.js @@ -0,0 +1,80 @@ +// Upscale a finished render with a model from ComfyUI's upscale_models; the result replaces the render. Never / button beside the rating / always. +import { composer as c } from "../../composer/composer.js"; +import { inPlace } from "../../shell/place.js"; +import { genUnitId, isGenerative, unitRoot } from "../../shell/scenes.js"; +import { UPSCALED, keyOf, swap, upscale } from "./job.js"; + +export default { + id: "upscale", + mount: "timeline.actions", + needs: ["project", "api", "selection"], + setup({ host, app }) { + const p = app.project, sel = app.selection; + const put = inPlace(host), busy = new Set(); + const render = (id) => ((p.project || {}).scene_renders || {})[id]; + const target = () => { // the picked clip's generation unit root, when it has a render + const open = p.project, sc = open && open.scenes.find((s) => s.id === sel.focus); + const root = sc && isGenerative(sc) ? unitRoot(open, genUnitId(sc)) : null; + const r = root && render(root.id); + return r && r.media ? r.media : null; + }; + + async function run(media) { + const model = p.pref("upscaleModel", ""); + if (!media || media.subfolder === UPSCALED) return; // never upscale twice + if (!model) return c.toast.warn({ text: "Upscale: pick a model in Settings ▸ Editor ▸ Upscale finished renders." }); + const key = keyOf(media), pid = p.project.id; + if (busy.has(key)) return; + busy.add(key); draw(); c.toast.info({ text: `Upscaling with ${model}…` }); + try { + const out = await upscale(app.api, media, model); + const swapped = await p.editFor(pid, (pr) => swap(pr, media, out) > 0); + if (!swapped) c.toast.warn({ text: "Upscaled, but that render was replaced meanwhile, so nothing was swapped." }); + else c.toast.good({ text: "Upscaled: the render now plays the larger video." }); + } catch (err) { c.toast.warn({ text: `Upscale failed: ${err.message}` }); } + busy.delete(key); draw(); + } + + function draw() { + const media = p.pref("upscaleMode", "never") === "button" ? target() : null; + if (!media) return put(); + const working = busy.has(keyOf(media)) || media.subfolder === UPSCALED; + put(c.button.sm({ label: busy.has(keyOf(media)) ? "Upscaling…" : media.subfolder === UPSCALED ? "Upscaled" : "Upscale", tone: "ghost", disabled: working, + title: "Upscale this video with the model picked in Settings ▸ Editor; it replaces the render", onClick: () => run(media) }).node); + } + + // "Always": a render that appears after the project was opened is upscaled. What was there on open is left alone. + let seen = null, openId = null; + const watch = () => { + const open = p.project; + if (!open) { seen = null; openId = null; return; } + const now = new Map(Object.entries(open.scene_renders || {}).filter(([, r]) => r && r.media).map(([id, r]) => [id, keyOf(r.media)])); + if (open.id !== openId || !seen) { openId = open.id; seen = now; return; } + if (p.pref("upscaleMode", "never") === "always") { + const fresh = new Map([...now].filter(([id, k]) => seen.get(id) !== k)); + for (const media of new Map([...fresh.keys()].map((id) => [fresh.get(id), open.scene_renders[id].media])).values()) run(media); + } + seen = now; + }; + + app.editorSettings.push(() => { + const page = c.region.stack({ gap: "md" }); + const row = (models) => page.set([ + c.label.section({ text: "Upscale finished renders" }), + c.hint.default({ text: "Runs an upscale model over a finished video and replaces the render with the result." }), + c.settingsRow.default({ label: "Upscale", hint: "Never, with a button beside the rating, or on every new render.", control: c.select.md({ label: "Upscale", value: p.pref("upscaleMode", "never"), + options: [{ value: "never", label: "Never" }, { value: "button", label: "When I press Upscale" }, { value: "always", label: "Always, when a render finishes" }], onChange: (v) => p.setPref("upscaleMode", v) }) }), + c.settingsRow.default({ label: "Model", hint: "From ComfyUI/models/upscale_models.", control: models === null ? c.text.sm({ text: "Loading…" }) + : c.select.md({ label: "Model", value: p.pref("upscaleModel", ""), onChange: (v) => p.setPref("upscaleModel", v), + options: [{ value: "", label: models.length ? "— pick a model —" : "Nothing in models/upscale_models" }, ...models.map((m) => ({ value: m, label: m })), + ...(p.pref("upscaleModel", "") && !models.includes(p.pref("upscaleModel", "")) ? [{ value: p.pref("upscaleModel", ""), label: `${p.pref("upscaleModel", "")} (missing)` }] : [])] }) }), + ]); + row(null); + app.api.upscaleModels().then((r) => row(r.models || [])).catch(() => page.set([c.banner.warn({ text: "Could not list upscale models." })])); + return page; + }); + + draw(); watch(); + return app.on(() => { watch(); draw(); }); + }, +}; diff --git a/app/ui/features/upscale/job.js b/app/ui/features/upscale/job.js new file mode 100644 index 00000000..2a346719 --- /dev/null +++ b/app/ui/features/upscale/job.js @@ -0,0 +1,27 @@ +// One upscale job from queue to result, and the swap of the new file under every render that plays the old one. Pure of the page. +export const keyOf = (m) => `${m.type || "output"}/${m.subfolder || ""}/${m.filename}`; +export const UPSCALED = "funpack_upscaled"; + +/** -> the upscaled media, or throws a sentence. `sleep` is injectable so a test need not wait. */ +export async function upscale(api, media, model, sleep = (ms) => new Promise((r) => setTimeout(r, ms))) { + const id = await api.queueUpscale(media, model); + let lost = 0; + for (;;) { + await sleep(2000); + const r = await api.upscaleResult(id); + if (!r) { lost = 0; continue; } + if (r.retry) { if (++lost < 30) continue; throw new Error("lost contact with ComfyUI"); } + if (r.gone) { if (++lost < 3) continue; throw new Error("the job left ComfyUI's queue without finishing (interrupted or ComfyUI restarted)"); } + if (r.error) throw new Error(r.error); + return r.videos[0]; + } +} + +/** One file backs a whole chain run, so every scene render and ghost that plays `from` gets `to`. -> how many were swapped. */ +export function swap(p, from, to) { + const same = (m) => !!m && keyOf(m) === keyOf(from); + let n = 0; + for (const [id, r] of Object.entries(p.scene_renders || {})) if (r && same(r.media)) { p.scene_renders[id] = { ...r, media: to }; n += 1; } + p.scene_ghosts = (p.scene_ghosts || []).map((g) => (same(g.media) ? (n += 1, { ...g, media: to }) : g)); + return n; +} diff --git a/app/ui/features/upscale/job.test.js b/app/ui/features/upscale/job.test.js new file mode 100644 index 00000000..8de856b5 --- /dev/null +++ b/app/ui/features/upscale/job.test.js @@ -0,0 +1,28 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { swap, upscale } from "./job.js"; + +const m = (f, sub = "") => ({ filename: f, subfolder: sub, type: "output" }); + +test("swap replaces the file under every render and ghost that plays it, and nothing else", () => { + const p = { scene_renders: { a: { media: m("x.mp4"), inSec: 1 }, b: { media: m("x.mp4") }, c: { media: m("y.mp4") } }, scene_ghosts: [{ id: "g", media: m("x.mp4") }] }; + assert.equal(swap(p, m("x.mp4"), m("up.mp4", "funpack_upscaled")), 3); + assert.equal(p.scene_renders.a.media.filename, "up.mp4"); assert.equal(p.scene_renders.a.inSec, 1); + assert.equal(p.scene_renders.c.media.filename, "y.mp4"); +}); + +test("upscale waits while queued, then returns the video; an error is thrown in words", async () => { + const seq = [null, null, { videos: [m("up.mp4", "funpack_upscaled")] }]; + const api = { queueUpscale: async () => "id", upscaleResult: async () => seq.shift() }; + assert.equal((await upscale(api, m("x.mp4"), "mdl", async () => {})).filename, "up.mp4"); + const bad = { queueUpscale: async () => "id", upscaleResult: async () => ({ error: "out of memory" }) }; + await assert.rejects(upscale(bad, m("x.mp4"), "mdl", async () => {}), /out of memory/); +}); + +test("a job that vanishes is reported after a few looks, a dropped reply is retried longer", async () => { + const gone = { queueUpscale: async () => "id", upscaleResult: async () => ({ gone: true }) }; + await assert.rejects(upscale(gone, m("x.mp4"), "mdl", async () => {}), /left ComfyUI's queue/); + let n = 0; + const flaky = { queueUpscale: async () => "id", upscaleResult: async () => (++n < 5 ? { retry: true } : { videos: [m("u.mp4")] }) }; + assert.equal((await upscale(flaky, m("x.mp4"), "mdl", async () => {})).filename, "u.mp4"); +}); diff --git a/app/ui/modules.js b/app/ui/modules.js index d1cc4c32..a94731d8 100644 --- a/app/ui/modules.js +++ b/app/ui/modules.js @@ -15,6 +15,7 @@ export default [ "./features/render/index.js", "./features/render/selected.js", "./features/rating/index.js", + "./features/upscale/index.js", "./features/status/index.js", "./features/audio/index.js", "./features/montage/index.js", diff --git a/app/ui/shell/api.js b/app/ui/shell/api.js index 665ea010..e1b97aa0 100644 --- a/app/ui/shell/api.js +++ b/app/ui/shell/api.js @@ -48,6 +48,34 @@ export const api = { if (!res.ok) throw new Error((payload && payload.why) || `HTTP ${res.status}`); return payload; }, + upscaleModels: () => call("GET", "/api/m/render/upscale_models"), + /** The upscale is an ordinary ComfyUI job: queued on /prompt, read back from /history. -> the prompt id. */ + async queueUpscale(media, model) { + const graph = { 1: { class_type: "FunPackUpscaleVideo", inputs: { filename: media.filename, subfolder: media.subfolder || "", type: media.type === "temp" ? "temp" : "output", upscale_model: model } } }; + const res = await fetch("/prompt", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ prompt: graph }) }); + const body = await res.json().catch(() => ({})); + if (!res.ok || !body.prompt_id) { + const nodes = Object.values(body.node_errors || {}).flatMap((n) => (n.errors || []).map((x) => x.message)); + throw new Error([(body.error || {}).message, ...nodes].filter(Boolean).join("; ") || `ComfyUI refused the job (HTTP ${res.status})`); + } + return body.prompt_id; + }, + /** null while it is queued or running, else {videos} | {error} | {gone} | {retry} (a dropped reply is not a failed job). */ + async upscaleResult(promptId) { + let entry = null; + try { const res = await fetch(`/history/${encodeURIComponent(promptId)}`); entry = res.ok ? (await res.json())[promptId] : null; } catch { return { retry: true }; } + if (!entry) { + try { + const q = await (await fetch("/queue")).json(); + return [...(q.queue_running || []), ...(q.queue_pending || [])].some((r) => r[1] === promptId) ? null : { gone: true }; + } catch { return { retry: true }; } + } + const st = entry.status || {}; + if (st.status_str === "error") { const m = (st.messages || []).find((x) => x[0] === "execution_error"); return { error: (m && m[1] && m[1].exception_message) || "failed inside ComfyUI" }; } + if (!st.completed) return null; + const videos = Object.values(entry.outputs || {}).flatMap((o) => o.videos || []); + return videos.length ? { videos } : { error: "no video came out, check the ComfyUI terminal" }; + }, expandPrompt: (body) => call("POST", "/api/prompt/expand", { ...body, seed: 1 }), packs: () => call("GET", "/api/packs"), pack: (action, body) => call("POST", `/api/packs/${action}`, body || {}), diff --git a/app/ui/shell/project.js b/app/ui/shell/project.js index 001a8e97..4d5e0573 100644 --- a/app/ui/shell/project.js +++ b/app/ui/shell/project.js @@ -450,6 +450,15 @@ export function createProject({ onChange, onError, onOpen, keepConsistent } = {} changed(); }, + /** Editor preferences that travel with the project (a new rental has an empty browser). */ + pref(key, fallback) { const s = project && project.editor_settings; return s && key in s ? s[key] : fallback; }, + setPref(key, value) { + if (!project || this.pref(key) === value) return; + (project.editor_settings ||= {})[key] = value; + scheduleSave(); + changed(); + }, + /** Attach what a run produced to the scene it was started from. */ setResult(id, result) { this.setScene(id, "result", result); },