olivv-cs-ComfyUI-FunPack/app/ui/features/upscale/index.js

// Upscale a finished render with a model from ComfyUI's upscale_models, or with NVIDIA RTX Super Resolution; the result replaces the render. Never / button beside the rating / always.
import { composer as c } from "../../composer/composer.js";
import { inPlace } from "../../shell/place.js";
import { genUnitId, isGenerative, unitRoot } from "../../shell/scenes.js";
import { derived, gaveUp, wentThrough } from "../../shell/renders.js";
import { RTX, RTX_DEFAULTS, keepsSize, keyOf, qualityLabel, swap, upscale } from "./job.js";

export default {
  id: "upscale",
  mount: "timeline.actions",
  needs: ["project", "api", "selection"],
  modules: ["render"],
  setup({ host, app }) {
    const p = app.project, sel = app.selection;
    const RETRIES = 480;                                  // 15 s apart: two hours of a generation
    const put = inPlace(host), busy = new Set(), heavy = new Set(), waited = new Map();       // waited: how often "Always" put a render off for a running generation
    const render = (id) => ((p.project || {}).scene_renders || {})[id];
    const target = () => {                               // the picked clip's generation unit root, when it has a render
      const open = p.project, sc = open && open.scenes.find((s) => s.id === sel.focus);
      const root = sc && isGenerative(sc) ? unitRoot(open, genUnitId(sc)) : null;
      const r = root && render(root.id);
      return r && r.media ? r.media : null;
    };

    const rtxOn = () => p.pref("upscaleEngine", "model") === "rtx";
    const rtxOpts = () => ({ ...RTX_DEFAULTS, ...(p.pref("upscaleRtx", {}) || {}) });
    // Pressed with nothing to upscale with: say what would fix it, and take the person there.
    async function suggest() {
      let all = []; try { all = await app.api.addons(); } catch { /* no add-on list: the model way alone */ }
      const vsr = all.find((a) => a.id === "nvidia_vsr");
      const items = [
        ...(vsr ? [vsr.state === "installed" ? { id: "editor", label: "Use NVIDIA RTX Super Resolution", hint: "It is installed: pick it as the Engine in Settings ▸ Editor." }
          : vsr.state === "pending" ? { id: "modules", label: "NVIDIA RTX Super Resolution is switched on", hint: "Restart ComfyUI to install it." }
          : { id: "modules", label: "Install NVIDIA RTX Super Resolution", hint: "Native upscaling, no model file needed. Switch it on in Settings ▸ Modules." }] : []),
        { id: "editor", label: "Download an upscale model", hint: "Put an upscale model (.safetensors) in ComfyUI/models/upscale_models, then pick it in Settings ▸ Editor." }];
      const pick = await c.modal.choice({ title: "Nothing to upscale with yet", items }).result;
      if (pick) app.openSettings(pick);
    }

    let told = false;                                   // "waits" is said once, however many renders wait
    const stillThere = (pid, key) => p.project && p.project.id === pid
      && Object.values(p.project.scene_renders || {}).some((r) => r && r.media && keyOf(r.media) === key);
    async function run(media, pressed = false, project = null) {
      let again = false;                                  // a retry is on its way: the wait count stays
      if (!p.project) return;
      if (!media || wentThrough(media, "upscale")) return;                    // never upscale twice, even after an interpolation in between
      const key = keyOf(media), pid = project || p.project.id;
      if (busy.has(key)) return;
      busy.add(key); draw();
      try {
        // RTX only counts when this machine has it on right now: a project saved elsewhere may still say "rtx"
        if (rtxOn() && !(await app.api.upscaleModels().then((r) => Boolean(r.rtx)).catch(() => false)))
          return void (gaveUp(media, "upscale"), c.toast.warn({ text: `${RTX} is not available on this machine (or is switched off in Settings ▸ Modules). Pick Upscale model as the Engine in Settings ▸ Editor.` }));
        const model = rtxOn() ? RTX : p.pref("upscaleModel", "");
        if (!model) return void (pressed ? suggest() : (gaveUp(media, "upscale"), c.toast.warn({ text: "Upscale: pick a model or NVIDIA RTX Super Resolution in Settings ▸ Editor." })));
        c.toast.info({ text: `Upscaling with ${model}…` });
        if (model !== RTX) { heavy.add(key); app.has.add("upscale.model"); }          // Generate waits: an upscale model shares ComfyUI's model memory
        const out = await upscale(app.api, media, model, undefined, rtxOn() ? rtxOpts() : undefined);
        derived(media, out, "upscale");
        waited.delete(key); if (!waited.size) told = false;
        const swapped = await p.editFor(pid, (pr) => swap(pr, media, out) > 0);
        if (!swapped) c.toast.warn({ text: "Upscaled, but that render was replaced meanwhile, so nothing was swapped." });
        else c.toast.good({ text: "Upscaled: the render now plays the larger video." });
      } catch (err) {
        if (err.stopped) c.toast.info({ text: "Upscale stopped. The render stays as it was." });
        else if (!pressed && /^ComfyUI is generating/.test(err.message || "") && (waited.get(key) || 0) < RETRIES) {       // Always: not lost, tried again when the generation is over
          if (!told) { told = true; c.toast.info({ text: "Upscale waits for the generation to finish." }); }
          waited.set(key, (waited.get(key) || 0) + 1); again = true;
          setTimeout(() => {                         // only for the same project, the same render and while Always is still on
            if (stillThere(pid, key) && p.pref("upscaleMode", "never") === "always") run(media, false, pid);
            else { waited.delete(key); if (!waited.size) told = false; }
          }, 15000);
        }
        else { waited.delete(key); if (!waited.size) told = false; gaveUp(media, "upscale"); c.toast.warn({ text: `Upscale failed: ${err.message}` }); }
      }
      finally { if (!again) { waited.delete(key); if (!waited.size) told = false; } busy.delete(key); heavy.delete(key); if (!heavy.size) app.has.delete("upscale.model"); draw(); }
    }

    function draw() {
      const media = p.pref("upscaleMode", "never") === "button" ? target() : null;
      if (!media) return put();
      const working = busy.has(keyOf(media)) || wentThrough(media, "upscale");
      put(c.button.sm({ label: busy.has(keyOf(media)) ? "Upscaling…" : wentThrough(media, "upscale") ? "Upscaled" : "Upscale", tone: "ghost", disabled: working,
        title: "Upscale this video with the engine picked in Settings ▸ Editor; it replaces the render", onClick: () => run(media, true) }).node);
    }

    // "Always": a render that appears after the project was opened is upscaled. What was there on open is left alone.
    let seen = null, obj = null;
    const watch = () => {
      const open = p.project;
      if (!open) { seen = null; obj = null; return; }
      const now = new Map(Object.entries(open.scene_renders || {}).filter(([, r]) => r && r.media).map(([id, r]) => [id, keyOf(r.media)]));
      if (open !== obj || !seen) { obj = open; seen = now; return; }       // another project, or undo/redo put an older copy back: not a new render
      if (p.pref("upscaleMode", "never") === "always") {
        const fresh = new Map([...now].filter(([id, k]) => seen.get(id) !== k));
        for (const media of new Map([...fresh.keys()].map((id) => [fresh.get(id), open.scene_renders[id].media])).values()) run(media);
      }
      seen = now;
    };

    app.editorSettings.push(() => {
      const page = c.region.stack({ gap: "md" });
      const set = (k, v) => { p.setPref("upscaleRtx", { ...rtxOpts(), [k]: v }); draw(); paint(info); };
      const gone = (v) => ({ value: v, label: `${v} (missing)` });
      const rtxRows = (r) => {
        const o = rtxOpts(), same = keepsSize(o.quality);
        return [
          c.settingsRow.default({ label: "Quality", hint: "Upscale is for normal videos, High bitrate for clean ones. Denoise and Deblur fix a video at its own size.", control: c.select.md({ label: "Quality", value: o.quality, onChange: (v) => set("quality", v),
            options: (r.rtx.qualities || []).map((q) => ({ value: q, label: qualityLabel(q) })) }) }),
          ...(same ? [c.hint.default({ text: "This quality keeps the video's size, so there is no scale." })] : [
            c.settingsRow.default({ label: "Size by", hint: "A multiplier of the video's size, or one fixed size.", control: c.select.md({ label: "Size by", value: o.resize, onChange: (v) => set("resize", v),
              options: [{ value: "scale", label: "Scale by multiplier" }, { value: "size", label: "Target dimensions" }] }) }),
            ...(o.resize === "scale"
              ? [c.settingsRow.default({ label: "Scale", hint: "1 to 4 times.", control: c.slider.readout({ label: "Scale", value: o.scale, min: 1, max: 4, step: 0.05, precision: 2, onCommit: (v) => set("scale", v) }) })]
              : [c.settingsRow.default({ label: "Width", hint: "In pixels, 64 to 8192. Rounded to a multiple of 8.", control: c.number.md({ label: "Width", value: o.width, min: 64, max: 8192, step: 8, precision: 0, onChange: (v) => set("width", v) }) }),
                 c.settingsRow.default({ label: "Height", hint: "In pixels, 64 to 8192. Rounded to a multiple of 8.", control: c.number.md({ label: "Height", value: o.height, min: 64, max: 8192, step: 8, precision: 0, onChange: (v) => set("height", v) }) })])]),
          c.settingsRow.default({ label: "Strength", hint: "How much of the effect to apply.", control: c.slider.readout({ label: "Strength", value: o.strength, min: 0, max: 1, step: 0.05, precision: 2, onCommit: (v) => set("strength", v) }) }),
          c.toggle.default({ label: "10-bit processing", hint: "More precise colour inside the upscale. The saved video stays 8-bit.", checked: o.tenbit, onChange: (v) => set("tenbit", v) }),
        ];
      };
      let info = null;
      const paint = (r) => {
        info = r;
        const chosen = p.pref("upscaleEngine", "model"), models = r === null ? null : r.models || [];
        page.set([
          c.label.section({ text: "Upscale finished renders" }),
          c.hint.default({ text: "Runs an upscale over a finished video and replaces the render with the result." }),
          c.settingsRow.default({ label: "Upscale", hint: "Never, with a button beside the rating, or on every new render.", control: c.select.md({ label: "Upscale", value: p.pref("upscaleMode", "never"),
            options: [{ value: "never", label: "Never" }, { value: "button", label: "When I press Upscale" }, { value: "always", label: "Always, when a render finishes" }], onChange: (v) => p.setPref("upscaleMode", v) }) }),
          c.settingsRow.default({ label: "Engine", hint: r && r.rtx ? "An upscale model from a file, or the RTX card's own upscaler." : r && r.rtx_offered ? "An upscale model from a file. NVIDIA RTX Super Resolution can be switched on in Settings ▸ Modules." : "An upscale model from a file.",
            control: c.select.md({ label: "Engine", value: chosen, onChange: (v) => { p.setPref("upscaleEngine", v); paint(info); },
              options: [{ value: "model", label: "Upscale model" }, ...(r && r.rtx ? [{ value: "rtx", label: RTX }] : chosen === "rtx" ? [{ value: "rtx", label: `${RTX} (not installed)` }] : [])] }) }),
          ...(chosen === "rtx" ? (r && r.rtx ? rtxRows(r) : [c.banner.warn({ text: r && !r.rtx_offered ? `${RTX} needs an NVIDIA RTX card on Windows or Linux, so it is not available here. Pick Upscale model.` : `${RTX} is not installed or is switched off: switch it on in Settings ▸ Modules and restart ComfyUI.` })])
            : [c.settingsRow.default({ label: "Model", hint: "From ComfyUI/models/upscale_models.", control: models === null ? c.text.sm({ text: "Loading…" })
              : c.select.md({ label: "Model", value: p.pref("upscaleModel", ""), onChange: (v) => p.setPref("upscaleModel", v),
                options: [{ value: "", label: models.length ? "— pick a model —" : "Nothing in models/upscale_models" }, ...models.map((m) => ({ value: m, label: m })),
                  ...(p.pref("upscaleModel", "") && !models.includes(p.pref("upscaleModel", "")) ? [gone(p.pref("upscaleModel", ""))] : [])] }) })]),
        ]);
      };
      paint(null);
      app.api.upscaleModels().then(paint).catch(() => page.set([c.banner.warn({ text: "Could not list upscale models." })]));
      return page;
    });

    draw(); watch();
    return app.on(() => { watch(); draw(); });
  },
};