Video input: prepare reference videos (size, crop, rate, skip, cut, every Nth, cap) per project or per scene

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BzJLJWVu7RZnbRqm3nYAnX

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Added (experimental)
 
+- **Video input** (Project and Scene tabs): reference videos from the Media bin can be got ready before the model sees them, like a video loader node: target width and height (fill, fit with bars, or stretch; rounded to a multiple), edge crop in percent, frame rate (project's, the video's own or a number), skip first N frames, cut last N, every Nth frame, frame cap. The sound is cut to match. One set per project and one per scene; a dropdown in Project settings says whose is used (Automatic: the project's when switched on, otherwise the scene's). Off by default, so existing projects load videos as before. Works for any model whose pipeline takes reference videos (MiniMax H3 today). Tested on made-up frames and an encoded file, not yet on a rental.
 - **Attention explorer** (needs a Taste key): each run tries a small attention change (sharper, as is or looser) per depth group of blocks, and your ratings decide which changes stay; how far a change goes learns from ratings too (or type your own). The Log names the exact blocks made sharper or looser. Also learns, per run: how strongly each group attends to your prompt ("Prompt focus"), and the clip's four time bands and its sound separately ("Clip regions"). A rating counts more for runs of a similar prompt, and a combination you liked on this prompt is reused (with some groups re-drawn). Zero extra model passes. Untested on a real H3.
 - **Noise merging** (Settings > Engine > Sampling): "Merge" can also run a whole second video beside yours from the second seed ("Second run": one extra pass at every step up to the last one named, blended at the steps you name), or mix the run's starting noise with a second seed's (no extra cost, variance kept). Or, at the steps you name, the step is answered a second time on the same picture with different noise (a second seed), and the two pictures are blended at your strength. Merge order picks which one is the base. The second seed is random each run unless you type one. Picture and sound are both merged (the sound can be left out with "Merge sound too"). "Second pass new noise" sets how much of the step's own noise the second pass swaps for the second seed's (1 all, lower keeps more of your own). Costs one extra model pass per merged step. Untested on a real H3.
 - **SLA rising sparsity** (diffusion model loader, `sla_sparsity_start`): skip fewer attention blocks on early steps and more on late ones, in a straight line up to `sla_sparsity`. Off at 0. One real H3 render (4 steps, 0.70 to 0.90): quality held, perhaps slightly better. Idea from GSRS (arXiv 2610.21129); only the schedule is taken, not its other parts.

app/ui/features/generate/inputs.js

@@ -3,6 +3,7 @@
 import { sceneReferences, wireReferences, routeFor } from "../../shell/reference_wiring.js";
 import { kinds, sounds } from "../../shell/bin.js";
 import { DRIVEN } from "../../shell/scenes.js";
+import { activeVideoInput, videoInputJson } from "../../shell/video_input.js";
 
 export const rolesAt = (slots, at) => (slots || []).flatMap((s) => (s.roles || []).filter((r) => r.at === at).map((role) => ({ slot: s, role })));
 
@@ -48,6 +49,9 @@ export async function buildInputs({ project, scene, slots, expand, nodeInfo, fra
   };
   const { overrides, unwired, left } = wireReferences(plan.ids, slots, source ? [source.slot.id] : [], plan.pins, (id) => kinds.get(id), routeOf, (id) => sounds.has(id));
   for (const [id, fields] of Object.entries(overrides)) raw[id] = { ...raw[id], ...fields };
+  // A reference video is got ready as the project (or the scene) says; with nothing switched on it loads as before.
+  const prepare = activeVideoInput(project, scene);
+  if (prepare) for (const [id, fields] of Object.entries(overrides)) if (fields.full && fields.media_id && kinds.get(fields.media_id) === "video") raw[id].prepare = videoInputJson(prepare, project, scene);
   // A picked start picture that no slot takes would otherwise be ignored without a word.
   const ignored = !source && scene.source_image && project.generation_mode !== "t2v";
   if (ignored) left.push({ id: scene.source_image, at: "assets.source_image", why: "this pipeline starts from no picture" });

app/ui/features/generate/inputs.test.js

@@ -123,3 +123,23 @@ test("a video reference reaches the model's video input from a preset that says
   assert.equal(r.inputs.ref1.full, true);
   learn([]);
 });
+
+test("a reference video is prepared as the project says, else as the scene says, and untouched when nothing is on", async () => {
+  const { learn } = await import("../../shell/bin.js");
+  learn([{ id: "vid", name: "v.mp4", kind: "video" }]);
+  const s = [{ id: "ref1", node: "Loader", roles: [{ at: "assets.reference_1", input: "media_id", wireTo: { slot: "r2v", input: "ref_images.ref_image_0" } }] }, { id: "r2v", node: "Model", inputs: {} }];
+  const info = { Model: { inputs: { "ref_videos.ref_video_0": "IMAGE" }, outputs: [] }, Loader: { inputs: {}, outputs: ["IMAGE", "IMAGE", "AUDIO"] } };
+  const run = async (project, scene) => (await buildInputs({ project: { video: {}, frame_rate: 24, ...project }, scene: { text: "x", references: ["vid"], ...scene }, slots: s, expand: async () => null, nodeInfo: async (c) => info[c] })).inputs.ref1;
+  assert.equal((await run({}, {})).prepare, undefined);
+  const scene = { video_input: { on: true, skip_first: 5, every_nth: 2 } };
+  const sc = JSON.parse((await run({}, scene)).prepare);
+  assert.equal(sc.skip_first, 5);
+  assert.equal(sc.every_nth, 2);
+  assert.equal(sc.fps, 24);          // "the project's FPS"
+  const both = { video_input: { on: true, skip_first: 9, fps_from: "source" } };
+  assert.equal(JSON.parse((await run(both, scene)).prepare).skip_first, 9);          // auto: the project's wins when on
+  assert.equal(JSON.parse((await run(both, scene)).prepare).fps, 0);
+  assert.equal(JSON.parse((await run({ ...both, video_input_from: "scene" }, scene)).prepare).skip_first, 5);
+  assert.equal((await run({ video_input_from: "project", video_input: { on: false } }, scene)).prepare, undefined);
+  learn([]);
+});

app/ui/features/inspector/hub.test.js

@@ -47,3 +47,24 @@ test("typing in the Prompt box, then pressing a button in the panel, does not re
   assert.ok(button.isConnected, "the pressed button is still there for its mouse-up");
   host.remove();
 });
+
+test("the scene's Video input card switches on, shows its controls, and says when the project's settings win", async () => {
+  let fire;
+  const calls = [];
+  const doc = { id: "P", frame_rate: 25, num_frames_per_scene: 49, scenes: [{ id: "a", text: "x" }], scene_renders: {} };
+  const p = { project: doc, get scenes() { return doc.scenes; }, get selected() { return doc.scenes[0]; }, selectedId: "a",
+    setText: () => {}, setScene: (id, key, value) => { calls.push([key, value]); doc.scenes[0][key] = value; fire("change"); }, edit: (fn) => fn(doc) };
+  const host = document.createElement("div");
+  document.body.append(host);
+  hub.setup({ host, app: { dom: createDom(), project: p, on: (fn) => { fire = fn; return () => {}; }, api: {}, say: () => {} } });
+  assert.ok(host.textContent.includes("Video input"));
+  assert.ok(!host.textContent.includes("Every Nth frame"));
+  const sw = [...host.querySelectorAll("input[role=switch]")].find((i) => i.closest(".cx-toggle-row").textContent.includes("Prepare video inputs"));
+  sw.checked = true; sw.dispatchEvent(new window.Event("change", { bubbles: true }));
+  assert.equal(calls[0][0], "video_input");
+  assert.equal(calls[0][1].on, true);
+  assert.ok(host.textContent.includes("Every Nth frame") && host.textContent.includes("Skip first") && host.textContent.includes("Cut last"));
+  doc.video_input = { on: true }; fire("change");
+  assert.ok(host.textContent.includes("project's video input settings are used"));
+  host.remove();
+});

app/ui/features/inspector/index.js

@@ -35,7 +35,7 @@ export default {
     function draw(force) {      // not on every keystroke: that would rebuild the box being typed in
       const sc = p.selected, open = p.project, slots = app.pipeline && (app.pipeline.view || app.pipeline.slots)();
       learnGrid(app.api, slots).then((learned) => { if (learned) draw(true); });      // the length arrows follow the model's frame grid once its node is described
-      const next = `${tab}|${open && open.id}|${p.selectedId}|${JSON.stringify(gridOf(slots))}|${Object.keys((app.pipeline && app.pipeline.skipped && app.pipeline.skipped()) || {}).join()}|${tab === "project" && open ? [open.num_frames_per_scene, open.frame_rate, JSON.stringify(open.video || {}), open.postfix_enabled !== false, open.generation_mode, open.export_size_from, open.width, open.height, open.scenes.map((s) => s.frames_mode === "custom"), Object.keys(open.scene_renders || {}), open.scenes.length] : ""}|${tab === "scene" && sc && open ? [effFrames(sc, open), effFps(sc, open), p.scenes.indexOf(sc), sc.frames_mode, sc.fps_mode, p.scenes.length, sc.source_image, (sc.references || []).join(), JSON.stringify(open.fixed_references || {}), sc.source_in, sc.source_dur, sc.removed_from_plan, (sc.source || {}).type, sc.excluded, typing() ? shownText : (shownText = sc.text), ((open.scene_renders || {})[sc.id] || {}).durationSec, Boolean(((open.scene_renders || {})[sc.id] || {}).media), bin.version, ...sections.map((s) => { try { return s.key(sc, open); } catch { return ""; } })] : ""}`;
+      const next = `${tab}|${open && open.id}|${p.selectedId}|${JSON.stringify(gridOf(slots))}|${Object.keys((app.pipeline && app.pipeline.skipped && app.pipeline.skipped()) || {}).join()}|${tab === "project" && open ? [open.num_frames_per_scene, open.frame_rate, JSON.stringify(open.video || {}), JSON.stringify(open.video_input || {}), open.video_input_from, open.postfix_enabled !== false, open.generation_mode, open.export_size_from, open.width, open.height, open.scenes.map((s) => s.frames_mode === "custom"), Object.keys(open.scene_renders || {}), open.scenes.length] : ""}|${tab === "scene" && sc && open ? [effFrames(sc, open), effFps(sc, open), p.scenes.indexOf(sc), sc.frames_mode, sc.fps_mode, p.scenes.length, sc.source_image, (sc.references || []).join(), JSON.stringify(open.fixed_references || {}), sc.source_in, sc.source_dur, JSON.stringify(sc.video_input || {}), JSON.stringify(open.video_input || {}), open.video_input_from, sc.removed_from_plan, (sc.source || {}).type, sc.excluded, typing() ? shownText : (shownText = sc.text), ((open.scene_renders || {})[sc.id] || {}).durationSec, Boolean(((open.scene_renders || {})[sc.id] || {}).media), bin.version, ...sections.map((s) => { try { return s.key(sc, open); } catch { return ""; } })] : ""}`;
       titled();
       if (!force && next === key) return;
       key = next;

app/ui/features/inspector/project.js

@@ -3,6 +3,7 @@ import { composer as c } from "../../composer/composer.js";
 import { DRIVEN, isVideoClip, segments, snapFrames } from "../../shell/scenes.js";
 import { gridOf } from "../../shell/frame_grid.js";
 import { card } from "./scene.js";
+import { projectVideoInput } from "./videoinput.js";
 
 const STARTS = [{ value: "i2v", label: "From an image" }, { value: "t2v", label: "From a prompt" }];
 
@@ -73,6 +74,7 @@ export function projectRows(p, slots, api) {
       c.field.default({ label: "Final render size", hint: "The size of the exported video: the first clip's, the project's, or any rendered clip's own.", control: c.select.md({ label: "Final render size", options: sizes(open), value: open.export_size_from || "",
         onChange: (v) => p.setField("export_size_from", v) }) }),
       c.field.default({ label: "Start shots", hint: "From an image: a scene uses its picture, or the last frame of the clip before it. From a prompt: every scene starts without a picture. (MiniMax H3 takes only the picture's shape.)", control: c.select.md({ label: "Start shots", options: STARTS, value: open.generation_mode === "t2v" ? "t2v" : "i2v", onChange: (v) => p.setField("generation_mode", v) }) })),
+    ...projectVideoInput(p),
     c.label.section({ text: "Prompt" }),
     card(c.field.default({ label: "Anchor", hint: "Put in front of every scene's prompt.", control: c.textarea.md({ label: "Anchor", value: p.anchor, rows: 2, onInput: (v) => p.setAnchor(v) }) }),
       c.field.default({ label: "Negative prompt", hint: "What to steer away from. A model that ignores negative prompts (MiniMax H3) uses it only with Shadow negative or Negative erase on.", control: c.textarea.md({ label: "Negative prompt", value: p.negative, rows: 2, onInput: (v) => p.setNegative(v) }) }),

app/ui/features/inspector/scene.js

@@ -4,6 +4,7 @@ import { namesOf, pick } from "./mediapick.js";
 import { effFrames, effFps, genUnitId, isSubclip, seconds, snapFrames } from "../../shell/scenes.js";
 import { gridOf } from "../../shell/frame_grid.js";
 import { restoreToPlan } from "../../shell/edits.js";
+import { sceneVideoInput } from "./videoinput.js";
 
 const SOURCES = [{ value: "image", label: "Image · i2v anchor" }, { value: "carry", label: "From generated frame" }, { value: "video", label: "Video clip" }];
 const MODES = [{ value: "project", label: "Project default" }, { value: "timeline", label: "Timeline trim" }, { value: "custom", label: "Custom" }];
@@ -63,6 +64,7 @@ export function sceneRows(p, app) {
     card(mediaField(app, p, sc, { label: "Aspect ratio source", hint: "The scene's picture (or a video's first frame). Its shape sets this scene's aspect ratio (the project's Width/Height set the scale). A model that starts from a picture, such as LTX image-to-video, also begins the clip from it; the prompt enhancer can read it.",
       key: "source_image", multiple: false, kinds: ["image", "video"], empty: "— choose aspect ratio source —" }),
     mediaField(app, p, sc, { label: "References", key: "references", multiple: true, kinds: ["image", "video", "audio"], empty: "+ Add reference" })),
+    ...sceneVideoInput(p, sc),
     c.label.section({ text: "Length" }),
     card(c.field.row({ fields: [
       lengthField(p, sc, "Frames", "frames_mode", "frames", effFrames(sc, open), gridOf(app.pipeline && (app.pipeline.view || app.pipeline.slots)())),

app/ui/features/inspector/videoinput.js

@@ -0,0 +1,52 @@
+// The "Video input" card: how a picked reference video is got ready (size, crop, rate, skip, cut, every Nth, cap).
+// One card for the project and one per scene; the project's dropdown says whose is used.
+import { composer as c } from "../../composer/composer.js";
+import { card } from "./scene.js";
+import { BLANK, FITS, FROM, RATES, activeVideoInput } from "../../shell/video_input.js";
+
+/** `use`: the settings shown; `save(next)` stores them. `size`: the project's width/height, offered as a one-tap target. */
+function settingsCard(use, save, size) {
+  const set = (key) => (v) => save({ ...use, [key]: v });
+  const n = (label, key, { min = 0, max = 100000, hint, step = 1 } = {}) => c.field.default({ label, hint, control: c.number.md({ label, min, max, step, precision: 0, value: use[key], onChange: set(key) }) });
+  return card(
+    c.toggle.default({ label: "Prepare video inputs", hint: "Size, crop, frame rate and which frames a picked reference video gives the model. Off: the video loads whole at 24 fps.", checked: use.on, onChange: set("on") }),
+    ...(use.on ? [
+      c.label.section({ text: "Frames" }),
+      c.field.row({ fields: [n("Skip first", "skip_first", { hint: "Frames dropped from the start." }), n("Cut last", "cut_last", { hint: "Frames dropped from the end." })] }),
+      c.field.row({ fields: [n("Every Nth frame", "every_nth", { min: 1, max: 1000, hint: "1 keeps every frame." }), n("Frame cap", "frame_cap", { hint: "0 = no limit." })] }),
+      c.field.row({ fields: [c.field.default({ label: "Frame rate", control: c.select.md({ label: "Frame rate", options: RATES, value: use.fps_from, onChange: set("fps_from") }) }),
+        use.fps_from === "custom" ? n("FPS", "fps", { min: 1, max: 1000 }) : null].filter(Boolean) }),
+      c.label.section({ text: "Picture" }),
+      c.field.row({ fields: [n("Width", "width", { max: 16384, hint: "0 keeps the video's own." }), n("Height", "height", { max: 16384, hint: "0 keeps the video's own." })] }),
+      size ? c.button.sm({ label: `Use the project's size (${size[0]}×${size[1]})`, tone: "ghost", onClick: () => save({ ...use, width: size[0], height: size[1] }) }) : null,
+      c.field.row({ fields: [c.field.default({ label: "When the shape differs", control: c.select.md({ label: "When the shape differs", options: FITS, value: use.fit, onChange: set("fit") }) }),
+        n("Round to a multiple of", "multiple_of", { min: 1, max: 256, hint: "Most models want 32." })] }),
+      c.label.section({ text: "Crop the edges first (%)" }),
+      c.field.row({ fields: [n("Left", "crop_left", { max: 90 }), n("Right", "crop_right", { max: 90 })] }),
+      c.field.row({ fields: [n("Top", "crop_top", { max: 90 }), n("Bottom", "crop_bottom", { max: 90 })] }),
+    ] : []));
+}
+
+export function projectVideoInput(p) {
+  const open = p.project, use = { ...BLANK, ...(open.video_input || {}) };
+  const v = open.video || {}, size = [v.width || v.target_width || open.width, v.height || v.target_height || open.height];
+  return [
+    c.label.section({ text: "Video input" }),
+    card(c.field.default({ label: "Reference videos use", hint: "Where the preparation comes from. Automatic: the project's when it is on, otherwise each scene's own.",
+      control: c.select.md({ label: "Reference videos use", options: FROM, value: open.video_input_from || "auto", onChange: (x) => p.setField("video_input_from", x) }) })),
+    settingsCard(use, (next) => p.setField("video_input", next), size[0] && size[1] ? size : null),
+  ];
+}
+
+export function sceneVideoInput(p, sc) {
+  const open = p.project, use = { ...BLANK, ...(sc.video_input || {}) }, live = activeVideoInput(open, sc);
+  const from = open.video_input_from || "auto";
+  return [
+    c.label.section({ text: "Video input" }),
+    from === "project" || (from === "auto" && (open.video_input || {}).on)
+      ? c.hint.default({ text: "The project's video input settings are used; this scene's own are ignored. Change that in Project settings." })
+      : null,
+    settingsCard(use, (next) => p.setScene(sc.id, "video_input", next), null),
+    live ? null : c.hint.default({ text: "Switched off: reference videos load whole at 24 fps." }),
+  ].filter(Boolean);
+}

app/ui/shell/video_input.js

@@ -0,0 +1,24 @@
+// How a picked reference video is got ready for the model (the loader choices: size, crop, rate, skip, cut, every Nth, cap).
+// The project keeps one set and every scene its own; `video_input_from` says whose is used.
+import { effFps } from "./scenes.js";
+
+export const FROM = [{ value: "auto", label: "Automatic (the project's when on, else the scene's)" }, { value: "project", label: "The project's" }, { value: "scene", label: "Each scene's own" }];
+export const FITS = [{ value: "crop", label: "Fill (crop the overflow)" }, { value: "pad", label: "Fit (black bars)" }, { value: "stretch", label: "Stretch" }];
+export const RATES = [{ value: "project", label: "The project's FPS" }, { value: "source", label: "The video's own" }, { value: "custom", label: "A number" }];
+
+export const BLANK = { on: false, fit: "crop", fps_from: "project", fps: 24, skip_first: 0, cut_last: 0, every_nth: 1, frame_cap: 0, width: 0, height: 0, multiple_of: 32, crop_left: 0, crop_right: 0, crop_top: 0, crop_bottom: 0 };
+
+/** The settings that apply to this scene, or null when none is switched on (the video then loads as it always did). */
+export function activeVideoInput(project, scene) {
+  const own = { ...BLANK, ...((project || {}).video_input || {}) }, mine = { ...BLANK, ...((scene || {}).video_input || {}) };
+  const from = (project || {}).video_input_from || "auto";
+  const use = from === "project" ? own : from === "scene" ? mine : own.on ? own : mine;
+  return use.on ? use : null;
+}
+
+/** The JSON the loader node takes. */
+export function videoInputJson(use, project, scene) {
+  const fps = use.fps_from === "source" ? 0 : use.fps_from === "custom" ? use.fps : effFps(scene, project);
+  const { on, fps_from, ...rest } = use;
+  return JSON.stringify({ ...rest, fps });
+}

core/projects.py

@@ -146,6 +146,23 @@ def _clean_source(raw) -> dict:
     return out
 
 
+#: How a picked reference video is got ready for the model (the loader choices): per project and per scene.
+#: `on` switches it; every number is a whole count or percent, `fit` is a word. Anything else is dropped.
+VIDEO_INPUT_FITS = ("crop", "pad", "stretch")
+VIDEO_INPUT_LIMITS = {"fps": (0, 1000), "skip_first": (0, 10 ** 6), "cut_last": (0, 10 ** 6), "every_nth": (1, 1000),
+                      "frame_cap": (0, 10 ** 6), "width": (0, 16384), "height": (0, 16384), "multiple_of": (1, 256),
+                      "crop_left": (0, 90), "crop_right": (0, 90), "crop_top": (0, 90), "crop_bottom": (0, 90)}
+
+
+def _clean_video_input(raw) -> dict:
+    d = raw if isinstance(raw, dict) else {}
+    out = {"on": _bool(d.get("on")), "fit": d.get("fit") if d.get("fit") in VIDEO_INPUT_FITS else "crop",
+           "fps_from": d.get("fps_from") if d.get("fps_from") in ("project", "source", "custom") else "project"}
+    for key, (lo, hi) in VIDEO_INPUT_LIMITS.items():
+        out[key] = _int(d.get(key), lo, lo, hi)
+    return out
+
+
 #: What a person can say about a result: it was good, or it was not.
 #:
 #: Binary because that is all anything downstream reads. dev proved it on H3 --
@@ -204,6 +221,8 @@ class Scene:
     height: int | None = None
     #: How the scene's latent is born (carry / image / video clip / ...).
     source: dict = field(default_factory=lambda: _clean_source(None))
+    #: How this scene's reference videos are prepared; used when the project says "scene" (or "auto" and its own is off).
+    video_input: dict = field(default_factory=lambda: _clean_video_input(None))
     excluded: bool = False
     #: Timeline cuts of one generated clip share the root scene's id; only the root
     #: (cut_offset_frames == 0) owns prompt, rating and source.
@@ -265,6 +284,7 @@ class Scene:
             width=_int(d.get("width"), None, 1, MAX_SETTING),
             height=_int(d.get("height"), None, 1, MAX_SETTING),
             source=_clean_source(d.get("source")),
+            video_input=_clean_video_input(d.get("video_input")),
             excluded=_bool(d.get("excluded")),
             gen_unit_id=guide if is_token(guide) else None,
             cut_offset_frames=_int(d.get("cut_offset_frames"), 0, 0, 10 ** 7),
@@ -438,6 +458,10 @@ class Project:
     continuity_settings: dict = field(default_factory=dict)
     #: Media ids pinned to a slot for every scene: { media id: slot number 1-16 }.
     fixed_references: dict = field(default_factory=dict)
+    #: Reference-video preparation for the whole project, and whose wins: "auto" = the project's when it is on,
+    #: otherwise the scene's; "project" / "scene" = that one only.
+    video_input: dict = field(default_factory=lambda: _clean_video_input(None))
+    video_input_from: str = "auto"
     generation_meta: dict = field(default_factory=dict)
     #: What each scene last rendered ({media, inSec, promptId}) and the removed scenes whose
     #: clip still previews: a reload shows the timeline the person left.
@@ -501,6 +525,8 @@ class Project:
             guide_settings=_blob(d.get("guide_settings"), dict),
             continuity_settings=_blob(d.get("continuity_settings"), dict),
             fixed_references=_pins(d.get("fixed_references")),
+            video_input=_clean_video_input(d.get("video_input")),
+            video_input_from=d.get("video_input_from") if d.get("video_input_from") in ("auto", "project", "scene") else "auto",
             generation_meta=_blob(d.get("generation_meta"), dict),
             scene_renders=_blob(d.get("scene_renders"), dict),
             scene_variants=_blob(d.get("scene_variants"), dict),

core/tests/test_projects.py

@@ -603,3 +603,15 @@ def test_a_template_never_carries_variables_and_an_old_one_loses_them():
     from core import projects
     kept = projects._clean_templates([{"name": "old", "scenes": ["a"], "variables": [{"name": "v", "value": "1"}]}])
     assert kept == [{"name": "old", "scenes": ["a"]}]
+
+
+def test_video_input_is_cleaned_and_survives_a_round_trip():
+    from core.projects import Project
+    p = Project.from_dict({"video_input": {"on": True, "skip_first": 7, "fit": "pad", "crop_left": 500, "every_nth": 0, "bogus": 1},
+                           "video_input_from": "scene", "scenes": [{"video_input": {"on": "yes", "fit": "weird"}}]})
+    assert p.video_input["on"] is True and p.video_input["skip_first"] == 7 and p.video_input["fit"] == "pad"
+    assert p.video_input["crop_left"] == 0 and p.video_input["every_nth"] == 1 and "bogus" not in p.video_input
+    assert p.scenes[0].video_input["on"] is False and p.scenes[0].video_input["fit"] == "crop"
+    again = Project.from_dict(p.to_dict())
+    assert again.video_input == p.video_input and again.video_input_from == "scene"
+    assert Project.from_dict({"video_input_from": "x"}).video_input_from == "auto"

core/tests/test_shipped_nodes.py

@@ -87,7 +87,7 @@ FROZEN = {
         "model", "lora_name", "strength_model", "clip", "strength_clip",
     ],
     "FunUIVAELoader": ["vae_name", "dtype"],
-    "FunUILoadMedia": ["media_id", "blank_width", "blank_height", "full"],
+    "FunUILoadMedia": ["media_id", "blank_width", "blank_height", "full", "prepare"],
 }
 
 

modules/media/load/nodes.py

@@ -10,10 +10,13 @@ a run queues, the same way a scene's prompt text lands on CLIPTextEncode's
 `text`), not from someone typing a file name into this node by hand.
 """
 
+import json
+
 import torch
 from comfy_api.latest import InputImpl, io
 
 from ..._core import log, media as media_mod
+from . import prepare as prep
 
 
 #: Frame rate the H3 reference-video input takes.
@@ -68,13 +71,18 @@ class FunUILoadMedia(io.ComfyNode):
                 io.Boolean.Input(
                     "full", default=False, optional=True,
                     tooltip="Read the whole video and its sound. Off: only the first frame, which is all a picture input needs."),
+                io.String.Input(
+                    "prepare", default="", optional=True,
+                    tooltip="With full on: how to get the video ready, as JSON. Keys: fps, skip_first, cut_last, every_nth, frame_cap, "
+                            "width, height, fit (crop|pad|stretch), crop_left/right/top/bottom (percent), multiple_of. "
+                            "Empty: all frames at 24 fps, size untouched."),
             ],
             outputs=[io.Image.Output(display_name="image"), io.Image.Output(display_name="frames"),
                      io.Audio.Output(display_name="audio")],
         )
 
     @classmethod
-    def execute(cls, media_id: str, blank_width: int = 0, blank_height: int = 0, full: bool = False) -> io.NodeOutput:
+    def execute(cls, media_id: str, blank_width: int = 0, blank_height: int = 0, full: bool = False, prepare: str = "") -> io.NodeOutput:
         mid = (media_id or "").strip()
         if not mid and blank_width > 0 and blank_height > 0:
             # A text-only start: the picture exists only to give the pipeline a SHAPE (see the H3 preset).
@@ -119,6 +127,15 @@ class FunUILoadMedia(io.ComfyNode):
             return io.NodeOutput(first, first, _SILENCE)
 
         fps = float(components.frame_rate or REF_VIDEO_FPS)
+        if (prepare or "").strip():
+            try:
+                options = json.loads(prepare)
+                frames, out_fps, start, span = prep.prepare(images, fps, options if isinstance(options, dict) else {})
+            except (ValueError, TypeError) as exc:           # bad JSON, or a cut that leaves nothing
+                raise RuntimeError(f"FunUI Load Media: {name!r} cannot be prepared ({exc}).") from exc
+            sound = prep.cut_audio(components.audio, start, span) if components.audio else _SILENCE
+            log.info("FunUI Load Media", f"loaded {name}: {frames.shape[0]} frames {frames.shape[2]}x{frames.shape[1]} at {out_fps:g} fps")
+            return io.NodeOutput(frames[:1], frames, sound)
         # Whatever the source rate, the model wants 24 fps: pick the frame showing at each 1/24 s.
         count = max(1, int(images.shape[0] / fps * REF_VIDEO_FPS))
         index = torch.clamp((torch.arange(count) * fps / REF_VIDEO_FPS).long(), max=images.shape[0] - 1)

modules/media/load/prepare.py

@@ -0,0 +1,112 @@
+"""Getting a loaded video ready for a model: the choices ComfyUI's video loaders (VHS Load Video and the like) offer.
+
+Pure tensor functions with no node or file in them, so they are tested on made-up frames. The order is the
+loaders': pick the frames at the target rate, drop the first and last ones, take every Nth, cap the count;
+and in space: crop the margins, fit to the target size, round to a multiple the model wants.
+"""
+
+import torch
+import torch.nn.functional as F
+
+FITS = ("crop", "pad", "stretch")
+
+#: The keys a preparation understands, with what each does when left at its default.
+DEFAULTS = {
+    "fps": 0,            # target frame rate; 0 keeps the video's own
+    "skip_first": 0,     # frames dropped from the start (at the target rate)
+    "cut_last": 0,       # frames dropped from the end
+    "every_nth": 1,      # keep one frame in N
+    "frame_cap": 0,      # at most this many frames; 0 = all
+    "width": 0,          # target size; 0 keeps the (cropped) video's own
+    "height": 0,
+    "fit": "crop",       # how a different shape meets the target: crop the overflow, pad the gap, or stretch
+    "crop_left": 0,      # percent of the width/height cut off each side before fitting
+    "crop_right": 0,
+    "crop_top": 0,
+    "crop_bottom": 0,
+    "multiple_of": 1,    # round the final size down to a multiple of this (models want 16 / 32)
+}
+
+
+def pick_frames(count, src_fps, fps=0, skip_first=0, cut_last=0, every_nth=1, frame_cap=0):
+    """Indices into the source frames, and the frame rate they play at.
+
+    -> (indices list, out_fps, start_sec, span_sec): the last two say which stretch of the soundtrack goes with them.
+    """
+    src_fps = float(src_fps) if src_fps and src_fps > 0 else 24.0
+    out_fps = float(fps) if fps and fps > 0 else src_fps
+    n = max(1, int(count / src_fps * out_fps + 1e-6)) if out_fps != src_fps else count
+    index = [min(count - 1, int(i * src_fps / out_fps + 1e-6)) for i in range(n)] if out_fps != src_fps else list(range(count))
+    lo, hi = max(0, int(skip_first)), len(index) - max(0, int(cut_last))
+    kept = index[lo:max(lo, hi)]
+    if not kept:
+        raise ValueError(f"nothing is left: the video has {len(index)} frames at {out_fps:g} fps and {lo} are skipped and {max(0, int(cut_last))} cut")
+    start, span = lo / out_fps, len(kept) / out_fps
+    nth = max(1, int(every_nth))
+    kept = kept[::nth]
+    if frame_cap and frame_cap > 0:
+        kept = kept[:int(frame_cap)]
+        span = min(span, len(kept) * nth / out_fps)
+    return kept, out_fps / nth, start, span
+
+
+def _round(value, multiple):
+    return max(multiple, int(value) // multiple * multiple) if multiple > 1 else max(1, int(value))
+
+
+def shape(images, width=0, height=0, fit="crop", crop_left=0, crop_right=0, crop_top=0, crop_bottom=0, multiple_of=1):
+    """Crop the margins, fit to the target size, round to the multiple. images: [N, H, W, C] floats 0..1."""
+    n, h, w, c = images.shape
+    left, right = int(w * min(max(crop_left, 0), 90) / 100), int(w * min(max(crop_right, 0), 90) / 100)
+    top, bottom = int(h * min(max(crop_top, 0), 90) / 100), int(h * min(max(crop_bottom, 0), 90) / 100)
+    if left + right >= w or top + bottom >= h:
+        raise ValueError("the crop leaves no picture")
+    if left or right or top or bottom:
+        images = images[:, top:h - bottom, left:w - right]
+        h, w = images.shape[1:3]
+    tw, th = (width or w), (height or h)
+    if bool(width) != bool(height):          # one side given: the other follows the shape
+        tw, th = (width, round(width * h / w)) if width else (round(height * w / h), height)
+    multiple = max(1, int(multiple_of))
+    tw, th = _round(tw, multiple), _round(th, multiple)
+    if (tw, th) == (w, h):
+        return images
+    if fit not in FITS:
+        fit = "crop"
+
+    def resize(x, size):
+        t = x.permute(0, 3, 1, 2)
+        t = F.interpolate(t, size=size, mode="bilinear", align_corners=False, antialias=size[0] < t.shape[2] or size[1] < t.shape[3])
+        return t.permute(0, 2, 3, 1).clamp(0, 1)
+
+    if fit == "stretch":
+        return resize(images, (th, tw))
+    scale = max(tw / w, th / h) if fit == "crop" else min(tw / w, th / h)
+    sh, sw = max(1, round(h * scale)), max(1, round(w * scale))
+    scaled = resize(images, (sh, sw))
+    if fit == "crop":
+        y, x = (sh - th) // 2, (sw - tw) // 2
+        return scaled[:, y:y + th, x:x + tw]
+    out = torch.zeros((n, th, tw, c), dtype=scaled.dtype)
+    y, x = (th - sh) // 2, (tw - sw) // 2
+    out[:, y:y + sh, x:x + sw] = scaled
+    return out
+
+
+def prepare(images, src_fps, options=None):
+    """The whole job. -> (frames, out_fps, start_sec, span_sec)."""
+    o = {**DEFAULTS, **{k: v for k, v in (options or {}).items() if k in DEFAULTS}}
+    index, out_fps, start, span = pick_frames(images.shape[0], src_fps, o["fps"], o["skip_first"], o["cut_last"], o["every_nth"], o["frame_cap"])
+    frames = shape(images[torch.tensor(index)], o["width"], o["height"], o["fit"], o["crop_left"], o["crop_right"],
+                   o["crop_top"], o["crop_bottom"], o["multiple_of"])
+    return frames, out_fps, start, span
+
+
+def cut_audio(audio, start, span):
+    """The soundtrack's matching stretch (audio dict with waveform [1, C, T] and sample_rate)."""
+    rate = audio["sample_rate"]
+    wave = audio["waveform"]
+    if wave.shape[-1] <= 1:
+        return audio
+    a = min(wave.shape[-1] - 1, int(start * rate))
+    return {"waveform": wave[..., a:a + max(1, int(span * rate))], "sample_rate": rate}

modules/media/load/tests/test_prepare.py

@@ -0,0 +1,103 @@
+"""Video preparation: the loader choices (rate, skip, cut, every Nth, cap, crop, size), on made-up frames and a real file."""
+
+import json
+
+import pytest
+import torch
+
+from modules.media.load import prepare as prep
+
+
+def _clip(n=48, h=40, w=80):
+    """Frame i is filled with the value i/255, so which frames survived can be read off the pixels."""
+    return torch.stack([torch.full((h, w, 3), i / 255.0) for i in range(n)])
+
+
+def _ids(frames):
+    return [round(float(f[0, 0, 0]) * 255) for f in frames]
+
+
+def test_frames_are_picked_in_loader_order():
+    frames, fps, start, span = prep.prepare(_clip(), 24, {"skip_first": 4, "cut_last": 4, "every_nth": 2, "frame_cap": 10})
+    assert _ids(frames) == [4, 6, 8, 10, 12, 14, 16, 18, 20, 22]
+    assert fps == 12 and start == pytest.approx(4 / 24) and span == pytest.approx(20 / 24)
+
+
+def test_target_rate_picks_the_frame_showing_at_each_tick():
+    frames, fps, *_ = prep.prepare(_clip(48), 48, {"fps": 24})
+    assert fps == 24 and _ids(frames) == list(range(0, 48, 2))
+
+
+def test_nothing_left_is_refused():
+    with pytest.raises(ValueError, match="nothing is left"):
+        prep.prepare(_clip(10), 24, {"skip_first": 6, "cut_last": 4})
+
+
+def test_crop_fit_fills_the_target_and_loses_the_overflow():
+    out = prep.shape(_clip(2, 40, 80), 32, 32, "crop")
+    assert out.shape == (2, 32, 32, 3)
+
+
+def test_pad_fit_keeps_everything_with_black_bars():
+    out = prep.shape(_clip(1, 40, 80) + 0.5, 64, 64, "pad")
+    assert out.shape == (1, 64, 64, 3)
+    assert float(out[0, 0, 0, 0]) == 0.0 and float(out[0, 32, 32, 0]) > 0.4
+
+
+def test_stretch_and_one_sided_target_and_multiple():
+    assert prep.shape(_clip(1, 40, 80), 64, 64, "stretch").shape[1:3] == (64, 64)
+    assert prep.shape(_clip(1, 40, 80), 64, 0).shape[1:3] == (32, 64)          # height follows the shape
+    assert prep.shape(_clip(1, 50, 90), 0, 0, multiple_of=32).shape[1:3] == (32, 64)
+
+
+def test_margin_crop_and_a_crop_that_removes_everything():
+    assert prep.shape(_clip(1, 40, 80), crop_left=25, crop_right=25).shape[1:3] == (40, 40)
+    with pytest.raises(ValueError, match="no picture"):
+        prep.shape(_clip(1, 4, 4), crop_left=50, crop_right=50)
+
+
+def test_sound_follows_the_picked_stretch():
+    wave = torch.arange(48000, dtype=torch.float32).reshape(1, 1, -1)
+    cut = prep.cut_audio({"waveform": wave, "sample_rate": 48000}, 0.25, 0.5)
+    assert cut["waveform"].shape[-1] == 24000 and float(cut["waveform"][0, 0, 0]) == 12000
+
+
+# --- the node, on a real encoded file --------------------------------------------------------
+
+@pytest.fixture
+def stored(comfyui, tmp_path, monkeypatch):
+    from core import config, media
+    monkeypatch.setattr(config, "MEDIA_DIR", tmp_path / "media")
+    import av
+    path = tmp_path / "clip.mp4"
+    with av.open(str(path), "w") as out:
+        stream = out.add_stream("libx264", rate=24)
+        stream.width, stream.height, stream.pix_fmt = 64, 32, "yuv420p"
+        for i in range(48):
+            frame = av.VideoFrame.from_ndarray(torch.full((32, 64, 3), i * 5, dtype=torch.uint8).numpy(), format="rgb24")
+            for packet in stream.encode(frame):
+                out.mux(packet)
+        for packet in stream.encode():
+            out.mux(packet)
+    return media.save_upload("clip.mp4", path.read_bytes())
+
+
+def test_the_node_prepares_a_real_video(stored):
+    from modules.media.load.nodes import FunUILoadMedia
+    options = {"skip_first": 8, "cut_last": 8, "every_nth": 2, "width": 32, "height": 32, "fit": "crop", "multiple_of": 16}
+    first, frames, _audio = FunUILoadMedia.execute(stored["id"], full=True, prepare=json.dumps(options)).result
+    assert frames.shape == (16, 32, 32, 3) and first.shape[0] == 1
+
+
+def test_no_options_keeps_the_old_behaviour(stored):
+    from modules.media.load.nodes import FunUILoadMedia
+    _first, frames, _audio = FunUILoadMedia.execute(stored["id"], full=True).result
+    assert frames.shape[0] == 48 and frames.shape[1:3] == (32, 64)
+
+
+def test_a_cut_that_leaves_nothing_is_said_in_plain_words(stored):
+    from modules.media.load.nodes import FunUILoadMedia
+    with pytest.raises(RuntimeError, match="cannot be prepared"):
+        FunUILoadMedia.execute(stored["id"], full=True, prepare=json.dumps({"skip_first": 40, "cut_last": 20})).result
+    with pytest.raises(RuntimeError, match="cannot be prepared"):
+        FunUILoadMedia.execute(stored["id"], full=True, prepare="{not json").result