LTX image-to-video: the first reference picture is the start when none is picked

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BzJLJWVu7RZnbRqm3nYAnX

app/ui/features/generate/inputs.js

@@ -20,12 +20,17 @@ export async function buildInputs({ project, scene, slots, expand, nodeInfo, fra
   for (const slot of (slots || []).filter((s) => s.node === "FunUIShotCamera")) put(slot, "project", project.id || "");      // what the node keeps per project (the words focus infusion draws on), also for pipelines saved before it was asked for
   const source = rolesAt(slots, "assets.source_image")[0];
   const plan = sceneReferences(scene, project.fixed_references);
-  // A pipeline that takes a whole video where it takes a picture (video-to-video): with no start picked, the first VIDEO
-  // among the references is the one it works from, and is not also sent as a reference. It is never blanked by "From a prompt".
-  const takesVideo = Boolean(source && source.role.video_frames);
-  const fromRefs = takesVideo && !scene.source_image ? plan.ids.find((id) => kinds.get(id) === "video") : null;
-  if (fromRefs) plan.ids = plan.ids.filter((id) => id !== fromRefs);
-  const startId = takesVideo ? scene.source_image || fromRefs || "" : project.generation_mode === "t2v" ? "" : scene.source_image;
+  // A pipeline that takes whole media where it takes a picture (LTX): with no start picked, the first reference is the start:
+  // a VIDEO first (it is then not also sent as a reference: nothing takes it there), else the first picture (which stays a
+  // reference too, so an identity input still gets it). A video-only pipeline (video-to-video) looks at videos alone and is
+  // never blanked by "From a prompt".
+  const takesVideo = Boolean(source && source.role.video_frames), videoOnly = Boolean(source && source.role.video_only);
+  const wanted = (id) => kinds.get(id) === "video" || (!videoOnly && kinds.get(id) === "image");
+  const blank = project.generation_mode === "t2v" && !videoOnly;
+  const fromRefs = takesVideo && !scene.source_image && !blank
+    ? plan.ids.find((id) => kinds.get(id) === "video") || (videoOnly ? null : plan.ids.find((id) => wanted(id))) : null;
+  if (fromRefs && kinds.get(fromRefs) === "video") plan.ids = plan.ids.filter((id) => id !== fromRefs);
+  const startId = takesVideo ? (blank ? "" : scene.source_image || fromRefs || "") : project.generation_mode === "t2v" ? "" : scene.source_image;
   if (source) put(source.slot, "media_id", startId || "");    // "from a prompt" starts every shot without a picture
   // A pipeline that can take a whole video where it takes a picture (LTX) says so on the role: a picked VIDEO then
   // supplies its frames (prepared as Video input says); the preset wires the frames output where the picture goes (a
@@ -80,7 +85,7 @@ export async function buildInputs({ project, scene, slots, expand, nodeInfo, fra
     if (!(body && typeof body.text === "string")) failedExpand = true;
   }
   // Features that add inputs of their own (a Chat comment for the enhancer, say) say so through a hook: { inputs: {slot: {name: value}}, notes: [text] }.
-  const noVideo = takesVideo && !startId ? ["This pipeline works from a video: pick one as the start, or add it under References."] : [];
+  const noVideo = takesVideo && videoOnly && !startId ? ["This pipeline works from a video: pick one as the start, or add it under References."] : [];
   const notes = failedExpand ? ["The prompt could not be expanded (shortcuts, $variables, anchor, postfix): the text was sent as typed."] : [];
   notes.push(...noVideo);
   for (const hook of hooks) {

app/ui/features/generate/inputs.test.js

@@ -162,7 +162,7 @@ test("a video picked as the start feeds its frames to what the picture fed, when
 test("a video-in pipeline takes the first video among the references as its start, and says so when there is none", async () => {
   const { learn } = await import("../../shell/bin.js");
   learn([{ id: "vid", name: "v.mp4", kind: "video" }, { id: "pic", name: "p.png", kind: "image" }]);
-  const slots = [{ id: "src", node: "FunUILoadMedia", roles: [{ at: "assets.source_image", input: "media_id", video_frames: true }], inputs: {} }, { id: "a", node: "X", inputs: { image: ["src", 0] } }];
+  const slots = [{ id: "src", node: "FunUILoadMedia", roles: [{ at: "assets.source_image", input: "media_id", video_frames: true, video_only: true }], inputs: {} }, { id: "a", node: "X", inputs: { image: ["src", 0] } }];
   const run = (scene, project = {}) => buildInputs({ project: { video: {}, frame_rate: 25, ...project }, scene: { text: "x", ...scene }, slots, expand: async () => null });
   const r = await run({ references: ["pic", "vid"] }, { generation_mode: "t2v" });
   assert.equal(r.inputs.src.media_id, "vid");
@@ -171,3 +171,15 @@ test("a video-in pipeline takes the first video among the references as its star
   assert.ok((await run({ references: ["pic"] })).notes.some((n) => /works from a video/.test(n)));
   learn([]);
 });
+
+test("an LTX image-to-video pipeline starts from the first reference picture when none is picked, and keeps it as a reference", async () => {
+  const { learn } = await import("../../shell/bin.js");
+  learn([{ id: "vid", name: "v.mp4", kind: "video" }, { id: "pic", name: "p.png", kind: "image" }, { id: "pic2", name: "q.png", kind: "image" }]);
+  const slots = [{ id: "src", node: "FunUILoadMedia", roles: [{ at: "assets.source_image", input: "media_id", video_frames: true }], inputs: {} }];
+  const run = (scene, project = {}) => buildInputs({ project: { video: {}, frame_rate: 25, ...project }, scene: { text: "x", ...scene }, slots, expand: async () => null });
+  assert.equal((await run({ references: ["pic", "pic2"] })).inputs.src.media_id, "pic");
+  assert.equal((await run({ references: ["pic", "vid"] })).inputs.src.media_id, "vid");          // a video is the start when there is one
+  assert.equal((await run({ source_image: "pic2", references: ["pic"] })).inputs.src.media_id, "pic2");          // a picked start wins
+  assert.equal((await run({ references: ["pic"] }, { generation_mode: "t2v" })).inputs.src.media_id, "");          // "From a prompt": no picture
+  learn([]);
+});

modules/models/ltx/pipeline.py

@@ -129,7 +129,8 @@ def _direct_video_to_video():
     slots[at:at + 1] = [
         # video_frames: the whole video is fed, not its first frame (see inputs.js).
         {"id": "source_image", "group": "Reference media", "node": "FunUILoadMedia",
-         "roles": [{"at": "assets.source_image", "input": "media_id", "video_frames": True}], "inputs": {"media_id": ""}},
+         "roles": [{"at": "assets.source_image", "input": "media_id", "video_frames": True, "video_only": True}],
+         "inputs": {"media_id": ""}},
         {"id": "size", "group": "Preparation", "node": "GetImageSize", "inputs": {"image": ["source_image", 1]}},
         {"id": "encode", "group": "Preparation", "node": "VAEEncode", "inputs": {"pixels": ["source_image", 1], "vae": ["vae", 0]}},
         {"id": "audio_latent", "group": "Preparation", "node": "LTXVEmptyLatentAudio",