From e4111cf74a98bbbb5878d6473d7c31ee465ea1ec Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 08:39:08 +0000 Subject: [PATCH] LTX image-to-video: the first reference picture is the start when none is picked Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01BzJLJWVu7RZnbRqm3nYAnX --- app/ui/features/generate/inputs.js | 19 ++++++++++++------- app/ui/features/generate/inputs.test.js | 14 +++++++++++++- modules/models/ltx/pipeline.py | 3 ++- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/app/ui/features/generate/inputs.js b/app/ui/features/generate/inputs.js index 85175eb4..95d28ec2 100644 --- a/app/ui/features/generate/inputs.js +++ b/app/ui/features/generate/inputs.js @@ -20,12 +20,17 @@ export async function buildInputs({ project, scene, slots, expand, nodeInfo, fra for (const slot of (slots || []).filter((s) => s.node === "FunUIShotCamera")) put(slot, "project", project.id || ""); // what the node keeps per project (the words focus infusion draws on), also for pipelines saved before it was asked for const source = rolesAt(slots, "assets.source_image")[0]; const plan = sceneReferences(scene, project.fixed_references); - // A pipeline that takes a whole video where it takes a picture (video-to-video): with no start picked, the first VIDEO - // among the references is the one it works from, and is not also sent as a reference. It is never blanked by "From a prompt". - const takesVideo = Boolean(source && source.role.video_frames); - const fromRefs = takesVideo && !scene.source_image ? plan.ids.find((id) => kinds.get(id) === "video") : null; - if (fromRefs) plan.ids = plan.ids.filter((id) => id !== fromRefs); - const startId = takesVideo ? scene.source_image || fromRefs || "" : project.generation_mode === "t2v" ? "" : scene.source_image; + // A pipeline that takes whole media where it takes a picture (LTX): with no start picked, the first reference is the start: + // a VIDEO first (it is then not also sent as a reference: nothing takes it there), else the first picture (which stays a + // reference too, so an identity input still gets it). A video-only pipeline (video-to-video) looks at videos alone and is + // never blanked by "From a prompt". + const takesVideo = Boolean(source && source.role.video_frames), videoOnly = Boolean(source && source.role.video_only); + const wanted = (id) => kinds.get(id) === "video" || (!videoOnly && kinds.get(id) === "image"); + const blank = project.generation_mode === "t2v" && !videoOnly; + const fromRefs = takesVideo && !scene.source_image && !blank + ? plan.ids.find((id) => kinds.get(id) === "video") || (videoOnly ? null : plan.ids.find((id) => wanted(id))) : null; + if (fromRefs && kinds.get(fromRefs) === "video") plan.ids = plan.ids.filter((id) => id !== fromRefs); + const startId = takesVideo ? (blank ? "" : scene.source_image || fromRefs || "") : project.generation_mode === "t2v" ? "" : scene.source_image; if (source) put(source.slot, "media_id", startId || ""); // "from a prompt" starts every shot without a picture // A pipeline that can take a whole video where it takes a picture (LTX) says so on the role: a picked VIDEO then // supplies its frames (prepared as Video input says); the preset wires the frames output where the picture goes (a @@ -80,7 +85,7 @@ export async function buildInputs({ project, scene, slots, expand, nodeInfo, fra if (!(body && typeof body.text === "string")) failedExpand = true; } // Features that add inputs of their own (a Chat comment for the enhancer, say) say so through a hook: { inputs: {slot: {name: value}}, notes: [text] }. - const noVideo = takesVideo && !startId ? ["This pipeline works from a video: pick one as the start, or add it under References."] : []; + const noVideo = takesVideo && videoOnly && !startId ? ["This pipeline works from a video: pick one as the start, or add it under References."] : []; const notes = failedExpand ? ["The prompt could not be expanded (shortcuts, $variables, anchor, postfix): the text was sent as typed."] : []; notes.push(...noVideo); for (const hook of hooks) { diff --git a/app/ui/features/generate/inputs.test.js b/app/ui/features/generate/inputs.test.js index b83c2f21..e3b81918 100644 --- a/app/ui/features/generate/inputs.test.js +++ b/app/ui/features/generate/inputs.test.js @@ -162,7 +162,7 @@ test("a video picked as the start feeds its frames to what the picture fed, when test("a video-in pipeline takes the first video among the references as its start, and says so when there is none", async () => { const { learn } = await import("../../shell/bin.js"); learn([{ id: "vid", name: "v.mp4", kind: "video" }, { id: "pic", name: "p.png", kind: "image" }]); - const slots = [{ id: "src", node: "FunUILoadMedia", roles: [{ at: "assets.source_image", input: "media_id", video_frames: true }], inputs: {} }, { id: "a", node: "X", inputs: { image: ["src", 0] } }]; + const slots = [{ id: "src", node: "FunUILoadMedia", roles: [{ at: "assets.source_image", input: "media_id", video_frames: true, video_only: true }], inputs: {} }, { id: "a", node: "X", inputs: { image: ["src", 0] } }]; const run = (scene, project = {}) => buildInputs({ project: { video: {}, frame_rate: 25, ...project }, scene: { text: "x", ...scene }, slots, expand: async () => null }); const r = await run({ references: ["pic", "vid"] }, { generation_mode: "t2v" }); assert.equal(r.inputs.src.media_id, "vid"); @@ -171,3 +171,15 @@ test("a video-in pipeline takes the first video among the references as its star assert.ok((await run({ references: ["pic"] })).notes.some((n) => /works from a video/.test(n))); learn([]); }); + +test("an LTX image-to-video pipeline starts from the first reference picture when none is picked, and keeps it as a reference", async () => { + const { learn } = await import("../../shell/bin.js"); + learn([{ id: "vid", name: "v.mp4", kind: "video" }, { id: "pic", name: "p.png", kind: "image" }, { id: "pic2", name: "q.png", kind: "image" }]); + const slots = [{ id: "src", node: "FunUILoadMedia", roles: [{ at: "assets.source_image", input: "media_id", video_frames: true }], inputs: {} }]; + const run = (scene, project = {}) => buildInputs({ project: { video: {}, frame_rate: 25, ...project }, scene: { text: "x", ...scene }, slots, expand: async () => null }); + assert.equal((await run({ references: ["pic", "pic2"] })).inputs.src.media_id, "pic"); + assert.equal((await run({ references: ["pic", "vid"] })).inputs.src.media_id, "vid"); // a video is the start when there is one + assert.equal((await run({ source_image: "pic2", references: ["pic"] })).inputs.src.media_id, "pic2"); // a picked start wins + assert.equal((await run({ references: ["pic"] }, { generation_mode: "t2v" })).inputs.src.media_id, ""); // "From a prompt": no picture + learn([]); +}); diff --git a/modules/models/ltx/pipeline.py b/modules/models/ltx/pipeline.py index 2d14b61f..8ac349b8 100644 --- a/modules/models/ltx/pipeline.py +++ b/modules/models/ltx/pipeline.py @@ -129,7 +129,8 @@ def _direct_video_to_video(): slots[at:at + 1] = [ # video_frames: the whole video is fed, not its first frame (see inputs.js). {"id": "source_image", "group": "Reference media", "node": "FunUILoadMedia", - "roles": [{"at": "assets.source_image", "input": "media_id", "video_frames": True}], "inputs": {"media_id": ""}}, + "roles": [{"at": "assets.source_image", "input": "media_id", "video_frames": True, "video_only": True}], + "inputs": {"media_id": ""}}, {"id": "size", "group": "Preparation", "node": "GetImageSize", "inputs": {"image": ["source_image", 1]}}, {"id": "encode", "group": "Preparation", "node": "VAEEncode", "inputs": {"pixels": ["source_image", 1], "vae": ["vae", 0]}}, {"id": "audio_latent", "group": "Preparation", "node": "LTXVEmptyLatentAudio",