feat(ltx): 'Anchor as guide' pipeline preset (stock LTXVAddGuide at frame 0, empty latent, guides cropped before decode)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
digital-garbage
co-authored by
Claude Sonnet 5.5
Claude Sonnet 5.5
parent
0f9a2ac98e
commit
1343486f87
@@ -16,7 +16,7 @@ the conditioning, the sound's length and the saved video agree on.
|
||||
DISTILLED_SIGMAS = "1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0"
|
||||
|
||||
|
||||
def _base(image_to_video: bool):
|
||||
def _base(image_to_video: bool, as_guide: bool = False):
|
||||
video = {"at": "project.video", "input": "frame_rate", "label": "FPS", "drives": "fps"}
|
||||
latent_source = ["latent", 0]
|
||||
model_source, positive_source, negative_source = ["modifiers", 0], ["conditioning", 0], ["conditioning", 1]
|
||||
@@ -51,13 +51,29 @@ def _base(image_to_video: bool):
|
||||
"inputs": {"media_id": ""}},
|
||||
{"id": "split", "group": "Preparation", "node": "LTXVSeparateAVLatent",
|
||||
"inputs": {"av_latent": ["latent", 0]}},
|
||||
{"id": "anchor", "group": "Preparation", "node": "LTXVImgToVideoInplace",
|
||||
"inputs": {"vae": ["model", 2], "image": ["source_image", 0], "latent": ["split", 0],
|
||||
"strength": 1.0, "bypass": False}},
|
||||
{"id": "join", "group": "Preparation", "node": "LTXVConcatAVLatent",
|
||||
"inputs": {"video_latent": ["anchor", 0], "audio_latent": ["split", 1]}},
|
||||
]
|
||||
if as_guide:
|
||||
# The picture steers through guide attention at frame 0 and is NOT the clip's first frame:
|
||||
# the latent stays empty, so the opening is free to move. Strength is the slot's own number.
|
||||
slots += [
|
||||
{"id": "anchor", "group": "Preparation", "node": "LTXVAddGuide",
|
||||
"inputs": {"positive": ["conditioning", 0], "negative": ["conditioning", 1], "vae": ["model", 2],
|
||||
"latent": ["split", 0], "image": ["source_image", 0], "frame_idx": 0, "strength": 0.35}},
|
||||
{"id": "join", "group": "Preparation", "node": "LTXVConcatAVLatent",
|
||||
"inputs": {"video_latent": ["anchor", 2], "audio_latent": ["split", 1]}},
|
||||
]
|
||||
guided = (["anchor", 0], ["anchor", 1])
|
||||
else:
|
||||
slots += [
|
||||
{"id": "anchor", "group": "Preparation", "node": "LTXVImgToVideoInplace",
|
||||
"inputs": {"vae": ["model", 2], "image": ["source_image", 0], "latent": ["split", 0],
|
||||
"strength": 1.0, "bypass": False}},
|
||||
{"id": "join", "group": "Preparation", "node": "LTXVConcatAVLatent",
|
||||
"inputs": {"video_latent": ["anchor", 0], "audio_latent": ["split", 1]}},
|
||||
]
|
||||
guided = (["conditioning", 0], ["conditioning", 1])
|
||||
latent_source = ["join", 0]
|
||||
positive_source, negative_source = guided
|
||||
slots += [
|
||||
{"id": "settings", "group": "Preparation", "node": "FunPackModifierSettings", "inputs": {"settings": "{}"}},
|
||||
{"id": "modifiers", "group": "Preparation", "node": "FunPackLoadModifiers",
|
||||
@@ -68,8 +84,8 @@ def _base(image_to_video: bool):
|
||||
# A reference face carried into the clip. The picture is the scene's first reference
|
||||
# (assets.reference_1): with none picked this node passes everything through untouched.
|
||||
{"id": "identity", "group": "Preparation", "node": "FunPackIdentityTransfer",
|
||||
"inputs": {"model": ["modifiers", 0], "vae": ["model", 2], "positive": ["conditioning", 0],
|
||||
"negative": ["conditioning", 1], "latent": latent_source, "source_id": 2.0,
|
||||
"inputs": {"model": ["modifiers", 0], "vae": ["model", 2], "positive": positive_source,
|
||||
"negative": negative_source, "latent": latent_source, "source_id": 2.0,
|
||||
"phase_scale": 1.0, "id_strength": 1.0, "arcface_mode": "auto_adjust"}},
|
||||
{"id": "identity_source", "group": "Reference media", "node": "FunPackLoadMedia",
|
||||
"roles": [{"at": "assets.reference_1", "input": "media_id",
|
||||
@@ -87,8 +103,9 @@ def _base(image_to_video: bool):
|
||||
"seed": 0, "steps": 8, "cfg": 1.0, "sampler_name": "euler", "scheduler": "normal",
|
||||
"denoise": 1.0}},
|
||||
|
||||
*_uncrop(as_guide),
|
||||
{"id": "decode", "group": "Render", "node": "FunPackDecode", "inputs": {
|
||||
"samples": ["sampler", 0], "vae": ["model", 2], "model": ["model", 0], "audio_vae": ["audio_vae", 0]}},
|
||||
"samples": ["rejoin", 0] if as_guide else ["sampler", 0], "vae": ["model", 2], "model": ["model", 0], "audio_vae": ["audio_vae", 0]}},
|
||||
{"id": "video", "group": "Render", "node": "CreateVideo",
|
||||
"roles": [{"at": "project.video", "input": "fps", "label": "FPS", "drives": "fps"}],
|
||||
"inputs": {"images": ["decode", 0], "fps": 25.0, "audio": ["decode", 1]}},
|
||||
@@ -98,9 +115,24 @@ def _base(image_to_video: bool):
|
||||
return slots
|
||||
|
||||
|
||||
def _uncrop(as_guide):
|
||||
"""A guide sits in the latent as extra frames: take them off again before the picture is decoded."""
|
||||
if not as_guide:
|
||||
return []
|
||||
return [
|
||||
{"id": "unjoin", "group": "Render", "node": "LTXVSeparateAVLatent", "inputs": {"av_latent": ["sampler", 0]}},
|
||||
{"id": "crop", "group": "Render", "node": "LTXVCropGuides",
|
||||
"inputs": {"positive": ["identity", 1], "negative": ["identity", 2], "latent": ["unjoin", 0]}},
|
||||
{"id": "rejoin", "group": "Render", "node": "LTXVConcatAVLatent",
|
||||
"inputs": {"video_latent": ["crop", 2], "audio_latent": ["unjoin", 1]}},
|
||||
]
|
||||
|
||||
|
||||
def presets():
|
||||
return [{"id": "ltx23_text_to_video", "title": "LTX 2.3 · Text to Video", "slots": _base(False)},
|
||||
{"id": "ltx23_image_to_video", "title": "LTX 2.3 · Image to Video", "slots": _base(True)}]
|
||||
{"id": "ltx23_image_to_video", "title": "LTX 2.3 · Image to Video", "slots": _base(True)},
|
||||
{"id": "ltx23_anchor_guide", "title": "LTX 2.3 · Anchor as guide (image steers, opening stays free)",
|
||||
"slots": _base(True, True)}]
|
||||
|
||||
|
||||
PROVIDES = {"pipeline_presets": presets}
|
||||
|
||||
@@ -122,7 +122,7 @@ def test_the_empty_latent_node_passes_the_audio_vae_to_a_provider_that_asks_and_
|
||||
assert seen == {"plain": True, "wants": ("A", 30.0)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pid", ["ltx23_text_to_video", "ltx23_image_to_video"])
|
||||
@pytest.mark.parametrize("pid", ["ltx23_text_to_video", "ltx23_image_to_video", "ltx23_anchor_guide"])
|
||||
def test_the_pipelines_build_a_real_graph(registered, pid):
|
||||
from core import graph
|
||||
from modules.models.ltx.pipeline import presets
|
||||
@@ -143,7 +143,7 @@ def test_the_pipelines_build_a_real_graph(registered, pid):
|
||||
def test_the_presets_are_offered_through_the_registry(registered):
|
||||
from core import routes
|
||||
found = {preset["id"] for _spec, make in routes.modules().providers("pipeline_presets") for preset in make()}
|
||||
assert {"ltx23_text_to_video", "ltx23_image_to_video"} <= found
|
||||
assert {"ltx23_text_to_video", "ltx23_image_to_video", "ltx23_anchor_guide"} <= found
|
||||
|
||||
|
||||
def test_the_picture_is_found_in_both_shapes_a_sampling_latent_takes(comfyui):
|
||||
|
||||
Reference in New Issue
Block a user