diff --git a/modules/models/ltx/pipeline.py b/modules/models/ltx/pipeline.py index 529a8ab5..81924131 100644 --- a/modules/models/ltx/pipeline.py +++ b/modules/models/ltx/pipeline.py @@ -16,7 +16,7 @@ the conditioning, the sound's length and the saved video agree on. DISTILLED_SIGMAS = "1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0" -def _base(image_to_video: bool): +def _base(image_to_video: bool, as_guide: bool = False): video = {"at": "project.video", "input": "frame_rate", "label": "FPS", "drives": "fps"} latent_source = ["latent", 0] model_source, positive_source, negative_source = ["modifiers", 0], ["conditioning", 0], ["conditioning", 1] @@ -51,13 +51,29 @@ def _base(image_to_video: bool): "inputs": {"media_id": ""}}, {"id": "split", "group": "Preparation", "node": "LTXVSeparateAVLatent", "inputs": {"av_latent": ["latent", 0]}}, - {"id": "anchor", "group": "Preparation", "node": "LTXVImgToVideoInplace", - "inputs": {"vae": ["model", 2], "image": ["source_image", 0], "latent": ["split", 0], - "strength": 1.0, "bypass": False}}, - {"id": "join", "group": "Preparation", "node": "LTXVConcatAVLatent", - "inputs": {"video_latent": ["anchor", 0], "audio_latent": ["split", 1]}}, ] + if as_guide: + # The picture steers through guide attention at frame 0 and is NOT the clip's first frame: + # the latent stays empty, so the opening is free to move. Strength is the slot's own number. + slots += [ + {"id": "anchor", "group": "Preparation", "node": "LTXVAddGuide", + "inputs": {"positive": ["conditioning", 0], "negative": ["conditioning", 1], "vae": ["model", 2], + "latent": ["split", 0], "image": ["source_image", 0], "frame_idx": 0, "strength": 0.35}}, + {"id": "join", "group": "Preparation", "node": "LTXVConcatAVLatent", + "inputs": {"video_latent": ["anchor", 2], "audio_latent": ["split", 1]}}, + ] + guided = (["anchor", 0], ["anchor", 1]) + else: + slots += [ + {"id": "anchor", "group": "Preparation", "node": "LTXVImgToVideoInplace", + "inputs": {"vae": ["model", 2], "image": ["source_image", 0], "latent": ["split", 0], + "strength": 1.0, "bypass": False}}, + {"id": "join", "group": "Preparation", "node": "LTXVConcatAVLatent", + "inputs": {"video_latent": ["anchor", 0], "audio_latent": ["split", 1]}}, + ] + guided = (["conditioning", 0], ["conditioning", 1]) latent_source = ["join", 0] + positive_source, negative_source = guided slots += [ {"id": "settings", "group": "Preparation", "node": "FunPackModifierSettings", "inputs": {"settings": "{}"}}, {"id": "modifiers", "group": "Preparation", "node": "FunPackLoadModifiers", @@ -68,8 +84,8 @@ def _base(image_to_video: bool): # A reference face carried into the clip. The picture is the scene's first reference # (assets.reference_1): with none picked this node passes everything through untouched. {"id": "identity", "group": "Preparation", "node": "FunPackIdentityTransfer", - "inputs": {"model": ["modifiers", 0], "vae": ["model", 2], "positive": ["conditioning", 0], - "negative": ["conditioning", 1], "latent": latent_source, "source_id": 2.0, + "inputs": {"model": ["modifiers", 0], "vae": ["model", 2], "positive": positive_source, + "negative": negative_source, "latent": latent_source, "source_id": 2.0, "phase_scale": 1.0, "id_strength": 1.0, "arcface_mode": "auto_adjust"}}, {"id": "identity_source", "group": "Reference media", "node": "FunPackLoadMedia", "roles": [{"at": "assets.reference_1", "input": "media_id", @@ -87,8 +103,9 @@ def _base(image_to_video: bool): "seed": 0, "steps": 8, "cfg": 1.0, "sampler_name": "euler", "scheduler": "normal", "denoise": 1.0}}, + *_uncrop(as_guide), {"id": "decode", "group": "Render", "node": "FunPackDecode", "inputs": { - "samples": ["sampler", 0], "vae": ["model", 2], "model": ["model", 0], "audio_vae": ["audio_vae", 0]}}, + "samples": ["rejoin", 0] if as_guide else ["sampler", 0], "vae": ["model", 2], "model": ["model", 0], "audio_vae": ["audio_vae", 0]}}, {"id": "video", "group": "Render", "node": "CreateVideo", "roles": [{"at": "project.video", "input": "fps", "label": "FPS", "drives": "fps"}], "inputs": {"images": ["decode", 0], "fps": 25.0, "audio": ["decode", 1]}}, @@ -98,9 +115,24 @@ def _base(image_to_video: bool): return slots +def _uncrop(as_guide): + """A guide sits in the latent as extra frames: take them off again before the picture is decoded.""" + if not as_guide: + return [] + return [ + {"id": "unjoin", "group": "Render", "node": "LTXVSeparateAVLatent", "inputs": {"av_latent": ["sampler", 0]}}, + {"id": "crop", "group": "Render", "node": "LTXVCropGuides", + "inputs": {"positive": ["identity", 1], "negative": ["identity", 2], "latent": ["unjoin", 0]}}, + {"id": "rejoin", "group": "Render", "node": "LTXVConcatAVLatent", + "inputs": {"video_latent": ["crop", 2], "audio_latent": ["unjoin", 1]}}, + ] + + def presets(): return [{"id": "ltx23_text_to_video", "title": "LTX 2.3 · Text to Video", "slots": _base(False)}, - {"id": "ltx23_image_to_video", "title": "LTX 2.3 · Image to Video", "slots": _base(True)}] + {"id": "ltx23_image_to_video", "title": "LTX 2.3 · Image to Video", "slots": _base(True)}, + {"id": "ltx23_anchor_guide", "title": "LTX 2.3 · Anchor as guide (image steers, opening stays free)", + "slots": _base(True, True)}] PROVIDES = {"pipeline_presets": presets} diff --git a/modules/models/ltx/tests/test_ltx.py b/modules/models/ltx/tests/test_ltx.py index eca43e35..54894932 100644 --- a/modules/models/ltx/tests/test_ltx.py +++ b/modules/models/ltx/tests/test_ltx.py @@ -122,7 +122,7 @@ def test_the_empty_latent_node_passes_the_audio_vae_to_a_provider_that_asks_and_ assert seen == {"plain": True, "wants": ("A", 30.0)} -@pytest.mark.parametrize("pid", ["ltx23_text_to_video", "ltx23_image_to_video"]) +@pytest.mark.parametrize("pid", ["ltx23_text_to_video", "ltx23_image_to_video", "ltx23_anchor_guide"]) def test_the_pipelines_build_a_real_graph(registered, pid): from core import graph from modules.models.ltx.pipeline import presets @@ -143,7 +143,7 @@ def test_the_pipelines_build_a_real_graph(registered, pid): def test_the_presets_are_offered_through_the_registry(registered): from core import routes found = {preset["id"] for _spec, make in routes.modules().providers("pipeline_presets") for preset in make()} - assert {"ltx23_text_to_video", "ltx23_image_to_video"} <= found + assert {"ltx23_text_to_video", "ltx23_image_to_video", "ltx23_anchor_guide"} <= found def test_the_picture_is_found_in_both_shapes_a_sampling_latent_takes(comfyui):