diff --git a/CHANGELOG.md b/CHANGELOG.md index 2433288c..a115feb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ ### Added (experimental) -- **LTX 2.3 · Video to Video, direct** preset (UNTESTED): no IC-LoRA. The picked video is encoded and sampled again from part-way down the schedule, so it keeps its layout and motion. How much changes is the sigmas slot (default `0.725, 0.421875, 0.0`; start higher to change more). Size and length are the video's own, set by Video input. The graph validates; never run on a GPU. +- **LTX 2.3 · Video to Video, direct** preset (UNTESTED): no IC-LoRA. The picked video is encoded and sampled again from part-way down the schedule, so it keeps its layout and motion. How much changes is the sigmas slot (default `0.909375, 0.725, 0.421875, 0.0`; start higher to change more, lower to keep more). Size and length are the video's own, set by Video input. The graph validates; never run on a GPU. - **LTX presets load separate files** (diffusion model, video VAE, audio VAE, text encoder) instead of one checkpoint. - **LTX 2.3 · Video to Video, IC-LoRA** preset (UNTESTED): the wiring of ComfyUI's own LTX 2.3 IC-LoRA template (stock LoRA loader, Get IC-LoRA Parameters, the video as a guide at strength 1). Pick the IC-LoRA file in the pipeline, then pick a video as the scene's start picture; Video input prepares it. The graph builds and validates; never run on a GPU. - **Video input** (Project and Scene tabs): reference videos (and, on LTX, a video picked as the scene start, fed as frames) from the Media bin can be got ready before the model sees them, like a video loader node: target width and height (fill, fit with bars, or stretch; rounded to a multiple), edge crop in percent, frame rate (project's, the video's own or a number), skip first N frames, cut last N, every Nth frame, frame cap. The sound is cut to match. One set per project and one per scene; a dropdown in Project settings says whose is used (Automatic: the project's when switched on, otherwise the scene's). Off by default, so existing projects load videos as before. Works for any model whose pipeline takes reference videos (MiniMax H3 today). Tested on made-up frames and an encoded file, not yet on a rental. diff --git a/modules/models/ltx/pipeline.py b/modules/models/ltx/pipeline.py index f0f2a7bc..2d14b61f 100644 --- a/modules/models/ltx/pipeline.py +++ b/modules/models/ltx/pipeline.py @@ -139,7 +139,7 @@ def _direct_video_to_video(): ] for x in slots: if x["id"] == "sigmas": - x["inputs"]["sigmas"] = "0.725, 0.421875, 0.0" # the last two steps of the distilled schedule: about 70% re-noised + x["inputs"]["sigmas"] = "0.909375, 0.725, 0.421875, 0.0" # the last three steps of the distilled schedule: about 90% re-noised return slots diff --git a/modules/models/ltx/tests/test_v2v_end_to_end.py b/modules/models/ltx/tests/test_v2v_end_to_end.py index 184a9cd4..ea7fb3ef 100644 --- a/modules/models/ltx/tests/test_v2v_end_to_end.py +++ b/modules/models/ltx/tests/test_v2v_end_to_end.py @@ -118,12 +118,20 @@ def test_direct_video_to_video_end_to_end(clip_id, real_tiny_ltx, monkeypatch): slots = upgrade(old) prepare = json.dumps({"skip_first": 1, "cut_last": 0, "every_nth": 1, "multiple_of": 32, "fps": 25}) - slots, problems = graph.override(slots, {"source_image": {"media_id": clip_id, "full": True, "prepare": prepare}}) + slots, problems = graph.override(slots, {"source_image": {"media_id": clip_id, "full": True, "prepare": prepare}, + "positive": {"text": "a red fox runs"}}) assert problems == [] prompt, problems = graph.build(slots) unset = ("needs 'ckpt_name'", "needs 'model_name'", "needs 'vae_name'", "needs 'clip_name1'", "is '', which is not one of") assert [p for p in problems if not any(u in p for u in unset)] == [] + # the typed prompt is what the sampler's positive input comes from, through the conditioning node and nothing else + node = prompt["sampler"]["inputs"]["positive"] + while prompt[node[0]]["class_type"] != "CLIPTextEncode": + node = next(v for v in prompt[node[0]]["inputs"].values() if isinstance(v, list) and prompt[v[0]]["class_type"] in ("LTXVConditioning", "CLIPTextEncode")) + assert prompt[node[0]]["inputs"]["text"] == "a red fox runs" + assert prompt["sampler"]["inputs"]["latent"][0] == "latent" and prompt["sampler"]["inputs"]["denoise"] == 1.0 + vae, audio_vae = StubVae(), StubAudioVae() run, done = _run(prompt, {"model": (real_tiny_ltx, ""), "vae": (vae, ""), "audio_vae": (audio_vae, ""), "clip": (None, "")}) run("latent") @@ -140,9 +148,13 @@ def test_direct_video_to_video_end_to_end(clip_id, real_tiny_ltx, monkeypatch): from modules.sampling.sampler.nodes import FunUISampler run("sigmas") sig = done["sigmas"][0] - assert [round(float(x), 3) for x in sig] == [0.725, 0.422, 0.0] # the tail of the distilled schedule + assert [round(float(x), 3) for x in sig] == [0.909, 0.725, 0.422, 0.0] # the tail of the distilled schedule start = done["latent"][0] out = FunUISampler.execute(real_tiny_ltx, [[ctx, {}]], [[ctx, {}]], start, 1, 2, 1.0, "euler", "normal", sigmas=sig).result[0] + # the prompt reaches the sampler: a different text embedding, same seed and latent, gives a different video AND sound + other = FunUISampler.execute(real_tiny_ltx, [[torch.randn(1, 5, 48), {}]], [[ctx, {}]], start, 1, 2, 1.0, "euler", "normal", sigmas=sig).result[0] + assert not torch.allclose(other["samples"].unbind()[0], out["samples"].unbind()[0]) + assert not torch.allclose(other["samples"].unbind()[1], out["samples"].unbind()[1]) got_video, got_audio = out["samples"].unbind() assert got_video.shape == video.shape and got_audio.shape == audio.shape assert torch.isfinite(got_video).all() and not torch.allclose(got_video, video) # it was denoised, not passed through