Merge pull request #484 from Lightricks/pr-2026-05-11
Update ComfyUI-LTXVideo - 2026-05-11
This commit is contained in:
Michael Kupchick
@@ -54,6 +54,7 @@ LTX-2.3 Workflows:
|
||||
* [`IC-LoRA distilled model depth + human pose + edges`](./example_workflows/2.3/LTX-2.3_ICLoRA_Union_Control_Distilled.json)
|
||||
* [`IC-LoRA distilled model I2V motion tracking`](./example_workflows/2.3/LTX-2.3_ICLoRA_Motion_Track_Distilled.json)
|
||||
* [`IC-LoRA distilled model HDR`](./example_workflows/2.3/LTX-2.3_ICLoRA_HDR_Distilled.json)
|
||||
* [`IC-LoRA distilled model Lipdub; two stages (with upsampling)`](./example_workflows/2.3/LTX-2.3_ICLoRA_Lipdub_Two_Stage_Distilled.json)
|
||||
|
||||
Older Workflows (LTX-2.0):
|
||||
|
||||
@@ -91,6 +92,17 @@ We provide an **HDR IC-LoRA** that generates linear HDR video encoded in ARRI Lo
|
||||
- **SDR preview + raw HDR**: The node outputs both a Reinhard-tonemapped SDR preview and the raw linear HDR tensor for downstream use.
|
||||
- **EXR export**: Optionally writes the linear HDR frames as a 16/32-bit EXR image sequence. To enable EXR writing, set `OPENCV_IO_ENABLE_OPENEXR=1` in the environment before starting ComfyUI. The exported EXR sequence is best viewed in [DJV](https://github.com/grizzlypeak3d/DJV) (or [DJV for macOS](https://djv.en.uptodown.com/mac/download)).
|
||||
|
||||
## Lipdub IC-LoRA
|
||||
|
||||
We provide a **Lipdub IC-LoRA** that dubs or rephrases speech in video. Given a source video and a text prompt containing the desired dialogue, it generates new lip movements and audio that match the target text while preserving the speaker's identity.
|
||||
|
||||
### Key Features
|
||||
|
||||
- **Multilingual dubbing**: Translate speech into another language - the model regenerates lips and audio to match.
|
||||
- **Same-language rephrasing**: Change what the speaker says while keeping the original language.
|
||||
- **Two-stage pipeline**: Stage 1 generates the video and audio at base resolution; Stage 2 upscales while freezing the audio.
|
||||
- **Speaker identity preservation**: Reference audio tokens provide speaker context so the generated voice stays consistent.
|
||||
|
||||
## Required Models
|
||||
|
||||
Download the following models:
|
||||
@@ -116,6 +128,7 @@ Download the following models:
|
||||
* [`ltx-2.3-22b-ic-lora-union-control-ref0.5.safetensors`](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-Union-Control/blob/main/ltx-2.3-22b-ic-lora-union-control-ref0.5.safetensors)
|
||||
* [`ltx-2.3-22b-ic-lora-motion-track-control-ref0.5.safetensors`](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-Motion-Track-Control/blob/main/ltx-2.3-22b-ic-lora-motion-track-control-ref0.5.safetensors)
|
||||
* [`ltx-2.3-22b-ic-lora-hdr-0.9.safetensors`](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-HDR)
|
||||
* [`ltx-2.3-22b-ic-lora-lipdub-0.9.safetensors`](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-LipDub)
|
||||
* [`ltx-2-19b-ic-lora-detailer.safetensors`](https://huggingface.co/Lightricks/LTX-2-19b-IC-LoRA-Detailer/blob/main/ltx-2-19b-ic-lora-detailer.safetensors)
|
||||
* [`ltx-2-19b-ic-lora-pose-control.safetensors`](https://huggingface.co/Lightricks/LTX-2-19b-IC-LoRA-Pose-Control/blob/main/ltx-2-19b-ic-lora-pose-control.safetensors)
|
||||
* [`ltx-2-19b-lora-camera-control-dolly-in.safetensors`](https://huggingface.co/Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-In/blob/main/ltx-2-19b-lora-camera-control-dolly-in.safetensors)
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
from . import internal # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
from .conditioning_loader import LTXVLoadConditioning
|
||||
from .conditioning_saver import LTXVSaveConditioning
|
||||
from .decoder_noise import DecoderNoise
|
||||
@@ -18,6 +21,7 @@ from .iclora import (
|
||||
LTXAddVideoICLoRAGuide,
|
||||
LTXAddVideoICLoRAGuideAdvanced,
|
||||
LTXICLoRALoaderModelOnly,
|
||||
LTXVSetAudioRefTokens,
|
||||
)
|
||||
from .latent_norm import (
|
||||
LTXVAdainLatent,
|
||||
@@ -107,6 +111,7 @@ NODE_CLASS_MAPPINGS = {
|
||||
"LTXAddVideoICLoRAGuide": LTXAddVideoICLoRAGuide,
|
||||
"LTXAddVideoICLoRAGuideAdvanced": LTXAddVideoICLoRAGuideAdvanced,
|
||||
"LTXICLoRALoaderModelOnly": LTXICLoRALoaderModelOnly,
|
||||
"LTXVSetAudioRefTokens": LTXVSetAudioRefTokens,
|
||||
"LTXVLoadConditioning": LTXVLoadConditioning,
|
||||
"LTXVSaveConditioning": LTXVSaveConditioning,
|
||||
"LTXVDrawTracks": LTXVDrawTracks,
|
||||
|
||||
@@ -1485,9 +1485,32 @@
|
||||
true
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5121,
|
||||
"type": "Note",
|
||||
"pos": [
|
||||
-504.4601702780707,
|
||||
2965.4285923622097
|
||||
],
|
||||
"size": [
|
||||
210,
|
||||
88
|
||||
],
|
||||
"flags": {},
|
||||
"order": 10,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"in case of OOM use tile_size of 768\ntemporal size 16"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 4851,
|
||||
"type": "VAEDecode",
|
||||
"type": "VAEDecodeTiled",
|
||||
"pos": [
|
||||
-518.1273243034602,
|
||||
3125.1032336912526
|
||||
@@ -1511,6 +1534,42 @@
|
||||
"name": "vae",
|
||||
"type": "VAE",
|
||||
"link": 13619
|
||||
},
|
||||
{
|
||||
"localized_name": "tile_size",
|
||||
"name": "tile_size",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "tile_size"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "overlap",
|
||||
"name": "overlap",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "overlap"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "temporal_size",
|
||||
"name": "temporal_size",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "temporal_size"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "temporal_overlap",
|
||||
"name": "temporal_overlap",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "temporal_overlap"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
@@ -1526,8 +1585,14 @@
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.14.1",
|
||||
"Node name for S&R": "VAEDecode"
|
||||
}
|
||||
"Node name for S&R": "VAEDecodeTiled"
|
||||
},
|
||||
"widgets_values": [
|
||||
768,
|
||||
256,
|
||||
8,
|
||||
4
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5120,
|
||||
@@ -2814,6 +2879,10 @@
|
||||
},
|
||||
"4851": {
|
||||
"inputs": {
|
||||
"tile_size": 768,
|
||||
"overlap": 256,
|
||||
"temporal_size": 8,
|
||||
"temporal_overlap": 4,
|
||||
"samples": [
|
||||
"5013",
|
||||
2
|
||||
@@ -2823,9 +2892,9 @@
|
||||
2
|
||||
]
|
||||
},
|
||||
"class_type": "VAEDecode",
|
||||
"class_type": "VAEDecodeTiled",
|
||||
"_meta": {
|
||||
"title": "VAE Decode"
|
||||
"title": "VAE Decode (Tiled)"
|
||||
}
|
||||
},
|
||||
"5011": {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fa96b9f0b71a388b30b24b64734779b35562d6ab2b8cd377781a5d5a4a388b10
|
||||
size 5226932
|
||||
@@ -1,9 +1,12 @@
|
||||
import logging
|
||||
|
||||
import comfy
|
||||
import comfy.samplers
|
||||
import comfy.sd
|
||||
import comfy.utils
|
||||
import comfy_extras.nodes_lt as nodes_lt
|
||||
import folder_paths
|
||||
import node_helpers
|
||||
import torch
|
||||
from comfy_api.latest import io
|
||||
|
||||
@@ -518,3 +521,83 @@ class LTXICLoRALoaderModelOnly(io.ComfyNode):
|
||||
model, None, lora, strength_model, 0
|
||||
)
|
||||
return io.NodeOutput(model_lora, latent_downscale_factor)
|
||||
|
||||
|
||||
def _patchify_audio_latent(latent):
|
||||
"""Reshape audio latent (b, c, t, f) → ref_audio token dict (b, t, c*f)."""
|
||||
b, c, t, f = latent.shape
|
||||
ref_tokens = latent.permute(0, 2, 1, 3).reshape(b, t, c * f)
|
||||
return {"tokens": ref_tokens}
|
||||
|
||||
|
||||
@comfy_node(name="LTXVSetAudioRefTokens")
|
||||
class LTXVSetAudioRefTokens(io.ComfyNode):
|
||||
"""Provide speaker identity context for audio generation.
|
||||
|
||||
Patchifies the audio latent and attaches it as reference tokens on both
|
||||
positive and negative conditioning. The model prepends these tokens with
|
||||
negative temporal positions so they serve as identity context without
|
||||
being part of the generated output.
|
||||
|
||||
Also outputs a frozen copy of the audio latent (noise_mask=0) for
|
||||
direct use in stage 2 without re-encoding.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="LTXVSetAudioRefTokens",
|
||||
display_name=NODES_DISPLAY_NAME_PREFIX + " Set Audio Ref Tokens",
|
||||
category="Lightricks/IC-LoRA",
|
||||
description=(
|
||||
"Provides speaker identity context for audio generation by attaching "
|
||||
"reference audio tokens to the conditioning. The tokens are prepended "
|
||||
"with negative temporal positions so the model treats them as context "
|
||||
"rather than generation targets."
|
||||
),
|
||||
inputs=[
|
||||
io.Conditioning.Input(
|
||||
"positive",
|
||||
tooltip="Positive conditioning to attach the reference audio tokens to.",
|
||||
),
|
||||
io.Conditioning.Input(
|
||||
"negative",
|
||||
tooltip="Negative conditioning to attach the reference audio tokens to.",
|
||||
),
|
||||
io.Latent.Input(
|
||||
"audio_latent",
|
||||
tooltip="Encoded audio latent from LTXV Audio VAE Encode.",
|
||||
),
|
||||
],
|
||||
outputs=[
|
||||
io.Conditioning.Output(
|
||||
display_name="positive",
|
||||
tooltip="Positive conditioning with reference audio tokens attached.",
|
||||
),
|
||||
io.Conditioning.Output(
|
||||
display_name="negative",
|
||||
tooltip="Negative conditioning with reference audio tokens attached.",
|
||||
),
|
||||
io.Latent.Output(
|
||||
display_name="frozen_audio",
|
||||
tooltip="Audio latent with noise_mask=0, fully frozen during denoising.",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, positive, negative, audio_latent) -> io.NodeOutput:
|
||||
latent = audio_latent["samples"]
|
||||
ref_audio = _patchify_audio_latent(latent)
|
||||
positive = node_helpers.conditioning_set_values(
|
||||
positive, {"ref_audio": ref_audio}
|
||||
)
|
||||
negative = node_helpers.conditioning_set_values(
|
||||
negative, {"ref_audio": ref_audio}
|
||||
)
|
||||
|
||||
frozen = audio_latent.copy()
|
||||
b, c, t, f = latent.shape
|
||||
frozen["noise_mask"] = torch.zeros((b, 1, t, 1), dtype=torch.float32)
|
||||
|
||||
return io.NodeOutput(positive, negative, frozen)
|
||||
|
||||
Reference in New Issue
Block a user