nodes/Lightricks-ComfyUI-LTXVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #398 from Lightricks/pr-2026-01-29

Browse Source 
Update ComfyUI-LTXVideo - 2026-01-29
This commit is contained in:
Michael Kupchick
2026-01-29 18:46:54 +02:00
committed by GitHub
parent d153ca3f78 f82614dc6d
commit 49add6dddb
7 changed files with 14675 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+20 -2
View File
@@ -43,9 +43,9 @@ The nodes will appear in your node menu under the “LTXVideo” category. Requi
The ComfyUI-LTXVideo installation includes several example workflows.
You can see them all at:
'''
```
ComfyUI/custom_nodes/ComfyUI-LTXVideo/example_workflows/
'''
```
* [`Text to video full model`](./example_workflows/LTX-2_T2V_Full_wLora.json)
* [`Text to video distilled model (Fast)`](./example_workflows/LTX-2_T2V_Distilled_wLora.json)
@@ -53,6 +53,23 @@ ComfyUI/custom_nodes/ComfyUI-LTXVideo/example_workflows/
* [`Image to video distilled model (Fast)`](./example_workflows/LTX-2_I2V_Distilled_wLora.json)
* [`Video to video detailer`](./example_workflows/LTX-2_V2V_Detailer.json)
* [`IC-LoRA distilled model (depth + human pose + edges)`](./example_workflows/LTX-2_ICLoRA_All_Distilled.json)
* [`IC-LoRA distilled model with downscaled reference latents`](./example_workflows/LTX-2_ICLoRA_All_Distilled_ref0.5.json)
## Union IC-LoRA Model
We introduce a new **Union IC-LoRA** model that combines depth, pose, and edge control conditions into a single unified LoRA.
### Key Features
- **Unified Control**: A single LoRA that supports multiple control conditions (depth, human pose, or edges).
- **Downsampled Latent Processing**: The union LoRA operates on a downsampled latent size, which reduces memory usage and significantly speeds up inference while maintaining quality.
### How It Works
The union LoRA is trained to understand and respond to all three control signals (depth maps, pose skeletons, and edge maps) within a single model. The model learns to:
1. **Parse multiple conditions**: Identify which control signals are present in the input
2. **Process at reduced resolution**: Work on downsampled latents to improve efficiency
## Required Models
@@ -77,6 +94,7 @@ Download the following models:
* [`Gemma 3`](https://huggingface.co/google/gemma-3-12b-it-qat-q4_0-unquantized)
**LoRAs** Choose and download to `COMFYUI_ROOT_FOLDER/models/loras` folder.
* [`ltx-2-19b-ic-lora-union-ref0.5.safetensors`](https://huggingface.co/Lightricks/LTX-2-19b-IC-LoRA-Union-Control/blob/main/ltx-2-19b-ic-lora-union-control-ref0.5.safetensors) *(Union LoRA - supports depth, pose, and edges in one model)*
* [`ltx-2-19b-ic-lora-canny-control.safetensors`](https://huggingface.co/Lightricks/LTX-2-19b-IC-LoRA-Canny-Control/blob/main/ltx-2-19b-ic-lora-canny-control.safetensors)
* [`ltx-2-19b-ic-lora-depth-control.safetensors`](https://huggingface.co/Lightricks/LTX-2-19b-IC-LoRA-Depth-Control/blob/main/ltx-2-19b-ic-lora-depth-control.safetensors)
* [`ltx-2-19b-ic-lora-detailer.safetensors`](https://huggingface.co/Lightricks/LTX-2-19b-IC-LoRA-Detailer/blob/main/ltx-2-19b-ic-lora-detailer.safetensors)
__init__.py
+7
View File
@@ -1,3 +1,5 @@
from .conditioning_loader import LTXVLoadConditioning
from .conditioning_saver import LTXVSaveConditioning
from .decoder_noise import DecoderNoise
from .dynamic_conditioning import DynamicConditioning
from .easy_samplers import (
@@ -11,6 +13,7 @@ from .gemma_api_conditioning import GemmaAPITextEncode
from .gemma_encoder import LTXVGemmaCLIPModelLoader, LTXVGemmaEnhancePrompt
from .guide import LTXVAddGuideAdvanced
from .guiders import GuiderParametersNode, MultimodalGuiderNode
from .iclora import LTXAddVideoICLoRAGuide, LTXICLoRALoaderModelOnly
from .latent_norm import (
LTXVAdainLatent,
LTXVPerStepAdainPatcher,
@@ -91,6 +94,10 @@ NODE_CLASS_MAPPINGS = {
"LowVRAMCheckpointLoader": LowVRAMCheckpointLoader,
"LowVRAMAudioVAELoader": LowVRAMAudioVAELoader,
"LowVRAMLatentUpscaleModelLoader": LowVRAMLatentUpscaleModelLoader,
"LTXAddVideoICLoRAGuide": LTXAddVideoICLoRAGuide,
"LTXICLoRALoaderModelOnly": LTXICLoRALoaderModelOnly,
"LTXVLoadConditioning": LTXVLoadConditioning,
"LTXVSaveConditioning": LTXVSaveConditioning,
}
# Consistent display names between static and dynamic node mappings in nodes_registry.py,
conditioning_loader.py
+82
View File
@@ -0,0 +1,82 @@
import hashlib
from pathlib import Path
from typing import Any
import folder_paths
import safetensors
import torch
from comfy_api.latest import io
from .nodes_registry import comfy_node
@comfy_node(name="LTXVLoadConditioning")
class LTXVLoadConditioning(io.ComfyNode):
@classmethod
def define_schema(cls) -> io.Schema:
files = folder_paths.get_filename_list("embeddings")
if not files:
files = [""]
return io.Schema(
node_id="LTXVLoadConditioning",
display_name="🅛🅣🅧 LTXV Load Conditioning",
category="lightricks/LTXV",
inputs=[
io.Combo.Input("file_name", options=sorted(files)),
io.Combo.Input("device", options=["cpu", "gpu"]),
],
outputs=[
io.Conditioning.Output(),
],
)
@classmethod
def execute(cls, file_name: str, device: str) -> io.NodeOutput:
file_path = folder_paths.get_full_path("embeddings", file_name)
if not Path(file_path).exists():
raise FileNotFoundError(f"Conditioning file not found: {file_path}")
target_device = "cpu"
if device == "gpu":
target_device = "cuda" if torch.cuda.is_available() else "cpu"
conditioning: list[list[Any]] = []
with safetensors.safe_open(
file_path, framework="pt", device=target_device
) as f:
tensor_keys = [k for k in f.keys() if k.startswith("conditioning_data_")]
for tensor_key in sorted(tensor_keys):
idx = tensor_key.replace("conditioning_data_", "")
tensor = f.get_tensor(tensor_key)
options: dict[str, Any] = {}
mask_key = f"attention_mask_{idx}"
if mask_key in f.keys():
options["attention_mask"] = f.get_tensor(mask_key)
conditioning.append([tensor, options])
if not conditioning:
raise ValueError(f"No conditioning data found in file: {file_name}")
return io.NodeOutput(conditioning)
@classmethod
def fingerprint_inputs(cls, file_name: str, device: str) -> str:
file_path = folder_paths.get_full_path("embeddings", file_name)
with open(file_path, "rb") as f:
return hashlib.sha256(f.read()).hexdigest()
@classmethod
def validate_inputs(cls, file_name: str, device: str) -> bool | str:
if not file_name:
return "No files found. Please save a conditioning first."
try:
file_path = folder_paths.get_full_path("embeddings", file_name)
if not Path(file_path).exists():
return f"File not found: {file_name}"
except Exception:
return f"Invalid file: {file_name}"
return True
conditioning_saver.py
+69
View File
@@ -0,0 +1,69 @@
from datetime import datetime
from pathlib import Path
import comfy.utils
import folder_paths
import torch
from comfy_api.latest import io, ui
from .nodes_registry import comfy_node
@comfy_node(name="LTXVSaveConditioning")
class LTXVSaveConditioning(io.ComfyNode):
@classmethod
def define_schema(cls) -> io.Schema:
return io.Schema(
node_id="LTXVSaveConditioning",
display_name="🅛🅣🅧 LTXV Save Conditioning",
category="lightricks/LTXV",
inputs=[
io.Conditioning.Input("conditioning"),
io.String.Input("filename", default="conditioning"),
io.Combo.Input("dtype", options=["bfloat16", "float16"]),
],
is_output_node=True,
)
@classmethod
def execute(cls, conditioning: list, filename: str, dtype: str) -> io.NodeOutput:
if not conditioning or len(conditioning) == 0:
raise ValueError("Conditioning is empty")
embeddings_folder = Path(folder_paths.get_folder_paths("embeddings")[0])
embeddings_folder.mkdir(parents=True, exist_ok=True)
sanitized_filename = "".join(
c for c in filename if c.isalnum() or c in ("_", "-", ".")
)
if not sanitized_filename:
sanitized_filename = "conditioning"
output_path = embeddings_folder / f"{sanitized_filename}.safetensors"
target_dtype = torch.bfloat16 if dtype == "bfloat16" else torch.float16
tensors_to_save: dict[str, torch.Tensor] = {}
for idx, (cond_tensor, cond_options) in enumerate(conditioning):
tensor_converted = cond_tensor.to(dtype=target_dtype).contiguous()
tensors_to_save[f"conditioning_data_{idx}"] = tensor_converted
if "attention_mask" in cond_options:
mask = cond_options["attention_mask"].contiguous()
tensors_to_save[f"attention_mask_{idx}"] = mask
metadata = {
"num_conditionings": str(len(conditioning)),
"dtype": dtype,
"created_at": str(datetime.now()),
}
comfy.utils.save_torch_file(
tensors_to_save, str(output_path), metadata=metadata
)
file_size_mb = output_path.stat().st_size / (1024 * 1024)
return io.NodeOutput(
ui=ui.Text(f"Saved: {output_path.name} ({file_size_mb:.2f} MB)")
)
example_workflows/LTX-2_ICLoRA_All_Distilled_ref0.5.json
+14193
View File
File diff suppressed because it is too large Load Diff
iclora.py
+276
View File
@@ -0,0 +1,276 @@
import logging
import comfy
import comfy.sd
import comfy_extras.nodes_lt as nodes_lt
import folder_paths
from comfy_api.latest import io
from .latents import LTXVDilateLatent
from .nodes_registry import NODES_DISPLAY_NAME_PREFIX, comfy_node
@comfy_node(name="LTXAddVideoICLoRAGuide")
class LTXAddVideoICLoRAGuide(io.ComfyNode):
PATCHIFIER = nodes_lt.SymmetricPatchifier(1, start_end=True)
@classmethod
def define_schema(cls):
return io.Schema(
node_id="LTXAddVideoICLoRAGuide",
display_name=NODES_DISPLAY_NAME_PREFIX + " Add Video IC-LoRA Guide",
category="Lightricks/IC-LoRA",
description=(
"Adds one or more conditioning frames starting at the specified frame index. "
"Supports both single images and multi-frame videos. "
"The latent_downscale_factor resizes input to a fraction of the target size "
"(1 = original, 2 = half, 3 = third, etc.) for IC-LoRA on small grids."
),
inputs=[
io.Conditioning.Input("positive"),
io.Conditioning.Input("negative"),
io.Vae.Input("vae"),
io.Latent.Input(
"latent",
tooltip="Video-only latent to condition. Must be a 5D video latent (batch, channels, frames, height, width).",
),
io.Image.Input("image"),
io.Int.Input(
"frame_idx",
default=0,
min=-9999,
max=9999,
tooltip=(
"Frame index to start the conditioning at. "
"For single-frame videos, any frame_idx value is acceptable. "
"For videos, frame_idx must be 1 modulo 8, otherwise it will be rounded "
"down to the nearest 1 modulo 8. Negative values are counted from the end of the video."
),
),
io.Float.Input(
"strength",
default=1.0,
min=0.0,
max=1.0,
step=0.01,
),
io.Float.Input(
"latent_downscale_factor",
default=1.0,
min=1.0,
max=10.0,
step=1.0,
tooltip="For IC-LoRA on small grid. 1 means original size, 2 means half size, 3 means third, etc.",
),
io.Combo.Input(
"crop",
options=["disabled", "center"],
default="disabled",
tooltip="Crop mode when resizing. 'center' crops to fit, 'disabled' stretches to fit.",
),
io.Boolean.Input(
"use_tiled_encode",
default=False,
tooltip="Enable tiled VAE encoding for large resolutions/long videos to reduce memory usage.",
),
io.Int.Input(
"tile_size",
default=256,
min=64,
max=512,
step=32,
tooltip="Spatial tile size for tiled encoding. Only used when use_tiled_encode is enabled.",
),
io.Int.Input(
"tile_overlap",
default=64,
min=16,
max=256,
step=16,
tooltip="Overlap between tiles for tiled encoding. Only used when use_tiled_encode is enabled.",
),
],
outputs=[
io.Conditioning.Output("positive"),
io.Conditioning.Output("negative"),
io.Latent.Output("latent"),
],
)
@classmethod
def encode(
cls,
vae,
latent_width,
latent_height,
images,
scale_factors,
latent_downscale_factor,
crop,
use_tiled_encode,
tile_size,
tile_overlap,
):
time_scale_factor, width_scale_factor, height_scale_factor = scale_factors
num_frames_to_keep = (
(images.shape[0] - 1) // time_scale_factor
) * time_scale_factor + 1
images = images[:num_frames_to_keep]
# Divide target size by latent_downscale_factor
target_width = int(latent_width * width_scale_factor / latent_downscale_factor)
target_height = int(
latent_height * height_scale_factor / latent_downscale_factor
)
pixels = comfy.utils.common_upscale(
images.movedim(-1, 1),
target_width,
target_height,
"bilinear",
crop=crop,
).movedim(1, -1)
encode_pixels = pixels[:, :, :, :3]
if use_tiled_encode:
guide_latent = vae.encode_tiled(
encode_pixels,
tile_x=tile_size,
tile_y=tile_size,
overlap=tile_overlap,
)
else:
guide_latent = vae.encode(encode_pixels)
return encode_pixels, guide_latent
@classmethod
def execute(
cls,
positive,
negative,
vae,
latent,
image,
frame_idx,
strength,
latent_downscale_factor,
crop,
use_tiled_encode,
tile_size,
tile_overlap,
) -> io.NodeOutput:
scale_factors = vae.downscale_index_formula
latent_image = latent["samples"]
noise_mask = nodes_lt.get_noise_mask(latent)
_, _, latent_length, latent_height, latent_width = latent_image.shape
image, guide_latent = cls.encode(
vae,
latent_width,
latent_height,
image,
scale_factors,
latent_downscale_factor,
crop,
use_tiled_encode,
tile_size,
tile_overlap,
)
guide_mask = None
# Dilate the latent if latent_downscale_factor > 1
if latent_downscale_factor > 1:
if (
latent_width % latent_downscale_factor != 0
or latent_height % latent_downscale_factor != 0
):
raise ValueError(
f"Latent spatial size {latent_width}x{latent_height} must be divisible by latent_downscale_factor {latent_downscale_factor}"
)
dilated = LTXVDilateLatent().dilate_latent(
{"samples": guide_latent},
horizontal_scale=int(latent_downscale_factor),
vertical_scale=int(latent_downscale_factor),
)[0]
guide_mask = dilated["noise_mask"]
guide_latent = dilated["samples"]
frame_idx, latent_idx = nodes_lt.LTXVAddGuide.get_latent_index(
positive, latent_length, len(image), frame_idx, scale_factors
)
assert (
latent_idx + guide_latent.shape[2] <= latent_length
), "Conditioning frames exceed the length of the latent sequence."
positive, negative, latent_image, noise_mask = (
nodes_lt.LTXVAddGuide.append_keyframe(
positive,
negative,
frame_idx,
latent_image,
noise_mask,
guide_latent,
strength,
scale_factors,
guide_mask=guide_mask,
latent_downscale_factor=latent_downscale_factor,
)
)
return io.NodeOutput(
positive, negative, {"samples": latent_image, "noise_mask": noise_mask}
)
@comfy_node(name="LTXICLoRALoaderModelOnly")
class LTXICLoRALoaderModelOnly(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="LTXICLoRALoaderModelOnly",
display_name=NODES_DISPLAY_NAME_PREFIX + " IC-LoRA Loader Model Only",
category="Lightricks/IC-LoRA",
description="Loads a LoRA model and extracts the latent_downscale_factor from the safetensors metadata.",
inputs=[
io.Model.Input("model"),
io.Combo.Input(
"lora_name",
options=folder_paths.get_filename_list("loras"),
),
io.Float.Input(
"strength_model",
default=1.0,
min=-100.0,
max=100.0,
step=0.01,
),
],
outputs=[
io.Model.Output("model"),
io.Float.Output("latent_downscale_factor"),
],
)
@classmethod
def execute(cls, model, lora_name, strength_model) -> io.NodeOutput:
lora_path = folder_paths.get_full_path_or_raise("loras", lora_name)
# Load lora and extract metadata
lora, metadata = comfy.utils.load_torch_file(
lora_path, safe_load=True, return_metadata=True
)
# Extract latent_downscale_factor from metadata
try:
latent_downscale_factor = float(metadata["reference_downscale_factor"])
except (KeyError, ValueError, TypeError):
latent_downscale_factor = 1.0
logging.warning(
"Failed to extract reference_downscale_factor from metadata for %s, using 1.0",
lora_path,
)
if strength_model == 0:
return io.NodeOutput(model, latent_downscale_factor)
model_lora, _ = comfy.sd.load_lora_for_models(
model, None, lora, strength_model, 0
)
return io.NodeOutput(model_lora, latent_downscale_factor)
nodes_registry.py
+28
View File
@@ -1,4 +1,5 @@
import re
from functools import wraps
from typing import Callable, Optional, Type
NODE_CLASS_MAPPINGS = {}
@@ -36,6 +37,29 @@ def register_node(node_class: Type, name: str, description: str) -> None:
NODE_DISPLAY_NAME_MAPPINGS[name] = description
def _is_v3_node(node_class: Type) -> bool:
"""Check if the node class is a v3 node (has define_schema method)."""
return hasattr(node_class, "define_schema") and callable(
getattr(node_class, "define_schema")
)
def _wrap_define_schema(node_class: Type, display_name: str) -> None:
"""Wrap the define_schema method to inject the display_name into the schema."""
original_define_schema = node_class.define_schema
@classmethod
@wraps(original_define_schema.__func__)
def wrapped_define_schema(cls):
schema = original_define_schema.__func__(cls)
# Only set display_name if not already set in the schema
if schema.display_name is None:
schema.display_name = display_name
return schema
node_class.define_schema = wrapped_define_schema
def comfy_node(
node_class: Optional[Type] = None,
name: Optional[str] = None,
@@ -82,6 +106,10 @@ def comfy_node(
description = _format_description(description, name, experimental, deprecated)
# For v3 nodes, wrap define_schema to inject the display_name
if _is_v3_node(node_class):
_wrap_define_schema(node_class, description)
register_node(node_class, name, description)
return node_class