Compare commits

..
Author SHA1 Message Date
github-actions[bot] d164816f8a Automated PR - 2026-09-22 2026-09-22 17:39:48 +00:00
DoronGenzelHass cb1f8204dc Merge pull request #564 from Lightricks/pr-2026-09-22
Update ComfyUI-LTXVideo - 2026-09-22
2026-09-22 11:27:55 +03:00
github-actions[bot] 35364168d9 Automated PR - 2026-09-22 2026-09-22 08:20:29 +00:00
Noa Lichtenstein dfb2786749 Merge pull request #560 from Lightricks/pr-2026-09-17
Update ComfyUI-LTXVideo - 2026-09-17
2026-09-17 18:25:06 -04:00
github-actions[bot] 63a7f34e64 Automated PR - 2026-09-17 2026-09-17 22:05:55 +00:00
24 changed files with 1631 additions and 2267 deletions
+2 -2
View File
@@ -86,13 +86,13 @@ The union LoRA is trained to understand and respond to both control signals (dep
## HDR IC-LoRA
We provide an **HDR IC-LoRA** that generates linear HDR video encoded in ARRI LogC3, enabling workflows that output high-dynamic-range content suitable for grading and EXR export.
We provide an **HDR IC-LoRA** that upgrades SDR video into ACEScct HDR (same working space as pipelines `hdr_ic_lora`), suitable for grading, EXR export, and HLG masters. Use the **2.5** graph: [`LTX-2.5_ICLoRA_HDR_Distilled.json`](./example_workflows/2.5/LTX-2.5_ICLoRA_HDR_Distilled.json). The 2.3 workflow remains as a legacy LogC3 example.
### Key Features
- **Linear HDR output**: The LoRA produces frames in LogC3-compressed space; the `LTXVHDRDecodePostprocess` node decodes these back to linear HDR values.
- **SDR preview + raw HDR**: The node outputs both a Reinhard-tonemapped SDR preview and the raw linear HDR tensor for downstream use.
- **EXR export**: Optionally writes the linear HDR frames as a 16/32-bit EXR image sequence. To enable EXR writing, set `OPENCV_IO_ENABLE_OPENEXR=1` in the environment before starting ComfyUI. The exported EXR sequence is best viewed in [DJV](https://github.com/grizzlypeak3d/DJV) (or [DJV for macOS](https://djv.en.uptodown.com/mac/download)).
- **EXR export**: Optionally writes the linear HDR frames as a 16/32-bit EXR image sequence via **OpenImageIO** (`pip install openimageio`, also listed in `requirements.txt`). The exported EXR sequence is best viewed in [DJV](https://github.com/grizzlypeak3d/DJV) (or [DJV for macOS](https://djv.en.uptodown.com/mac/download)).
## Dub-It IC-LoRA
+11 -5
View File
@@ -14,7 +14,13 @@ from .gemma_api_conditioning import GemmaAPITextEncode
from .gemma_encoder import LTXVGemmaCLIPModelLoader, LTXVGemmaEnhancePrompt
from .guide import LTXVAddGuideAdvanced, LTXVAddGuideAdvancedAttention
from .guiders import GuiderParametersNode, MultimodalGuiderNode
from .hdr import LTXVHDRDecodePostprocess
from .hdr_nodes import (
LTXVHDRDecodePostprocess,
LTXVLoadEXRSequence,
LTXVSaveHLG,
LTXVSDRToHDRWorkingSpace,
LTXVVAEForceFloat32,
)
from .iclora import (
LTXAddVideoICLoRAGuide,
LTXAddVideoICLoRAGuideAdvanced,
@@ -28,7 +34,6 @@ from .latent_norm import (
LTXVStatNormLatent,
)
from .latents import (
LTXVAddLatentGuide,
LTXVImgToVideoConditionOnly,
LTXVSelectLatents,
LTXVSetVideoLatentNoiseMasks,
@@ -40,7 +45,6 @@ from .low_vram_loaders import (
LowVRAMLatentUpscaleModelLoader,
)
from .masks import LTXVPreprocessMasks
from .matte_refine import LTXRefineAlphaMatte
from .nodes_registry import NODE_CLASS_MAPPINGS as RUNTIME_NODE_CLASS_MAPPINGS
from .nodes_registry import (
NODE_DISPLAY_NAME_MAPPINGS as RUNTIME_NODE_DISPLAY_NAME_MAPPINGS,
@@ -70,7 +74,6 @@ NODE_CLASS_MAPPINGS = {
"LTXVLinearOverlapLatentTransition": LinearOverlapLatentTransition,
"LTXVAddGuideAdvanced": LTXVAddGuideAdvanced,
"LTXVAddGuideAdvancedAttention": LTXVAddGuideAdvancedAttention,
"LTXVAddLatentGuide": LTXVAddLatentGuide,
"LTXVAdainLatent": LTXVAdainLatent,
"LTXVImgToVideoConditionOnly": LTXVImgToVideoConditionOnly,
"LTXVPerStepAdainPatcher": LTXVPerStepAdainPatcher,
@@ -119,9 +122,12 @@ NODE_CLASS_MAPPINGS = {
"LTXVInpaintPreprocess": LTXVInpaintPreprocess,
"LTXVLaplacianPyramidBlend": LTXVLaplacianPyramidBlend,
"LTXVHDRDecodePostprocess": LTXVHDRDecodePostprocess,
"LTXVSDRToHDRWorkingSpace": LTXVSDRToHDRWorkingSpace,
"LTXVLoadEXRSequence": LTXVLoadEXRSequence,
"LTXVVAEForceFloat32": LTXVVAEForceFloat32,
"LTXVSaveHLG": LTXVSaveHLG,
"LTXVAudioOnlyModel": LTXVAudioOnlyModel,
"LTXVAudioOnlyEmptyVideoLatent": LTXVAudioOnlyEmptyVideoLatent,
"LTXRefineAlphaMatte": LTXRefineAlphaMatte,
}
# Consistent display names between static and dynamic node mappings in nodes_registry.py,
-202
View File
@@ -1,202 +0,0 @@
"""Shared matte → RGBA / over-plate helpers for Refine and Over Background."""
from __future__ import annotations
import math
from typing import Any
import torch
import torch.nn.functional as F
def nchw_mask(mask: torch.Tensor) -> torch.Tensor:
mask = mask.to(dtype=torch.float32)
if mask.ndim == 2:
mask = mask.unsqueeze(0)
return mask
def rgb_frames(image: torch.Tensor) -> torch.Tensor:
if image.ndim == 3:
image = image.unsqueeze(0)
return image[..., :3]
def frames_from_video(video: Any) -> torch.Tensor:
if hasattr(video, "get_components"):
images = video.get_components().images
else:
images = video
return rgb_frames(images)
def luma_mask(image: torch.Tensor) -> torch.Tensor:
"""IMAGE matte → MASK. 4-channel uses alpha; otherwise the red channel
(same as Comfy ImageToMask `red`, which matches a grayscale RGB matte)."""
if image.ndim == 3:
image = image.unsqueeze(0)
image = image.to(dtype=torch.float32)
if image.shape[-1] >= 4:
return image[..., 3].clamp(0.0, 1.0)
return image[..., 0].clamp(0.0, 1.0)
def mask_from_matte(
matte: torch.Tensor | None,
mask: torch.Tensor | None,
) -> torch.Tensor:
if matte is not None:
return luma_mask(matte)
if mask is None:
raise ValueError("Connect matte frames (IMAGE) or a MASK.")
return nchw_mask(mask)
def resize_rgb(image: torch.Tensor, height: int, width: int) -> torch.Tensor:
if image.shape[1] == height and image.shape[2] == width:
return image
x = image.movedim(-1, 1)
x = F.interpolate(x, size=(height, width), mode="bilinear", align_corners=False)
return x.movedim(1, -1)
def resize_mask(mask: torch.Tensor, height: int, width: int) -> torch.Tensor:
if mask.shape[-2:] == (height, width):
return mask
return F.interpolate(
mask.unsqueeze(1),
size=(height, width),
mode="bilinear",
align_corners=False,
).squeeze(1)
def match_length(plate: torch.Tensor, frames: int, length: str) -> torch.Tensor:
count = plate.shape[0]
if count == frames:
return plate
if count > frames:
return plate[:frames]
if length == "loop" and count > 1:
reps = int(math.ceil(frames / count))
return plate.repeat(reps, *([1] * (plate.ndim - 1)))[:frames]
extra = frames - count
return torch.cat([plate, plate[-1:].expand(extra, *([-1] * (plate.ndim - 1)))], dim=0)
def match_mask_to_rgb(
alpha: torch.Tensor,
frames: int,
height: int,
width: int,
length: str,
) -> torch.Tensor:
if alpha.shape[-2:] != (height, width):
alpha = resize_mask(alpha, height, width)
if alpha.shape[0] == 1 and frames > 1:
return alpha.expand(frames, -1, -1)
if alpha.shape[0] != frames:
return match_length(alpha.unsqueeze(-1), frames, length).squeeze(-1)
return alpha
def fit_plate(plate: torch.Tensor, height: int, width: int, fit: str) -> torch.Tensor:
plate = plate[..., :3]
n, h, w, _ = plate.shape
if h == height and w == width:
return plate
if fit == "stretch":
return resize_rgb(plate, height, width)
if fit == "contain":
scale = min(height / h, width / w)
else:
scale = max(height / h, width / w)
nh = max(1, int(round(h * scale)))
nw = max(1, int(round(w * scale)))
if fit == "contain":
nh = min(nh, height)
nw = min(nw, width)
else:
nh = max(nh, height)
nw = max(nw, width)
x = resize_rgb(plate, nh, nw)
if fit == "contain":
canvas = torch.zeros((n, height, width, 3), device=x.device, dtype=x.dtype)
top = (height - nh) // 2
left = (width - nw) // 2
canvas[:, top : top + nh, left : left + nw] = x
return canvas
top = max(0, (nh - height) // 2)
left = max(0, (nw - width) // 2)
cropped = x[:, top : top + height, left : left + width]
if cropped.shape[1] != height or cropped.shape[2] != width:
return resize_rgb(plate, height, width)
return cropped
def pad_even(image: torch.Tensor) -> torch.Tensor:
"""H.264 yuv420p cannot encode odd width or height."""
n, h, w, c = image.shape
ph = h + (h % 2)
pw = w + (w % 2)
if ph == h and pw == w:
return image
out = image.new_zeros((n, ph, pw, c))
out[:, :h, :w] = image
return out
def resolve_plate(
background: str,
height: int,
width: int,
frames: int,
fit: str,
length: str,
device: torch.device,
dtype: torch.dtype,
background_image: torch.Tensor | None = None,
background_video: Any | None = None,
) -> torch.Tensor | None:
if background == "none":
return None
if background == "image":
if background_image is None:
raise ValueError(
"background=image but no still is loaded. Use Load plate on this node, "
"or connect background_image."
)
plate = rgb_frames(background_image).to(device=device, dtype=dtype)
else:
if background_video is None:
raise ValueError(
"background=video but no clip is loaded. Use Load plate on this node, "
"or connect background_video."
)
plate = frames_from_video(background_video).to(device=device, dtype=dtype)
plate = fit_plate(plate, height, width, fit)
return match_length(plate, frames, length)
def over_composite(
fg: torch.Tensor,
alpha: torch.Tensor,
plate: torch.Tensor | None,
) -> torch.Tensor:
fg = fg[..., :3].contiguous()
if plate is None:
return pad_even(fg)
if fg.shape[0] == 1 and plate.shape[0] > 1:
fg = fg.expand(plate.shape[0], -1, -1, -1)
alpha = alpha.expand(plate.shape[0], -1, -1)
a = alpha.unsqueeze(-1).to(device=fg.device, dtype=fg.dtype)
out = fg * a + plate * (1.0 - a)
return pad_even(out.clamp(0.0, 1.0).contiguous())
def join_rgba(rgb: torch.Tensor, alpha: torch.Tensor) -> torch.Tensor:
"""Straight RGBA. Alpha 1 = keep subject (does not invert like Join Image with Alpha)."""
rgb = rgb[..., :3]
a = alpha.unsqueeze(-1).to(device=rgb.device, dtype=rgb.dtype).clamp(0.0, 1.0)
return torch.cat([rgb, a], dim=-1).contiguous()
+90 -9
View File
@@ -9,6 +9,76 @@ from comfy_api.latest import io
from .nodes_registry import comfy_node
# Pipeline HDR IC-LoRA scene embeddings (`hdr_ic_lora._load_video_context`).
_PIPELINE_VIDEO_KEYS = ("video_context", "video_prompt_embeds")
_PIPELINE_AUDIO_KEYS = ("audio_context", "audio_prompt_embeds")
# LTX-AV text encode layout: video 4096 + audio 2048 on the last dim.
_VIDEO_CONTEXT_DIM = 4096
_AUDIO_CONTEXT_DIM = 2048
_AV_CONTEXT_DIM = _VIDEO_CONTEXT_DIM + _AUDIO_CONTEXT_DIM
def _unsqueeze_batch(t: torch.Tensor, *, source: str, kind: str) -> torch.Tensor:
"""Ensure conditioning tensors are ``[B, T, D]`` (pipeline files are often ``[T, D]``)."""
if t.ndim == 2:
return t.unsqueeze(0)
if t.ndim == 3:
return t
raise ValueError(
f"{source}: {kind} embedding must be 2D [T, D] or 3D [B, T, D]; "
f"got shape {tuple(t.shape)}"
)
def _load_pipeline_scene_embedding(f: Any, source: str) -> list[list[Any]]:
"""Load ``video_context`` (+ optional ``audio_context``) into Comfy CONDITIONING.
Matches ``ltx_pipelines.hdr_ic_lora``: the safetensors replaces text encoding.
For AV Comfy models, video (4096) and audio (2048) contexts are concatenated on
the last dim — same layout as a fully processed LTX-AV text encode.
"""
keys = list(f.keys())
video_key = next((k for k in _PIPELINE_VIDEO_KEYS if k in keys), None)
if video_key is None:
return []
video = _unsqueeze_batch(f.get_tensor(video_key), source=source, kind=video_key)
video_dim = int(video.shape[-1])
audio_key = next((k for k in _PIPELINE_AUDIO_KEYS if k in keys), None)
if video_dim == _AV_CONTEXT_DIM:
# Already video∥audio on the last dim (rare export); use as-is.
tensor = video
elif video_dim == _VIDEO_CONTEXT_DIM:
if audio_key is None:
raise ValueError(
f"{source}: {video_key} last dim is {_VIDEO_CONTEXT_DIM} but no "
f"audio_context / audio_prompt_embeds found. AV conditioning needs "
f"{_VIDEO_CONTEXT_DIM}+{_AUDIO_CONTEXT_DIM}="
f"{_AV_CONTEXT_DIM} on the last dim."
)
audio = _unsqueeze_batch(f.get_tensor(audio_key), source=source, kind=audio_key)
audio_dim = int(audio.shape[-1])
if audio_dim != _AUDIO_CONTEXT_DIM:
raise ValueError(
f"{source}: {audio_key} last dim must be {_AUDIO_CONTEXT_DIM}; "
f"got {audio_dim} with shape {tuple(audio.shape)}"
)
if audio.shape[:-1] != video.shape[:-1]:
raise ValueError(
f"{source}: audio/video context shape mismatch "
f"(excluding last dim): {tuple(audio.shape)} vs {tuple(video.shape)}"
)
tensor = torch.cat([video, audio], dim=-1)
else:
raise ValueError(
f"{source}: {video_key} last dim must be {_VIDEO_CONTEXT_DIM} "
f"(or pre-concatenated {_AV_CONTEXT_DIM}); got {video_dim} with shape "
f"{tuple(video.shape)}. Check that video/audio keys are not swapped."
)
return [[tensor, {}]]
@comfy_node(name="LTXVLoadConditioning")
class LTXVLoadConditioning(io.ComfyNode):
@@ -21,6 +91,11 @@ class LTXVLoadConditioning(io.ComfyNode):
node_id="LTXVLoadConditioning",
display_name="🅛🅣🅧 LTXV Load Conditioning",
category="lightricks/LTXV",
description=(
"Load precomputed text conditioning from models/embeddings. "
"Supports Comfy conditioning_data_* files and pipeline scene "
"embeddings (video_context / audio_context), e.g. HDR IC-LoRA."
),
inputs=[
io.Combo.Input("file_name", options=sorted(files)),
io.Combo.Input("device", options=["cpu", "gpu"]),
@@ -47,19 +122,25 @@ class LTXVLoadConditioning(io.ComfyNode):
) as f:
tensor_keys = [k for k in f.keys() if k.startswith("conditioning_data_")]
for tensor_key in sorted(tensor_keys):
idx = tensor_key.replace("conditioning_data_", "")
tensor = f.get_tensor(tensor_key)
if tensor_keys:
for tensor_key in sorted(tensor_keys):
idx = tensor_key.replace("conditioning_data_", "")
tensor = f.get_tensor(tensor_key)
options: dict[str, Any] = {}
mask_key = f"attention_mask_{idx}"
if mask_key in f.keys():
options["attention_mask"] = f.get_tensor(mask_key)
options: dict[str, Any] = {}
mask_key = f"attention_mask_{idx}"
if mask_key in f.keys():
options["attention_mask"] = f.get_tensor(mask_key)
conditioning.append([tensor, options])
conditioning.append([tensor, options])
else:
conditioning = _load_pipeline_scene_embedding(f, source=file_name)
if not conditioning:
raise ValueError(f"No conditioning data found in file: {file_name}")
raise ValueError(
f"No conditioning data found in file: {file_name}. "
"Expected conditioning_data_* or video_context / video_prompt_embeds."
)
return io.NodeOutput(conditioning)
+8 -8
View File
@@ -12,7 +12,7 @@ from comfy_extras.nodes_lt import EmptyLTXVLatentVideo, LTXVAddGuide, LTXVCropGu
from .guide import blur_internal
from .latent_norm import LTXVAdainLatent
from .latents import LTXVAddLatentGuide, LTXVSelectLatents
from .latents import LTXVSelectLatents, add_latent_guide
from .nodes_registry import comfy_node
@@ -248,7 +248,7 @@ class LTXVBaseSampler:
positive,
negative,
latents,
) = LTXVAddLatentGuide().generate(
) = add_latent_guide(
vae=vae,
positive=positive,
negative=negative,
@@ -446,7 +446,7 @@ class LTXVExtendSampler:
positive,
negative,
new_latents,
) = LTXVAddLatentGuide().generate(
) = add_latent_guide(
vae=vae,
positive=positive,
negative=negative,
@@ -504,7 +504,7 @@ class LTXVExtendSampler:
positive,
negative,
new_latents,
) = LTXVAddLatentGuide().generate(
) = add_latent_guide(
vae=vae,
positive=positive,
negative=negative,
@@ -519,7 +519,7 @@ class LTXVExtendSampler:
positive,
negative,
new_latents,
) = LTXVAddLatentGuide().generate(
) = add_latent_guide(
vae=vae,
positive=positive,
negative=negative,
@@ -552,7 +552,7 @@ class LTXVExtendSampler:
positive,
negative,
denoised_output_latents,
) = LTXVAddLatentGuide().generate(
) = add_latent_guide(
vae=vae,
positive=positive,
negative=negative,
@@ -756,7 +756,7 @@ class LTXVInContextSampler:
positive,
negative,
new_latents,
) = LTXVAddLatentGuide().generate(
) = add_latent_guide(
vae=vae,
positive=positive,
negative=negative,
@@ -792,7 +792,7 @@ class LTXVInContextSampler:
positive,
negative,
new_latents,
) = LTXVAddLatentGuide().generate(
) = add_latent_guide(
vae=vae,
positive=positive,
negative=negative,
@@ -1119,7 +1119,7 @@
"outputs": [],
"properties": {},
"widgets_values": [
"Set OPENCV_IO_ENABLE_OPENEXR=1 in your shell before launching ComfyUI, otherwise save_exr will fail"
"EXR export requires OpenImageIO (pip install openimageio). Same backend as ltx-pipelines."
],
"color": "#432",
"bgcolor": "#653"
@@ -1407,6 +1407,16 @@
"type": "IMAGE",
"link": 13664
},
{
"localized_name": "transfer",
"name": "transfer",
"shape": 7,
"type": "COMBO",
"widget": {
"name": "transfer"
},
"link": null
},
{
"localized_name": "exposure",
"name": "exposure",
@@ -1427,6 +1437,16 @@
},
"link": null
},
{
"localized_name": "exr_color_space",
"name": "exr_color_space",
"shape": 7,
"type": "COMBO",
"widget": {
"name": "exr_color_space"
},
"link": null
},
{
"localized_name": "output_dir",
"name": "output_dir",
@@ -1478,8 +1498,10 @@
"Node name for S&R": "LTXVHDRDecodePostprocess"
},
"widgets_values": [
"logc3",
7.1,
true,
"linear",
"output/hdr_exr3",
"frame",
true
@@ -3086,8 +3108,10 @@
},
"5114": {
"inputs": {
"transfer": "logc3",
"exposure": 7.1,
"save_exr": true,
"exr_color_space": "linear",
"output_dir": "output/hdr_exr3",
"filename_prefix": "frame",
"half_precision": true,
@@ -7636,7 +7636,7 @@
"Node name for S&R": "ManualSigmas"
},
"widgets_values": [
"0.85, 0.7250, 0.4219, 0.0"
"0.909375, 0.725, 0.421875, 0.0"
]
},
{
@@ -5314,7 +5314,7 @@
"ver": "0.14.1"
},
"widgets_values": [
"0.85, 0.7250, 0.4219, 0.0"
"0.909375, 0.725, 0.421875, 0.0"
]
},
{
@@ -6019,7 +6019,7 @@
"ver": "0.14.1"
},
"widgets_values": [
"0.85, 0.7250, 0.4219, 0.0"
"0.909375, 0.725, 0.421875, 0.0"
]
},
{
+1 -1
View File
@@ -98,6 +98,6 @@ Two-stage is the better default when you care about spatial detail. Single-stage
| Ingredients | 1 | — | Reference sheet | Cast / props / location from one image |
| Motion Track | 1 | — | Image + drawn tracks | Directed motion from a still |
| Inpaint two-stage | 2 | 2× spatial | Video + mask (+ frozen audio) | Replacing a region |
| Outpaint two-stage | 2 | 2× spatial | Video + mask (+ frozen audio) | Extending the frame |
| Outpaint two-stage | 2 | 2× spatial | Video + target size (+ frozen audio) | Extending the frame |
Python / pipeline equivalents of these ideas live in [pipeline-selection.md](https://github.com/Lightricks/LTX-2/blob/main/packages/ltx-pipelines/docs/pipeline-selection.md) in the LTX-2 repo.
+56 -16
View File
@@ -4,6 +4,7 @@ Replaces the CLIP encoding step entirely using an external API.
"""
import io
import json
import logging
import pickle
@@ -23,14 +24,18 @@ UPDATE_MESSAGE = (
INVALID_API_KEY_MESSAGE = (
"Invalid API key. Please generate a new API key at: https://console.ltx.video/"
)
MISSING_MODEL_ID_MESSAGE = "Model ID cannot be identified from the provided model file"
MISSING_MODEL_SELECTOR_MESSAGE = (
"Cannot identify the text encoder from the provided model file."
)
MODEL_ID_METADATA_KEY = "encrypted_wandb_properties"
GEMMA_SOURCE_CHECKPOINT_METADATA_KEY = "gemma_source_checkpoint"
MODEL_FOLDERS = ("checkpoints", "diffusion_models")
def model_filename_list() -> list[str]:
"""Every model the node can read a model id from, in folder order and de-duplicated.
"""Every model the node can read a text-encoder selector from, de-duplicated.
A name present in both folders is listed once; resolve_model_path resolves
it the same way round, so the entry always refers to the same file the
@@ -55,17 +60,49 @@ def resolve_model_path(ckpt_name: str) -> str:
)
def extract_model_id(ckpt_name: str) -> str:
model_id_key = "encrypted_wandb_properties"
def parse_gemma_source_checkpoint(raw: str) -> dict[str, str]:
"""Parse the ``gemma_source_checkpoint`` safetensors metadata JSON string."""
try:
obj = json.loads(raw)
except json.JSONDecodeError as exc:
raise ValueError(MISSING_MODEL_SELECTOR_MESSAGE) from exc
if not isinstance(obj, dict):
raise ValueError(MISSING_MODEL_SELECTOR_MESSAGE)
ltx_version, gemma_version = obj.get("ltx_version"), obj.get("gemma_version")
if not ltx_version or not gemma_version:
raise ValueError(MISSING_MODEL_SELECTOR_MESSAGE)
return {"ltx_version": str(ltx_version), "gemma_version": str(gemma_version)}
def prompt_embedding_selector_from_metadata(
metadata: dict[str, str] | None,
) -> dict[str, str | dict[str, str]]:
"""Build the XOR selector for ``/v1/prompt-embedding``.
LTX-2.5+ checkpoints identify the text encoder via ``gemma_source_checkpoint``
(``ltx_version`` + ``gemma_version``). Pre-2.5 checkpoints use
``encrypted_wandb_properties`` as ``model_id``. Prefer the 2.5+ selector when
both are present: OS 2.5 files may still carry an unusable ``model_id``.
"""
if metadata:
raw_gemma = metadata.get(GEMMA_SOURCE_CHECKPOINT_METADATA_KEY)
if raw_gemma:
return {"model": parse_gemma_source_checkpoint(raw_gemma)}
model_id = metadata.get(MODEL_ID_METADATA_KEY)
if model_id:
return {"model_id": model_id}
raise ValueError(MISSING_MODEL_SELECTOR_MESSAGE)
def extract_prompt_embedding_selector(
ckpt_name: str,
) -> dict[str, str | dict[str, str]]:
with safe_open(
resolve_model_path(ckpt_name),
framework="pt",
device="cpu",
) as f:
metadata = f.metadata()
if not metadata or model_id_key not in metadata:
raise ValueError(MISSING_MODEL_ID_MESSAGE)
return metadata[model_id_key]
return prompt_embedding_selector_from_metadata(f.metadata())
@comfy_node(name="GemmaAPITextEncode")
@@ -74,8 +111,8 @@ class GemmaAPITextEncode:
Encodes text prompts using the LTX Video API, returning CONDITIONING for LTX-2 models.
This node replaces the local CLIP encoding step by sending the prompt to an external API
for processing. It requires an API key and automatically extracts the model ID from the
checkpoint file metadata.
for processing. It requires an API key and selects the text encoder from checkpoint
metadata: ``model`` (LTX-2.5+) or ``model_id`` (LTX-2.3).
Inputs:
- api_key: Authentication key for the LTX Video API
@@ -117,9 +154,8 @@ class GemmaAPITextEncode:
"ckpt_name": (
model_filename_list(),
{
"tooltip": "The model to read the API model id from. "
"Either a checkpoint or a diffusion model; it must "
"carry the model id in its safetensors metadata."
"tooltip": "The model to read the API text-encoder selector from. "
"Either a checkpoint or a diffusion model."
},
),
},
@@ -142,14 +178,18 @@ class GemmaAPITextEncode:
if not ckpt_name or not ckpt_name.strip():
raise ValueError("Model path is required")
model_id = extract_model_id(ckpt_name)
selector = extract_prompt_embedding_selector(ckpt_name)
payload = {
"prompt": prompt,
"model_id": model_id,
"enhance_prompt": enhance_prompt,
**selector,
}
if "model_id" in selector:
selector_desc = f"model_id: {str(selector['model_id'])[:50]}..."
else:
selector_desc = f"model: {selector['model']}"
logger.info(
f"Calling API to encode prompt: {prompt[:50]}... with model_id: {model_id[:50]}..."
f"Calling API to encode prompt: {prompt[:50]}... with {selector_desc}"
)
try:
response = requests.post(
-245
View File
@@ -1,245 +0,0 @@
"""HDR utilities and ComfyUI node for HDR IC-LoRA inference.
Provides HDR compression/decompression transforms and a single ComfyUI node:
- LTXVHDRDecodePostprocess: Decompress VAE output, tonemap for preview,
and optionally save raw linear HDR frames as EXR.
"""
import logging
import os
import torch
from torch import Tensor
from .nodes_registry import comfy_node
logger = logging.getLogger("LTXVideo.hdr")
# ---------------------------------------------------------------------------
# HDR Transform Classes (ported from ltx_core/hdr.py)
# ---------------------------------------------------------------------------
class LogC3:
"""ARRI LogC3 (EI 800) HDR compression.
Maps linear [0, inf) -> [0, 1] via the camera log curve, then scales to
[-1, 1] for VAE input.
"""
A = 5.555556
B = 0.052272
C = 0.247190
D = 0.385537
E = 5.367655
F = 0.092809
CUT = 0.010591
def compress(self, hdr: Tensor) -> Tensor:
x = torch.clamp(hdr, min=0.0)
log_part = self.C * torch.log10(self.A * x + self.B) + self.D
lin_part = self.E * x + self.F
logc = torch.where(x >= self.CUT, log_part, lin_part)
logc = torch.clamp(logc, 0.0, 1.0)
return logc * 2.0 - 1.0
def decompress(self, z: Tensor) -> Tensor:
logc = torch.clamp((z + 1.0) / 2.0, 0.0, 1.0)
cut_log = self.E * self.CUT + self.F
lin_from_log = (torch.pow(10.0, (logc - self.D) / self.C) - self.B) / self.A
lin_from_lin = (logc - self.F) / self.E
return torch.where(logc >= cut_log, lin_from_log, lin_from_lin)
# ---------------------------------------------------------------------------
# Transform registry — singleton (stateless)
# ---------------------------------------------------------------------------
_LOGC3 = LogC3()
def _hdr_decompress(decoded_01: Tensor) -> Tensor:
"""Decompress VAE-decoded image from [0,1] to linear HDR [0, inf).
ComfyUI's VAE decode returns images in [0, 1] via ``(raw + 1) / 2``.
This function reverses that and applies LogC3 HDR decompression.
"""
raw = decoded_01.float() * 2.0 - 1.0
return _LOGC3.decompress(raw)
def _linear_to_srgb(x: Tensor) -> Tensor:
"""Convert linear [0, 1] to sRGB [0, 1]."""
return torch.where(
x <= 0.0031308,
12.92 * x,
1.055 * torch.pow(x.clamp(min=0.0031308), 1.0 / 2.4) - 0.055,
).clamp(0.0, 1.0)
# ---------------------------------------------------------------------------
# ComfyUI Node
# ---------------------------------------------------------------------------
@comfy_node(name="LTXVHDRDecodePostprocess")
class LTXVHDRDecodePostprocess:
"""Decompress HDR from VAE output, tonemap for preview, optionally save EXR.
Place after VAE Decode in an HDR IC-LoRA workflow. Recovers linear HDR
values from the compressed latent space and tonemaps them to SDR for
display. When ``save_exr`` is enabled, also writes the raw linear HDR
frames as an EXR image sequence.
make sure to set OPENCV_IO_ENABLE_OPENEXR=1 environment in the command line # Must be set before cv2 import
Outputs:
tonemapped: SDR preview [0, 1] after Reinhard tonemap + sRGB gamma.
hdr_linear: Raw linear HDR values [0, inf) for further processing.
"""
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"image": ("IMAGE",),
},
"optional": {
"exposure": (
"FLOAT",
{
"default": 0.0,
"min": -10.0,
"max": 10.0,
"step": 0.1,
"display": "slider",
"tooltip": (
"Exposure in stops (EV). 0 = no change, "
"+1 = 2x brighter, -1 = half brightness."
),
},
),
"save_exr": (
"BOOLEAN",
{
"default": False,
"tooltip": "Save raw linear HDR frames as EXR sequence.",
},
),
"output_dir": (
"STRING",
{
"default": "output/hdr_exr",
"tooltip": (
"Directory for EXR frames (relative to ComfyUI "
"output directory, or absolute path)."
),
},
),
"filename_prefix": (
"STRING",
{"default": "frame"},
),
"half_precision": (
"BOOLEAN",
{
"default": True,
"tooltip": (
"Save EXR as float16 (half). Smaller files, "
"negligible quality loss for most workflows."
),
},
),
},
}
RETURN_TYPES = ("IMAGE", "IMAGE")
RETURN_NAMES = ("tonemapped", "hdr_linear")
OUTPUT_NODE = True
FUNCTION = "postprocess"
CATEGORY = "Lightricks/HDR"
DESCRIPTION = (
"Decompresses VAE-decoded output from HDR IC-LoRA (LogC3) and applies "
"Reinhard tonemapping. Place after VAE Decode. 'tonemapped' is the "
"SDR preview; 'hdr_linear' is raw linear HDR for downstream use. "
"Enable 'save_exr' to write an EXR image sequence."
"if save_exr is enabled, make sure to set OPENCV_IO_ENABLE_OPENEXR=1 environment in the command line"
)
def postprocess(
self,
image: torch.Tensor,
exposure: float = 0.0,
save_exr: bool = False,
output_dir: str = "output/hdr_exr",
filename_prefix: str = "frame",
half_precision: bool = True,
) -> tuple:
hdr = _hdr_decompress(image)
hdr = torch.clamp(hdr, min=0.0, max=1e4)
# Reinhard tonemap with exposure (stops → linear multiplier)
exposure_mult = 2.0**exposure
hdr_exposed = hdr * exposure_mult
tonemapped_linear = (hdr_exposed / (1.0 + hdr_exposed)).clamp(0.0, 1.0)
tonemapped = _linear_to_srgb(tonemapped_linear)
if save_exr:
assert os.environ.get("OPENCV_IO_ENABLE_OPENEXR") == "1", (
"EXR output is enabled (save_exr = TRUE), but OpenCV does not support EXR by default. "
"To enable it, set the environment variable OPENCV_IO_ENABLE_OPENEXR=1 before starting ComfyUI, then restart. "
"Alternatively, disable EXR output or switch to PNG/JPG."
)
self._save_exr_frames(hdr, output_dir, filename_prefix, half_precision)
return (tonemapped, hdr)
@staticmethod
def _save_exr_frames(
hdr_image: torch.Tensor,
output_dir: str,
filename_prefix: str,
half_precision: bool,
) -> None:
try:
import cv2
except ImportError:
logger.error(
"opencv-python is required for EXR export. "
"Install with: pip install opencv-python"
)
return
import folder_paths
import numpy as np
if not os.path.isabs(output_dir):
output_dir = os.path.join(folder_paths.get_output_directory(), output_dir)
os.makedirs(output_dir, exist_ok=True)
frames = hdr_image.cpu().numpy()
exr_type = (
cv2.IMWRITE_EXR_TYPE_HALF if half_precision else cv2.IMWRITE_EXR_TYPE_FLOAT
)
params = [
cv2.IMWRITE_EXR_TYPE,
exr_type,
cv2.IMWRITE_EXR_COMPRESSION,
cv2.IMWRITE_EXR_COMPRESSION_ZIP,
]
for i in range(frames.shape[0]):
frame_bgr = frames[i][:, :, ::-1].astype(np.float32).copy()
path = os.path.join(output_dir, f"{filename_prefix}_{i:05d}.exr")
cv2.imwrite(path, frame_bgr, params)
logger.info(
"Saved %d EXR frame(s) to %s (%s)",
frames.shape[0],
output_dir,
"float16" if half_precision else "float32",
)
+251
View File
@@ -0,0 +1,251 @@
"""ACEScct working-space + primaries math for HDR Comfy paths.
Kept in lockstep with ``ltx_core.hdr`` / ``ltx_core.color.primaries`` on
ltx-2-internal: ACEScct curve constants match ``ltx_core.hdr``, and primaries
matrices / EXR chromaticities come from ``colour-science`` with Bradford CAT
(same source OpenColorIO-Config-ACES uses for its utility CLFs).
``colour-science`` is required (same as ``ltx_core.color.primaries``).
Used by HDR Comfy nodes in ``hdr_nodes`` (EXR ↔ VAE, SDR→HDR IC-LoRA ingest
via ``SDRColorSpace`` / sRGB EOTF → ACEScct). File I/O helpers live in ``hdr_io``.
"""
from __future__ import annotations
import enum
import logging
import colour
import torch
from torch import Tensor
logger = logging.getLogger("LTXVideo.hdr_color")
# ACEScct (AMPAS S-2016-001) — same constants as ``ltx_core.hdr``
_ACESCCT_A_LIN = 10.5402377416545
_ACESCCT_B_LIN = 0.0729055341958355
_ACESCCT_X_BRK = 0.0078125
_ACESCCT_Y_BRK = 0.155251141552511
_ACESCCT_LOG_M = 17.52
_ACESCCT_LOG_B = 9.72
_CAT = "Bradford"
_CS_ACESCG = colour.RGB_COLOURSPACES["ACEScg"]
_CS_REC709 = colour.RGB_COLOURSPACES["ITU-R BT.709"]
_CS_REC2020 = colour.RGB_COLOURSPACES["ITU-R BT.2020"]
def _rgb_to_rgb(src: colour.RGB_Colourspace, dst: colour.RGB_Colourspace) -> Tensor:
"""Linear RGB→RGB matrix from colour-science, as float32."""
return torch.as_tensor(
colour.matrix_RGB_to_RGB(src, dst, chromatic_adaptation_transform=_CAT),
dtype=torch.float32,
)
def _exr_chroma(cs: colour.RGB_Colourspace) -> tuple[float, ...]:
"""EXR ``chromaticities`` order: Rxy, Gxy, Bxy, Wxy."""
r, g, b = cs.primaries
w = cs.whitepoint
return (
float(r[0]),
float(r[1]),
float(g[0]),
float(g[1]),
float(b[0]),
float(b[1]),
float(w[0]),
float(w[1]),
)
_ACESCG_TO_SRGB = _rgb_to_rgb(_CS_ACESCG, _CS_REC709)
_SRGB_TO_ACESCG = torch.linalg.inv(_ACESCG_TO_SRGB.double()).to(torch.float32)
_REC709_TO_2020 = _rgb_to_rgb(_CS_REC709, _CS_REC2020)
_ACESCG_TO_2020 = _rgb_to_rgb(_CS_ACESCG, _CS_REC2020)
_CHROMA_REC709 = _exr_chroma(_CS_REC709)
_CHROMA_ACESCG = _exr_chroma(_CS_ACESCG)
_MATRICES_SOURCE = "colour-science"
# Public aliases (parity with ``ltx_core.color.primaries`` exports).
ACESCG_TO_SRGB = _ACESCG_TO_SRGB
SRGB_TO_ACESCG = _SRGB_TO_ACESCG
REC709_TO_2020 = _REC709_TO_2020
ACESCG_TO_2020 = _ACESCG_TO_2020
_CHROMATICITIES = {
"rec709": _CHROMA_REC709,
"acescg": _CHROMA_ACESCG,
}
class Primaries(enum.Enum):
"""Authoring / working-space colour primaries (linear light)."""
REC709 = "rec709"
ACESCG = "acescg"
@property
def exr_chromaticities(self) -> tuple[float, ...]:
"""EXR ``chromaticities`` header (R/G/B/W x,y)."""
return _CHROMATICITIES[self.value]
@property
def matrix_to_rec2020(self) -> Tensor:
"""3x3 linear map from this basis to Rec.2020 (for HLG)."""
return _REC709_TO_2020 if self is Primaries.REC709 else _ACESCG_TO_2020
def to(self, target: Primaries, video: Tensor) -> Tensor:
"""Convert linear light from this basis into ``target``. Identity when same."""
if self is target:
return video
if (self, target) == (Primaries.REC709, Primaries.ACESCG):
return _apply_matrix(video, _SRGB_TO_ACESCG)
if (self, target) == (Primaries.ACESCG, Primaries.REC709):
return _apply_matrix(video, _ACESCG_TO_SRGB)
raise ValueError(f"No primaries conversion from {self!r} to {target!r}.")
def to_rec2020(self, video: Tensor) -> Tensor:
"""Linear light in this basis → Rec.2020 (HLG master space)."""
return _apply_matrix(video, self.matrix_to_rec2020)
class HDRColorSpace(enum.Enum):
"""Source / EXR authoring colour space (matches pipelines ``--hdr``)."""
SRGB_LINEAR = "srgb_linear"
ACESCG = "acescg"
ACESCCT = "acescct"
@property
def is_log_working(self) -> bool:
return self is HDRColorSpace.ACESCCT
@property
def source_primaries(self) -> Primaries:
if self in (HDRColorSpace.ACESCG, HDRColorSpace.ACESCCT):
return Primaries.ACESCG
return Primaries.REC709
def exr_output_tags(self) -> tuple[Primaries, str]:
"""``(primaries, colorSpace tag)`` for EXR headers."""
if self is HDRColorSpace.ACESCCT:
return Primaries.ACESCG, "ACEScct"
if self is HDRColorSpace.ACESCG:
return Primaries.ACESCG, "ACEScg"
return Primaries.REC709, "sRGB"
def _apply_matrix(video: Tensor, mat: Tensor) -> Tensor:
"""Apply 3x3 on channel-last ``[..., H, W, 3]`` or channel-first ``[..., 3, H, W]``.
Comfy IMAGE tensors are channel-last; ``ltx_core`` often uses channel-first.
"""
mat = mat.to(device=video.device, dtype=video.dtype)
if video.shape[-1] == 3:
return video @ mat.T
if video.shape[-3] == 3:
return torch.einsum("...chw,dc->...dhw", video, mat)
raise ValueError(f"Expected RGB in last or -3 dim; got shape {tuple(video.shape)}")
# sRGB EOTF (IEC 61966-2-1) — same constants as ``ltx_core.hdr``.
_SRGB_A = 0.055
_SRGB_LINEAR_THRESHOLD = 0.04045
_SRGB_LINEAR_SLOPE = 12.92
_SRGB_GAMMA = 2.4
def compress_acescct(hdr: Tensor) -> Tensor:
"""Linear HDR [0, ∞) → ACEScct [0, 1]."""
x = torch.clamp(hdr, min=0.0)
log_part = (torch.log2(torch.clamp(x, min=1e-12)) + _ACESCCT_LOG_B) / _ACESCCT_LOG_M
lin_part = _ACESCCT_A_LIN * x + _ACESCCT_B_LIN
return torch.clamp(torch.where(x > _ACESCCT_X_BRK, log_part, lin_part), 0.0, 1.0)
def decompress_acescct(ct: Tensor) -> Tensor:
"""ACEScct [0, 1] → linear ACEScg [0, ∞)."""
ct = torch.clamp(ct, 0.0, 1.0)
lin_from_log = torch.pow(2.0, ct * _ACESCCT_LOG_M - _ACESCCT_LOG_B)
lin_from_lin = (ct - _ACESCCT_B_LIN) / _ACESCCT_A_LIN
return torch.where(ct > _ACESCCT_Y_BRK, lin_from_log, lin_from_lin)
def srgb_eotf_to_linear(srgb_01: Tensor) -> Tensor:
"""sRGB-encoded [0, 1] → display-linear Rec.709/sRGB (IEC 61966-2-1 EOTF)."""
x = torch.clamp(srgb_01.float(), 0.0, 1.0)
return torch.where(
x <= _SRGB_LINEAR_THRESHOLD,
x / _SRGB_LINEAR_SLOPE,
torch.pow((x + _SRGB_A) / (1.0 + _SRGB_A), _SRGB_GAMMA),
)
def to_acescct_working_space(linear_rgb: Tensor, source_primaries: Primaries) -> Tensor:
"""Scene-linear RGB → ACEScct [0, 1] (channel-last or channel-first)."""
native = source_primaries.to(Primaries.ACESCG, linear_rgb.float())
return compress_acescct(native.clamp(min=0.0))
def acescct_to_linear(
working: Tensor, out_primaries: Primaries = Primaries.REC709
) -> Tensor:
"""ACEScct [0, 1] → scene-linear in ``out_primaries``."""
linear_acescg = decompress_acescct(working.float())
return Primaries.ACESCG.to(out_primaries, linear_acescg).clamp(min=0.0)
class SDRColorSpace(enum.Enum):
"""SDR ingest colour space for mapping float RGB into ACEScct.
Matches pipelines ``SDRColorSpace`` / ``hdr_ic_lora`` IDT:
* ``srgb_gamma`` — display-referred sRGB (apply EOTF, then ACEScct).
* ``srgb`` — scene-linear Rec.709 / sRGB primaries (no EOTF).
* ``acescg`` — scene-linear ACEScg → ACEScct.
* ``acescct`` — already ACEScct log codes; clamp passthrough.
"""
SRGB = "srgb"
SRGB_GAMMA = "srgb_gamma"
ACESCG = "acescg"
ACESCCT = "acescct"
def to_working_space(self, video: Tensor) -> Tensor:
"""Map SDR float RGB into ACEScct ``[0, 1]`` (channel-last or channel-first)."""
if not video.is_floating_point():
raise ValueError(
f"SDRColorSpace.to_working_space expects float RGB; got dtype={video.dtype}. "
"Normalize integer pixels before calling."
)
video = video.float()
if self is SDRColorSpace.ACESCCT:
return video.clamp(0.0, 1.0)
if self is SDRColorSpace.SRGB_GAMMA:
return to_acescct_working_space(
srgb_eotf_to_linear(video), Primaries.REC709
)
if self is SDRColorSpace.SRGB:
return to_acescct_working_space(video, Primaries.REC709)
if self is SDRColorSpace.ACESCG:
return to_acescct_working_space(video, Primaries.ACESCG)
raise ValueError(f"Unsupported SDR color space: {self!r}")
def images_to_acescct(images: Tensor, color_space: HDRColorSpace) -> Tensor:
"""Convert loaded EXR frames ``[F,H,W,3]`` into ACEScct ``[0,1]`` for the VAE."""
images = images.float()
if color_space.is_log_working:
return images.clamp(0.0, 1.0)
return to_acescct_working_space(images, color_space.source_primaries)
def sdr_images_to_acescct(images: Tensor, color_space: SDRColorSpace) -> Tensor:
"""Convert SDR frames ``[F,H,W,C]`` into ACEScct ``[0,1]`` for HDR IC-LoRA."""
rgb = images[..., :3]
working = color_space.to_working_space(rgb)
if images.shape[-1] > 3:
return torch.cat([working, images[..., 3:].float()], dim=-1)
return working
+394
View File
@@ -0,0 +1,394 @@
"""HDR file I/O helpers: EXR read/write, HLG encode, path utilities.
Shared by Comfy HDR nodes (ingest + delivery). Color math lives in ``hdr_color``;
node classes live in ``hdr_nodes``.
"""
from __future__ import annotations
import logging
import math
import os
import re
from pathlib import Path
import folder_paths
import numpy as np
import torch
from torch import Tensor
from .hdr_color import Primaries
logger = logging.getLogger("LTXVideo.hdr_io")
# BT.2020 / HLG (ARIB STD-B67) encode constants — from ltx_core.color.hlg
_HLG_A, _HLG_B, _HLG_C = 0.17883277, 0.28466892, 0.55991073
_AV_COLOR_PRIMARIES_BT2020 = 9
_AV_COLOR_TRC_ARIB_STD_B67 = 18
_AV_COLORSPACE_BT2020_NCL = 9
_AV_COLOR_RANGE_MPEG = 1
_KR_2020, _KG_2020, _KB_2020 = 0.2627, 0.6780, 0.0593
_BT2020_RGB_TO_YUV = torch.tensor(
[
[_KR_2020, _KG_2020, _KB_2020],
[-_KR_2020 / 1.8814, -_KG_2020 / 1.8814, 0.5],
[0.5, -_KG_2020 / 1.4746, -_KB_2020 / 1.4746],
],
dtype=torch.float32,
)
def _resolve_path(path: str) -> Path:
p = Path(path).expanduser()
if p.is_absolute() and p.exists():
return p
# Prefer Comfy input dir for relative paths.
candidate = Path(folder_paths.get_input_directory()) / path
if candidate.exists():
return candidate
if p.exists():
return p.resolve()
raise FileNotFoundError(
f"HDR path not found: {path!r} (tried {candidate} and {p.resolve()})"
)
def _natural_sort_key(name: str) -> tuple:
"""Sort key so ``frame_9.exr`` precedes ``frame_10.exr`` (not lexicographic)."""
parts = re.split(r"(\d+)", name)
key: list = []
for part in parts:
if part.isdigit():
key.append((0, int(part)))
else:
key.append((1, part.casefold()))
return tuple(key)
def _list_exr_files(path: Path) -> list[Path]:
"""Resolve a still ``.exr`` or an EXR folder to a sorted frame list."""
if path.is_file():
if path.suffix.lower() != ".exr":
raise ValueError(f"Expected a .exr file; got {path}")
return [path]
if path.is_dir():
# Case-insensitive match (Linux is case-sensitive; macOS usually is not).
files = sorted(
{p for p in path.iterdir() if p.is_file() and p.suffix.lower() == ".exr"},
key=lambda p: _natural_sort_key(p.name),
)
if not files:
raise RuntimeError(f"No .exr frames in {path}")
return files
raise ValueError(f"Path must be a .exr file or a directory of EXRs; got {path}")
def _silent_audio(num_frames: int, frame_rate: float, sample_rate: int = 44100) -> dict:
"""Stereo silence long enough for the LTX audio VAE mel path.
EXR plates have no soundtrack; V2V still freezes an audio latent. A 0-length
waveform crashes ``torchaudio.functional.resample`` / ``VAEEncodeAudio``.
"""
if frame_rate <= 0:
raise ValueError(f"frame_rate must be > 0; got {frame_rate}")
duration_s = max(float(num_frames) / float(frame_rate), 1.0)
# Keep a comfortable floor for mel/STFT (typical n_fft is 1024–2048).
n_samples = max(int(math.ceil(duration_s * sample_rate)), sample_rate, 8192)
return {
"waveform": torch.zeros(1, 2, n_samples, dtype=torch.float32),
"sample_rate": int(sample_rate),
}
def _read_exr(path: Path) -> Tensor:
"""Read one EXR as float32 ``[H,W,3]`` (unbounded linear or log codes as stored).
Requires OpenImageIO (same as ``ltx_pipelines`` media I/O). No OpenCV fallback.
"""
try:
import OpenImageIO as oiio
except ImportError as e:
raise RuntimeError(
"Reading EXR requires OpenImageIO. Install with: pip install openimageio"
) from e
buf = oiio.ImageBuf(str(path))
pixels = buf.get_pixels(oiio.FLOAT)
if pixels is None or buf.has_error:
raise RuntimeError(f"OpenImageIO failed on '{path}': {buf.geterror()}")
arr = np.ascontiguousarray(np.asarray(pixels, dtype=np.float32))
if arr.ndim == 2:
arr = np.repeat(arr[..., None], 3, axis=-1)
return torch.from_numpy(np.ascontiguousarray(arr[..., :3]))
def _save_exr_frame(
frame_hwc: Tensor,
path: Path,
*,
half_precision: bool,
primaries: Primaries | None,
color_space_tag: str | None,
) -> None:
"""Write one EXR via OpenImageIO (same backend as ``ltx_pipelines``).
Optional ``primaries`` / ``color_space_tag`` set EXR header attributes when
provided; ``exr_color_space=linear`` may omit them for legacy untagged plates.
"""
try:
import OpenImageIO as oiio
except ImportError as e:
raise RuntimeError(
"Saving EXR requires OpenImageIO. Install with: pip install openimageio"
) from e
rgb = frame_hwc.detach().float().cpu().numpy()
path.parent.mkdir(parents=True, exist_ok=True)
h, w, _ = rgb.shape
fmt = oiio.HALF if half_precision else oiio.FLOAT
spec = oiio.ImageSpec(w, h, 3, fmt)
spec.channelnames = ("R", "G", "B")
spec.attribute("compression", "zip")
if primaries is not None:
spec.attribute(
"chromaticities",
"float[8]",
primaries.exr_chromaticities,
)
if color_space_tag is not None:
spec.attribute("colorSpace", color_space_tag)
out = oiio.ImageOutput.create(str(path))
if out is None:
raise RuntimeError(
f"OpenImageIO cannot create writer for {path}. "
"Ensure OpenImageIO is built with OpenEXR support."
)
try:
if not out.open(str(path), spec):
raise RuntimeError(
f"Failed to access EXR output file at '{path}': {out.geterror()}"
)
# OIIO converts to HALF from float32 buffer when spec is HALF.
if not out.write_image(np.ascontiguousarray(rgb, dtype=np.float32)):
raise RuntimeError(f"Failed to write EXR '{path}': {out.geterror()}")
finally:
out.close()
def _hlg_inverse_oetf_scalar(v: float) -> float:
return (
(v * v) / 3.0 if v <= 0.5 else (math.exp((v - _HLG_C) / _HLG_A) + _HLG_B) / 12.0
)
def _linear_to_hlg_signal(
rgb_fhwc: Tensor,
primaries: Primaries = Primaries.REC709,
white_signal: float = 0.75,
) -> Tensor:
"""Scene-linear RGB ``[F,H,W,3]`` in ``primaries`` → HLG signal in BT.2020.
Maps with ``primaries.matrix_to_rec2020`` then clamps (same as
``ltx_core.color.hlg.HlgGpuConverter``). Prefer ACEScg→Rec.2020 directly
over ACEScg→Rec.709→Rec.2020 so OOG negatives are not crushed early.
"""
white_x = _hlg_inverse_oetf_scalar(white_signal)
roll_k = white_x / (1.0 - white_x)
mat = primaries.matrix_to_rec2020.to(device=rgb_fhwc.device, dtype=rgb_fhwc.dtype)
lin = torch.nan_to_num((rgb_fhwc @ mat.T).clamp(min=0.0), nan=0.0, neginf=0.0)
x = torch.where(
lin <= 1.0,
lin * white_x,
1.0 - (1.0 - white_x) * torch.exp(-roll_k * (lin - 1.0)),
)
sig = torch.where(
x <= 1.0 / 12.0,
torch.sqrt((3.0 * x).clamp(min=0.0)),
_HLG_A * torch.log((12.0 * x - _HLG_B).clamp(min=1e-12)) + _HLG_C,
)
return sig.clamp(0.0, 1.0)
def _rgb_hlg_to_yuv420p10(
rgb_fhwc: Tensor,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""HLG RGB ``[F,H,W,3]`` → planar uint16 Y,U,V (limited / BT.2020)."""
if rgb_fhwc.shape[-2] % 2 or rgb_fhwc.shape[-3] % 2:
raise ValueError(f"HLG encode needs even H and W; got {tuple(rgb_fhwc.shape)}")
rgb = rgb_fhwc.movedim(-1, -3).contiguous() # F,3,H,W
mat = _BT2020_RGB_TO_YUV.to(device=rgb.device, dtype=rgb.dtype)
pixels = rgb.movedim(-3, -1)
yuv = pixels.flatten(-3, -2) @ mat.T
yuv = yuv.unflatten(-2, rgb.shape[-2:]).movedim(-1, -3)
y = yuv[:, :1]
uv = torch.nn.functional.avg_pool2d(
yuv[:, 1:3].contiguous(), kernel_size=2, stride=2
)
# MPEG limited 10-bit
y = y.mul(219 * 4).add_(16 * 4)
uv = uv.mul(224 * 4).add_(128 * 4)
y16 = y[:, 0].round_().clamp_(0, 1023).to(torch.uint16).cpu().numpy()
u16 = uv[:, 0].round_().clamp_(0, 1023).to(torch.uint16).cpu().numpy()
v16 = uv[:, 1].round_().clamp_(0, 1023).to(torch.uint16).cpu().numpy()
return y16, u16, v16
def _encode_hlg_mp4(
frames_linear: Tensor,
out_path: Path,
fps: float,
audio: dict | None = None,
*,
primaries: Primaries = Primaries.REC709,
) -> None:
"""Encode scene-linear ``[F,H,W,3]`` (in ``primaries``) to BT.2020/HLG/10-bit HEVC.
Optional Comfy ``AUDIO`` dict ``{waveform, sample_rate}`` is muxed as AAC.
The AAC stream is registered *before* video packets (PyAV requires this).
"""
from fractions import Fraction
import av
hlg = _linear_to_hlg_signal(frames_linear.float(), primaries=primaries)
y, u, v = _rgb_hlg_to_yuv420p10(hlg)
f, height, width = int(y.shape[0]), int(y.shape[1]), int(y.shape[2])
out_path.parent.mkdir(parents=True, exist_ok=True)
container = av.open(str(out_path), mode="w", options={"movflags": "+faststart"})
audio_stream = None
try:
stream = container.add_stream(
"libx265", rate=Fraction(fps).limit_denominator(1000)
)
stream.width = width
stream.height = height
stream.pix_fmt = "yuv420p10le"
stream.codec_tag = "hvc1"
threads = max(1, min(os.cpu_count() or 8, 16))
stream.options = {
"crf": "12",
"preset": "medium",
"x265-params": (
"colorprim=bt2020:transfer=arib-std-b67:colormatrix=bt2020nc:range=limited:"
f"repeat-headers=1:info=0:pools={threads}:frame-threads=4"
),
}
ctx = stream.codec_context
ctx.thread_count = threads
ctx.color_primaries = _AV_COLOR_PRIMARIES_BT2020
ctx.color_trc = _AV_COLOR_TRC_ARIB_STD_B67
ctx.colorspace = _AV_COLORSPACE_BT2020_NCL
ctx.color_range = _AV_COLOR_RANGE_MPEG
if audio is not None:
audio_stream = _prepare_aac_stream(container, audio)
for i in range(f):
frame = av.VideoFrame(width, height, "yuv420p10le")
for plane, src in zip(frame.planes, (y[i], u[i], v[i]), strict=True):
dest = np.frombuffer(plane, dtype=np.uint16).reshape(
plane.height, plane.line_size // 2
)
dest[:, : src.shape[1]] = src
frame.colorspace = _AV_COLORSPACE_BT2020_NCL
frame.color_range = _AV_COLOR_RANGE_MPEG
for packet in stream.encode(frame):
container.mux(packet)
for packet in stream.encode():
container.mux(packet)
if audio is not None and audio_stream is not None:
_mux_comfy_audio_aac(container, audio_stream, audio)
finally:
container.close()
def _comfy_audio_stereo_n2(audio: dict) -> tuple[Tensor, int]:
"""Return stereo int16-ready float samples ``[N, 2]`` and sample rate."""
waveform = audio.get("waveform")
sample_rate = int(audio.get("sample_rate", 0) or 0)
if waveform is None or sample_rate <= 0:
raise ValueError("AUDIO must include waveform and sample_rate")
samples = waveform.detach().float().cpu()
if samples.ndim == 3:
samples = samples[0]
if samples.ndim != 2:
raise ValueError(
f"Expected AUDIO waveform [C,N] or [B,C,N]; got {tuple(waveform.shape)}"
)
if samples.shape[0] == 2:
samples_cn = samples
elif samples.shape[1] == 2:
samples_cn = samples.T
elif samples.shape[0] == 1:
samples_cn = samples.repeat(2, 1)
else:
raise ValueError(
f"AUDIO must be mono or stereo; got shape {tuple(samples.shape)}"
)
return samples_cn.T.contiguous(), sample_rate
def _prepare_aac_stream(container, audio: dict):
"""Register a stereo AAC stream before any packets are muxed."""
from fractions import Fraction
_, sample_rate = _comfy_audio_stereo_n2(audio)
audio_stream = container.add_stream("aac", rate=sample_rate)
audio_stream.codec_context.sample_rate = sample_rate
audio_stream.codec_context.layout = "stereo"
audio_stream.codec_context.time_base = Fraction(1, sample_rate)
return audio_stream
def _mux_comfy_audio_aac(container, audio_stream, audio: dict) -> None:
"""Mux Comfy AUDIO onto an already-added AAC stream (after video flush)."""
import av
samples_n2, sample_rate = _comfy_audio_stereo_n2(audio)
samples_i16 = (samples_n2.clamp(-1.0, 1.0) * 32767.0).to(torch.int16)
frame_in = av.AudioFrame.from_ndarray(
samples_i16.reshape(1, -1).numpy(),
format="s16",
layout="stereo",
)
frame_in.sample_rate = sample_rate
cc = audio_stream.codec_context
resampler = av.audio.resampler.AudioResampler(
format=cc.format or "fltp",
layout=cc.layout or "stereo",
rate=cc.sample_rate or sample_rate,
)
next_pts = 0
for rframe in resampler.resample(frame_in):
if rframe.pts is None:
rframe.pts = next_pts
next_pts += rframe.samples
for packet in audio_stream.encode(rframe):
container.mux(packet)
for rframe in resampler.resample(None):
if rframe.pts is None:
rframe.pts = next_pts
next_pts += rframe.samples
for packet in audio_stream.encode(rframe):
container.mux(packet)
for packet in audio_stream.encode(None):
container.mux(packet)
def _trim_to_8k1(images: Tensor) -> Tensor:
"""Keep longest legal ``8k+1`` prefix (VAE temporal scale 8)."""
n = images.shape[0]
legal = ((n - 1) // 8) * 8 + 1
if legal < 1:
raise ValueError(f"Need at least 1 frame; got {n}")
if legal != n:
logger.warning(
"Trimmed EXR sequence from %d to %d frames (must be 8k+1)", n, legal
)
return images[:legal]
+675
View File
@@ -0,0 +1,675 @@
"""ComfyUI HDR nodes: ingest, decode postprocess, and delivery.
Stage-oriented layout (shared ACEScct working space):
- Ingest: ``LTXVSDRToHDRWorkingSpace``, ``LTXVLoadEXRSequence``
- Runtime: ``LTXVVAEForceFloat32``
- Delivery: ``LTXVHDRDecodePostprocess``, ``LTXVSaveHLG``
Color math: ``hdr_color``. File I/O helpers: ``hdr_io``.
"""
from __future__ import annotations
import logging
import os
from pathlib import Path
import folder_paths
import torch
from torch import Tensor
from .hdr_color import (
HDRColorSpace,
Primaries,
SDRColorSpace,
compress_acescct,
images_to_acescct,
sdr_images_to_acescct,
)
from .hdr_io import (
_encode_hlg_mp4,
_list_exr_files,
_read_exr,
_resolve_path,
_save_exr_frame,
_silent_audio,
_trim_to_8k1,
)
from .nodes_registry import comfy_node
logger = logging.getLogger("LTXVideo.hdr_nodes")
_EXR_COLOR_SPACE_CHOICES = ["linear", "acescct", "srgb_linear", "acescg"]
_SDR_COLOR_SPACE_CHOICES = [c.value for c in SDRColorSpace]
_COLOR_SPACE_CHOICES = [c.value for c in HDRColorSpace]
class LogC3:
"""ARRI LogC3 (EI 800) HDR compression.
Maps linear [0, inf) -> [0, 1] via the camera log curve, then scales to
[-1, 1] for VAE input.
"""
A = 5.555556
B = 0.052272
C = 0.247190
D = 0.385537
E = 5.367655
F = 0.092809
CUT = 0.010591
def compress(self, hdr: Tensor) -> Tensor:
x = torch.clamp(hdr, min=0.0)
log_part = self.C * torch.log10(self.A * x + self.B) + self.D
lin_part = self.E * x + self.F
logc = torch.where(x >= self.CUT, log_part, lin_part)
logc = torch.clamp(logc, 0.0, 1.0)
return logc * 2.0 - 1.0
def decompress(self, z: Tensor) -> Tensor:
logc = torch.clamp((z + 1.0) / 2.0, 0.0, 1.0)
cut_log = self.E * self.CUT + self.F
lin_from_log = (torch.pow(10.0, (logc - self.D) / self.C) - self.B) / self.A
lin_from_lin = (logc - self.F) / self.E
return torch.where(logc >= cut_log, lin_from_log, lin_from_lin)
# ---------------------------------------------------------------------------
# Transform registry — singleton (stateless)
# ---------------------------------------------------------------------------
_LOGC3 = LogC3()
def _hdr_decompress(decoded_01: Tensor, transfer: str = "acescct") -> Tensor:
"""Decompress VAE-decoded ``[0,1]`` image to linear HDR ``[0, inf)``.
ComfyUI's VAE decode returns images in ``[0, 1]`` via ``(raw + 1) / 2``.
* ``acescct`` — ACEScct working codes (SDR→HDR IC-LoRA + native HDR).
* ``logc3`` — legacy LogC3 SDR→HDR LoRA only.
"""
transfer = (transfer or "acescct").lower()
if transfer == "acescct":
from .hdr_color import decompress_acescct
# Decoded IMAGE is already ACEScct working-space [0, 1].
return decompress_acescct(decoded_01.float())
if transfer != "logc3":
raise ValueError(
f"Unsupported HDR transfer {transfer!r}; use 'logc3' or 'acescct'."
)
raw = decoded_01.float() * 2.0 - 1.0
return _LOGC3.decompress(raw)
def _linear_to_srgb(x: Tensor) -> Tensor:
"""Convert linear [0, 1] to sRGB [0, 1]."""
return torch.where(
x <= 0.0031308,
12.92 * x,
1.055 * torch.pow(x.clamp(min=0.0031308), 1.0 / 2.4) - 0.055,
).clamp(0.0, 1.0)
# ---------------------------------------------------------------------------
# ComfyUI Nodes
# ---------------------------------------------------------------------------
@comfy_node(name="LTXVSDRToHDRWorkingSpace")
class LTXVSDRToHDRWorkingSpace:
"""Map SDR float RGB into ACEScct ``[0, 1]`` for HDR IC-LoRA conditioning.
Place **before** resize when the source is display sRGB so
the EOTF is not applied to bilinear-filtered codes.
Default ``srgb_gamma`` matches typical ``LoadVideo`` / MP4 decode.
"""
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"image": ("IMAGE",),
"color_space": (
_SDR_COLOR_SPACE_CHOICES,
{
"default": "srgb_gamma",
"tooltip": (
"SDR ingest IDT. srgb_gamma = display sRGB (EOTF then "
"ACEScct); srgb = scene-linear Rec.709; acescg / acescct "
"for float EXR-style plates."
),
},
),
},
}
RETURN_TYPES = ("IMAGE",)
RETURN_NAMES = ("acescct",)
FUNCTION = "convert"
CATEGORY = "Lightricks/HDR"
DESCRIPTION = (
"SDR → ACEScct working space for HDR IC-LoRA (pipelines SDRColorSpace). "
"Wire LoadVideo images through this before resize / IC-LoRA guide."
)
def convert(self, image: torch.Tensor, color_space: str = "srgb_gamma") -> tuple:
space = SDRColorSpace(color_space)
return (sdr_images_to_acescct(image, space),)
def _exr_payload_for_color_space(
image: Tensor,
hdr_linear: Tensor,
transfer: str,
exr_color_space: str,
) -> tuple[Tensor, Primaries | None, str | None]:
"""Build EXR RGB frames and optional primaries / colorSpace tags.
``linear`` keeps the legacy 2.3 behaviour (raw decompressed buffer, no tags).
Other values match pipelines ``--hdr`` EXR authoring.
"""
space = (exr_color_space or "linear").lower()
transfer = (transfer or "acescct").lower()
rgb_in = image[..., :3].float()
hdr = hdr_linear[..., :3].float()
if space == "linear":
return hdr, None, None
if space == "acescct":
if transfer == "acescct":
frames = rgb_in.clamp(0.0, 1.0)
else:
# LogC3 linear is treated as Rec.709 scene-linear for ACEScct authoring.
frames = compress_acescct(
Primaries.REC709.to(Primaries.ACESCG, hdr).clamp(min=0.0)
)
return frames, Primaries.ACESCG, "ACEScct"
if space == "acescg":
if transfer == "acescct":
frames = hdr # decompress_acescct → ACEScg linear
else:
frames = Primaries.REC709.to(Primaries.ACESCG, hdr).clamp(min=0.0)
return frames, Primaries.ACESCG, "ACEScg"
if space == "srgb_linear":
if transfer == "acescct":
frames = Primaries.ACESCG.to(Primaries.REC709, hdr).clamp(min=0.0)
else:
frames = hdr.clamp(min=0.0)
return frames, Primaries.REC709, "sRGB"
raise ValueError(
f"Unsupported exr_color_space {exr_color_space!r}; "
f"use one of {_EXR_COLOR_SPACE_CHOICES}."
)
@comfy_node(name="LTXVHDRDecodePostprocess")
class LTXVHDRDecodePostprocess:
"""Decompress HDR from VAE output, tonemap for preview, optionally save EXR.
Place after VAE Decode. Recovers linear HDR from the VAE ``[0,1]`` image and
tonemaps to SDR for display. When ``save_exr`` is enabled, writes an EXR
sequence (legacy ``linear``, or tagged ``acescct`` / ``srgb_linear`` /
``acescg``).
For BT.2020/HLG masters, wire ``hdr_linear`` into ``LTXVSaveHLG``.
Outputs:
tonemapped: SDR preview [0, 1] after Reinhard tonemap + sRGB gamma.
hdr_linear: Raw linear HDR values [0, inf) for further processing.
With ``transfer=acescct`` this is ACEScg scene-linear; with
``logc3`` it is LogC3/Rec.709-style scene-linear.
"""
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"image": ("IMAGE",),
},
"optional": {
"transfer": (
["acescct", "logc3"],
{
"default": "acescct",
"tooltip": (
"Working-space curve of the VAE decode. "
"Use acescct for SDR→HDR IC-LoRA (after "
"LTXVSDRToHDRWorkingSpace) and for native HDR EXR; "
"logc3 only for legacy LogC3 HDR LoRA checkpoints."
),
},
),
"exposure": (
"FLOAT",
{
"default": 0.0,
"min": -10.0,
"max": 10.0,
"step": 0.1,
"display": "slider",
"tooltip": (
"Exposure in stops (EV). 0 = no change, "
"+1 = 2x brighter, -1 = half brightness. "
"Affects the tonemapped preview only."
),
},
),
"save_exr": (
"BOOLEAN",
{
"default": False,
"tooltip": "Save HDR frames as an EXR sequence.",
},
),
"exr_color_space": (
_EXR_COLOR_SPACE_CHOICES,
{
"default": "acescct",
"tooltip": (
"EXR contents when save_exr is on. "
"Default 'acescct' matches pipelines hdr_ic_lora "
"ACEScct log masters. 'acescg' / 'srgb_linear' are "
"tagged linear; 'linear' is legacy raw decompress "
"(untagged). All paths use OpenImageIO."
),
},
),
"output_dir": (
"STRING",
{
"default": "output/hdr_exr",
"tooltip": (
"Directory for EXR frames (relative to ComfyUI "
"output directory, or absolute path)."
),
},
),
"filename_prefix": (
"STRING",
{"default": "frame"},
),
"half_precision": (
"BOOLEAN",
{
"default": True,
"tooltip": (
"Save EXR as float16 (half). Smaller files, "
"negligible quality loss for most workflows."
),
},
),
},
}
RETURN_TYPES = ("IMAGE", "IMAGE")
RETURN_NAMES = ("tonemapped", "hdr_linear")
OUTPUT_NODE = True
FUNCTION = "postprocess"
CATEGORY = "Lightricks/HDR"
DESCRIPTION = (
"Decompresses VAE-decoded HDR (ACEScct or legacy LogC3), tonemaps for "
"preview, and optionally writes EXR (linear / ACEScct / sRGB linear / "
"ACEScg). Wire hdr_linear into LTXVSaveHLG for a BT.2020/HLG master."
)
def postprocess(
self,
image: torch.Tensor,
transfer: str = "acescct",
exposure: float = 0.0,
save_exr: bool = False,
exr_color_space: str = "acescct",
output_dir: str = "output/hdr_exr",
filename_prefix: str = "frame",
half_precision: bool = True,
) -> tuple:
hdr = _hdr_decompress(image, transfer=transfer)
hdr = torch.clamp(hdr, min=0.0, max=1e4)
# Reinhard tonemap with exposure (stops → linear multiplier)
exposure_mult = 2.0**exposure
hdr_exposed = hdr * exposure_mult
tonemapped_linear = (hdr_exposed / (1.0 + hdr_exposed)).clamp(0.0, 1.0)
tonemapped = _linear_to_srgb(tonemapped_linear)
if save_exr:
frames, primaries, tag = _exr_payload_for_color_space(
image, hdr, transfer, exr_color_space
)
self._save_exr_frames_tagged(
frames,
output_dir,
filename_prefix,
half_precision,
primaries=primaries,
color_space_tag=tag,
)
return (tonemapped, hdr)
@staticmethod
def _resolve_exr_dir(output_dir: str) -> str:
import folder_paths
if not os.path.isabs(output_dir):
output_dir = os.path.join(folder_paths.get_output_directory(), output_dir)
os.makedirs(output_dir, exist_ok=True)
return output_dir
@classmethod
def _save_exr_frames_tagged(
cls,
frames: torch.Tensor,
output_dir: str,
filename_prefix: str,
half_precision: bool,
*,
primaries: Primaries | None,
color_space_tag: str | None,
) -> None:
output_dir = cls._resolve_exr_dir(output_dir)
for i, frame in enumerate(frames):
_save_exr_frame(
frame,
Path(output_dir) / f"{filename_prefix}_{i:05d}.exr",
half_precision=half_precision,
primaries=primaries,
color_space_tag=color_space_tag,
)
logger.info(
"Saved %d EXR frame(s) to %s (%s%s)",
frames.shape[0],
output_dir,
"float16" if half_precision else "float32",
f", {color_space_tag}" if color_space_tag else ", untagged linear",
)
@comfy_node(name="LTXVLoadEXRSequence")
class LTXVLoadEXRSequence:
"""Load an EXR still or folder and convert to ACEScct ``[0,1]`` IMAGE for the VAE.
Declares the source colour space the same way as pipelines ``--hdr``:
``srgb_linear`` / ``acescg`` (compress to ACEScct) or ``acescct`` (passthrough).
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"path": (
"STRING",
{
"default": "",
"tooltip": (
"Path to a .exr still or a directory of *.exr frames. "
"I2V: single still is fine. V2V: use a folder with 8k+1 frames. "
"Relative paths are resolved under ComfyUI's input folder."
),
},
),
"color_space": (_COLOR_SPACE_CHOICES, {"default": "srgb_linear"}),
"frame_rate": (
"FLOAT",
{
"default": 24.0,
"min": 1.0,
"max": 120.0,
"step": 0.001,
"tooltip": "Required for EXR folders (no container fps).",
},
),
},
"optional": {
"frame_start": ("INT", {"default": 0, "min": 0, "max": 100000}),
"frame_cap": (
"INT",
{
"default": 0,
"min": 0,
"max": 100000,
"tooltip": "Max frames to load (0 = all). Still trimmed to 8k+1.",
},
),
"trim_to_8k1": (
"BOOLEAN",
{
"default": True,
"tooltip": "Trim to longest 8k+1 prefix required by the video VAE.",
},
),
},
}
RETURN_TYPES = ("IMAGE", "FLOAT", "INT", "INT", "INT", "AUDIO")
RETURN_NAMES = ("images", "frame_rate", "width", "height", "frame_count", "audio")
FUNCTION = "load"
CATEGORY = "Lightricks/HDR"
DESCRIPTION = (
"Native HDR input: load EXR → ACEScct [0,1] frames for IC-LoRA / VAE encode. "
"Also emits silent AUDIO matching duration so V2V graphs that freeze audio still wire. "
"Native EXR path only (not SDR→HDR IC-LoRA ingest). "
"Downstream: set LTXVPreprocess img_compression=0 — Python --hdr EXR conditioning "
"skips JPEG/CRF; compressing ACEScct codes darkens HDR highlights."
)
def load(
self,
path: str,
color_space: str,
frame_rate: float,
frame_start: int = 0,
frame_cap: int = 0,
trim_to_8k1: bool = True,
):
root = _resolve_path(path)
files = _list_exr_files(root)
files = files[frame_start:]
if frame_cap > 0:
files = files[:frame_cap]
if not files:
raise ValueError("No EXR frames left after frame_start / frame_cap.")
cs = HDRColorSpace(color_space)
frames = [_read_exr(f) for f in files]
shapes = {tuple(f.shape) for f in frames}
if len(shapes) != 1:
raise ValueError(
"EXR frames must share the same H×W×C; got "
+ ", ".join(str(s) for s in sorted(shapes))
)
images = torch.stack(frames, dim=0)
images = images_to_acescct(images, cs)
if trim_to_8k1:
images = _trim_to_8k1(images)
h, w = int(images.shape[1]), int(images.shape[2])
n = int(images.shape[0])
if n < 1:
raise ValueError(f"No EXR frames loaded from {root}")
audio = _silent_audio(n, float(frame_rate))
logger.info(
"Loaded %d EXR frame(s) from %s as %s → ACEScct (%dx%d @ %.3f fps); "
"silent audio %d samples @ %d Hz",
n,
root if root.is_dir() else (root.parent if root.is_file() else root),
cs.value,
w,
h,
frame_rate,
int(audio["waveform"].shape[-1]),
int(audio["sample_rate"]),
)
return (images, float(frame_rate), w, h, n, audio)
@comfy_node(name="LTXVVAEForceFloat32")
class LTXVVAEForceFloat32:
"""Force the video VAE to encode/decode in float32 (native HDR requirement).
This node does **not** clone the VAE into a separate float32 copy for the
output. It mutates the same object that was passed in (``vae_dtype`` and
``first_stage_model`` weights). The output socket is that same instance.
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"vae": ("VAE",),
}
}
RETURN_TYPES = ("VAE",)
FUNCTION = "patch"
CATEGORY = "Lightricks/HDR"
DESCRIPTION = (
"Forces float32 on the video VAE (vae_dtype + first_stage_model) for "
"native HDR. IMPORTANT: this does NOT clone a float32 VAE for the "
"output — it mutates the input object in place and returns the same "
"instance. If you also wire that loader's VAE (or this output) into "
"another branch, those nodes share the mutated model: they may see "
"float16 or float32 depending on execution order (a race with the "
"original dtype). Prefer a single chain VAELoader → this node → all "
"encode/decode / IC-LoRA guide uses."
)
def patch(self, vae):
# In-place: Comfy Module.to(dtype) mutates weights; a wrapper copy would
# still share first_stage_model and waste peak memory for no isolation.
vae.vae_dtype = torch.float32
model = getattr(vae, "first_stage_model", None)
if model is not None:
try:
model.to(dtype=torch.float32)
except Exception as exc:
raise RuntimeError(
"LTXVVAEForceFloat32 could not cast VAE weights to float32 "
f"({type(exc).__name__}: {exc}). Native HDR requires float32."
) from exc
logger.info("VAE dtype forced to float32 for native HDR")
return (vae,)
@comfy_node(name="LTXVSaveHLG")
class LTXVSaveHLG:
"""Encode scene-linear ``hdr_linear`` frames to a BT.2020 / HLG 10-bit HEVC master.
Wire the ``hdr_linear`` output of ``LTXVHDRDecodePostprocess`` here (not the
tonemapped preview). Matches pipelines ``encode_video`` HLG delivery.
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"hdr_linear": (
"IMAGE",
{
"tooltip": (
"Scene-linear HDR from LTXVHDRDecodePostprocess. "
"With transfer=acescct this is ACEScg linear — leave "
"linear_primaries=acescg. For logc3, use rec709."
),
},
),
"frame_rate": (
"FLOAT",
{"default": 24.0, "min": 1.0, "max": 120.0, "step": 0.001},
),
"filename_prefix": ("STRING", {"default": "hdr/ltxv_hlg"}),
},
"optional": {
"linear_primaries": (
["acescg", "rec709"],
{
"default": "acescg",
"tooltip": (
"Primaries of hdr_linear. ACEScct decode "
"(SDR→HDR IC-LoRA or native HDR) → acescg. "
"Legacy LogC3 → rec709."
),
},
),
"audio": (
"AUDIO",
{
"tooltip": (
"Optional track to mux into the HLG MP4 (e.g. Decode AUDIO). "
"Omit for video-only."
),
},
),
},
}
RETURN_TYPES = ()
OUTPUT_NODE = True
FUNCTION = "save"
CATEGORY = "Lightricks/HDR"
DESCRIPTION = (
"Writes a BT.2020/HLG 10-bit HEVC master from scene-linear hdr_linear "
"(from LTXVHDRDecodePostprocess). Maps linear_primaries → Rec.2020 "
"directly (no Rec.709 hop), then HLG. Optional AUDIO is muxed as AAC."
)
def save(
self,
hdr_linear: Tensor,
frame_rate: float,
filename_prefix: str,
linear_primaries: str = "acescg",
audio: dict | None = None,
):
if hdr_linear.ndim != 4 or hdr_linear.shape[-1] < 3:
raise ValueError(f"Expected IMAGE [F,H,W,C]; got {tuple(hdr_linear.shape)}")
# ACEScg/Rec.709 → Rec.2020 inside HLG encode (Primaries.to_rec2020 /
# matrix_to_rec2020). No Rec.709 intermediate that would crush OOG early.
linear = hdr_linear[..., :3].float()
prim = (linear_primaries or "acescg").lower()
if prim == "acescg":
src_primaries = Primaries.ACESCG
elif prim == "rec709":
src_primaries = Primaries.REC709
else:
raise ValueError(
f"Unsupported linear_primaries {linear_primaries!r}; use acescg or rec709."
)
# Even dims for 4:2:0
h, w = linear.shape[1], linear.shape[2]
if h % 2 or w % 2:
linear = linear[:, : h - (h % 2), : w - (w % 2), :]
full_output_folder, filename, counter, _subfolder, _ = (
folder_paths.get_save_image_path(
filename_prefix,
folder_paths.get_output_directory(),
linear.shape[2],
linear.shape[1],
)
)
hlg_path = Path(full_output_folder) / f"{filename}_{counter:05d}_hlg.mp4"
_encode_hlg_mp4(
linear,
hlg_path,
fps=float(frame_rate),
audio=audio,
primaries=src_primaries,
)
logger.info(
"Wrote HLG 10-bit master to %s%s",
hlg_path,
" (with audio)" if audio is not None else "",
)
return {}
+44 -21
View File
@@ -13,6 +13,15 @@ from comfy_api.latest import io
from .latents import LTXVDilateLatent
from .nodes_registry import NODES_DISPLAY_NAME_PREFIX, comfy_node
FRAME_IDX_TOOLTIP = (
"RoPE position of the reference, in pixel frames.\n"
"0 aligns the reference with the generated clip. Use for video references.\n"
">0 places the reference at a specific pixel-frame position in the clip.\n"
"-1 places the reference outside the generated frame range, while still using it "
"for attention conditioning. Use for single-image references.\n"
"For multi-frame references, >0 values are rounded down to 1 mod 8."
)
@comfy_node(name="LTXAddVideoICLoRAGuide")
class LTXAddVideoICLoRAGuide(io.ComfyNode):
@@ -44,12 +53,7 @@ class LTXAddVideoICLoRAGuide(io.ComfyNode):
default=0,
min=-9999,
max=9999,
tooltip=(
"Frame index to start the conditioning at. "
"For single-frame videos, any frame_idx value is acceptable. "
"For videos, frame_idx must be 1 modulo 8, otherwise it will be rounded "
"down to the nearest 1 modulo 8. Negative values are counted from the end of the video."
),
tooltip=FRAME_IDX_TOOLTIP,
),
io.Float.Input(
"strength",
@@ -101,6 +105,35 @@ class LTXAddVideoICLoRAGuide(io.ComfyNode):
],
)
@classmethod
def resolve_frame_idx(
cls,
positive,
frame_idx,
latent_length,
image_length,
guide_latent,
scale_factors,
):
"""Map frame_idx to the position append_keyframe should use.
A negative value is a RoPE position before the clip, not an index counted back
from its end, so it bypasses get_latent_index. That is what lets a one-frame
reference sit at -1: adjacent to the first generated frame without landing in
its slot, which is the only position a one-frame guide can occupy that makes it
indistinguishable from the frame being generated. The length assert is
meaningless there -- the guide is outside the clip by construction.
"""
if frame_idx < 0:
return frame_idx
frame_idx, latent_idx = nodes_lt.LTXVAddGuide.get_latent_index(
positive, latent_length, image_length, frame_idx, scale_factors
)
assert (
latent_idx + guide_latent.shape[2] <= latent_length
), "Conditioning frames exceed the length of the latent sequence."
return frame_idx
@classmethod
def encode(
cls,
@@ -219,12 +252,9 @@ class LTXAddVideoICLoRAGuide(io.ComfyNode):
guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
)
frame_idx, latent_idx = nodes_lt.LTXVAddGuide.get_latent_index(
positive, latent_length, len(image), frame_idx, scale_factors
frame_idx = cls.resolve_frame_idx(
positive, frame_idx, latent_length, len(image), guide_latent, scale_factors
)
assert (
latent_idx + guide_latent.shape[2] <= latent_length
), "Conditioning frames exceed the length of the latent sequence."
positive, negative, latent_image, noise_mask = (
nodes_lt.LTXVAddGuide.append_keyframe(
@@ -292,11 +322,7 @@ class LTXAddVideoICLoRAGuideAdvanced(LTXAddVideoICLoRAGuide):
default=0,
min=-9999,
max=9999,
tooltip=(
"Frame index to start the conditioning at. The value is rounded to the "
"nearest frame and wrapped modulo the number of video frames. Negative "
"values are counted from the end of the video before wrapping."
),
tooltip=FRAME_IDX_TOOLTIP,
),
io.Float.Input("strength", default=1.0, min=0.0, max=1.0, step=0.01),
io.Float.Input(
@@ -420,12 +446,9 @@ class LTXAddVideoICLoRAGuideAdvanced(LTXAddVideoICLoRAGuide):
guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
)
frame_idx, latent_idx = nodes_lt.LTXVAddGuide.get_latent_index(
positive, latent_length, len(image), frame_idx, scale_factors
frame_idx = cls.resolve_frame_idx(
positive, frame_idx, latent_length, len(image), guide_latent, scale_factors
)
assert (
latent_idx + guide_latent.shape[2] <= latent_length
), "Conditioning frames exceed the length of the latent sequence."
positive, negative, latent_image, noise_mask = (
nodes_lt.LTXVAddGuide.append_keyframe(
+13
View File
@@ -53,6 +53,19 @@ def append_guide_attention_entry(
return _set_guide_attention_entries(conditioning, entries)
def set_last_guide_attention_strength(conditioning, strength):
"""Override the attention strength of the most recently appended guide entry.
Separate from a guide's denoise strength: one says how clean the reference is
kept, the other how strongly the rest of the sequence may attend to it.
"""
entries = _get_guide_attention_entries(conditioning)
if not entries:
return conditioning
updated = [*entries[:-1], {**entries[-1], "strength": strength}]
return _set_guide_attention_entries(conditioning, updated)
def normalize_mask(mask):
"""Normalize a ComfyUI MASK to (1, 1, F, H, W) for downstream processing.
+40 -94
View File
@@ -7,6 +7,7 @@ import torch
from comfy.ldm.lightricks.vae.audio_vae import LATENT_DOWNSAMPLE_FACTOR
from comfy.nested_tensor import NestedTensor
from .iclora_attention import set_last_guide_attention_strength
from .nodes_registry import comfy_node
@@ -397,102 +398,47 @@ class LTXVDilateLatent:
return (latent,)
@comfy_node(name="LTXVAddLatentGuide")
class LTXVAddLatentGuide:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"vae": ("VAE",),
"positive": ("CONDITIONING",),
"negative": ("CONDITIONING",),
"latent": ("LATENT",),
"guiding_latent": ("LATENT",),
"latent_idx": (
"INT",
{
"default": 0,
"min": -9999,
"max": 9999,
"step": 1,
"tooltip": "Latent index to start the conditioning at. Can be negative to"
"indicate that the conditioning is on the frames before the latent.",
},
),
"strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0}),
}
}
def add_latent_guide(
*,
vae,
positive,
negative,
latent,
guiding_latent,
latent_idx,
strength,
attention_mask=None,
):
"""Pin an already-encoded latent as a guide, via ComfyUI's native node.
RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
RETURN_NAMES = ("positive", "negative", "latent")
ComfyUI ships LTXVAddLatentGuide itself (Comfy-Org/ComfyUI#16176), so the
implementation lives there rather than being duplicated here; this is only the
call-shape adapter the pack's samplers want -- keyword arguments and a plain
tuple instead of a NodeOutput.
CATEGORY = "ltxtricks"
FUNCTION = "generate"
DESCRIPTION = "Adds a keyframe or a video segment at a specific frame index."
def generate(
self, vae, positive, negative, latent, guiding_latent, latent_idx, strength
):
noise_mask = nodes_lt.get_noise_mask(latent)
latent = latent["samples"]
guide = guiding_latent["samples"]
# Record original (pre-dilation) guide latent shape for spatial mask downsampling
guide_orig_shape = list(guide.shape[2:]) # [F, H_small, W_small]
assert (
latent.shape[4] % guide.shape[4] == 0
and latent.shape[3] % guide.shape[3] == 0
), "The ratio of the height and width of the latents and optional_guiding_latents must be an integer"
guiding_latent = LTXVDilateLatent().dilate_latent(
guiding_latent,
horizontal_scale=latent.shape[4] // guide.shape[4],
vertical_scale=latent.shape[3] // guide.shape[3],
)[0]
guide = guiding_latent["samples"]
guide_mask = guiding_latent.get("noise_mask", None)
# Pre-filter token count = product of dilated spatial dims
# (before grid_mask filtering removes padding positions)
iclora_tokens_added = guide.shape[2] * guide.shape[3] * guide.shape[4]
scale_factors = vae.downscale_index_formula
if latent_idx <= 0:
frame_idx = latent_idx * scale_factors[0]
else:
frame_idx = 1 + (latent_idx - 1) * scale_factors[0]
positive, negative, latent, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
positive=positive,
negative=negative,
frame_idx=frame_idx,
latent_image=latent,
noise_mask=noise_mask,
guiding_latent=guide,
strength=strength,
scale_factors=scale_factors,
guide_mask=guide_mask,
)
# Track this guide in guide_attention_entries for per-reference attention control.
from .iclora_attention import append_guide_attention_entry
positive = append_guide_attention_entry(
positive, iclora_tokens_added, guide_orig_shape
)
negative = append_guide_attention_entry(
negative, iclora_tokens_added, guide_orig_shape
)
return (
positive,
negative,
{"samples": latent, "noise_mask": noise_mask},
)
Deliberately does not forward pixel_frame_offset: that input exists only in
Lightricks' ComfyUI fork, and the pack has to keep working against a stock
install.
"""
positive, negative, latent = nodes_lt.LTXVAddLatentGuide.execute(
positive=positive,
negative=negative,
vae=vae,
latent=latent,
guiding_latent=guiding_latent,
latent_idx=latent_idx,
strength=strength,
attention_mask=attention_mask,
)
if attention_mask is None:
# The core node records `strength` as the guide's attention strength too, so a
# weakly pinned guide would also become weakly attended. The implementation this
# replaced always left attention at full strength, and the samplers calling this
# pass strengths well below 1.0 (0.15 in the tiled sampler, 0.5 when extending),
# so keep the two decoupled rather than silently reweighting their attention.
positive = set_last_guide_attention_strength(positive, 1.0)
negative = set_last_guide_attention_strength(negative, 1.0)
return positive, negative, latent
@comfy_node(name="LTXVImgToVideoConditionOnly")
-514
View File
@@ -1,514 +0,0 @@
"""User-facing matte cleanup after IC-LoRA alpha generation."""
from __future__ import annotations
import base64
from io import BytesIO
from pathlib import Path
import folder_paths
import torch
import torch.nn.functional as F
from comfy_api.latest import InputImpl, io
from PIL import Image
from .alpha_composite import (
join_rgba,
match_length,
match_mask_to_rgb,
over_composite,
mask_from_matte,
resolve_plate,
resize_rgb,
rgb_frames,
)
from .nodes_registry import NODES_DISPLAY_NAME_PREFIX, comfy_node
_LIVE_MAX_SIDE = 320
_LIVE_MAX_FRAMES = 240
_PLATE_CACHE: dict[tuple[str, int], torch.Tensor] = {}
def _live_b64(frame: torch.Tensor, jpeg: bool = False) -> str:
t = frame.detach().float().cpu()
if t.ndim == 2:
img = Image.fromarray((t.clamp(0, 1) * 255).byte().numpy(), mode="L")
fmt, kw = "PNG", {"compress_level": 6}
else:
img = Image.fromarray((t[..., :3].clamp(0, 1) * 255).byte().numpy(), mode="RGB")
fmt, kw = ("JPEG", {"quality": 60}) if jpeg else ("PNG", {"compress_level": 6})
buf = BytesIO()
img.save(buf, format=fmt, **kw)
return base64.b64encode(buf.getvalue()).decode("ascii")
def _index_time(tensor: torch.Tensor, idx: torch.Tensor) -> torch.Tensor:
return tensor.index_select(0, idx.to(device=tensor.device))
def _live_clip(
source: torch.Tensor,
rgb: torch.Tensor,
plate: torch.Tensor | None = None,
) -> tuple[list[str], list[str], list[str], float, int]:
n_source = int(source.shape[0])
n, h, w = n_source, int(source.shape[-2]), int(source.shape[-1])
if n > _LIVE_MAX_FRAMES:
idx = torch.linspace(0, n - 1, _LIVE_MAX_FRAMES, device=source.device).round().long()
source = _index_time(source, idx)
rgb = _index_time(rgb, idx)
if plate is not None:
plate = _index_time(plate, idx)
n = int(source.shape[0])
scale = min(1.0, _LIVE_MAX_SIDE / float(max(h, w)))
if scale < 1.0:
nh = max(1, int(round(h * scale)))
nw = max(1, int(round(w * scale)))
source = F.interpolate(
source.unsqueeze(1), size=(nh, nw), mode="bilinear", align_corners=False
).squeeze(1)
rgb = resize_rgb(rgb, nh, nw)
if plate is not None:
plate = resize_rgb(plate, nh, nw)
source_cpu = source.detach().float().cpu()
rgb_cpu = rgb.detach().float().cpu()
masks = [_live_b64(source_cpu[i], jpeg=False) for i in range(n)]
rgbs = [_live_b64(rgb_cpu[i], jpeg=True) for i in range(n)]
bgs: list[str] = []
if plate is not None:
plate_cpu = plate.detach().float().cpu()
bgs = [_live_b64(plate_cpu[i], jpeg=True) for i in range(n)]
return masks, rgbs, bgs, float(scale), n_source
def _expand_mask(mask: torch.Tensor, pixels: int) -> torch.Tensor:
"""Square dilate (positive) or erode (negative) in pixel units.
Replicate-pad before the pool so the border matches the live preview (which
clamps coordinates) instead of max_pool2d's zero fill.
"""
if pixels == 0:
return mask
x = mask.unsqueeze(1)
pad = abs(int(pixels))
k = pad * 2 + 1
if pixels < 0:
x = 1.0 - x
x = F.pad(x, (pad, pad, 0, 0), mode="replicate")
x = F.max_pool2d(x, kernel_size=(1, k), stride=1)
x = F.pad(x, (0, 0, pad, pad), mode="replicate")
x = F.max_pool2d(x, kernel_size=(k, 1), stride=1)
if pixels < 0:
x = 1.0 - x
return x.squeeze(1)
def _gaussian_blur(mask: torch.Tensor, sigma: float) -> torch.Tensor:
if sigma <= 0:
return mask
radius = max(1, int(round(float(sigma) * 3.0)))
k = radius * 2 + 1
coords = torch.arange(k, device=mask.device, dtype=mask.dtype) - radius
g = torch.exp(-(coords * coords) / (2.0 * sigma * sigma))
g = g / g.sum()
x = mask.unsqueeze(1)
x = F.pad(x, (0, 0, radius, radius), mode="replicate")
x = F.conv2d(x, g.view(1, 1, k, 1))
x = F.pad(x, (radius, radius, 0, 0), mode="replicate")
x = F.conv2d(x, g.view(1, 1, 1, k))
return x.squeeze(1)
def _gaussian_blur_rgb(image: torch.Tensor, sigma: float) -> torch.Tensor:
"""NHWC RGB, same separable blur as `_gaussian_blur`."""
if sigma <= 0:
return image
n, h, w, c = image.shape
x = image.movedim(-1, 1).reshape(n * c, h, w)
return _gaussian_blur(x, sigma).reshape(n, c, h, w).movedim(1, -1)
_UNMIX_FLOOR = 0.1
_UNMIX_GAIN_FLOOR = 0.25
def _uniform_clamp(rgb: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
"""Move `rgb` toward `target` as far as stays in gamut, scaling all channels
together. Per-channel clamping would clip one channel first and shift hue."""
delta = target - rgb
big = torch.full_like(delta, 1e6)
to_high = torch.where(delta > 0, (1.0 - rgb) / delta.clamp_min(1e-6), big)
to_low = torch.where(delta < 0, rgb / (-delta).clamp_min(1e-6), big)
t = torch.minimum(to_high, to_low).amin(dim=-1, keepdim=True).clamp(0.0, 1.0)
return rgb + t * delta
def _unmix_color(
rgb: torch.Tensor,
alpha: torch.Tensor,
amount: float,
radius: float,
) -> torch.Tensor:
"""Recover straight foreground color: C = αF + (1-α)B.
Estimates B from low-alpha pixels, then F = (C - (1-α)B) / α, which is already
identity at α=1. Two guards keep a bad α from turning into a colored fringe:
the divisor is floored at `_UNMIX_GAIN_FLOOR` so background removal is never
amplified more than ~3x, and the result is pulled back into gamut uniformly
instead of per channel. Below `_UNMIX_FLOOR` the correction fades out, since
α is then too small to solve for F at all. Amount 0 is identity.
"""
if amount <= 0:
return rgb
a = alpha.clamp(0.0, 1.0)
bg_w = (1.0 - a) * (1.0 - a)
sigma = max(0.5, float(radius))
num = _gaussian_blur_rgb(rgb * bg_w.unsqueeze(-1), sigma)
den = _gaussian_blur(bg_w, sigma).unsqueeze(-1).clamp_min(1e-4)
background = (num / den).clamp(0.0, 1.0)
safe = a.unsqueeze(-1).clamp_min(_UNMIX_GAIN_FLOOR)
foreground = _uniform_clamp(rgb, (rgb - (1.0 - safe) * background) / safe)
gate = (a / _UNMIX_FLOOR).clamp(0.0, 1.0).unsqueeze(-1)
mix = float(amount) * gate
return (rgb * (1.0 - mix) + foreground * mix).clamp(0.0, 1.0)
def _clip_levels(mask: torch.Tensor, black_clip: float, white_clip: float) -> torch.Tensor:
lo = float(black_clip)
hi = 1.0 - float(white_clip)
if lo <= 0.0 and hi >= 1.0:
return mask
if hi <= lo:
hi = lo + 1e-4
return ((mask - lo) / (hi - lo)).clamp(0.0, 1.0)
def _refine_mask(
mask: torch.Tensor,
invert: bool,
expand: int,
feather: float,
black_clip: float,
white_clip: float,
mix: float,
) -> torch.Tensor:
work = mask
if invert:
work = 1.0 - work
raw = work
mix = min(1.0, max(0.0, float(mix)))
if mix <= 0.0:
return raw.clamp(0.0, 1.0)
work = _expand_mask(work, int(expand))
work = _clip_levels(work, black_clip, white_clip)
work = _gaussian_blur(work, float(feather))
if mix < 1.0:
work = torch.lerp(raw, work, mix)
return work.clamp(0.0, 1.0)
def _load_plate_file(name: str) -> torch.Tensor:
"""Read a still or video from Comfy's input folder as NHWC RGB."""
name = (name or "").strip().replace("\\", "/")
parts = Path(name).parts
if not name or Path(name).is_absolute() or ".." in parts:
raise ValueError(f"Invalid plate file: {name!r}")
exists = getattr(folder_paths, "exists_annotated_filepath", None)
if callable(exists) and not exists(name):
raise ValueError(f"Plate file not found: {name}")
path = folder_paths.get_annotated_filepath(name)
try:
mtime = Path(path).stat().st_mtime_ns
except OSError:
mtime = 0
key = (str(path), int(mtime))
cached = _PLATE_CACHE.get(key)
if cached is not None:
return cached
images = InputImpl.VideoFromFile(path).get_components().images
if images is None or int(images.shape[0]) == 0:
raise ValueError(f"Could not read plate file: {name}")
frames = rgb_frames(images)
_PLATE_CACHE.clear()
_PLATE_CACHE[key] = frames
return frames
@comfy_node(name="LTXRefineAlphaMatte")
class LTXRefineAlphaMatte(io.ComfyNode):
"""Post-process a generated alpha matte. Identity at default widget values."""
@classmethod
def define_schema(cls):
return io.Schema(
node_id="LTXRefineAlphaMatte",
display_name=NODES_DISPLAY_NAME_PREFIX + " Refine Alpha Matte",
category="Lightricks/IC-LoRA",
description=(
"Cleanup for IC-LoRA alpha mattes, plus optional over-plate composite. "
"Defaults leave the matte and RGB unchanged. Connect Decode via Get Video "
"Components to `matte` (white = subject); `mask` still works. Expand is "
"Shift Edge (negative = choke). Feather blurs the silhouette. Black/white "
"clip is Resolve Clean Black / Clean White. Unmix pulls old-background "
"color out of mixed edges. `rgba` is straight alpha. `over` is "
"the opaque composite when background is image/video. Use **Load plate** "
"on this node (no extra Load Image / Load Video nodes). Live preview uses "
"the same plate as Queue Prompt."
),
inputs=[
io.Image.Input(
"matte",
optional=True,
tooltip="Matte frames (white = subject). Connect Get Video Components from Decode. Optional if `mask` is connected.",
),
io.Image.Input(
"image",
optional=True,
tooltip="Source RGB for unmix, RGBA, and over. Gray placeholder if unconnected.",
),
io.Boolean.Input(
"invert",
default=False,
label_on="inverted",
label_off="normal",
tooltip="Swap subject and background.",
),
io.Int.Input(
"expand",
display_name="expand (px)",
default=0,
min=-128,
max=128,
tooltip="Shift Edge in pixels. Positive grows the subject, negative chokes it (kills halos).",
),
io.Float.Input(
"feather",
default=0.0,
min=0.0,
max=64.0,
step=0.5,
tooltip="Gaussian blur of the silhouette in pixels. 2–8 is a typical AE feather.",
),
io.Float.Input(
"black_clip",
display_name="clean black",
default=0.0,
min=0.0,
max=0.5,
step=0.01,
tooltip="Crush near-transparent gray specks in the background to 0.",
),
io.Float.Input(
"white_clip",
display_name="clean white",
default=0.0,
min=0.0,
max=0.5,
step=0.01,
tooltip="Fill near-opaque gray holes in the subject to 1.",
),
io.Float.Input(
"mix",
default=1.0,
min=0.0,
max=1.0,
step=0.01,
tooltip="0 = raw matte (after invert), 1 = fully refined.",
),
io.Float.Input(
"unmix",
default=0.0,
min=0.0,
max=1.0,
step=0.01,
tooltip=(
"Decontaminate edge color. 0 = original RGB. 1 = fully unmix mixed "
"pixels so they keep the subject color, not the old background."
),
),
io.Float.Input(
"unmix_radius",
display_name="unmix radius",
default=12.0,
min=1.0,
max=64.0,
step=0.5,
tooltip=(
"How far (pixels) to look for background color when unmixing. "
"Raise for long hair/fur; lower if nearby objects tint the edge."
),
),
io.Combo.Input(
"background",
options=["none", "image", "video"],
default="none",
tooltip=(
"none = original RGB on `over`. image / video = the file from "
"**Load plate** (or a wired background_image / background_video)."
),
),
io.Combo.Input(
"fit",
options=["cover", "contain", "stretch"],
default="cover",
tooltip="How the plate fills the frame: cover crops, contain letterboxes, stretch distorts.",
),
io.Combo.Input(
"length",
options=["hold", "loop"],
default="hold",
tooltip="If the plate is shorter than the clip: hold last frame or loop.",
),
io.String.Input(
"plate_file",
default="",
socketless=True,
tooltip="Set by Load plate. Filename in Comfy's input folder.",
),
io.Mask.Input(
"mask",
optional=True,
advanced=True,
tooltip="Optional MASK. Used only if you do not connect `matte`.",
),
io.Image.Input(
"background_image",
optional=True,
lazy=True,
advanced=True,
tooltip="Optional still from another node. Load plate does not need this.",
),
io.Video.Input(
"background_video",
optional=True,
lazy=True,
advanced=True,
tooltip="Optional clip from another node. Load plate does not need this.",
),
],
outputs=[
io.Image.Output(
"over",
tooltip="Opaque composite over the plate (or original RGB when background=none). Width/height padded to even for H.264; rgba/mask/image keep the source size.",
),
io.Image.Output(
"rgba",
tooltip="Straight RGBA from the refined matte (alpha 1 = keep). Use for WebM / EXR.",
),
io.Mask.Output("mask", tooltip="Refined matte. White = keep."),
io.Image.Output(
"image",
tooltip="Source RGB after color unmix. Same as input when unmix is 0.",
),
],
)
@classmethod
def check_lazy_status(
cls,
matte=None,
image=None,
invert=False,
expand=0,
feather=0.0,
black_clip=0.0,
white_clip=0.0,
mix=1.0,
unmix=0.0,
unmix_radius=12.0,
background="none",
fit="cover",
length="hold",
plate_file="",
mask=None,
background_image=None,
background_video=None,
**_kwargs,
):
needed = []
if background == "image" and background_image is None and not plate_file:
needed.append("background_image")
if background == "video" and background_video is None and not plate_file:
needed.append("background_video")
return needed
@classmethod
def execute(
cls,
matte: torch.Tensor | None = None,
image: torch.Tensor | None = None,
invert: bool = False,
expand: int = 0,
feather: float = 0.0,
black_clip: float = 0.0,
white_clip: float = 0.0,
mix: float = 1.0,
unmix: float = 0.0,
unmix_radius: float = 12.0,
background: str = "none",
fit: str = "cover",
length: str = "hold",
plate_file: str = "",
mask: torch.Tensor | None = None,
background_image: torch.Tensor | None = None,
background_video=None,
**_kwargs,
) -> io.NodeOutput:
source = mask_from_matte(matte, mask)
work = _refine_mask(
source,
invert=invert,
expand=int(expand),
feather=float(feather),
black_clip=float(black_clip),
white_clip=float(white_clip),
mix=float(mix),
)
n, h, w = work.shape
if image is None:
rgb = torch.full((n, h, w, 3), 0.5, device=work.device, dtype=work.dtype)
else:
rgb = resize_rgb(rgb_frames(image).to(device=work.device, dtype=work.dtype), h, w)
if rgb.shape[0] != n:
rgb = match_length(rgb, n, length)
work = match_mask_to_rgb(work, n, h, w, length)
source = match_mask_to_rgb(source, n, h, w, length)
unmixed = _unmix_color(rgb, work, float(unmix), float(unmix_radius))
if background != "none" and plate_file:
loaded = None
if background == "image" and background_image is None:
loaded = _load_plate_file(plate_file)
background_image = loaded
elif background == "video" and background_video is None:
loaded = _load_plate_file(plate_file)
background_video = loaded
plate = resolve_plate(
background,
h,
w,
n,
fit,
length,
unmixed.device,
unmixed.dtype,
background_image=background_image,
background_video=background_video,
)
rgba = join_rgba(unmixed, work)
over = over_composite(unmixed, work, plate)
live_mask, live_rgb, live_bg, live_scale, live_source = _live_clip(source, rgb, plate)
return io.NodeOutput(
over,
rgba,
work,
unmixed.contiguous(),
ui={
"live_mask": live_mask,
"live_rgb": live_rgb,
"live_bg": live_bg,
"live_scale": [live_scale],
"live_source_frames": [live_source],
},
)
+2 -3
View File
@@ -10,7 +10,6 @@ from kornia.geometry.transform.pyramid import (
build_pyramid,
find_next_powerof_two,
is_powerof_two,
pad,
)
from torch import Tensor
@@ -28,7 +27,7 @@ def _pad_for_laplacian(image: torch.Tensor) -> tuple[torch.Tensor, tuple[int, in
if not (is_powerof_two(h) and is_powerof_two(w)):
pad_right = find_next_powerof_two(w) - w
pad_down = find_next_powerof_two(h) - h
image = pad(image, (0, pad_right, 0, pad_down), "reflect")
image = F.pad(image, (0, pad_right, 0, pad_down), mode="reflect")
return image, (pad_right, pad_down)
@@ -41,7 +40,7 @@ def _gaussian_pyramid(
h, w = images.shape[2], images.shape[3]
if not (is_powerof_two(w) and is_powerof_two(h)):
padding = (0, find_next_powerof_two(w) - w, 0, find_next_powerof_two(h) - h)
images = pad(images, padding, border_type)
images = F.pad(images, padding, mode=border_type)
return build_pyramid(images, max_level, border_type, align_corners)
+4
View File
@@ -1,6 +1,10 @@
# Required for HDR primaries (same as ltx-core); no baked-matrix fallback.
colour-science>=0.4.4
diffusers
einops
huggingface_hub>=0.25.2
kornia
ninja~=1.11.1.4
# Required for HDR EXR read/write (same as ltx-pipelines); no OpenCV EXR fallback.
openimageio
transformers[timm]>=4.50.0
+3 -3
View File
@@ -5,7 +5,7 @@ import torch
from comfy_extras.nodes_custom_sampler import SamplerCustomAdvanced
from comfy_extras.nodes_lt import LTXVAddGuide, LTXVCropGuides
from .latents import LTXVAddLatentGuide, LTXVSelectLatents
from .latents import LTXVSelectLatents, add_latent_guide
from .nodes_registry import comfy_node
@@ -233,7 +233,7 @@ class LTXVTiledSampler:
tile_positive,
tile_negative,
tile_latents,
) = LTXVAddLatentGuide().generate(
) = add_latent_guide(
vae=vae,
positive=tile_positive,
negative=tile_negative,
@@ -249,7 +249,7 @@ class LTXVTiledSampler:
tile_positive,
tile_negative,
tile_latents,
) = LTXVAddLatentGuide().generate(
) = add_latent_guide(
vae=vae,
positive=tile_positive,
negative=tile_negative,
+9 -2
View File
@@ -97,6 +97,11 @@ class LTXVInpaintPreprocess(io.ComfyNode):
"""Composites images with a green (#66FF00) background where mask is active.
If the mask has a single frame it is broadcast to match the video length.
The fill is always SDR display-encoded ``#66FF00`` (``_BG_COLOR_RGB``), even
when ``images`` are already ACEScct. Native HDR inpaint IC-LoRA was trained
with that pixel-space key; converting the green into ACEScct changes the
VAE latent cue and breaks inpainting.
"""
@classmethod
@@ -105,8 +110,10 @@ class LTXVInpaintPreprocess(io.ComfyNode):
node_id="LTXVInpaintPreprocess",
category="Lightricks/image_processing",
description=(
"Composites images with a green background where mask is "
"active, for inpainting conditioning."
"Composites images with a green (#66FF00) background where mask "
"is active, for inpainting conditioning. Always paints SDR "
"#66FF00 (not ACEScct-converted) — that key matches IC-LoRA "
"training even on native HDR / ACEScct plates."
),
inputs=[
io.Image.Input(
File diff suppressed because it is too large Load Diff