Merge pull request #583 from Lightricks/pr-2026-10-20
ComfyUI-LTXVideo 1.1.0
This commit is contained in:
Alex Artyomov
@@ -16,8 +16,6 @@ from .guiders import GuiderParametersNode, MultimodalGuiderNode
|
||||
from .hdr_nodes import (
|
||||
LTXVHDRDecodePostprocess,
|
||||
LTXVLoadEXRSequence,
|
||||
LTXVSaveHLG,
|
||||
LTXVSDRToHDRWorkingSpace,
|
||||
LTXVVAEForceFloat32,
|
||||
)
|
||||
from .iclora import (
|
||||
@@ -106,10 +104,8 @@ NODE_CLASS_MAPPINGS = {
|
||||
"LTXVInpaintPreprocess": LTXVInpaintPreprocess,
|
||||
"LTXVLaplacianPyramidBlend": LTXVLaplacianPyramidBlend,
|
||||
"LTXVHDRDecodePostprocess": LTXVHDRDecodePostprocess,
|
||||
"LTXVSDRToHDRWorkingSpace": LTXVSDRToHDRWorkingSpace,
|
||||
"LTXVLoadEXRSequence": LTXVLoadEXRSequence,
|
||||
"LTXVVAEForceFloat32": LTXVVAEForceFloat32,
|
||||
"LTXVSaveHLG": LTXVSaveHLG,
|
||||
"LTXVAudioOnlyModel": LTXVAudioOnlyModel,
|
||||
"LTXVAudioOnlyEmptyVideoLatent": LTXVAudioOnlyEmptyVideoLatent,
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+728
-910
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -60,7 +60,7 @@ Upscale prompts should describe **look and style** (lighting, palette, film stoc
|
||||
| -------- | ------------ |
|
||||
| [LTX-2.5_V2V_TiledFusion_Upscale.json](./LTX-2.5_V2V_TiledFusion_Upscale.json) | **V2V upscale.** Preprocess resizes the guide to `output_size` (FullHD / 4K / 8K), then 8 fused steps with the detail refiner. Tiles come from `tile_size` (HD). |
|
||||
| [LTX-2.5_V2V_TiledFusion_Native_4K_8K.json](./LTX-2.5_V2V_TiledFusion_Native_4K_8K.json) | **V2V native / dynamic 4K/8K.** Stage 1 at `initial_canvas_size` FullHD (8 steps, Day-To-Night IC-LoRA). Latent ×2 (×4 when `output_size` is 8K), then 3 fused steps with a fresh prompt and the original clip resized to output. Optional second IC-LoRA on stage 2. |
|
||||
| [LTX-2.5_V2V_TiledFusion_SDR_to_HDR.json](./LTX-2.5_V2V_TiledFusion_SDR_to_HDR.json) | Same recipe as Native, with the HDR IC-LoRA. SDR MP4 → ACEScct, then `LTXVHDRDecodePostprocess` (EXR + HLG). |
|
||||
| [LTX-2.5_V2V_TiledFusion_SDR_to_HDR.json](./LTX-2.5_V2V_TiledFusion_SDR_to_HDR.json) | Same recipe as Native, with the HDR IC-LoRA. SDR MP4 → ACEScct; EXR via `LTXVHDRDecodePostprocess`, HLG via core Create Video / Save Video. |
|
||||
| [LTX-2.5_V2V_TiledFusion_HDR_to_HDR.json](./LTX-2.5_V2V_TiledFusion_HDR_to_HDR.json) | **Native HDR→HDR refine.** EXR still (ACEScct, float32 VAE, `img_compression` 0) with the detail refiner instead of the HDR IC-LoRA. Same 4K/8K tiled ladder; EXR + HLG out. |
|
||||
|
||||
## Which workflow should I use?
|
||||
@@ -137,17 +137,17 @@ Two-stage is the better default when you care about spatial detail. Single-stage
|
||||
| Tiled Fusion SDR→HDR | 2 | 2× or 4× latent | Same as native, HDR IC-LoRA | SDR→ACEScct 4K/8K; EXR + HLG |
|
||||
| Tiled Fusion HDR→HDR | 2 | 2× or 4× latent | HDR EXR still + refine IC-LoRA | Native ACEScct refine; EXR + HLG |
|
||||
|
||||
Native HDR I/O: [LTX-2.5_V2V_TiledFusion_HDR_to_HDR.json](./LTX-2.5_V2V_TiledFusion_HDR_to_HDR.json) — `LTXVLoadEXRSequence` → ACEScct → `LTXVVAEForceFloat32` → refine IC-LoRA tiled fusion → `LTXVHDRDecodePostprocess` (`transfer=acescct`, optional EXR) → `LTXVSaveHLG` from `hdr_linear`. `img_compression = 0`. Same idea as pipelines `--hdr` on Distilled / IC-LoRA.
|
||||
Native HDR I/O: [LTX-2.5_V2V_TiledFusion_HDR_to_HDR.json](./LTX-2.5_V2V_TiledFusion_HDR_to_HDR.json) — `LTXVLoadEXRSequence` → ACEScct → `LTXVVAEForceFloat32` → refine IC-LoRA tiled fusion → EXR via `LTXVHDRDecodePostprocess` (`transfer=acescct`, EXR=`acescg`); HLG via Convert Image Color Space (`HDR ACEScct` → `HDR`) → Create Video (`HDR`) → Save Video. `img_compression = 0`. Same idea as pipelines `--hdr` on Distilled / IC-LoRA.
|
||||
|
||||
SDR→HDR IC-LoRA (separate from native EXR HDR): [LTX-2.3_ICLoRA_HDR_Distilled.json](../2.3/LTX-2.3_ICLoRA_HDR_Distilled.json) for a single-stage pass, or [LTX-2.5_V2V_TiledFusion_SDR_to_HDR.json](./LTX-2.5_V2V_TiledFusion_SDR_to_HDR.json) for the tiled 4K/8K ladder. `LoadVideo` → `LTXVSDRToHDRWorkingSpace` (`srgb_gamma`) → HDR IC-LoRA guide → float32 VAE decode → `LTXVHDRDecodePostprocess` (`transfer=acescct`, EXR=`acescct`) → HLG. Sampler **euler**; original audio frozen/muxed. Same idea as pipelines `hdr_ic_lora --text-embeddings`.
|
||||
SDR→HDR IC-LoRA (separate from native EXR HDR): [LTX-2.5_V2V_TiledFusion_SDR_to_HDR.json](./LTX-2.5_V2V_TiledFusion_SDR_to_HDR.json) (tiled 4K/8K ladder): `LoadVideo` → Convert Image Color Space (`sRGB` → `HDR ACEScct`) → HDR IC-LoRA guide → float32 VAE decode → EXR via `LTXVHDRDecodePostprocess` (`transfer=acescct`, EXR=`acescg`); HLG via Convert Image Color Space (`HDR ACEScct` → `HDR`) → Create Video (`HDR`) → Save Video. Sampler **euler**; original audio frozen/muxed. Same idea as pipelines `hdr_ic_lora --text-embeddings`. For a single-stage pass, [LTX-2.3_ICLoRA_HDR_Distilled.json](../2.3/LTX-2.3_ICLoRA_HDR_Distilled.json) is the older LogC3 graph: `LTXVHDRDecodePostprocess` (`transfer=logc3`) writes a linear EXR, without the core HDR nodes.
|
||||
|
||||
For native HDR, set **`img_compression = 0`** (Python `--hdr` EXR conditioning skips JPEG/CRF; CRF on ACEScct darkens highlights). Turn **prompt enhancer off** when matching goldens. I2V stage-2 distilled sigmas should be `0.909375, 0.725, 0.421875, 0.0` (not `0.85, …`). Inpaint keeps its own 2-step stage-2 schedule.
|
||||
|
||||
HDR graphs live under [`HDR_workflows/`](./HDR_workflows/).
|
||||
HDR graphs live under [`HDR_workflows/`](./HDR_workflows/). They use core Convert Image Color Space and HDR Create Video / Save Video, so they need ComfyUI v0.38.0 or newer.
|
||||
|
||||
Native HDR I/O: `LTXVLoadEXRSequence` → float32 VAE → `LTXVHDRDecodePostprocess` (`transfer=acescct`, optional EXR) → `LTXVSaveHLG` from `hdr_linear`. Same idea as pipelines `--hdr` on Distilled / IC-LoRA.
|
||||
Native HDR I/O: `LTXVLoadEXRSequence` → float32 VAE → EXR via `LTXVHDRDecodePostprocess` (`transfer=acescct`, EXR=`acescg`); HLG via Convert Image Color Space (`HDR ACEScct` → `HDR`) → Create Video (`HDR`) → Save Video. Same idea as pipelines `--hdr` on Distilled / IC-LoRA.
|
||||
|
||||
SDR→HDR IC-LoRA (separate from native EXR HDR): [LTX-2.5_ICLoRA_SDR_to_HDR_Distilled.json](./HDR_workflows/LTX-2.5_ICLoRA_SDR_to_HDR_Distilled.json) — `LoadVideo` → `LTXVSDRToHDRWorkingSpace` (`srgb_gamma`) → `Resize Image/Mask` (scale to multiple 32) → **fixed scene embeddings** (`LTXVLoadConditioning`, not a free-text prompt) → HDR IC-LoRA guide → float32 VAE decode → `LTXVHDRDecodePostprocess` (`transfer=acescct`, EXR=`acescg`) → HLG. Sampler **euler**; original audio frozen/muxed. Same idea as pipelines `hdr_ic_lora --text-embeddings`. Conditioning fps snaps to 60 when source fps is above 30 (playback fps unchanged for decode/export).
|
||||
SDR→HDR IC-LoRA (separate from native EXR HDR): [LTX-2.5_ICLoRA_SDR_to_HDR_Distilled.json](./HDR_workflows/LTX-2.5_ICLoRA_SDR_to_HDR_Distilled.json) — `LoadVideo` → Convert Image Color Space (`sRGB` → `HDR ACEScct`) → `Resize Image/Mask` (scale to multiple 32) → **fixed scene embeddings** (`LTXVLoadConditioning`, not a free-text prompt) → HDR IC-LoRA guide → float32 VAE decode → EXR via `LTXVHDRDecodePostprocess` (`transfer=acescct`, EXR=`acescg`); HLG via Convert Image Color Space (`HDR ACEScct` → `HDR`) → Create Video (`HDR`) → Save Video. Sampler **euler**; original audio frozen/muxed. Same idea as pipelines `hdr_ic_lora --text-embeddings`. Conditioning fps snaps to 60 when source fps is above 30 (playback fps unchanged for decode/export).
|
||||
|
||||
For native HDR, set **`img_compression = 0`** (Python `--hdr` EXR conditioning skips JPEG/CRF; CRF on ACEScct darkens highlights). Turn **prompt enhancer off** when matching goldens. I2V stage-2 distilled sigmas should be `0.909375, 0.725, 0.421875, 0.0` (not `0.85, …`). Inpaint keeps its own 2-step stage-2 schedule.
|
||||
|
||||
|
||||
+2
-79
@@ -7,8 +7,8 @@ matrices / EXR chromaticities come from ``colour-science`` with Bradford CAT
|
||||
|
||||
``colour-science`` is required (same as ``ltx_core.color.primaries``).
|
||||
|
||||
Used by HDR Comfy nodes in ``hdr_nodes`` (EXR ↔ VAE, SDR→HDR IC-LoRA ingest
|
||||
via ``SDRColorSpace`` / sRGB EOTF → ACEScct). File I/O helpers live in ``hdr_io``.
|
||||
Used by HDR Comfy nodes in ``hdr_nodes`` (EXR ↔ VAE ACEScct). SDR→ACEScct ingest
|
||||
uses Comfy Convert Image Color Space. File I/O helpers live in ``hdr_io``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -33,7 +33,6 @@ _ACESCCT_LOG_B = 9.72
|
||||
_CAT = "Bradford"
|
||||
_CS_ACESCG = colour.RGB_COLOURSPACES["ACEScg"]
|
||||
_CS_REC709 = colour.RGB_COLOURSPACES["ITU-R BT.709"]
|
||||
_CS_REC2020 = colour.RGB_COLOURSPACES["ITU-R BT.2020"]
|
||||
|
||||
|
||||
def _rgb_to_rgb(src: colour.RGB_Colourspace, dst: colour.RGB_Colourspace) -> Tensor:
|
||||
@@ -62,8 +61,6 @@ def _exr_chroma(cs: colour.RGB_Colourspace) -> tuple[float, ...]:
|
||||
|
||||
_ACESCG_TO_SRGB = _rgb_to_rgb(_CS_ACESCG, _CS_REC709)
|
||||
_SRGB_TO_ACESCG = torch.linalg.inv(_ACESCG_TO_SRGB.double()).to(torch.float32)
|
||||
_REC709_TO_2020 = _rgb_to_rgb(_CS_REC709, _CS_REC2020)
|
||||
_ACESCG_TO_2020 = _rgb_to_rgb(_CS_ACESCG, _CS_REC2020)
|
||||
_CHROMA_REC709 = _exr_chroma(_CS_REC709)
|
||||
_CHROMA_ACESCG = _exr_chroma(_CS_ACESCG)
|
||||
_MATRICES_SOURCE = "colour-science"
|
||||
@@ -71,8 +68,6 @@ _MATRICES_SOURCE = "colour-science"
|
||||
# Public aliases (parity with ``ltx_core.color.primaries`` exports).
|
||||
ACESCG_TO_SRGB = _ACESCG_TO_SRGB
|
||||
SRGB_TO_ACESCG = _SRGB_TO_ACESCG
|
||||
REC709_TO_2020 = _REC709_TO_2020
|
||||
ACESCG_TO_2020 = _ACESCG_TO_2020
|
||||
|
||||
_CHROMATICITIES = {
|
||||
"rec709": _CHROMA_REC709,
|
||||
@@ -91,11 +86,6 @@ class Primaries(enum.Enum):
|
||||
"""EXR ``chromaticities`` header (R/G/B/W x,y)."""
|
||||
return _CHROMATICITIES[self.value]
|
||||
|
||||
@property
|
||||
def matrix_to_rec2020(self) -> Tensor:
|
||||
"""3x3 linear map from this basis to Rec.2020 (for HLG)."""
|
||||
return _REC709_TO_2020 if self is Primaries.REC709 else _ACESCG_TO_2020
|
||||
|
||||
def to(self, target: Primaries, video: Tensor) -> Tensor:
|
||||
"""Convert linear light from this basis into ``target``. Identity when same."""
|
||||
if self is target:
|
||||
@@ -106,10 +96,6 @@ class Primaries(enum.Enum):
|
||||
return _apply_matrix(video, _ACESCG_TO_SRGB)
|
||||
raise ValueError(f"No primaries conversion from {self!r} to {target!r}.")
|
||||
|
||||
def to_rec2020(self, video: Tensor) -> Tensor:
|
||||
"""Linear light in this basis → Rec.2020 (HLG master space)."""
|
||||
return _apply_matrix(video, self.matrix_to_rec2020)
|
||||
|
||||
|
||||
class HDRColorSpace(enum.Enum):
|
||||
"""Source / EXR authoring colour space (matches pipelines ``--hdr``)."""
|
||||
@@ -150,13 +136,6 @@ def _apply_matrix(video: Tensor, mat: Tensor) -> Tensor:
|
||||
raise ValueError(f"Expected RGB in last or -3 dim; got shape {tuple(video.shape)}")
|
||||
|
||||
|
||||
# sRGB EOTF (IEC 61966-2-1) — same constants as ``ltx_core.hdr``.
|
||||
_SRGB_A = 0.055
|
||||
_SRGB_LINEAR_THRESHOLD = 0.04045
|
||||
_SRGB_LINEAR_SLOPE = 12.92
|
||||
_SRGB_GAMMA = 2.4
|
||||
|
||||
|
||||
def compress_acescct(hdr: Tensor) -> Tensor:
|
||||
"""Linear HDR [0, ∞) → ACEScct [0, 1]."""
|
||||
x = torch.clamp(hdr, min=0.0)
|
||||
@@ -173,16 +152,6 @@ def decompress_acescct(ct: Tensor) -> Tensor:
|
||||
return torch.where(ct > _ACESCCT_Y_BRK, lin_from_log, lin_from_lin)
|
||||
|
||||
|
||||
def srgb_eotf_to_linear(srgb_01: Tensor) -> Tensor:
|
||||
"""sRGB-encoded [0, 1] → display-linear Rec.709/sRGB (IEC 61966-2-1 EOTF)."""
|
||||
x = torch.clamp(srgb_01.float(), 0.0, 1.0)
|
||||
return torch.where(
|
||||
x <= _SRGB_LINEAR_THRESHOLD,
|
||||
x / _SRGB_LINEAR_SLOPE,
|
||||
torch.pow((x + _SRGB_A) / (1.0 + _SRGB_A), _SRGB_GAMMA),
|
||||
)
|
||||
|
||||
|
||||
def to_acescct_working_space(linear_rgb: Tensor, source_primaries: Primaries) -> Tensor:
|
||||
"""Scene-linear RGB → ACEScct [0, 1] (channel-last or channel-first)."""
|
||||
native = source_primaries.to(Primaries.ACESCG, linear_rgb.float())
|
||||
@@ -197,55 +166,9 @@ def acescct_to_linear(
|
||||
return Primaries.ACESCG.to(out_primaries, linear_acescg).clamp(min=0.0)
|
||||
|
||||
|
||||
class SDRColorSpace(enum.Enum):
|
||||
"""SDR ingest colour space for mapping float RGB into ACEScct.
|
||||
|
||||
Matches pipelines ``SDRColorSpace`` / ``hdr_ic_lora`` IDT:
|
||||
|
||||
* ``srgb_gamma`` — display-referred sRGB (apply EOTF, then ACEScct).
|
||||
* ``srgb`` — scene-linear Rec.709 / sRGB primaries (no EOTF).
|
||||
* ``acescg`` — scene-linear ACEScg → ACEScct.
|
||||
* ``acescct`` — already ACEScct log codes; clamp passthrough.
|
||||
"""
|
||||
|
||||
SRGB = "srgb"
|
||||
SRGB_GAMMA = "srgb_gamma"
|
||||
ACESCG = "acescg"
|
||||
ACESCCT = "acescct"
|
||||
|
||||
def to_working_space(self, video: Tensor) -> Tensor:
|
||||
"""Map SDR float RGB into ACEScct ``[0, 1]`` (channel-last or channel-first)."""
|
||||
if not video.is_floating_point():
|
||||
raise ValueError(
|
||||
f"SDRColorSpace.to_working_space expects float RGB; got dtype={video.dtype}. "
|
||||
"Normalize integer pixels before calling."
|
||||
)
|
||||
video = video.float()
|
||||
if self is SDRColorSpace.ACESCCT:
|
||||
return video.clamp(0.0, 1.0)
|
||||
if self is SDRColorSpace.SRGB_GAMMA:
|
||||
return to_acescct_working_space(
|
||||
srgb_eotf_to_linear(video), Primaries.REC709
|
||||
)
|
||||
if self is SDRColorSpace.SRGB:
|
||||
return to_acescct_working_space(video, Primaries.REC709)
|
||||
if self is SDRColorSpace.ACESCG:
|
||||
return to_acescct_working_space(video, Primaries.ACESCG)
|
||||
raise ValueError(f"Unsupported SDR color space: {self!r}")
|
||||
|
||||
|
||||
def images_to_acescct(images: Tensor, color_space: HDRColorSpace) -> Tensor:
|
||||
"""Convert loaded EXR frames ``[F,H,W,3]`` into ACEScct ``[0,1]`` for the VAE."""
|
||||
images = images.float()
|
||||
if color_space.is_log_working:
|
||||
return images.clamp(0.0, 1.0)
|
||||
return to_acescct_working_space(images, color_space.source_primaries)
|
||||
|
||||
|
||||
def sdr_images_to_acescct(images: Tensor, color_space: SDRColorSpace) -> Tensor:
|
||||
"""Convert SDR frames ``[F,H,W,C]`` into ACEScct ``[0,1]`` for HDR IC-LoRA."""
|
||||
rgb = images[..., :3]
|
||||
working = color_space.to_working_space(rgb)
|
||||
if images.shape[-1] > 3:
|
||||
return torch.cat([working, images[..., 3:].float()], dim=-1)
|
||||
return working
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""HDR file I/O helpers: EXR read/write, HLG encode, path utilities.
|
||||
"""HDR file I/O helpers: EXR read/write, path utilities.
|
||||
|
||||
Shared by Comfy HDR nodes (ingest + delivery). Color math lives in ``hdr_color``;
|
||||
Shared by Comfy HDR nodes (ingest + EXR delivery). Color math lives in ``hdr_color``;
|
||||
node classes live in ``hdr_nodes``.
|
||||
"""
|
||||
|
||||
@@ -8,7 +8,6 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
@@ -21,22 +20,6 @@ from .hdr_color import Primaries
|
||||
|
||||
logger = logging.getLogger("LTXVideo.hdr_io")
|
||||
|
||||
# BT.2020 / HLG (ARIB STD-B67) encode constants — from ltx_core.color.hlg
|
||||
_HLG_A, _HLG_B, _HLG_C = 0.17883277, 0.28466892, 0.55991073
|
||||
_AV_COLOR_PRIMARIES_BT2020 = 9
|
||||
_AV_COLOR_TRC_ARIB_STD_B67 = 18
|
||||
_AV_COLORSPACE_BT2020_NCL = 9
|
||||
_AV_COLOR_RANGE_MPEG = 1
|
||||
_KR_2020, _KG_2020, _KB_2020 = 0.2627, 0.6780, 0.0593
|
||||
_BT2020_RGB_TO_YUV = torch.tensor(
|
||||
[
|
||||
[_KR_2020, _KG_2020, _KB_2020],
|
||||
[-_KR_2020 / 1.8814, -_KG_2020 / 1.8814, 0.5],
|
||||
[0.5, -_KG_2020 / 1.4746, -_KB_2020 / 1.4746],
|
||||
],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_path(path: str) -> Path:
|
||||
p = Path(path).expanduser()
|
||||
@@ -176,211 +159,6 @@ def _save_exr_frame(
|
||||
out.close()
|
||||
|
||||
|
||||
def _hlg_inverse_oetf_scalar(v: float) -> float:
|
||||
return (
|
||||
(v * v) / 3.0 if v <= 0.5 else (math.exp((v - _HLG_C) / _HLG_A) + _HLG_B) / 12.0
|
||||
)
|
||||
|
||||
|
||||
def _linear_to_hlg_signal(
|
||||
rgb_fhwc: Tensor,
|
||||
primaries: Primaries = Primaries.REC709,
|
||||
white_signal: float = 0.75,
|
||||
) -> Tensor:
|
||||
"""Scene-linear RGB ``[F,H,W,3]`` in ``primaries`` → HLG signal in BT.2020.
|
||||
|
||||
Maps with ``primaries.matrix_to_rec2020`` then clamps (same as
|
||||
``ltx_core.color.hlg.HlgGpuConverter``). Prefer ACEScg→Rec.2020 directly
|
||||
over ACEScg→Rec.709→Rec.2020 so OOG negatives are not crushed early.
|
||||
"""
|
||||
white_x = _hlg_inverse_oetf_scalar(white_signal)
|
||||
roll_k = white_x / (1.0 - white_x)
|
||||
mat = primaries.matrix_to_rec2020.to(device=rgb_fhwc.device, dtype=rgb_fhwc.dtype)
|
||||
lin = torch.nan_to_num((rgb_fhwc @ mat.T).clamp(min=0.0), nan=0.0, neginf=0.0)
|
||||
x = torch.where(
|
||||
lin <= 1.0,
|
||||
lin * white_x,
|
||||
1.0 - (1.0 - white_x) * torch.exp(-roll_k * (lin - 1.0)),
|
||||
)
|
||||
sig = torch.where(
|
||||
x <= 1.0 / 12.0,
|
||||
torch.sqrt((3.0 * x).clamp(min=0.0)),
|
||||
_HLG_A * torch.log((12.0 * x - _HLG_B).clamp(min=1e-12)) + _HLG_C,
|
||||
)
|
||||
return sig.clamp(0.0, 1.0)
|
||||
|
||||
|
||||
def _rgb_hlg_to_yuv420p10(
|
||||
rgb_fhwc: Tensor,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""HLG RGB ``[F,H,W,3]`` → planar uint16 Y,U,V (limited / BT.2020)."""
|
||||
if rgb_fhwc.shape[-2] % 2 or rgb_fhwc.shape[-3] % 2:
|
||||
raise ValueError(f"HLG encode needs even H and W; got {tuple(rgb_fhwc.shape)}")
|
||||
rgb = rgb_fhwc.movedim(-1, -3).contiguous() # F,3,H,W
|
||||
mat = _BT2020_RGB_TO_YUV.to(device=rgb.device, dtype=rgb.dtype)
|
||||
pixels = rgb.movedim(-3, -1)
|
||||
yuv = pixels.flatten(-3, -2) @ mat.T
|
||||
yuv = yuv.unflatten(-2, rgb.shape[-2:]).movedim(-1, -3)
|
||||
y = yuv[:, :1]
|
||||
uv = torch.nn.functional.avg_pool2d(
|
||||
yuv[:, 1:3].contiguous(), kernel_size=2, stride=2
|
||||
)
|
||||
# MPEG limited 10-bit
|
||||
y = y.mul(219 * 4).add_(16 * 4)
|
||||
uv = uv.mul(224 * 4).add_(128 * 4)
|
||||
y16 = y[:, 0].round_().clamp_(0, 1023).to(torch.uint16).cpu().numpy()
|
||||
u16 = uv[:, 0].round_().clamp_(0, 1023).to(torch.uint16).cpu().numpy()
|
||||
v16 = uv[:, 1].round_().clamp_(0, 1023).to(torch.uint16).cpu().numpy()
|
||||
return y16, u16, v16
|
||||
|
||||
|
||||
def _encode_hlg_mp4(
|
||||
frames_linear: Tensor,
|
||||
out_path: Path,
|
||||
fps: float,
|
||||
audio: dict | None = None,
|
||||
*,
|
||||
primaries: Primaries = Primaries.REC709,
|
||||
) -> None:
|
||||
"""Encode scene-linear ``[F,H,W,3]`` (in ``primaries``) to BT.2020/HLG/10-bit HEVC.
|
||||
|
||||
Optional Comfy ``AUDIO`` dict ``{waveform, sample_rate}`` is muxed as AAC.
|
||||
The AAC stream is registered *before* video packets (PyAV requires this).
|
||||
"""
|
||||
from fractions import Fraction
|
||||
|
||||
import av
|
||||
|
||||
hlg = _linear_to_hlg_signal(frames_linear.float(), primaries=primaries)
|
||||
y, u, v = _rgb_hlg_to_yuv420p10(hlg)
|
||||
f, height, width = int(y.shape[0]), int(y.shape[1]), int(y.shape[2])
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
container = av.open(str(out_path), mode="w", options={"movflags": "+faststart"})
|
||||
audio_stream = None
|
||||
try:
|
||||
stream = container.add_stream(
|
||||
"libx265", rate=Fraction(fps).limit_denominator(1000)
|
||||
)
|
||||
stream.width = width
|
||||
stream.height = height
|
||||
stream.pix_fmt = "yuv420p10le"
|
||||
stream.codec_tag = "hvc1"
|
||||
threads = max(1, min(os.cpu_count() or 8, 16))
|
||||
stream.options = {
|
||||
"crf": "12",
|
||||
"preset": "medium",
|
||||
"x265-params": (
|
||||
"colorprim=bt2020:transfer=arib-std-b67:colormatrix=bt2020nc:range=limited:"
|
||||
f"repeat-headers=1:info=0:pools={threads}:frame-threads=4"
|
||||
),
|
||||
}
|
||||
ctx = stream.codec_context
|
||||
ctx.thread_count = threads
|
||||
ctx.color_primaries = _AV_COLOR_PRIMARIES_BT2020
|
||||
ctx.color_trc = _AV_COLOR_TRC_ARIB_STD_B67
|
||||
ctx.colorspace = _AV_COLORSPACE_BT2020_NCL
|
||||
ctx.color_range = _AV_COLOR_RANGE_MPEG
|
||||
|
||||
if audio is not None:
|
||||
audio_stream = _prepare_aac_stream(container, audio)
|
||||
|
||||
for i in range(f):
|
||||
frame = av.VideoFrame(width, height, "yuv420p10le")
|
||||
for plane, src in zip(frame.planes, (y[i], u[i], v[i]), strict=True):
|
||||
dest = np.frombuffer(plane, dtype=np.uint16).reshape(
|
||||
plane.height, plane.line_size // 2
|
||||
)
|
||||
dest[:, : src.shape[1]] = src
|
||||
frame.colorspace = _AV_COLORSPACE_BT2020_NCL
|
||||
frame.color_range = _AV_COLOR_RANGE_MPEG
|
||||
for packet in stream.encode(frame):
|
||||
container.mux(packet)
|
||||
for packet in stream.encode():
|
||||
container.mux(packet)
|
||||
|
||||
if audio is not None and audio_stream is not None:
|
||||
_mux_comfy_audio_aac(container, audio_stream, audio)
|
||||
finally:
|
||||
container.close()
|
||||
|
||||
|
||||
def _comfy_audio_stereo_n2(audio: dict) -> tuple[Tensor, int]:
|
||||
"""Return stereo int16-ready float samples ``[N, 2]`` and sample rate."""
|
||||
waveform = audio.get("waveform")
|
||||
sample_rate = int(audio.get("sample_rate", 0) or 0)
|
||||
if waveform is None or sample_rate <= 0:
|
||||
raise ValueError("AUDIO must include waveform and sample_rate")
|
||||
|
||||
samples = waveform.detach().float().cpu()
|
||||
if samples.ndim == 3:
|
||||
samples = samples[0]
|
||||
if samples.ndim != 2:
|
||||
raise ValueError(
|
||||
f"Expected AUDIO waveform [C,N] or [B,C,N]; got {tuple(waveform.shape)}"
|
||||
)
|
||||
if samples.shape[0] == 2:
|
||||
samples_cn = samples
|
||||
elif samples.shape[1] == 2:
|
||||
samples_cn = samples.T
|
||||
elif samples.shape[0] == 1:
|
||||
samples_cn = samples.repeat(2, 1)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"AUDIO must be mono or stereo; got shape {tuple(samples.shape)}"
|
||||
)
|
||||
return samples_cn.T.contiguous(), sample_rate
|
||||
|
||||
|
||||
def _prepare_aac_stream(container, audio: dict):
|
||||
"""Register a stereo AAC stream before any packets are muxed."""
|
||||
from fractions import Fraction
|
||||
|
||||
_, sample_rate = _comfy_audio_stereo_n2(audio)
|
||||
audio_stream = container.add_stream("aac", rate=sample_rate)
|
||||
audio_stream.codec_context.sample_rate = sample_rate
|
||||
audio_stream.codec_context.layout = "stereo"
|
||||
audio_stream.codec_context.time_base = Fraction(1, sample_rate)
|
||||
return audio_stream
|
||||
|
||||
|
||||
def _mux_comfy_audio_aac(container, audio_stream, audio: dict) -> None:
|
||||
"""Mux Comfy AUDIO onto an already-added AAC stream (after video flush)."""
|
||||
import av
|
||||
|
||||
samples_n2, sample_rate = _comfy_audio_stereo_n2(audio)
|
||||
samples_i16 = (samples_n2.clamp(-1.0, 1.0) * 32767.0).to(torch.int16)
|
||||
|
||||
frame_in = av.AudioFrame.from_ndarray(
|
||||
samples_i16.reshape(1, -1).numpy(),
|
||||
format="s16",
|
||||
layout="stereo",
|
||||
)
|
||||
frame_in.sample_rate = sample_rate
|
||||
|
||||
cc = audio_stream.codec_context
|
||||
resampler = av.audio.resampler.AudioResampler(
|
||||
format=cc.format or "fltp",
|
||||
layout=cc.layout or "stereo",
|
||||
rate=cc.sample_rate or sample_rate,
|
||||
)
|
||||
next_pts = 0
|
||||
for rframe in resampler.resample(frame_in):
|
||||
if rframe.pts is None:
|
||||
rframe.pts = next_pts
|
||||
next_pts += rframe.samples
|
||||
for packet in audio_stream.encode(rframe):
|
||||
container.mux(packet)
|
||||
for rframe in resampler.resample(None):
|
||||
if rframe.pts is None:
|
||||
rframe.pts = next_pts
|
||||
next_pts += rframe.samples
|
||||
for packet in audio_stream.encode(rframe):
|
||||
container.mux(packet)
|
||||
for packet in audio_stream.encode(None):
|
||||
container.mux(packet)
|
||||
|
||||
|
||||
def _trim_to_8k1(images: Tensor) -> Tensor:
|
||||
"""Keep longest legal ``8k+1`` prefix (VAE temporal scale 8)."""
|
||||
n = images.shape[0]
|
||||
|
||||
+6
-175
@@ -2,9 +2,9 @@
|
||||
|
||||
Stage-oriented layout (shared ACEScct working space):
|
||||
|
||||
- Ingest: ``LTXVSDRToHDRWorkingSpace``, ``LTXVLoadEXRSequence``
|
||||
- Ingest: ``LTXVLoadEXRSequence`` (SDR→ACEScct uses Comfy Convert Image Color Space)
|
||||
- Runtime: ``LTXVVAEForceFloat32``
|
||||
- Delivery: ``LTXVHDRDecodePostprocess``, ``LTXVSaveHLG``
|
||||
- Delivery: ``LTXVHDRDecodePostprocess`` (HLG video uses Comfy Create Video / Save Video)
|
||||
|
||||
Color math: ``hdr_color``. File I/O helpers: ``hdr_io``.
|
||||
"""
|
||||
@@ -15,20 +15,11 @@ import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import folder_paths
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from .hdr_color import (
|
||||
HDRColorSpace,
|
||||
Primaries,
|
||||
SDRColorSpace,
|
||||
compress_acescct,
|
||||
images_to_acescct,
|
||||
sdr_images_to_acescct,
|
||||
)
|
||||
from .hdr_color import HDRColorSpace, Primaries, compress_acescct, images_to_acescct
|
||||
from .hdr_io import (
|
||||
_encode_hlg_mp4,
|
||||
_list_exr_files,
|
||||
_read_exr,
|
||||
_resolve_path,
|
||||
@@ -41,7 +32,6 @@ from .nodes_registry import comfy_node
|
||||
logger = logging.getLogger("LTXVideo.hdr_nodes")
|
||||
|
||||
_EXR_COLOR_SPACE_CHOICES = ["linear", "acescct", "srgb_linear", "acescg"]
|
||||
_SDR_COLOR_SPACE_CHOICES = [c.value for c in SDRColorSpace]
|
||||
_COLOR_SPACE_CHOICES = [c.value for c in HDRColorSpace]
|
||||
|
||||
|
||||
@@ -119,48 +109,6 @@ def _linear_to_srgb(x: Tensor) -> Tensor:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@comfy_node(name="LTXVSDRToHDRWorkingSpace")
|
||||
class LTXVSDRToHDRWorkingSpace:
|
||||
"""Map SDR float RGB into ACEScct ``[0, 1]`` for HDR IC-LoRA conditioning.
|
||||
Place **before** resize when the source is display sRGB so
|
||||
the EOTF is not applied to bilinear-filtered codes.
|
||||
|
||||
Default ``srgb_gamma`` matches typical ``LoadVideo`` / MP4 decode.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE",),
|
||||
"color_space": (
|
||||
_SDR_COLOR_SPACE_CHOICES,
|
||||
{
|
||||
"default": "srgb_gamma",
|
||||
"tooltip": (
|
||||
"SDR ingest IDT. srgb_gamma = display sRGB (EOTF then "
|
||||
"ACEScct); srgb = scene-linear Rec.709; acescg / acescct "
|
||||
"for float EXR-style plates."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
RETURN_NAMES = ("acescct",)
|
||||
FUNCTION = "convert"
|
||||
CATEGORY = "Lightricks/HDR"
|
||||
DESCRIPTION = (
|
||||
"SDR → ACEScct working space for HDR IC-LoRA (pipelines SDRColorSpace). "
|
||||
"Wire LoadVideo images through this before resize / IC-LoRA guide."
|
||||
)
|
||||
|
||||
def convert(self, image: torch.Tensor, color_space: str = "srgb_gamma") -> tuple:
|
||||
space = SDRColorSpace(color_space)
|
||||
return (sdr_images_to_acescct(image, space),)
|
||||
|
||||
|
||||
def _exr_payload_for_color_space(
|
||||
image: Tensor,
|
||||
hdr_linear: Tensor,
|
||||
@@ -219,8 +167,6 @@ class LTXVHDRDecodePostprocess:
|
||||
sequence (legacy ``linear``, or tagged ``acescct`` / ``srgb_linear`` /
|
||||
``acescg``).
|
||||
|
||||
For BT.2020/HLG masters, wire ``hdr_linear`` into ``LTXVSaveHLG``.
|
||||
|
||||
Outputs:
|
||||
tonemapped: SDR preview [0, 1] after Reinhard tonemap + sRGB gamma.
|
||||
hdr_linear: Raw linear HDR values [0, inf) for further processing.
|
||||
@@ -241,8 +187,8 @@ class LTXVHDRDecodePostprocess:
|
||||
"default": "acescct",
|
||||
"tooltip": (
|
||||
"Working-space curve of the VAE decode. "
|
||||
"Use acescct for SDR→HDR IC-LoRA (after "
|
||||
"LTXVSDRToHDRWorkingSpace) and for native HDR EXR; "
|
||||
"Use acescct for SDR→HDR IC-LoRA (after Convert "
|
||||
"Image Color Space sRGB→HDR ACEScct) and for native HDR EXR; "
|
||||
"logc3 only for legacy LogC3 HDR LoRA checkpoints."
|
||||
),
|
||||
},
|
||||
@@ -317,7 +263,7 @@ class LTXVHDRDecodePostprocess:
|
||||
DESCRIPTION = (
|
||||
"Decompresses VAE-decoded HDR (ACEScct or legacy LogC3), tonemaps for "
|
||||
"preview, and optionally writes EXR (linear / ACEScct / sRGB linear / "
|
||||
"ACEScg). Wire hdr_linear into LTXVSaveHLG for a BT.2020/HLG master."
|
||||
"ACEScg)."
|
||||
)
|
||||
|
||||
def postprocess(
|
||||
@@ -558,118 +504,3 @@ class LTXVVAEForceFloat32:
|
||||
) from exc
|
||||
logger.info("VAE dtype forced to float32 for native HDR")
|
||||
return (vae,)
|
||||
|
||||
|
||||
@comfy_node(name="LTXVSaveHLG")
|
||||
class LTXVSaveHLG:
|
||||
"""Encode scene-linear ``hdr_linear`` frames to a BT.2020 / HLG 10-bit HEVC master.
|
||||
|
||||
Wire the ``hdr_linear`` output of ``LTXVHDRDecodePostprocess`` here (not the
|
||||
tonemapped preview). Matches pipelines ``encode_video`` HLG delivery.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"hdr_linear": (
|
||||
"IMAGE",
|
||||
{
|
||||
"tooltip": (
|
||||
"Scene-linear HDR from LTXVHDRDecodePostprocess. "
|
||||
"With transfer=acescct this is ACEScg linear — leave "
|
||||
"linear_primaries=acescg. For logc3, use rec709."
|
||||
),
|
||||
},
|
||||
),
|
||||
"frame_rate": (
|
||||
"FLOAT",
|
||||
{"default": 24.0, "min": 1.0, "max": 120.0, "step": 0.001},
|
||||
),
|
||||
"filename_prefix": ("STRING", {"default": "hdr/ltxv_hlg"}),
|
||||
},
|
||||
"optional": {
|
||||
"linear_primaries": (
|
||||
["acescg", "rec709"],
|
||||
{
|
||||
"default": "acescg",
|
||||
"tooltip": (
|
||||
"Primaries of hdr_linear. ACEScct decode "
|
||||
"(SDR→HDR IC-LoRA or native HDR) → acescg. "
|
||||
"Legacy LogC3 → rec709."
|
||||
),
|
||||
},
|
||||
),
|
||||
"audio": (
|
||||
"AUDIO",
|
||||
{
|
||||
"tooltip": (
|
||||
"Optional track to mux into the HLG MP4 (e.g. Decode AUDIO). "
|
||||
"Omit for video-only."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ()
|
||||
OUTPUT_NODE = True
|
||||
FUNCTION = "save"
|
||||
CATEGORY = "Lightricks/HDR"
|
||||
DESCRIPTION = (
|
||||
"Writes a BT.2020/HLG 10-bit HEVC master from scene-linear hdr_linear "
|
||||
"(from LTXVHDRDecodePostprocess). Maps linear_primaries → Rec.2020 "
|
||||
"directly (no Rec.709 hop), then HLG. Optional AUDIO is muxed as AAC."
|
||||
)
|
||||
|
||||
def save(
|
||||
self,
|
||||
hdr_linear: Tensor,
|
||||
frame_rate: float,
|
||||
filename_prefix: str,
|
||||
linear_primaries: str = "acescg",
|
||||
audio: dict | None = None,
|
||||
):
|
||||
if hdr_linear.ndim != 4 or hdr_linear.shape[-1] < 3:
|
||||
raise ValueError(f"Expected IMAGE [F,H,W,C]; got {tuple(hdr_linear.shape)}")
|
||||
|
||||
# ACEScg/Rec.709 → Rec.2020 inside HLG encode (Primaries.to_rec2020 /
|
||||
# matrix_to_rec2020). No Rec.709 intermediate that would crush OOG early.
|
||||
linear = hdr_linear[..., :3].float()
|
||||
prim = (linear_primaries or "acescg").lower()
|
||||
if prim == "acescg":
|
||||
src_primaries = Primaries.ACESCG
|
||||
elif prim == "rec709":
|
||||
src_primaries = Primaries.REC709
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported linear_primaries {linear_primaries!r}; use acescg or rec709."
|
||||
)
|
||||
|
||||
# Even dims for 4:2:0
|
||||
h, w = linear.shape[1], linear.shape[2]
|
||||
if h % 2 or w % 2:
|
||||
linear = linear[:, : h - (h % 2), : w - (w % 2), :]
|
||||
|
||||
full_output_folder, filename, counter, _subfolder, _ = (
|
||||
folder_paths.get_save_image_path(
|
||||
filename_prefix,
|
||||
folder_paths.get_output_directory(),
|
||||
linear.shape[2],
|
||||
linear.shape[1],
|
||||
)
|
||||
)
|
||||
hlg_path = Path(full_output_folder) / f"{filename}_{counter:05d}_hlg.mp4"
|
||||
_encode_hlg_mp4(
|
||||
linear,
|
||||
hlg_path,
|
||||
fps=float(frame_rate),
|
||||
audio=audio,
|
||||
primaries=src_primaries,
|
||||
)
|
||||
logger.info(
|
||||
"Wrote HLG 10-bit master to %s%s",
|
||||
hlg_path,
|
||||
" (with audio)" if audio is not None else "",
|
||||
)
|
||||
return {}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
[project]
|
||||
name = "ComfyUI-LTXVideo"
|
||||
version = "1.1.0"
|
||||
description = "Custom nodes for LTX-Video support in ComfyUI"
|
||||
requires-python = ">=3.10"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = [
|
||||
"colour-science>=0.4.4",
|
||||
"diffusers",
|
||||
"einops",
|
||||
"huggingface_hub>=0.25.2",
|
||||
"kornia",
|
||||
"ninja~=1.11.1.4",
|
||||
"openimageio",
|
||||
"transformers>=4.50.0",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Repository = "https://github.com/Lightricks/ComfyUI-LTXVideo"
|
||||
|
||||
[tool.comfy]
|
||||
PublisherId = "ltx-io"
|
||||
DisplayName = "ComfyUI-LTXVideo"
|
||||
requires-comfyui = ">=0.35.0"
|
||||
@@ -109,6 +109,14 @@ class STGFlag:
|
||||
skip_layers: List[int] = None
|
||||
|
||||
|
||||
def _unwrap_attention_tensor(v):
|
||||
# ComfyUI wraps q/k/v in AttentionTensorContainer before calling attention.
|
||||
# The skip path returns v as the attention output, so it must be a plain tensor.
|
||||
if isinstance(v, comfy.ldm.modules.attention.AttentionTensorContainer):
|
||||
return v.take()
|
||||
return v
|
||||
|
||||
|
||||
# context manager that replaces the attention function in a transformer block
|
||||
class PatchAttention(contextlib.AbstractContextManager):
|
||||
def __init__(self, attn_idx: Optional[Union[int, List[int]]] = None):
|
||||
@@ -144,14 +152,14 @@ class PatchAttention(contextlib.AbstractContextManager):
|
||||
def stg_attention(self, q, k, v, heads, *args, **kwargs):
|
||||
self.current_idx += 1
|
||||
if self.current_idx in self.attn_idx:
|
||||
return v
|
||||
return _unwrap_attention_tensor(v)
|
||||
else:
|
||||
return self.original_attention(q, k, v, heads, *args, **kwargs)
|
||||
|
||||
def stg_attention_masked(self, q, k, v, heads, *args, **kwargs):
|
||||
self.current_idx += 1
|
||||
if self.current_idx in self.attn_idx:
|
||||
return v
|
||||
return _unwrap_attention_tensor(v)
|
||||
else:
|
||||
return self.original_attention_masked(q, k, v, heads, *args, **kwargs)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user