Merge pull request #257 from Lightricks/pr-2025-07-16

Update ComfyUI-LTXVideo - 2025-07-16
This commit is contained in:
Michael Kupchick
2025-07-16 17:28:33 +03:00
committed by GitHub
22 changed files with 13285 additions and 368 deletions
+30
View File
@@ -12,6 +12,29 @@ ComfyUI-LTXVideo is a collection of custom nodes for ComfyUI, designed to provid
The model itself is supported in the core ComfyUI [code](https://github.com/comfyanonymous/ComfyUI/tree/master/comfy/ldm/lightricks).
The main LTXVideo repository can be found [here](https://github.com/Lightricks/LTX-Video).
# ⭐ 16.07.2025 - LTXV 0.9.8 Release ⭐
### 🚀 What's New
1. **LTXV 0.9.8 Model**<br>
The new model and its distilled variants offer improved prompt understanding and detail generation<br>
👉 [13B Distilled model](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-13b-0.9.8-distilled.safetensors)<br>
👉 [13B Distilled model 8-bit](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-13b-0.9.8-distilled-fp8.safetensors)<br>
👉 [2B from 13B Distilled model](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-2b-0.9.8-distilled.safetensors)<br>
👉 [2B from 13B Distilled model 8-bit](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-2b-0.9.8-distilled-fp8.safetensors)<br>
👉 [IC Lora Detailer](https://huggingface.co/Lightricks/LTX-Video-ICLoRA-detailer-13b-0.9.8/resolve/main/ltxv-098-ic-lora-detailer-comfyui.safetensors)<br>
2. **Autoregressive Generation**
Introducing new ComfyUI nodes that enable virtually infinite video generation. The new **LTXV Looping Sampler** node allows generation of videos with arbitrary length and consistent motion. **ICLoRAs** are supported as well—by providing guidance from existing videos (e.g., depth, pose, or Canny edges), you can generate long videos in a video-to-video manner.
👉 [Long Img2Video Generation Flow](#long-video-generation)
👉 [Long Video2Video Generation Flow](#long-video-generation)
3. **Detailer ICLoRA**
Introducing the **Detailer ICLoRA**, which enhances generated latents with fine details by applying a few additional diffusion steps. This results in significantly more detailed generations.
👉 [Detailer ICLoRA Flow](#video-upscaling)
# ⭐ 8.07.2025 - LTXVideo ICLora Release ⭐
### 🚀 What's New in LTXVideo ICLoRA
@@ -165,6 +188,13 @@ For portable ComfyUI installations, run
Note that to run the example workflows, you need to have some additional custom nodes, like [ComfyUI-VideoHelperSuite](https://github.com/kosinkadink/ComfyUI-VideoHelperSuite) and others, installed. You can do it by pressing "Install Missing Custom Nodes" button in ComfyUI Manager.
### Long Video Generation
🧩 [Image to Video Long Video](example_workflows/ltxv-13b-i2v-long-multi-prompt.json): Long video generation with support for multiple prompts along the video duration.
🧩 [Video to Video Long Video](example_workflows/ltxv-13b-v2v-long-depth.json): Long video-to-video generation. Given a guiding video—such as depth, pose, or edges—the flow generates a new video.
### Video Upscaling
🧩 [Video Upscaling](example_workflows/ltxv-13b-upscale.json): Upscales and adds fine details to any given video, increasing its spatial resolution by 2×.
### Easy to use multi scale generation workflows
🧩 [Image to video mixed](example_workflows/ltxv13b-i2v-mixed-multiscale.json): mixed flow with full and distilled model for best quality and speed trade-off.<br>
+4 -1
View File
@@ -10,6 +10,7 @@ from .guide import LTXVAddGuideAdvanced
from .latent_adain import LTXVAdainLatent
from .latent_upsampler import LTXVLatentUpsampler
from .latents import LTXVSelectLatents, LTXVSetVideoLatentNoiseMasks
from .looping_sampler import LTXVLoopingSampler, MultiPromptProvider
from .masks import LTXVPreprocessMasks
from .nodes_registry import NODE_CLASS_MAPPINGS as RUNTIME_NODE_CLASS_MAPPINGS
from .nodes_registry import (
@@ -24,7 +25,8 @@ from .stg import (
STGGuiderAdvancedNode,
STGGuiderNode,
)
from .tiled_sampler import LTXVLoopingSampler, LTXVTiledSampler, MultiPromptProvider
from .tiled_sampler import LTXVTiledSampler
from .tiled_vae_decode import LTXVTiledVAEDecode
from .tricks import NODE_CLASS_MAPPINGS as TRICKS_NODE_CLASS_MAPPINGS
from .tricks import NODE_DISPLAY_NAME_MAPPINGS as TRICKS_NODE_DISPLAY_NAME_MAPPINGS
from .utiltily_nodes import ImageToCPU
@@ -52,6 +54,7 @@ NODE_CLASS_MAPPINGS = {
"LTXVSetVideoLatentNoiseMasks": LTXVSetVideoLatentNoiseMasks,
"LTXVTiledSampler": LTXVTiledSampler,
"LTXVLoopingSampler": LTXVLoopingSampler,
"LTXVTiledVAEDecode": LTXVTiledVAEDecode,
"STGAdvancedPresets": STGAdvancedPresetsNode,
"STGGuiderAdvanced": STGGuiderAdvancedNode,
"STGGuiderNode": STGGuiderNode,
+95 -41
View File
@@ -5,12 +5,7 @@ import comfy_extras
import nodes
import torch
from comfy_extras.nodes_custom_sampler import SamplerCustomAdvanced
from comfy_extras.nodes_lt import (
EmptyLTXVLatentVideo,
LTXVAddGuide,
LTXVCropGuides,
LTXVImgToVideo,
)
from comfy_extras.nodes_lt import EmptyLTXVLatentVideo, LTXVAddGuide, LTXVCropGuides
from nodes import VAEEncode
from .guide import blur_internal
@@ -130,6 +125,10 @@ class LTXVBaseSampler:
crop="disabled",
crf=35,
blur=0,
optional_negative_index_latents=None,
optional_negative_index=-1,
optional_negative_index_strength=1.0,
optional_initialization_latents=None,
):
if optional_cond_images is not None:
@@ -167,26 +166,36 @@ class LTXVBaseSampler:
"Please use STGGuiderAdvanced."
)
if optional_cond_images is None:
if optional_initialization_latents is None:
(latents,) = EmptyLTXVLatentVideo().generate(width, height, num_frames, 1)
elif optional_cond_images.shape[0] == 1 and optional_cond_indices[0] == 0:
(
positive,
negative,
latents,
) = LTXVImgToVideo().generate(
positive=positive,
negative=negative,
vae=vae,
image=optional_cond_images[0].unsqueeze(0),
width=width,
height=height,
length=num_frames,
batch_size=1,
strength=strength,
)
else:
(latents,) = EmptyLTXVLatentVideo().generate(width, height, num_frames, 1)
latents = optional_initialization_latents
if (
optional_cond_images is not None
and optional_cond_images.shape[0] == 1
and optional_cond_indices[0] == 0
):
pixels = comfy.utils.common_upscale(
optional_cond_images[0].unsqueeze(0).movedim(-1, 1),
width,
height,
"bilinear",
"center",
).movedim(1, -1)
encode_pixels = pixels[:, :, :, :3]
t = vae.encode(encode_pixels)
latents["samples"][:, :, : t.shape[2]] = t
conditioning_latent_frames_mask = torch.ones(
(1, 1, latents["samples"].shape[2], 1, 1),
dtype=torch.float32,
device=latents["samples"].device,
)
conditioning_latent_frames_mask[:, :, : t.shape[2]] = 1.0 - strength
latents["noise_mask"] = conditioning_latent_frames_mask
elif optional_cond_images is not None:
for cond_image, cond_idx in zip(
optional_cond_images, optional_cond_indices
):
@@ -204,6 +213,21 @@ class LTXVBaseSampler:
strength=strength,
)
if optional_negative_index_latents is not None:
(
positive,
negative,
latents,
) = LTXVAddLatentGuide().generate(
vae=vae,
positive=positive,
negative=negative,
latent=latents,
guiding_latent=optional_negative_index_latents,
latent_idx=optional_negative_index,
strength=optional_negative_index_strength,
)
guider = copy.copy(guider)
guider.set_conds(positive, negative)
@@ -247,7 +271,7 @@ class LTXVExtendSampler:
"default": 80,
"min": -1,
"max": nodes.MAX_RESOLUTION,
"step": 8,
"step": 1,
"tooltip": "If -1, the number of frames will be based on the number of frames in the optional_guiding_latents.",
},
),
@@ -306,6 +330,7 @@ class LTXVExtendSampler:
guiding_strength=1.0,
optional_guiding_latents=None,
optional_reference_latents=None,
optional_initialization_latents=None,
adain_factor=0.0,
optional_negative_index_latents=None,
optional_negative_index=-1,
@@ -336,12 +361,19 @@ class LTXVExtendSampler:
latents, -overlap, -1
)
new_latents = EmptyLTXVLatentVideo().generate(
width=width * width_scale_factor,
height=height * height_scale_factor,
length=overlap * time_scale_factor + num_new_frames,
batch_size=1,
)[0]
if optional_initialization_latents is None:
new_latents = EmptyLTXVLatentVideo().generate(
width=width * width_scale_factor,
height=height * height_scale_factor,
length=overlap * time_scale_factor + num_new_frames,
batch_size=1,
)[0]
else:
new_latents = optional_initialization_latents
last_overlap_latents["samples"] = last_overlap_latents["samples"].to(
new_latents["samples"].device
)
(
positive,
@@ -486,8 +518,13 @@ class LTXVInContextSampler:
guiding_latents,
optional_cond_image=None,
num_frames=0,
optional_initialization_latents=None,
optional_negative_index_latents=None,
optional_negative_index=-1,
optional_negative_index_strength=1.0,
optional_cond_strength=1.0,
optional_guiding_strength=1.0,
):
try:
positive, negative = guider.raw_conds
except AttributeError:
@@ -504,12 +541,15 @@ class LTXVInContextSampler:
if num_frames != -1:
frames = (num_frames - 1) // time_scale_factor + 1
new_latents = EmptyLTXVLatentVideo().generate(
width=width * width_scale_factor,
height=height * height_scale_factor,
length=(frames - 1) * time_scale_factor + 1,
batch_size=1,
)[0]
if optional_initialization_latents is not None:
new_latents = optional_initialization_latents
else:
new_latents = EmptyLTXVLatentVideo().generate(
width=width * width_scale_factor,
height=height * height_scale_factor,
length=(frames - 1) * time_scale_factor + 1,
batch_size=1,
)[0]
if optional_cond_image is not None:
optional_cond_image = (
@@ -535,7 +575,7 @@ class LTXVInContextSampler:
latent=new_latents,
guiding_latent=cond_image_latent,
latent_idx=0,
strength=1.0,
strength=optional_cond_strength,
)
if optional_cond_image is not None:
@@ -553,9 +593,23 @@ class LTXVInContextSampler:
negative=negative,
latent=new_latents,
guiding_latent=guiding_latents,
latent_idx=0 if optional_cond_image is None else 1,
strength=1.0,
latent_idx=1 if optional_cond_image is not None else 0,
strength=optional_guiding_strength,
)
if optional_negative_index_latents is not None:
(
positive,
negative,
new_latents,
) = LTXVAddLatentGuide().generate(
vae=vae,
positive=positive,
negative=negative,
latent=new_latents,
guiding_latent=optional_negative_index_latents,
latent_idx=optional_negative_index,
strength=optional_negative_index_strength,
)
guider = copy.copy(guider)
guider.set_conds(positive, negative)
@@ -182,7 +182,7 @@
"Node name for S&R": "LTXVLatentUpsamplerModelLoader"
},
"widgets_values": [
"ltxv-spatial-upscaler-0.9.7.safetensors",
"ltxv-spatial-upscaler-0.9.8.safetensors",
true,
false
]
@@ -3113,7 +3113,7 @@
},
"1828": {
"inputs": {
"upscale_model": "ltxv-spatial-upscaler-0.9.7.safetensors",
"upscale_model": "ltxv-spatial-upscaler-0.9.8.safetensors",
"spatial_upsample": true,
"temporal_upsample": false
},
@@ -182,7 +182,7 @@
"Node name for S&R": "LTXVLatentUpsamplerModelLoader"
},
"widgets_values": [
"ltxv-spatial-upscaler-0.9.7.safetensors",
"ltxv-spatial-upscaler-0.9.8.safetensors",
true,
false
]
@@ -3056,7 +3056,7 @@
},
"1828": {
"inputs": {
"upscale_model": "ltxv-spatial-upscaler-0.9.7.safetensors",
"upscale_model": "ltxv-spatial-upscaler-0.9.8.safetensors",
"spatial_upsample": true,
"temporal_upsample": false
},
@@ -182,7 +182,7 @@
"Node name for S&R": "LTXVLatentUpsamplerModelLoader"
},
"widgets_values": [
"ltxv-spatial-upscaler-0.9.7.safetensors",
"ltxv-spatial-upscaler-0.9.8.safetensors",
true,
false
]
@@ -3501,7 +3501,7 @@
},
"1828": {
"inputs": {
"upscale_model": "ltxv-spatial-upscaler-0.9.7.safetensors",
"upscale_model": "ltxv-spatial-upscaler-0.9.8.safetensors",
"spatial_upsample": true,
"temporal_upsample": false
},
@@ -250,7 +250,7 @@
"Node name for S&R": "LTXVLatentUpsamplerModelLoader"
},
"widgets_values": [
"ltxv-spatial-upscaler-0.9.7.safetensors",
"ltxv-spatial-upscaler-0.9.8.safetensors",
true,
false
]
@@ -3275,7 +3275,7 @@
},
"1828": {
"inputs": {
"upscale_model": "ltxv-spatial-upscaler-0.9.7.safetensors",
"upscale_model": "ltxv-spatial-upscaler-0.9.8.safetensors",
"spatial_upsample": true,
"temporal_upsample": false
},
Binary file not shown.

After

Width:  |  Height:  |  Size: 80 KiB

+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d177d33a43e5f7b87dfa24726f55a0aa698f06d7dad255b759560d1b23e2e3be
size 1481235
+2 -2
View File
@@ -4272,7 +4272,7 @@
"Node name for S&R": "LTXVLatentUpsamplerModelLoader"
},
"widgets_values": [
"ltxv-spatial-upscaler-0.9.7.safetensors",
"ltxv-spatial-upscaler-0.9.8.safetensors",
true,
false
]
@@ -5749,7 +5749,7 @@
},
"1828": {
"inputs": {
"upscale_model": "ltxv-spatial-upscaler-0.9.7.safetensors",
"upscale_model": "ltxv-spatial-upscaler-0.9.8.safetensors",
"spatial_upsample": true,
"temporal_upsample": false
},
+2 -2
View File
@@ -214,7 +214,7 @@
"Node name for S&R": "LTXVLatentUpsamplerModelLoader"
},
"widgets_values": [
"ltxv-spatial-upscaler-0.9.7.safetensors",
"ltxv-spatial-upscaler-0.9.8.safetensors",
true,
false
]
@@ -2396,7 +2396,7 @@
},
"1828": {
"inputs": {
"upscale_model": "ltxv-spatial-upscaler-0.9.7.safetensors",
"upscale_model": "ltxv-spatial-upscaler-0.9.8.safetensors",
"spatial_upsample": true,
"temporal_upsample": false
},
+2 -2
View File
@@ -214,7 +214,7 @@
"Node name for S&R": "LTXVLatentUpsamplerModelLoader"
},
"widgets_values": [
"ltxv-spatial-upscaler-0.9.7.safetensors",
"ltxv-spatial-upscaler-0.9.8.safetensors",
true,
false
]
@@ -2397,7 +2397,7 @@
},
"1828": {
"inputs": {
"upscale_model": "ltxv-spatial-upscaler-0.9.7.safetensors",
"upscale_model": "ltxv-spatial-upscaler-0.9.8.safetensors",
"spatial_upsample": true,
"temporal_upsample": false
},
+2 -2
View File
@@ -243,7 +243,7 @@
"Node name for S&R": "LTXVLatentUpsamplerModelLoader"
},
"widgets_values": [
"ltxv-spatial-upscaler-0.9.7.safetensors",
"ltxv-spatial-upscaler-0.9.8.safetensors",
true,
false
]
@@ -2744,7 +2744,7 @@
},
"1828": {
"inputs": {
"upscale_model": "ltxv-spatial-upscaler-0.9.7.safetensors",
"upscale_model": "ltxv-spatial-upscaler-0.9.8.safetensors",
"spatial_upsample": true,
"temporal_upsample": false
},
@@ -214,7 +214,7 @@
"Node name for S&R": "LTXVLatentUpsamplerModelLoader"
},
"widgets_values": [
"ltxv-spatial-upscaler-0.9.7.safetensors",
"ltxv-spatial-upscaler-0.9.8.safetensors",
true,
false
]
@@ -2619,7 +2619,7 @@
},
"1828": {
"inputs": {
"upscale_model": "ltxv-spatial-upscaler-0.9.7.safetensors",
"upscale_model": "ltxv-spatial-upscaler-0.9.8.safetensors",
"spatial_upsample": true,
"temporal_upsample": false
},
File diff suppressed because it is too large Load Diff
@@ -116,7 +116,7 @@
"Node name for S&R": "LTXVLatentUpsamplerModelLoader"
},
"widgets_values": [
"ltxv-spatial-upscaler-0.9.7.safetensors",
"ltxv-spatial-upscaler-0.9.8.safetensors",
true,
false
]
@@ -3049,7 +3049,7 @@
},
"1828": {
"inputs": {
"upscale_model": "ltxv-spatial-upscaler-0.9.7.safetensors",
"upscale_model": "ltxv-spatial-upscaler-0.9.8.safetensors",
"spatial_upsample": true,
"temporal_upsample": false
},
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+16 -12
View File
@@ -37,20 +37,24 @@ class LTXVFilmGrain:
) -> Tuple[torch.Tensor]:
if grain_intensity < 0 or grain_intensity > 1:
raise ValueError("Grain intensity must be between 0 and 1.")
device = comfy.model_management.get_torch_device()
images = images.to(device)
grain = torch.randn_like(images, device=device)
grain[:, :, :, 0] *= 2
grain[:, :, :, 2] *= 3
grain = grain * saturation + grain[:, :, :, 1].unsqueeze(3).repeat(
1, 1, 1, 3
) * (1 - saturation)
grain = torch.zeros(images[0:1].shape, device=device)
# Blend the grain with the image
noised_images = images + grain_intensity * grain
noised_images.clamp_(0, 1)
noised_images = noised_images.to(comfy.model_management.intermediate_device())
# Process images in-place to reduce memory usage
for i in range(images.shape[0]):
# Generate grain for single image
torch.randn(grain.shape, device=device, out=grain)
grain[:, :, :, 0] *= 2
grain[:, :, :, 2] *= 3
grain = grain * saturation + grain[:, :, :, 1].unsqueeze(3).expand(
-1, -1, -1, 3
) * (1 - saturation)
return (noised_images,)
# Blend the grain with the image in-place
images[i : i + 1].add_(grain_intensity * grain)
images[i : i + 1].clamp_(0, 1)
images = images.to(comfy.model_management.intermediate_device())
return (images,)
+726
View File
@@ -0,0 +1,726 @@
import copy
from dataclasses import dataclass
import comfy
import torch
from .easy_samplers import LTXVBaseSampler, LTXVExtendSampler, LTXVInContextSampler
from .latents import LTXVSelectLatents
from .nodes_registry import comfy_node
@dataclass
class TileConfig:
"""Configuration for spatial tile processing."""
tile_latents: dict
tile_guiding_latents: dict
tile_negative_index_latents: dict
tile_cond_image: torch.Tensor
tile_height: int
tile_width: int
v: int
h: int
vertical_tiles: int
horizontal_tiles: int
first_seed: int
@dataclass
class SamplingConfig:
"""Configuration for sampling parameters."""
temporal_tile_size: int
temporal_overlap: int
temporal_overlap_cond_strength: float
guiding_strength: float
adain_factor: float
optional_negative_index: int
optional_negative_index_strength: float
optional_positive_conditionings: list
time_scale_factor: int
width_scale_factor: int
height_scale_factor: int
per_tile_seed_offsets: list
per_tile_use_negative_latents: list
@dataclass
class ModelConfig:
"""Configuration for model components."""
model: object
vae: object
noise: object
sampler: object
sigmas: object
guider: object
@comfy_node(
name="LTXVLoopingSampler",
)
class LTXVLoopingSampler:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"model": ("MODEL", {"tooltip": "The diffusion model to use."}),
"vae": ("VAE", {"tooltip": "The VAE to use."}),
"noise": ("NOISE", {"tooltip": "The noise to use."}),
"sampler": ("SAMPLER", {"tooltip": "The sampler to use."}),
"sigmas": ("SIGMAS", {"tooltip": "The sigmas to use."}),
"guider": (
"GUIDER",
{"tooltip": "The guider to use, must be a STGGuiderAdvanced."},
),
"latents": (
"LATENT",
{
"tooltip": "The latents to use for creating the long video, they can be guiding latents or empty latents when no guidance is used."
},
),
"guiding_strength": (
"FLOAT",
{
"default": 1.0,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "The strength of the conditioning on guiding latents, when optional_guiding_latents are provided.",
},
),
"temporal_tile_size": (
"INT",
{
"default": 80,
"min": 24,
"max": 1000,
"step": 8,
"tooltip": "The size of the temporal tile to use for the sampling, in pixel frames, in addition to the overlapping region.",
},
),
"temporal_overlap": (
"INT",
{
"default": 24,
"min": 16,
"max": 80,
"step": 8,
"tooltip": "The overlap between the temporal tiles, in pixel frames.",
},
),
"temporal_overlap_cond_strength": (
"FLOAT",
{
"default": 0.5,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "The strength of the conditioning on the latents from the previous temporal tile.",
},
),
"horizontal_tiles": (
"INT",
{
"default": 1,
"min": 1,
"max": 6,
"tooltip": "Number of horizontal spatial tiles.",
},
),
"vertical_tiles": (
"INT",
{
"default": 1,
"min": 1,
"max": 6,
"tooltip": "Number of vertical spatial tiles.",
},
),
"spatial_overlap": (
"INT",
{
"default": 1,
"min": 1,
"max": 8,
"tooltip": "Overlap between spatial tiles.",
},
),
},
"optional": {
"optional_cond_image": (
"IMAGE",
{
"tooltip": "The image to use for conditioning the first frame in the video (i2v setup). If not provided, the first frame will be unconditioned (t2v setup). The image will be resized to the size of the first frame."
},
),
"optional_guiding_latents": (
"LATENT",
{
"tooltip": "The latents to use for guiding the sampling, typically with an IC-LoRA."
},
),
"adain_factor": (
"FLOAT",
{
"default": 0.0,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "The strength of the AdaIn operation used to fix the statistics of each new generated temporal tile, to prevent accumulated oversaturation.",
},
),
"optional_positive_conditionings": (
"CONDITIONING",
{
"tooltip": "Optional way to provide changing positive prompts, one per temporal tile, using the MultiPromptProvider node."
},
),
"optional_negative_index_latents": (
"LATENT",
{
"tooltip": "Special optional latents to condition on a negative index before each new temporal tile as a way to provide long term context during video generation."
},
),
},
}
RETURN_TYPES = ("LATENT",)
RETURN_NAMES = ("denoised_output",)
FUNCTION = "sample"
CATEGORY = "sampling"
def _extract_latent_spatial_tile(self, latent_dict, v_start, v_end, h_start, h_end):
"""Extract spatial tile from a latent dictionary."""
if latent_dict is None:
return None
tile_samples = latent_dict["samples"][:, :, :, v_start:v_end, h_start:h_end]
return {"samples": tile_samples}
def _extract_spatial_tile(
self,
samples,
optional_guiding_latents,
optional_negative_index_latents,
optional_cond_image,
v_start,
v_end,
h_start,
h_end,
height_scale_factor,
width_scale_factor,
):
"""Extract spatial tiles from all inputs for a given spatial region."""
# Extract spatial tile from latents
tile_latents = self._extract_latent_spatial_tile(
{"samples": samples}, v_start, v_end, h_start, h_end
)
# Extract spatial tile from guiding latents if provided
tile_guiding_latents = self._extract_latent_spatial_tile(
optional_guiding_latents, v_start, v_end, h_start, h_end
)
# Extract spatial tile from negative index latents if provided
tile_negative_index_latents = self._extract_latent_spatial_tile(
optional_negative_index_latents, v_start, v_end, h_start, h_end
)
# Extract spatial tile from conditioning image if provided
tile_cond_image = None
if optional_cond_image is not None:
# Scale coordinates for image
img_h_start = v_start * height_scale_factor
img_h_end = v_end * height_scale_factor
img_w_start = h_start * width_scale_factor
img_w_end = h_end * width_scale_factor
# Extract image tile
tile_cond_image = optional_cond_image[
:, img_h_start:img_h_end, img_w_start:img_w_end, :
]
return (
tile_latents,
tile_guiding_latents,
tile_negative_index_latents,
tile_cond_image,
)
def _process_temporal_chunks(
self,
tile_config: TileConfig,
sampling_config: SamplingConfig,
model_config: ModelConfig,
):
"""Process all temporal chunks for a single spatial tile."""
chunk_index = 0
tile_out_latents = None
first_tile_out_latents = None
for i_temporal_tile, (start_index, end_index) in enumerate(
zip(
range(
0,
tile_config.tile_latents["samples"].shape[2]
+ sampling_config.temporal_tile_size
- sampling_config.temporal_overlap,
sampling_config.temporal_tile_size
- sampling_config.temporal_overlap,
),
range(
sampling_config.temporal_tile_size,
tile_config.tile_latents["samples"].shape[2]
+ sampling_config.temporal_tile_size
- sampling_config.temporal_overlap,
sampling_config.temporal_tile_size
- sampling_config.temporal_overlap,
),
)
):
if tile_config.tile_guiding_latents is not None:
guiding_latent_chunk = LTXVSelectLatents().select_latents(
tile_config.tile_guiding_latents,
start_index,
min(
end_index - 1,
tile_config.tile_guiding_latents["samples"].shape[2] - 1,
),
)[0]
else:
guiding_latent_chunk = None
latent_chunk = LTXVSelectLatents().select_latents(
tile_config.tile_latents,
start_index,
min(end_index - 1, tile_config.tile_latents["samples"].shape[2] - 1),
)[0]
print(
"Processing temporal chunk at index",
start_index,
"to",
min(end_index - 1, tile_config.tile_latents["samples"].shape[2] - 1),
)
seed_offset = self._get_per_tile_value(
sampling_config.per_tile_seed_offsets, i_temporal_tile
)
use_negative_latents = self._get_per_tile_value(
sampling_config.per_tile_use_negative_latents, i_temporal_tile
)
model_config.noise.seed = self._calculate_tile_seed(
tile_config.first_seed,
start_index,
tile_config.vertical_tiles,
tile_config.horizontal_tiles,
tile_config.v,
tile_config.h,
seed_offset,
)
# Handle optional positive conditionings
new_guider = self._prepare_guider_for_chunk(
model_config.guider,
sampling_config.optional_positive_conditionings,
chunk_index,
)
if start_index == 0:
if tile_config.tile_guiding_latents is not None:
tile_out_latents = LTXVInContextSampler().sample(
vae=model_config.vae,
guider=new_guider,
sampler=model_config.sampler,
sigmas=model_config.sigmas,
noise=model_config.noise,
guiding_latents=guiding_latent_chunk,
optional_cond_image=tile_config.tile_cond_image,
num_frames=-1,
optional_negative_index_latents=(
tile_config.tile_negative_index_latents
if use_negative_latents
else None
),
optional_negative_index=sampling_config.optional_negative_index,
optional_negative_index_strength=sampling_config.optional_negative_index_strength,
optional_initialization_latents=latent_chunk,
optional_cond_strength=sampling_config.temporal_overlap_cond_strength,
optional_guiding_strength=sampling_config.guiding_strength,
)[0]
else:
tile_out_latents = LTXVBaseSampler().sample(
model=model_config.model,
vae=model_config.vae,
noise=model_config.noise,
sampler=model_config.sampler,
sigmas=model_config.sigmas,
guider=new_guider,
num_frames=(
min(
sampling_config.temporal_tile_size,
tile_config.tile_latents["samples"].shape[2],
)
- 1
)
* sampling_config.time_scale_factor
+ 1,
width=tile_config.tile_width
* sampling_config.width_scale_factor,
height=tile_config.tile_height
* sampling_config.height_scale_factor,
optional_cond_images=tile_config.tile_cond_image,
optional_cond_indices="0",
crop="center",
crf=30,
strength=sampling_config.temporal_overlap_cond_strength,
optional_negative_index_latents=(
tile_config.tile_negative_index_latents
if use_negative_latents
else None
),
optional_negative_index=sampling_config.optional_negative_index,
optional_negative_index_strength=sampling_config.optional_negative_index_strength,
optional_initialization_latents=latent_chunk,
)[0]
first_tile_out_latents = copy.deepcopy(tile_out_latents)
else:
tile_out_latents = LTXVExtendSampler().sample(
model=model_config.model,
vae=model_config.vae,
sampler=model_config.sampler,
sigmas=model_config.sigmas,
noise=model_config.noise,
latents=tile_out_latents,
num_new_frames=(
latent_chunk["samples"].shape[2]
- sampling_config.temporal_overlap
)
* sampling_config.time_scale_factor,
frame_overlap=sampling_config.temporal_overlap
* sampling_config.time_scale_factor,
guider=new_guider,
strength=sampling_config.temporal_overlap_cond_strength,
guiding_strength=sampling_config.guiding_strength,
optional_guiding_latents=guiding_latent_chunk,
optional_reference_latents=first_tile_out_latents,
adain_factor=sampling_config.adain_factor,
optional_negative_index_latents=(
tile_config.tile_negative_index_latents
if use_negative_latents
else None
),
optional_negative_index=sampling_config.optional_negative_index,
optional_negative_index_strength=sampling_config.optional_negative_index_strength,
optional_initialization_latents=latent_chunk,
)[0]
chunk_index += 1
return tile_out_latents
def _create_spatial_weights(
self, tile_samples, v, h, horizontal_tiles, vertical_tiles, spatial_overlap
):
"""Create blending weights for spatial tiles."""
tile_weights = torch.ones_like(tile_samples, device=tile_samples.device)
# Apply horizontal blending weights
if h > 0: # Left overlap
h_blend = torch.linspace(0, 1, spatial_overlap, device=tile_samples.device)
tile_weights[:, :, :, :, :spatial_overlap] *= h_blend.view(1, 1, 1, 1, -1)
if h < horizontal_tiles - 1: # Right overlap
h_blend = torch.linspace(1, 0, spatial_overlap, device=tile_samples.device)
tile_weights[:, :, :, :, -spatial_overlap:] *= h_blend.view(1, 1, 1, 1, -1)
# Apply vertical blending weights
if v > 0: # Top overlap
v_blend = torch.linspace(0, 1, spatial_overlap, device=tile_samples.device)
tile_weights[:, :, :, :spatial_overlap, :] *= v_blend.view(1, 1, 1, -1, 1)
if v < vertical_tiles - 1: # Bottom overlap
v_blend = torch.linspace(1, 0, spatial_overlap, device=tile_samples.device)
tile_weights[:, :, :, -spatial_overlap:, :] *= v_blend.view(1, 1, 1, -1, 1)
return tile_weights
def _calculate_tile_seed(
self,
first_seed,
start_index,
vertical_tiles,
horizontal_tiles,
v,
h,
seed_offset,
):
"""Calculate the seed value for a specific temporal and spatial tile."""
return (
first_seed
+ start_index * (vertical_tiles * horizontal_tiles)
+ v * horizontal_tiles
+ h
+ seed_offset
)
def _get_per_tile_value(self, value_list, tile_index):
"""Get a value from a per-tile configuration list, using the last value if the list is shorter."""
return value_list[min(tile_index, len(value_list) - 1)]
def _parse_per_tile_config(self, config_string, default_value, converter_func):
"""Parse a comma-separated per-tile configuration string into a list with type conversion."""
if config_string == "":
config_string = default_value
return [converter_func(item.strip()) for item in config_string.split(",")]
def _prepare_guider_for_chunk(
self, guider, optional_positive_conditionings, chunk_index
):
"""Prepare the guider for a specific chunk, handling optional positive conditionings."""
if optional_positive_conditionings is not None:
new_guider = copy.copy(guider)
positive, negative = guider.raw_conds
# Use the conditioning at chunk_index, or the last one if we've run out
conditioning_index = min(
chunk_index, len(optional_positive_conditionings) - 1
)
new_guider.set_conds(
optional_positive_conditionings[conditioning_index],
negative,
)
new_guider.raw_conds = (
optional_positive_conditionings[conditioning_index],
negative,
)
return new_guider
else:
return guider
def sample(
self,
model,
vae,
noise,
sampler,
sigmas,
guider,
latents,
guiding_strength,
adain_factor,
temporal_tile_size,
temporal_overlap,
temporal_overlap_cond_strength,
horizontal_tiles,
vertical_tiles,
spatial_overlap,
optional_cond_image=None,
optional_guiding_latents=None,
optional_negative_index_latents=None,
optional_negative_index=-1,
optional_negative_index_strength=1.0,
optional_positive_conditionings=None,
per_tile_seed_offsets="0",
per_tile_use_negative_latents="1",
):
# Get dimensions and prepare for spatial tiling
samples = latents["samples"]
batch, channels, frames, height, width = samples.shape
time_scale_factor, width_scale_factor, height_scale_factor = (
vae.downscale_index_formula
)
temporal_tile_size = temporal_tile_size // time_scale_factor
temporal_overlap = temporal_overlap // time_scale_factor
first_seed = noise.seed
per_tile_seed_offsets = self._parse_per_tile_config(
per_tile_seed_offsets, "0", int
)
per_tile_use_negative_latents = self._parse_per_tile_config(
per_tile_use_negative_latents, "1", lambda x: bool(int(x))
)
if optional_guiding_latents is not None:
assert (
latents["samples"].shape[2]
== optional_guiding_latents["samples"].shape[2]
), "The number of frames in the latents and optional_guiding_latents must be the same"
# Calculate tile sizes with overlap
base_tile_height = (
height + (vertical_tiles - 1) * spatial_overlap
) // vertical_tiles
base_tile_width = (
width + (horizontal_tiles - 1) * spatial_overlap
) // horizontal_tiles
# Initialize output tensor and weight tensor
final_output = torch.zeros_like(samples, device=samples.device)
weights = torch.zeros_like(samples, device=samples.device)
if optional_cond_image is not None:
img_height = height * height_scale_factor
img_width = width * width_scale_factor
optional_cond_image = comfy.utils.common_upscale(
optional_cond_image.movedim(-1, 1),
img_width,
img_height,
"bicubic",
crop="center",
).movedim(1, -1)
img_batch, img_height, img_width, img_channels = optional_cond_image.shape
# Process each spatial tile
for v in range(vertical_tiles):
for h in range(horizontal_tiles):
# Calculate tile boundaries
h_start = h * (base_tile_width - spatial_overlap)
v_start = v * (base_tile_height - spatial_overlap)
# Adjust end positions for edge tiles
h_end = (
min(h_start + base_tile_width, width)
if h < horizontal_tiles - 1
else width
)
v_end = (
min(v_start + base_tile_height, height)
if v < vertical_tiles - 1
else height
)
# Calculate actual tile dimensions
tile_height = v_end - v_start
tile_width = h_end - h_start
print(f"Processing spatial tile at row {v}, col {h}:")
print(f" Position: ({v_start}:{v_end}, {h_start}:{h_end})")
print(f" Size: {tile_height}x{tile_width}")
# Extract spatial tiles from all inputs
(
tile_latents,
tile_guiding_latents,
tile_negative_index_latents,
tile_cond_image,
) = self._extract_spatial_tile(
samples,
optional_guiding_latents,
optional_negative_index_latents,
optional_cond_image,
v_start,
v_end,
h_start,
h_end,
height_scale_factor,
width_scale_factor,
)
# Process all temporal chunks for this spatial tile
tile_config = TileConfig(
tile_latents=tile_latents,
tile_guiding_latents=tile_guiding_latents,
tile_negative_index_latents=tile_negative_index_latents,
tile_cond_image=tile_cond_image,
tile_height=tile_height,
tile_width=tile_width,
v=v,
h=h,
vertical_tiles=vertical_tiles,
horizontal_tiles=horizontal_tiles,
first_seed=first_seed,
)
sampling_config = SamplingConfig(
temporal_tile_size=temporal_tile_size,
temporal_overlap=temporal_overlap,
temporal_overlap_cond_strength=temporal_overlap_cond_strength,
guiding_strength=guiding_strength,
adain_factor=adain_factor,
optional_negative_index=optional_negative_index,
optional_negative_index_strength=optional_negative_index_strength,
optional_positive_conditionings=optional_positive_conditionings,
time_scale_factor=time_scale_factor,
width_scale_factor=width_scale_factor,
height_scale_factor=height_scale_factor,
per_tile_seed_offsets=per_tile_seed_offsets,
per_tile_use_negative_latents=per_tile_use_negative_latents,
)
model_config = ModelConfig(
model=model,
vae=vae,
noise=noise,
sampler=sampler,
sigmas=sigmas,
guider=guider,
)
tile_out_latents = self._process_temporal_chunks(
tile_config,
sampling_config,
model_config,
)
# Create weight mask for this spatial tile
tile_weights = self._create_spatial_weights(
tile_latents["samples"],
v,
h,
horizontal_tiles,
vertical_tiles,
spatial_overlap,
)
# Add weighted tile to final output
final_output[:, :, :, v_start:v_end, h_start:h_end] += (
tile_out_latents["samples"].to(final_output.device) * tile_weights
)
weights[:, :, :, v_start:v_end, h_start:h_end] += tile_weights
# Normalize by weights
final_output = final_output / (weights + 1e-8)
out_latents = {"samples": final_output}
noise.seed = first_seed
return (out_latents,)
@comfy_node(
name="MultiPromptProvider",
)
class MultiPromptProvider:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"prompts": (
"STRING",
{
"multiline": True,
"dynamicPrompts": True,
"tooltip": "Prompts to encode, one per line. Each prompt will be encoded separately. Each prompt will be used in one temporal_tile in LTXVLoopingSampler.",
},
),
"clip": ("CLIP", {"tooltip": "CLIP model to encode the prompts."}),
},
}
RETURN_TYPES = ("CONDITIONING",)
RETURN_NAMES = ("conditionings",)
FUNCTION = "get_prompt_list"
CATEGORY = "prompt"
def get_prompt_list(self, prompts, clip):
prompt_list = prompts.split("|")
prompt_list = [prompt.strip() for prompt in prompt_list]
encoded_prompt_list = [
clip.encode_from_tokens_scheduled(clip.tokenize(prompt))
for prompt in prompt_list
]
return (encoded_prompt_list,)
-294
View File
@@ -5,7 +5,6 @@ import torch
from comfy_extras.nodes_custom_sampler import SamplerCustomAdvanced
from comfy_extras.nodes_lt import LTXVAddGuide, LTXVCropGuides
from .easy_samplers import LTXVBaseSampler, LTXVExtendSampler, LTXVInContextSampler
from .latents import LTXVAddLatentGuide, LTXVSelectLatents
from .nodes_registry import comfy_node
@@ -315,296 +314,3 @@ class LTXVTiledSampler:
denoised_output = denoised_output / (weights + 1e-8)
return {"samples": output}, {"samples": denoised_output}
@comfy_node(
name="LTXVLoopingSampler",
)
class LTXVLoopingSampler:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"model": ("MODEL", {"tooltip": "The diffusion model to use."}),
"vae": ("VAE", {"tooltip": "The VAE to use."}),
"noise": ("NOISE", {"tooltip": "The noise to use."}),
"sampler": ("SAMPLER", {"tooltip": "The sampler to use."}),
"sigmas": ("SIGMAS", {"tooltip": "The sigmas to use."}),
"guider": (
"GUIDER",
{"tooltip": "The guider to use, must be a STGGuiderAdvanced."},
),
"latents": (
"LATENT",
{
"tooltip": "The latents to use for creating the long video, they can be guiding latents or empty latents when no guidance is used."
},
),
"use_latents_to_guide": (
"BOOLEAN",
{
"default": True,
"tooltip": "Whether to use the latents to guide the sampling, such as with IC-LoRAs. If False, just the shape of the latents will be used.",
},
),
"guiding_strength": (
"FLOAT",
{
"default": 1.0,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "The strength of the conditioning on guiding latents, when use_latents_to_guide is True.",
},
),
"temporal_tile_size": (
"INT",
{
"default": 80,
"min": 24,
"max": 1000,
"step": 8,
"tooltip": "The size of the temporal tile to use for the sampling, in pixel frames, in addition to the overlapping region.",
},
),
"overlap": (
"INT",
{
"default": 24,
"min": 16,
"max": 80,
"step": 8,
"tooltip": "The overlap between the temporal tiles, in pixel frames.",
},
),
"overlap_cond_strength": (
"FLOAT",
{
"default": 0.5,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "The strength of the conditioning on the latents from the previous temporal tile.",
},
),
},
"optional": {
"optional_cond_image": (
"IMAGE",
{
"tooltip": "The image to use for conditioning the first frame in the video (i2v setup). If not provided, the first frame will be unconditioned (t2v setup). The image will be resized to the size of the first frame."
},
),
"adain_factor": (
"FLOAT",
{
"default": 0.0,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "The strength of the AdaIn operation used to fix the statistics of each new generated temporal tile, to prevent accumulated oversaturation.",
},
),
"optional_positive_conditionings": (
"CONDITIONING",
{
"tooltip": "Optional way to provide changing positive prompts, one per temporal tile, using the MultiPromptProvider node."
},
),
"optional_negative_index_latents": (
"LATENT",
{
"tooltip": "Special optional latents to condition on a negative index before each new temporal tile as a way to provide long term context during video generation."
},
),
"optional_negative_index": (
"INT",
{
"default": -1,
"min": -1000,
"max": 1000,
"tooltip": "The index in which to condition using the optional_negative_index_latents.",
},
),
"optional_negative_index_strength": (
"FLOAT",
{
"default": 1.0,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "The strength of the conditioning with the optional_negative_index_latents.",
},
),
},
}
RETURN_TYPES = ("LATENT",)
RETURN_NAMES = ("denoised_output",)
FUNCTION = "sample"
CATEGORY = "sampling"
def sample(
self,
model,
vae,
noise,
sampler,
sigmas,
guider,
latents,
use_latents_to_guide,
guiding_strength,
adain_factor,
temporal_tile_size,
overlap,
overlap_cond_strength,
optional_cond_image=None,
optional_negative_index_latents=None,
optional_negative_index=-1,
optional_negative_index_strength=1.0,
optional_positive_conditionings=None,
):
time_scale_factor, width_scale_factor, height_scale_factor = (
vae.downscale_index_formula
)
temporal_tile_size = temporal_tile_size // time_scale_factor
overlap = overlap // time_scale_factor
first_seed = noise.seed
chunk_index = 0
for start_index, end_index in zip(
range(
0,
latents["samples"].shape[2] + temporal_tile_size - overlap,
temporal_tile_size - overlap,
),
range(
temporal_tile_size,
latents["samples"].shape[2] + temporal_tile_size - overlap,
temporal_tile_size - overlap,
),
):
guiding_latent_chunk = LTXVSelectLatents().select_latents(
latents,
start_index,
min(end_index - 1, latents["samples"].shape[2] - 1),
)[0]
noise.seed = first_seed + start_index
# Handle optional positive conditionings
if optional_positive_conditionings is not None:
new_guider = copy.copy(guider)
positive, negative = guider.raw_conds
# Use the conditioning at chunk_index, or the last one if we've run out
conditioning_index = min(
chunk_index, len(optional_positive_conditionings) - 1
)
new_guider.set_conds(
optional_positive_conditionings[conditioning_index], negative
)
new_guider.raw_conds = (
optional_positive_conditionings[conditioning_index],
negative,
)
else:
new_guider = guider
if start_index == 0:
if use_latents_to_guide:
out_latents = LTXVInContextSampler().sample(
vae=vae,
guider=new_guider,
sampler=sampler,
sigmas=sigmas,
noise=noise,
guiding_latents=guiding_latent_chunk,
optional_cond_image=optional_cond_image,
num_frames=-1,
)[0]
else:
out_latents = LTXVBaseSampler().sample(
model=model,
vae=vae,
noise=noise,
sampler=sampler,
sigmas=sigmas,
guider=new_guider,
num_frames=(
min(temporal_tile_size, latents["samples"].shape[2]) - 1
)
* time_scale_factor
+ 1,
width=latents["samples"].shape[4] * width_scale_factor,
height=latents["samples"].shape[3] * height_scale_factor,
optional_cond_images=optional_cond_image,
optional_cond_indices="0",
strength=overlap_cond_strength,
)[0]
first_out_latents = copy.deepcopy(out_latents)
else:
out_latents = LTXVExtendSampler().sample(
model=model,
vae=vae,
sampler=sampler,
sigmas=sigmas,
noise=noise,
latents=out_latents,
num_new_frames=(guiding_latent_chunk["samples"].shape[2] - overlap)
* time_scale_factor,
frame_overlap=overlap * time_scale_factor,
guider=new_guider,
strength=overlap_cond_strength,
guiding_strength=guiding_strength,
optional_guiding_latents=(
guiding_latent_chunk if use_latents_to_guide else None
),
optional_reference_latents=first_out_latents,
adain_factor=adain_factor,
optional_negative_index_latents=optional_negative_index_latents,
optional_negative_index=optional_negative_index,
optional_negative_index_strength=optional_negative_index_strength,
)[0]
chunk_index += 1
noise.seed = first_seed
return (out_latents,)
@comfy_node(
name="MultiPromptProvider",
)
class MultiPromptProvider:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"prompts": (
"STRING",
{
"multiline": True,
"dynamicPrompts": True,
"tooltip": "Prompts to encode, one per line. Each prompt will be encoded separately. Each prompt will be used in one temporal_tile in LTXVLoopingSampler.",
},
),
"clip": ("CLIP", {"tooltip": "CLIP model to encode the prompts."}),
},
}
RETURN_TYPES = ("CONDITIONING",)
RETURN_NAMES = ("conditionings",)
FUNCTION = "get_prompt_list"
CATEGORY = "prompt"
def get_prompt_list(self, prompts, clip):
prompt_list = prompts.split("\n")
prompt_list = [prompt.strip() for prompt in prompt_list]
encoded_prompt_list = [
clip.encode_from_tokens_scheduled(clip.tokenize(prompt))
for prompt in prompt_list
]
return (encoded_prompt_list,)
+197
View File
@@ -0,0 +1,197 @@
import torch
from .nodes_registry import comfy_node
@comfy_node(
name="LTXVTiledVAEDecode",
)
class LTXVTiledVAEDecode:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"vae": ("VAE",),
"latents": ("LATENT",),
"horizontal_tiles": ("INT", {"default": 1, "min": 1, "max": 6}),
"vertical_tiles": ("INT", {"default": 1, "min": 1, "max": 6}),
"overlap": ("INT", {"default": 1, "min": 1, "max": 8}),
"last_frame_fix": ("BOOLEAN", {"default": False}),
},
}
RETURN_TYPES = ("IMAGE",)
RETURN_NAMES = ("image",)
FUNCTION = "decode"
CATEGORY = "latent"
def decode(
self,
vae,
latents,
horizontal_tiles,
vertical_tiles,
overlap,
last_frame_fix,
):
# Get the latent samples
samples = latents["samples"]
if last_frame_fix:
# Repeat the last frame along dimension 2 (frames)
# samples: [batch, channels, frames, height, width]
last_frame = samples[
:, :, -1:, :, :
] # shape: [batch, channels, 1, height, width]
samples = torch.cat([samples, last_frame], dim=2)
batch, channels, frames, height, width = samples.shape
time_scale_factor, width_scale_factor, height_scale_factor = (
vae.downscale_index_formula
)
image_frames = 1 + (frames - 1) * time_scale_factor
# Calculate output image dimensions
output_height = height * height_scale_factor
output_width = width * width_scale_factor
# Calculate tile sizes with overlap
base_tile_height = (height + (vertical_tiles - 1) * overlap) // vertical_tiles
base_tile_width = (width + (horizontal_tiles - 1) * overlap) // horizontal_tiles
# Initialize output tensor and weight tensor
# VAE decode returns images in format [batch, height, width, channels]
output = None
weights = None
# Process each tile
for v in range(vertical_tiles):
for h in range(horizontal_tiles):
# Calculate tile boundaries
h_start = h * (base_tile_width - overlap)
v_start = v * (base_tile_height - overlap)
# Adjust end positions for edge tiles
h_end = (
min(h_start + base_tile_width, width)
if h < horizontal_tiles - 1
else width
)
v_end = (
min(v_start + base_tile_height, height)
if v < vertical_tiles - 1
else height
)
# Calculate actual tile dimensions
tile_height = v_end - v_start
tile_width = h_end - h_start
print(f"Processing VAE decode tile at row {v}, col {h}:")
print(f" Position: ({v_start}:{v_end}, {h_start}:{h_end})")
print(f" Size: {tile_height}x{tile_width}")
# Extract tile
tile = samples[:, :, :, v_start:v_end, h_start:h_end]
# Create tile latents dict
tile_latents = {"samples": tile}
# Decode the tile
decoded_tile = vae.decode(tile_latents["samples"])
# Initialize output tensors on first tile
if output is None:
output = torch.zeros(
(
batch,
image_frames,
output_height,
output_width,
decoded_tile.shape[-1],
),
device=decoded_tile.device,
dtype=decoded_tile.dtype,
)
weights = torch.zeros(
(batch, image_frames, output_height, output_width, 1),
device=decoded_tile.device,
dtype=decoded_tile.dtype,
)
# Calculate output tile boundaries
out_h_start = v_start * height_scale_factor
out_h_end = v_end * height_scale_factor
out_w_start = h_start * width_scale_factor
out_w_end = h_end * width_scale_factor
# Create weight mask for this tile
tile_out_height = out_h_end - out_h_start
tile_out_width = out_w_end - out_w_start
tile_weights = torch.ones(
(batch, image_frames, tile_out_height, tile_out_width, 1),
device=decoded_tile.device,
dtype=decoded_tile.dtype,
)
# Calculate overlap regions in output space
overlap_out_h = overlap * height_scale_factor
overlap_out_w = overlap * width_scale_factor
# Apply horizontal blending weights
if h > 0: # Left overlap
h_blend = torch.linspace(
0, 1, overlap_out_w, device=decoded_tile.device
)
tile_weights[:, :, :, :overlap_out_w, :] *= h_blend.view(
1, 1, 1, -1, 1
)
if h < horizontal_tiles - 1: # Right overlap
h_blend = torch.linspace(
1, 0, overlap_out_w, device=decoded_tile.device
)
tile_weights[:, :, :, -overlap_out_w:, :] *= h_blend.view(
1, 1, 1, -1, 1
)
# Apply vertical blending weights
if v > 0: # Top overlap
v_blend = torch.linspace(
0, 1, overlap_out_h, device=decoded_tile.device
)
tile_weights[:, :, :overlap_out_h, :, :] *= v_blend.view(
1, 1, -1, 1, 1
)
if v < vertical_tiles - 1: # Bottom overlap
v_blend = torch.linspace(
1, 0, overlap_out_h, device=decoded_tile.device
)
tile_weights[:, :, -overlap_out_h:, :, :] *= v_blend.view(
1, 1, -1, 1, 1
)
# Add weighted tile to output
output[:, :, out_h_start:out_h_end, out_w_start:out_w_end, :] += (
decoded_tile * tile_weights
)
# Add weights to weight tensor
weights[
:, :, out_h_start:out_h_end, out_w_start:out_w_end, :
] += tile_weights
# Normalize by weights
output = output / (weights + 1e-8)
# Reshape output to match expected format [batch * frames, height, width, channels]
output = output.view(
batch * image_frames, output_height, output_width, output.shape[-1]
)
if last_frame_fix:
output = output[:-time_scale_factor, :, :]
return (output,)