From 683d46186a136bb5f5b52ebb12adbd91d086cab3 Mon Sep 17 00:00:00 2001 From: sync-bot Date: Thu, 15 Jan 2026 19:20:05 +0000 Subject: [PATCH] Automated PR - 2026-01-15 --- __init__.py | 2 + easy_samplers.py | 142 ++++++++++++++++++ .../LTX-2_I2V_Distilled_wLora.json | 2 +- example_workflows/LTX-2_I2V_Full_wLora.json | 2 +- .../LTX-2_ICLoRA_All_Distilled.json | 2 +- .../LTX-2_T2V_Distilled_wLora.json | 2 +- example_workflows/LTX-2_T2V_Full_wLora.json | 2 +- gemma_encoder.py | 8 +- guiders/multimodal_guider.py | 58 +++---- 9 files changed, 180 insertions(+), 40 deletions(-) diff --git a/__init__.py b/__init__.py index dfef2de..33c1d53 100644 --- a/__init__.py +++ b/__init__.py @@ -5,6 +5,7 @@ from .easy_samplers import ( LTXVBaseSampler, LTXVExtendSampler, LTXVInContextSampler, + LTXVNormalizingSampler, ) from .gemma_encoder import LTXVGemmaCLIPModelLoader, LTXVGemmaEnhancePrompt from .guide import LTXVAddGuideAdvanced @@ -61,6 +62,7 @@ NODE_CLASS_MAPPINGS = { "LTXVBaseSampler": LTXVBaseSampler, "LTXVInContextSampler": LTXVInContextSampler, "LTXVExtendSampler": LTXVExtendSampler, + "LTXVNormalizingSampler": LTXVNormalizingSampler, "LTXVPreprocessMasks": LTXVPreprocessMasks, "LTXVPatcherVAE": LTXVPatcherVAE, "LTXVPromptEnhancer": LTXVPromptEnhancer, diff --git a/easy_samplers.py b/easy_samplers.py index 84d7fd0..d44e77c 100644 --- a/easy_samplers.py +++ b/easy_samplers.py @@ -4,6 +4,9 @@ import comfy import comfy_extras import nodes import torch +from comfy.ldm.lightricks.av_model import LTXAVModel +from comfy.nested_tensor import NestedTensor +from comfy_api.latest import io from comfy_extras.nodes_custom_sampler import SamplerCustomAdvanced, SplitSigmas from comfy_extras.nodes_lt import EmptyLTXVLatentVideo, LTXVAddGuide, LTXVCropGuides @@ -907,3 +910,142 @@ class LinearOverlapLatentTransition: "batch_index": combined_batch_index, }, ) + + +@comfy_node(description="LTXV Normalizing Sampler") +class LTXVNormalizingSampler(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXVNormalizingSampler", + category="utility", + inputs=[ + io.Noise.Input("noise"), + io.Guider.Input("guider"), + io.Sampler.Input("sampler"), + io.Sigmas.Input("sigmas"), + io.Latent.Input("latent_image"), + io.String.Input( + "video_normalization_factors", default="1,1,1,1,1,1,1,1" + ), + io.String.Input( + "audio_normalization_factors", default="1,1,0.25,1,1,0.25,1,1" + ), + ], + outputs=[ + io.Latent.Output(display_name="denoised_output"), + ], + ) + + @classmethod + def execute( + cls, + noise, + guider, + sampler, + sigmas, + latent_image, + video_normalization_factors, + audio_normalization_factors, + ) -> io.NodeOutput: + + if ( + guider.model_patcher.model.diffusion_model.__class__.__name__ + != "LTXAVModel" + ): + raise ValueError() + + ltxav: LTXAVModel = guider.model_patcher.model.diffusion_model + + video_normalization_factors = video_normalization_factors.split(",") + audio_normalization_factors = audio_normalization_factors.split(",") + video_normalization_factors = [ + float(factor) for factor in video_normalization_factors + ] + audio_normalization_factors = [ + float(factor) for factor in audio_normalization_factors + ] + + # Extend normalization factors to match the length of sigmas + sigmas_len = len(sigmas) - 1 + if len(video_normalization_factors) < sigmas_len: + if len(video_normalization_factors) > 0: + video_normalization_factors.extend( + [video_normalization_factors[-1]] + * (sigmas_len - len(video_normalization_factors)) + ) + if len(audio_normalization_factors) < sigmas_len: + if len(audio_normalization_factors) > 0: + audio_normalization_factors.extend( + [audio_normalization_factors[-1]] + * (sigmas_len - len(audio_normalization_factors)) + ) + + # Calculate indices where both normalization factors are not 1.0 + sampling_split_indices = [ + i + 1 + for i, (v, a) in enumerate( + zip(video_normalization_factors, audio_normalization_factors) + ) + if v != 1.0 or a != 1.0 + ] + print("Sampling split indices: %s" % sampling_split_indices, flush=True) + + # Split sigmas according to sampling_split_indices + def split_by_indices(arr, indices): + """ + Splits arr into chunks according to indices (split points). + Indices are treated as starting a new chunk at each index in the list. + """ + if not indices: + return [arr] + split_points = sorted(set(indices)) + chunks = [] + prev = 0 + for idx in split_points: + if prev < idx: + chunks.append(arr[prev : idx + 1]) + prev = idx + if prev < len(arr): + chunks.append(arr[prev:]) + return chunks + + sigmas_chunks = split_by_indices(sigmas, sampling_split_indices) + print("Sigmas chunks: %s" % sigmas_chunks, flush=True) + + i = 0 + for sigmas_chunk in sigmas_chunks: + i += len(sigmas_chunk) - 1 + print("Sampling with sigmas %s" % (sigmas_chunk), flush=True) + (_, latent_image) = SamplerCustomAdvanced().execute( + noise=noise, + guider=guider, + sampler=sampler, + sigmas=sigmas_chunk, + latent_image=latent_image, + ) + video_samples, audio_samples = ltxav.separate_audio_and_video_latents( + latent_image["samples"].tensors, + None, + ) + if i - 1 < len(video_normalization_factors) and i - 1 < len( + audio_normalization_factors + ): + video_samples = video_samples * video_normalization_factors[i - 1] + audio_samples = audio_samples * audio_normalization_factors[i - 1] + latent_image["samples"] = NestedTensor( + ltxav.recombine_audio_and_video_latents( + video_samples, audio_samples + ) + ) + print( + "After %d steps, the latent image was normalized by %f and %f" + % ( + i, + video_normalization_factors[i - 1], + audio_normalization_factors[i - 1], + ), + flush=True, + ) + + return io.NodeOutput(latent_image) diff --git a/example_workflows/LTX-2_I2V_Distilled_wLora.json b/example_workflows/LTX-2_I2V_Distilled_wLora.json index 6075a08..6bb9864 100644 --- a/example_workflows/LTX-2_I2V_Distilled_wLora.json +++ b/example_workflows/LTX-2_I2V_Distilled_wLora.json @@ -565,7 +565,7 @@ "title": "Video Size", "properties": {}, "widgets_values": [ - "Width & height settings must be divisible by 32 + 1. \nFrame count must be divisible by 8 + 1. \n\n\nRunning with invalid parameters **will not cause errors**. Instead, the flow will silently choose the closest valid parameters. " + "Width & height settings must be divisible by 64. \nFrame count must be divisible by 8 + 1. \n\n\nRunning with invalid parameters **will not cause errors**. Instead, the flow will silently choose the closest valid parameters. " ], "color": "#432", "bgcolor": "#653" diff --git a/example_workflows/LTX-2_I2V_Full_wLora.json b/example_workflows/LTX-2_I2V_Full_wLora.json index 63b9611..4e71128 100644 --- a/example_workflows/LTX-2_I2V_Full_wLora.json +++ b/example_workflows/LTX-2_I2V_Full_wLora.json @@ -743,7 +743,7 @@ "title": "Video Size", "properties": {}, "widgets_values": [ - "Width & height settings must be divisible by 32 + 1. \nFrame count must be divisible by 8 + 1. \n\n\nRunning with invalid parameters **will not cause errors**. Instead, the flow will silently choose the closest valid parameters. " + "Width & height settings must be divisible by 64. \nFrame count must be divisible by 8 + 1. \n\n\nRunning with invalid parameters **will not cause errors**. Instead, the flow will silently choose the closest valid parameters. " ], "color": "#432", "bgcolor": "#653" diff --git a/example_workflows/LTX-2_ICLoRA_All_Distilled.json b/example_workflows/LTX-2_ICLoRA_All_Distilled.json index 2395d9d..faf2ab9 100644 --- a/example_workflows/LTX-2_ICLoRA_All_Distilled.json +++ b/example_workflows/LTX-2_ICLoRA_All_Distilled.json @@ -57,7 +57,7 @@ "title": "Video Size", "properties": {}, "widgets_values": [ - "Width & height settings must be divisible by 32 + 1. \n\nRunning with invalid parameters **will not cause errors**. Instead, the flow will silently choose the closest valid parameters. " + "Width & height settings must be divisible by 64. \n\nRunning with invalid parameters **will not cause errors**. Instead, the flow will silently choose the closest valid parameters. " ], "color": "#432", "bgcolor": "#653" diff --git a/example_workflows/LTX-2_T2V_Distilled_wLora.json b/example_workflows/LTX-2_T2V_Distilled_wLora.json index d0a1f00..b0a524d 100644 --- a/example_workflows/LTX-2_T2V_Distilled_wLora.json +++ b/example_workflows/LTX-2_T2V_Distilled_wLora.json @@ -567,7 +567,7 @@ "title": "Video Size", "properties": {}, "widgets_values": [ - "Width & height settings must be divisible by 32 + 1. \nFrame count must be divisible by 8 + 1. \n\n\nRunning with invalid parameters **will not cause errors**. Instead, the flow will silently choose the closest valid parameters. " + "Width & height settings must be divisible by 64. \nFrame count must be divisible by 8 + 1. \n\n\nRunning with invalid parameters **will not cause errors**. Instead, the flow will silently choose the closest valid parameters. " ], "color": "#432", "bgcolor": "#653" diff --git a/example_workflows/LTX-2_T2V_Full_wLora.json b/example_workflows/LTX-2_T2V_Full_wLora.json index 5a7f6a6..5f0fd32 100644 --- a/example_workflows/LTX-2_T2V_Full_wLora.json +++ b/example_workflows/LTX-2_T2V_Full_wLora.json @@ -520,7 +520,7 @@ "title": "Video Size", "properties": {}, "widgets_values": [ - "Width & height settings must be divisible by 32 + 1. \nFrame count must be divisible by 8 + 1. \n\n\nRunning with invalid parameters **will not cause errors**. Instead, the flow will silently choose the closest valid parameters. " + "Width & height settings must be divisible by 64. \nFrame count must be divisible by 8 + 1. \n\n\nRunning with invalid parameters **will not cause errors**. Instead, the flow will silently choose the closest valid parameters. " ], "color": "#432", "bgcolor": "#653" diff --git a/gemma_encoder.py b/gemma_encoder.py index 43031c6..3524ed7 100644 --- a/gemma_encoder.py +++ b/gemma_encoder.py @@ -1,5 +1,6 @@ import json import logging +from glob import glob from pathlib import Path from typing import Optional @@ -528,8 +529,11 @@ def find_matching_dir(root_path: str, pattern: str) -> str: """ Recursively search for files matching a glob pattern and return the parent directory of the first match. """ - - matches = list(Path(root_path).rglob(pattern)) + matches = [ + Path(p) + for p in glob(f"{root_path}/**", recursive=True) + if Path(p).match(pattern) + ] if not matches: raise FileNotFoundError( f"No files matching pattern '{pattern}' found under {root_path}" diff --git a/guiders/multimodal_guider.py b/guiders/multimodal_guider.py index fa2b604..441263b 100644 --- a/guiders/multimodal_guider.py +++ b/guiders/multimodal_guider.py @@ -85,6 +85,20 @@ class MultimodalGuider(comfy.samplers.CFGGuider): return list(stg_indexes) + def unpack_latents(self, x: torch.Tensor): + latent_shapes = ( + self.conds.get("positive", {})[0] + .get("model_conds", {}) + .get("latent_shapes", None) + .cond + ) + vx, ax = comfy.utils.unpack_latents(x, latent_shapes) + return vx, ax + + def pack_latents(self, vx: torch.Tensor, ax: torch.Tensor): + x, latent_shapes = comfy.utils.pack_latents([vx, ax]) + return x, latent_shapes + def predict_noise( self, x: torch.Tensor, @@ -92,7 +106,6 @@ class MultimodalGuider(comfy.samplers.CFGGuider): model_options: dict = {}, seed=None, ): - x_shape = x.shape # in CFGGuider.predict_noise, we call sampling_function(), which uses cfg_function() to compute pos & neg # but we'd rather do a single batch of sampling pos, neg, and perturbed, so we call calc_cond_batch([perturbed,pos,neg]) directly current_step = self.current_step @@ -115,16 +128,13 @@ class MultimodalGuider(comfy.samplers.CFGGuider): run_a2v = video_params.do_cross_attn(current_step) run_v2a = audio_params.do_cross_attn(current_step) - vx, ax = self.inner_model.diffusion_model.separate_audio_and_video_latents( - x, audio_length - ) + vx, ax = self.unpack_latents(x) if not run_vx: vx = self.last_denoised_v if not run_ax: ax = self.last_denoised_a - x = self.inner_model.diffusion_model.recombine_audio_and_video_latents( - vx, ax, x_shape - ) + x, _ = self.pack_latents(vx, ax) + if not run_vx and not run_ax: return x @@ -145,11 +155,7 @@ class MultimodalGuider(comfy.samplers.CFGGuider): del model_options["transformer_options"]["run_ax"] del model_options["transformer_options"]["a2v_cross_attn"] del model_options["transformer_options"]["v2a_cross_attn"] - v_noise_pred_pos, a_noise_pred_pos = ( - self.inner_model.diffusion_model.separate_audio_and_video_latents( - noise_pred_pos, audio_length - ) - ) + v_noise_pred_pos, a_noise_pred_pos = self.unpack_latents(noise_pred_pos) a_noise_pred_neg, v_noise_pred_neg = 0, 0 a_noise_pred_perturbed, v_noise_pred_perturbed = 0, 0 @@ -168,11 +174,7 @@ class MultimodalGuider(comfy.samplers.CFGGuider): timestep, model_options, )[0] - v_noise_pred_neg, a_noise_pred_neg = ( - self.inner_model.diffusion_model.separate_audio_and_video_latents( - noise_pred_neg, audio_length - ) - ) + v_noise_pred_neg, a_noise_pred_neg = self.unpack_latents(noise_pred_neg) finally: del model_options["transformer_options"]["run_vx"] del model_options["transformer_options"]["run_ax"] @@ -202,10 +204,8 @@ class MultimodalGuider(comfy.samplers.CFGGuider): timestep, model_options, )[0] - v_noise_pred_perturbed, a_noise_pred_perturbed = ( - self.inner_model.diffusion_model.separate_audio_and_video_latents( - noise_pred_perturbed, audio_length - ) + v_noise_pred_perturbed, a_noise_pred_perturbed = self.unpack_latents( + noise_pred_perturbed ) finally: self.stg_flag.do_skip = False @@ -229,10 +229,8 @@ class MultimodalGuider(comfy.samplers.CFGGuider): timestep, model_options, )[0] - v_noise_pred_modality, a_noise_pred_modality = ( - self.inner_model.diffusion_model.separate_audio_and_video_latents( - noise_pred_modality, audio_length - ) + v_noise_pred_modality, a_noise_pred_modality = self.unpack_latents( + noise_pred_modality ) finally: del model_options["transformer_options"]["a2v_cross_attn"] @@ -260,9 +258,7 @@ class MultimodalGuider(comfy.samplers.CFGGuider): else: ax = self.last_denoised_a - x = self.inner_model.diffusion_model.recombine_audio_and_video_latents( - vx, ax, x_shape - ) + x, _ = self.pack_latents(vx, ax) # normally this would be done in cfg_function, but we skipped # that for efficiency: we can compute the noise predictions in @@ -285,11 +281,7 @@ class MultimodalGuider(comfy.samplers.CFGGuider): } x = fn(args) - self.last_denoised_v, self.last_denoised_a = ( - self.inner_model.diffusion_model.separate_audio_and_video_latents( - x, audio_length - ) - ) + self.last_denoised_v, self.last_denoised_a = self.unpack_latents(x) return x