From 7f795600c830fc0cfd7fe3795bf7676676688f75 Mon Sep 17 00:00:00 2001 From: Shao Duan <46639847+shaoxiongduan@users.noreply.github.com> Date: Sat, 14 Feb 2026 03:53:11 +0800 Subject: [PATCH] [bugfix] Fixed ltx2 base cfg guidance (#1095) Co-authored-by: Will Lin --- examples/inference/basic/basic_ltx2.py | 10 ++- fastvideo/configs/sample/ltx2.py | 7 ++ fastvideo/models/dits/ltx2.py | 14 +++- fastvideo/pipelines/pipeline_batch_info.py | 7 ++ fastvideo/pipelines/stages/ltx2_denoising.py | 73 +++++++++++++++++--- 5 files changed, 99 insertions(+), 12 deletions(-) diff --git a/examples/inference/basic/basic_ltx2.py b/examples/inference/basic/basic_ltx2.py index b053e9364..8ee0ee09f 100644 --- a/examples/inference/basic/basic_ltx2.py +++ b/examples/inference/basic/basic_ltx2.py @@ -20,10 +20,10 @@ def main() -> None: # Uses FastVideo default sampling settings for LTX2 base. generator = VideoGenerator.from_pretrained( "Davids048/LTX2-Base-Diffusers", - num_gpus=1, + num_gpus=8, ) - output_path = "outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.1.mp4" + output_path = "outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.4.mp4" generator.generate_video( prompt=PROMPT, output_path=output_path, @@ -31,6 +31,12 @@ def main() -> None: num_frames=121, height=1088, width=1920, + # LTX2 uses these parameters for multi-modal CFG instead of guidance_scale + # ltx2_cfg_scale_video=3.0, + # ltx2_cfg_scale_audio=7.0, + # ltx2_modality_scale_video=3.0, + # ltx2_modality_scale_audio=3.0, + # ltx2_rescale_scale=0.7, ) generator.shutdown() diff --git a/fastvideo/configs/sample/ltx2.py b/fastvideo/configs/sample/ltx2.py index ab87361c8..4fae60709 100644 --- a/fastvideo/configs/sample/ltx2.py +++ b/fastvideo/configs/sample/ltx2.py @@ -9,6 +9,7 @@ class LTX2BaseSamplingParam(SamplingParam): """Default sampling parameters for LTX-2 base one-stage T2V. Values follow the official LTX-2 one-stage defaults. + Multi-modal CFG params are read by ``LTX2DenoisingStage``. """ seed: int = 10 @@ -37,6 +38,12 @@ class LTX2BaseSamplingParam(SamplingParam): "awkward pauses, incorrect timing, unnatural transitions, " "inconsistent framing, tilted camera, flat lighting, inconsistent " "tone, cinematic oversaturation, stylized filters, or AI artifacts.") + # Official LTX-2 multi-modal CFG defaults. + ltx2_cfg_scale_video: float = 3.0 + ltx2_cfg_scale_audio: float = 7.0 + ltx2_modality_scale_video: float = 3.0 + ltx2_modality_scale_audio: float = 3.0 + ltx2_rescale_scale: float = 0.7 @dataclass diff --git a/fastvideo/models/dits/ltx2.py b/fastvideo/models/dits/ltx2.py index 8ec5c5ab0..db9b9a21c 100644 --- a/fastvideo/models/dits/ltx2.py +++ b/fastvideo/models/dits/ltx2.py @@ -1534,6 +1534,7 @@ class BasicAVTransformerBlock(torch.nn.Module): audio: TransformerArgs | None, video_attention_mask: torch.Tensor | None = None, audio_attention_mask: torch.Tensor | None = None, + skip_cross_modal_attn: bool = False, ) -> tuple[TransformerArgs | None, TransformerArgs | None]: """Forward pass for transformer block. @@ -1542,6 +1543,8 @@ class BasicAVTransformerBlock(torch.nn.Module): audio: Audio transformer args video_attention_mask: SP padding attention mask for video [B, padded_seq_len] audio_attention_mask: SP padding attention mask for audio [B, padded_seq_len] + skip_cross_modal_attn: If True, skip A2V and V2A cross-modal + attention (used for the modality-isolated CFG pass). """ vx = video.x if video is not None else None ax = audio.x if audio is not None else None @@ -1585,7 +1588,7 @@ class BasicAVTransformerBlock(torch.nn.Module): mask=audio.context_mask, ) - if run_a2v or run_v2a: + if (run_a2v or run_v2a) and not skip_cross_modal_attn: vx_norm3 = torch.nn.functional.rms_norm(vx, (vx.shape[-1],), eps=self.norm_eps) ax_norm3 = torch.nn.functional.rms_norm(ax, (ax.shape[-1],), eps=self.norm_eps) @@ -2006,6 +2009,7 @@ class LTXModel(torch.nn.Module): audio: TransformerArgs | None, video_attention_mask: torch.Tensor | None = None, audio_attention_mask: torch.Tensor | None = None, + skip_cross_modal_attn: bool = False, ) -> tuple[TransformerArgs | None, TransformerArgs | None]: for block in self.transformer_blocks: video, audio = block( @@ -2013,6 +2017,7 @@ class LTXModel(torch.nn.Module): audio=audio, video_attention_mask=video_attention_mask, audio_attention_mask=audio_attention_mask, + skip_cross_modal_attn=skip_cross_modal_attn, ) return video, audio @@ -2039,6 +2044,7 @@ class LTXModel(torch.nn.Module): audio: Modality | None, video_attention_mask: torch.Tensor | None = None, audio_attention_mask: torch.Tensor | None = None, + skip_cross_modal_attn: bool = False, ) -> tuple[torch.Tensor | None, torch.Tensor | None]: """Forward pass through the LTX model. @@ -2047,6 +2053,9 @@ class LTXModel(torch.nn.Module): audio: Audio modality input video_attention_mask: SP padding attention mask for video [B, padded_seq_len] audio_attention_mask: SP padding attention mask for audio [B, padded_seq_len] + skip_cross_modal_attn: If True, skip A2V and V2A cross-modal + attention in all transformer blocks (modality-isolated + CFG pass). """ if os.getenv("LTX2_PIPELINE_DEBUG_LOG", "0") == "1": _debug_block_log_line( @@ -2070,6 +2079,7 @@ class LTXModel(torch.nn.Module): audio_args, video_attention_mask=video_attention_mask, audio_attention_mask=audio_attention_mask, + skip_cross_modal_attn=skip_cross_modal_attn, ) vx = ( @@ -2238,6 +2248,7 @@ class LTX2Transformer3DModel(CachableDiT): audio_encoder_hidden_states: torch.Tensor | None = None, audio_timestep: torch.Tensor | None = None, audio_encoder_attention_mask: torch.Tensor | None = None, + skip_cross_modal_attn: bool = False, **kwargs, ) -> torch.Tensor: if isinstance(encoder_hidden_states, list): @@ -2394,6 +2405,7 @@ class LTX2Transformer3DModel(CachableDiT): audio=audio_modality, video_attention_mask=video_attention_mask, audio_attention_mask=audio_attention_mask, + skip_cross_modal_attn=skip_cross_modal_attn, ) # Denoised prediction diff --git a/fastvideo/pipelines/pipeline_batch_info.py b/fastvideo/pipelines/pipeline_batch_info.py index fe3da6f90..e92898798 100644 --- a/fastvideo/pipelines/pipeline_batch_info.py +++ b/fastvideo/pipelines/pipeline_batch_info.py @@ -167,6 +167,13 @@ class ForwardBatch: eta: float = 0.0 sigmas: list[float] | None = None + # LTX-2 multi-modal CFG parameters + ltx2_cfg_scale_video: float = 1.0 + ltx2_cfg_scale_audio: float = 1.0 + ltx2_modality_scale_video: float = 1.0 + ltx2_modality_scale_audio: float = 1.0 + ltx2_rescale_scale: float = 0.0 + n_tokens: int | None = None # Other parameters that may be needed by specific schedulers diff --git a/fastvideo/pipelines/stages/ltx2_denoising.py b/fastvideo/pipelines/stages/ltx2_denoising.py index 17c90eb9c..707d8ae82 100644 --- a/fastvideo/pipelines/stages/ltx2_denoising.py +++ b/fastvideo/pipelines/stages/ltx2_denoising.py @@ -141,6 +141,7 @@ class LTX2DenoisingStage(PipelineStage): video_shape) else: token_count = 1 + timestep_template = torch.ones( (latents.shape[0], token_count), device=latents.device, @@ -209,12 +210,27 @@ class LTX2DenoisingStage(PipelineStage): device=latents.device, dtype=torch.float32, ) + # Multi-modal CFG parameters (per-stream scales). + cfg_scale_video = batch.ltx2_cfg_scale_video + cfg_scale_audio = batch.ltx2_cfg_scale_audio + modality_scale_video = batch.ltx2_modality_scale_video + modality_scale_audio = batch.ltx2_modality_scale_audio + rescale_scale = batch.ltx2_rescale_scale + + do_cfg = batch.do_classifier_free_guidance + logger.info( - "[LTX2] Denoising start: steps=%d dtype=%s guidance=%s " + "[LTX2] Denoising start: steps=%d dtype=%s " + "cfg_video=%.1f cfg_audio=%.1f mod_video=%.1f " + "mod_audio=%.1f rescale=%.2f " "sigmas_shape=%s latents_shape=%s", batch.num_inference_steps, target_dtype, - batch.guidance_scale, + cfg_scale_video, + cfg_scale_audio, + modality_scale_video, + modality_scale_audio, + rescale_scale, tuple(sigmas.shape), tuple(latents.shape), ) @@ -235,6 +251,7 @@ class LTX2DenoisingStage(PipelineStage): attn_metadata=None, forward_batch=batch, ): + # Pass 1: Full conditioning (text + cross-modal) pos_outputs = self.transformer( hidden_states=latents.to(target_dtype), encoder_hidden_states=prompt_embeds, @@ -250,8 +267,8 @@ class LTX2DenoisingStage(PipelineStage): pos_denoised = pos_outputs pos_audio = None - # Only run negative pass if CFG is enabled - if batch.do_classifier_free_guidance: + if do_cfg: + # Pass 2: Unconditioned text (negative prompt) neg_outputs = self.transformer( hidden_states=latents.to(target_dtype), encoder_hidden_states=neg_prompt_embeds, @@ -266,11 +283,48 @@ class LTX2DenoisingStage(PipelineStage): else: neg_denoised = neg_outputs neg_audio = None - pos_denoised = pos_denoised + (batch.guidance_scale - 1) * ( - pos_denoised - neg_denoised) - if pos_audio is not None and neg_audio is not None: - pos_audio = pos_audio + (batch.guidance_scale - - 1) * (pos_audio - neg_audio) + + # Pass 3: Modality-isolated (skip cross-modal attn) + mod_outputs = self.transformer( + hidden_states=latents.to(target_dtype), + encoder_hidden_states=prompt_embeds, + encoder_attention_mask=prompt_mask, + timestep=timestep, + audio_hidden_states=audio_latents, + audio_encoder_hidden_states=audio_context_p, + audio_timestep=audio_timestep, + skip_cross_modal_attn=True, + ) + if isinstance(mod_outputs, tuple): + mod_denoised, mod_audio = mod_outputs + else: + mod_denoised = mod_outputs + mod_audio = None + + # Multi-modal guidance formula per stream. + vid = (pos_denoised + (cfg_scale_video - 1) * + (pos_denoised - neg_denoised) + + (modality_scale_video - 1) * + (pos_denoised - mod_denoised)) + aud = None + if pos_audio is not None: + aud = (pos_audio + (cfg_scale_audio - 1) * + (pos_audio - neg_audio) + + (modality_scale_audio - 1) * + (pos_audio - mod_audio)) + + # Guidance rescaling (prevents saturation). + if rescale_scale > 0: + f_v = pos_denoised.std() / vid.std() + f_v = rescale_scale * f_v + (1 - rescale_scale) + vid = vid * f_v + if aud is not None: + f_a = pos_audio.std() / aud.std() + f_a = rescale_scale * f_a + (1 - rescale_scale) + aud = aud * f_a + + pos_denoised = vid + pos_audio = aud sigma_value = sigma.to(torch.float32) if isinstance( sigma, torch.Tensor) else torch.tensor( @@ -281,6 +335,7 @@ class LTX2DenoisingStage(PipelineStage): dt = sigma_next - sigma velocity = ((latents.float() - pos_denoised.float()) / sigma_value).to(latents.dtype) + latents = (latents.float() + velocity.float() * dt).to( latents.dtype) if pos_audio is not None and audio_latents is not None: