diff --git a/examples/inference/basic/basic_ltx2_base.py b/examples/inference/basic/basic_ltx2_base.py new file mode 100644 index 000000000..9632b4b43 --- /dev/null +++ b/examples/inference/basic/basic_ltx2_base.py @@ -0,0 +1,54 @@ +from fastvideo import VideoGenerator + + +PROMPT = ( + "A warm sunny backyard. The camera starts in a tight cinematic close-up " + "of a woman and a man in their 30s, facing each other with serious " + "expressions. The woman, emotional and dramatic, says softly, \"That's " + "it... Dad's lost it. And we've lost Dad.\" The man exhales, slightly " + "annoyed: \"Stop being so dramatic, Jess.\" A beat. He glances aside, " + "then mutters defensively, \"He's just having fun.\" The camera slowly " + "pans right, revealing the grandfather in the garden wearing enormous " + "butterfly wings, waving his arms in the air like he's trying to take " + "off. He shouts, \"Wheeeew!\" as he flaps his wings with full commitment. " + "The woman covers her face, on the verge of tears. The tone is deadpan, " + "absurd, and quietly tragic." +) + +official_negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +def main() -> None: + # Uses FastVideo default sampling settings for LTX2 base. + generator = VideoGenerator.from_pretrained( + "converted_weights/ltx2-base", + num_gpus=1, + ) + + output_path = "outputs_video/ltx2_basic/output_ltx2_base_t2v_1088_1920_1.0.mp4" + generator.generate_video( + prompt=PROMPT, + output_path=output_path, + save_video=True, + num_frames=121, + height=1088, + width=1920, + fps=24, + negative_prompt=official_negative_prompt, + guidance_scale=3.0, + ) + generator.shutdown() + + +if __name__ == "__main__": + main() diff --git a/fastvideo/configs/sample/ltx2.py b/fastvideo/configs/sample/ltx2.py index 4be8d40bd..ab87361c8 100644 --- a/fastvideo/configs/sample/ltx2.py +++ b/fastvideo/configs/sample/ltx2.py @@ -5,10 +5,44 @@ from fastvideo.configs.sample.base import SamplingParam @dataclass -class LTX2SamplingParam(SamplingParam): - """Default sampling parameters for LTX-2 distilled T2V. +class LTX2BaseSamplingParam(SamplingParam): + """Default sampling parameters for LTX-2 base one-stage T2V. + + Values follow the official LTX-2 one-stage defaults. """ + seed: int = 10 + num_frames: int = 121 + height: int = 512 + width: int = 768 + fps: int = 24 + num_inference_steps: int = 40 + guidance_scale: float = 3.0 + # Copied/following official LTX-2 DEFAULT_NEGATIVE_PROMPT. + negative_prompt: str = ( + "blurry, out of focus, overexposed, underexposed, low contrast, " + "washed out colors, excessive noise, grainy texture, poor lighting, " + "flickering, motion blur, distorted proportions, unnatural skin " + "tones, deformed facial features, asymmetrical face, missing facial " + "features, extra limbs, disfigured hands, wrong hand count, " + "artifacts around text, inconsistent perspective, camera shake, " + "incorrect depth of field, background too sharp, background clutter, " + "distracting reflections, harsh shadows, inconsistent lighting " + "direction, color banding, cartoonish rendering, 3D CGI look, " + "unrealistic materials, uncanny valley effect, incorrect ethnicity, " + "wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, " + "robotic voice, echo, background noise, off-sync audio, incorrect " + "dialogue, added dialogue, repetitive speech, jittery movement, " + "awkward pauses, incorrect timing, unnatural transitions, " + "inconsistent framing, tilted camera, flat lighting, inconsistent " + "tone, cinematic oversaturation, stylized filters, or AI artifacts.") + + +@dataclass +class LTX2DistilledSamplingParam(SamplingParam): + """Default sampling parameters for LTX-2 distilled one-stage T2V.""" + seed: int = 10 num_frames: int = 121 height: int = 1024 @@ -18,3 +52,7 @@ class LTX2SamplingParam(SamplingParam): guidance_scale: float = 1.0 # No default negative_prompt for distilled models negative_prompt: str = "" + + +# Backward compatibility alias. +LTX2SamplingParam = LTX2DistilledSamplingParam diff --git a/fastvideo/configs/sample/registry.py b/fastvideo/configs/sample/registry.py new file mode 100644 index 000000000..45fbb8b57 --- /dev/null +++ b/fastvideo/configs/sample/registry.py @@ -0,0 +1,206 @@ +# SPDX-License-Identifier: Apache-2.0 +import os +from collections.abc import Callable +from typing import Any + +from fastvideo.configs.sample.hunyuan import (FastHunyuanSamplingParam, + HunyuanSamplingParam) +from fastvideo.configs.sample.hunyuan15 import Hunyuan15_480P_SamplingParam, Hunyuan15_720P_SamplingParam +from fastvideo.configs.sample.hyworld import HYWorld_SamplingParam +from fastvideo.configs.sample.stepvideo import StepVideoT2VSamplingParam + +from fastvideo.configs.sample.cosmos import Cosmos_Predict2_2B_Video2World_SamplingParam +from fastvideo.configs.sample.cosmos2_5 import Cosmos_Predict2_5_2B_Diffusers_SamplingParam +from fastvideo.configs.sample.ltx2 import (LTX2BaseSamplingParam, + LTX2DistilledSamplingParam) + +# isort: off +from fastvideo.configs.sample.wan import ( + FastWanT2V480P_SamplingParam, + Wan2_1_Fun_1_3B_InP_SamplingParam, + Wan2_2_I2V_A14B_SamplingParam, + Wan2_2_T2V_A14B_SamplingParam, + Wan2_2_TI2V_5B_SamplingParam, + WanI2V_14B_480P_SamplingParam, + WanI2V_14B_720P_SamplingParam, + WanT2V_1_3B_SamplingParam, + WanT2V_14B_SamplingParam, + Wan2_1_Fun_1_3B_Control_SamplingParam, + SelfForcingWan2_1_T2V_1_3B_480P_SamplingParam, + SelfForcingWan2_2_T2V_A14B_480P_SamplingParam, + MatrixGame2_SamplingParam, +) +from fastvideo.configs.sample.turbodiffusion import ( + TurboDiffusionT2V_1_3B_SamplingParam, + TurboDiffusionT2V_14B_SamplingParam, + TurboDiffusionI2V_A14B_SamplingParam, +) +# isort: on +from fastvideo.logger import init_logger +from fastvideo.utils import (maybe_download_model_index, + verify_model_config_and_directory) + +logger = init_logger(__name__) +# Registry maps specific model weights to their config classes +SAMPLING_PARAM_REGISTRY: dict[str, Any] = { + "FastVideo/FastHunyuan-diffusers": FastHunyuanSamplingParam, + "hunyuanvideo-community/HunyuanVideo": HunyuanSamplingParam, + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v": + Hunyuan15_480P_SamplingParam, + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v": + Hunyuan15_720P_SamplingParam, + "FastVideo/HY-WorldPlay-Bidirectional-Diffusers": HYWorld_SamplingParam, + "FastVideo/stepvideo-t2v-diffusers": StepVideoT2VSamplingParam, + + # Wan2.1 + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers": WanT2V_1_3B_SamplingParam, + "Wan-AI/Wan2.1-T2V-14B-Diffusers": WanT2V_14B_SamplingParam, + "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers": WanI2V_14B_480P_SamplingParam, + "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers": WanI2V_14B_720P_SamplingParam, + "weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers": + Wan2_1_Fun_1_3B_InP_SamplingParam, + "IRMChen/Wan2.1-Fun-1.3B-Control-Diffusers": + Wan2_1_Fun_1_3B_Control_SamplingParam, + + # Wan2.2 + "Wan-AI/Wan2.2-TI2V-5B-Diffusers": Wan2_2_TI2V_5B_SamplingParam, + "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers": + Wan2_2_TI2V_5B_SamplingParam, + "Wan-AI/Wan2.2-T2V-A14B-Diffusers": Wan2_2_T2V_A14B_SamplingParam, + "Wan-AI/Wan2.2-I2V-A14B-Diffusers": Wan2_2_I2V_A14B_SamplingParam, + + # FastWan2.1 + "FastVideo/FastWan2.1-T2V-1.3B-Diffusers": FastWanT2V480P_SamplingParam, + + # FastWan2.2 + "FastVideo/FastWan2.2-TI2V-5B-Diffusers": Wan2_2_TI2V_5B_SamplingParam, + + # Causal Self-Forcing Wan2.1 + "wlsaidhi/SFWan2.1-T2V-1.3B-Diffusers": + SelfForcingWan2_1_T2V_1_3B_480P_SamplingParam, + + # Causal Self-Forcing Wan2.2 + "rand0nmr/SFWan2.2-T2V-A14B-Diffusers": + SelfForcingWan2_2_T2V_A14B_480P_SamplingParam, + "FastVideo/SFWan2.2-I2V-A14B-Preview-Diffusers": + SelfForcingWan2_2_T2V_A14B_480P_SamplingParam, + + # Cosmos2 + "nvidia/Cosmos-Predict2-2B-Video2World": + Cosmos_Predict2_2B_Video2World_SamplingParam, + + # Cosmos2.5 + "KyleShao/Cosmos-Predict2.5-2B-Diffusers": + Cosmos_Predict2_5_2B_Diffusers_SamplingParam, + + # MatrixGame2.0 models + "FastVideo/Matrix-Game-2.0-Base-Diffusers": MatrixGame2_SamplingParam, + "FastVideo/Matrix-Game-2.0-GTA-Diffusers": MatrixGame2_SamplingParam, + "FastVideo/Matrix-Game-2.0-TempleRun-Diffusers": MatrixGame2_SamplingParam, + + # TurboDiffusion models + "loayrashid/TurboWan2.1-T2V-1.3B-Diffusers": + TurboDiffusionT2V_1_3B_SamplingParam, + "loayrashid/TurboWan2.1-T2V-14B-Diffusers": + TurboDiffusionT2V_14B_SamplingParam, + "loayrashid/TurboWan2.2-I2V-A14B-Diffusers": + TurboDiffusionI2V_A14B_SamplingParam, + + # LTX-2 models + "Lightricks/LTX-2": LTX2BaseSamplingParam, + "FastVideo/LTX2-base": LTX2BaseSamplingParam, + "FastVideo/LTX2-Distilled-Diffusers": LTX2DistilledSamplingParam, + + # Add other specific weight variants +} + +# For determining pipeline type from model ID +SAMPLING_PARAM_DETECTOR: dict[str, Callable[[str], bool]] = { + "hunyuan": + lambda id: "hunyuan" in id.lower(), + "hunyuan15": + lambda id: "hunyuan15" in id.lower(), + "hyworld": + lambda id: "hyworld" in id.lower(), + "wanpipeline": + lambda id: "wanpipeline" in id.lower(), + "wanimagetovideo": + lambda id: "wanimagetovideo" in id.lower(), + "stepvideo": + lambda id: "stepvideo" in id.lower(), + "wandmdpipeline": + lambda id: "wandmdpipeline" in id.lower(), + "wancausaldmdpipeline": + lambda id: "wancausaldmdpipeline" in id.lower(), + "matrixgame": + lambda id: "matrixgame" in id.lower() or "matrix-game" in id.lower(), + "turbodiffusion": + lambda id: "turbodiffusion" in id.lower() or "turbowan" in id.lower(), + "cosmos25": + lambda id: "cosmos2_5" in id.lower(), + "cosmos": + lambda id: "cosmos" in id.lower() and "2_5" not in id.lower(), + "ltx2": + lambda id: "ltx2" in id.lower() or "ltx-2" in id.lower(), + # Add other pipeline architecture detectors +} + +# Fallback configs when exact match isn't found but architecture is detected +SAMPLING_FALLBACK_PARAM: dict[str, Any] = { + "hunyuan": + HunyuanSamplingParam, # Base Hunyuan config as fallback for any Hunyuan variant + "hunyuan15": + Hunyuan15_480P_SamplingParam, # Base Hunyuan15 config as fallback for any Hunyuan15 variant + "hyworld": + HYWorld_SamplingParam, # HYWorld-specific config as fallback for any HYWorld variant + "wanpipeline": + WanT2V_1_3B_SamplingParam, # Base Wan config as fallback for any Wan variant + "wanimagetovideo": WanI2V_14B_480P_SamplingParam, + "wandmdpipeline": FastWanT2V480P_SamplingParam, + "wancausaldmdpipeline": SelfForcingWan2_1_T2V_1_3B_480P_SamplingParam, + "stepvideo": StepVideoT2VSamplingParam, + "matrixgame": MatrixGame2_SamplingParam, + "turbodiffusion": + TurboDiffusionT2V_1_3B_SamplingParam, # Default to T2V for fallback + "cosmos25": Cosmos_Predict2_5_2B_Diffusers_SamplingParam, + "cosmos": Cosmos_Predict2_2B_Video2World_SamplingParam, + "ltx2": LTX2BaseSamplingParam, + # Other fallbacks by architecture +} + + +def get_sampling_param_cls_for_name(pipeline_name_or_path: str) -> Any | None: + """Get the appropriate sampling param for specific pretrained weights.""" + # First try exact match for specific weights + if pipeline_name_or_path in SAMPLING_PARAM_REGISTRY: + return SAMPLING_PARAM_REGISTRY[pipeline_name_or_path] + + # Try partial matches (for local paths that might include the weight ID) + for registered_id, config_class in SAMPLING_PARAM_REGISTRY.items(): + if registered_id in pipeline_name_or_path: + return config_class + + matrixgame_patterns = ["Matrix-Game", "Skywork--Matrix-Game", "matrixgame"] + for pattern in matrixgame_patterns: + if pattern.lower() in pipeline_name_or_path.lower(): + return MatrixGame2_SamplingParam + + if os.path.exists(pipeline_name_or_path): + config = verify_model_config_and_directory(pipeline_name_or_path) + else: + config = maybe_download_model_index(pipeline_name_or_path) + + pipeline_name = config["_class_name"] + + # If no match, try to use the fallback config + fallback_config = None + # Try to determine pipeline architecture for fallback + for pipeline_type, detector in SAMPLING_PARAM_DETECTOR.items(): + if detector(pipeline_name.lower()): + fallback_config = SAMPLING_FALLBACK_PARAM.get(pipeline_type) + break + + logger.warning( + "No match found for pipeline %s, using fallback sampling param %s.", + pipeline_name_or_path, fallback_config) + return fallback_config diff --git a/fastvideo/pipelines/stages/ltx2_denoising.py b/fastvideo/pipelines/stages/ltx2_denoising.py index 98d04f6e1..a1dbeac4a 100644 --- a/fastvideo/pipelines/stages/ltx2_denoising.py +++ b/fastvideo/pipelines/stages/ltx2_denoising.py @@ -46,6 +46,7 @@ def _ltx2_sigmas( stretch: bool = True, terminal: float = 0.1, ) -> torch.Tensor: + # Copied/following official LTX-2 scheduler (LTX2Scheduler.execute). tokens = math.prod( latent.shape[2:]) if latent is not None else MAX_SHIFT_ANCHOR sigmas = torch.linspace(1.0, @@ -137,6 +138,7 @@ class LTX2DenoisingStage(PipelineStage): latent=None, device=latents.device, ) + logger.info("[LTX2] Using computed sigma schedule") if hasattr(self.transformer, "patchifier"): video_shape = VideoLatentShape.from_torch_shape(latents.shape) token_count = self.transformer.patchifier.get_token_count( diff --git a/run_ltx2_no_guidance_compare.py b/run_ltx2_no_guidance_compare.py new file mode 100644 index 000000000..3ea5403dd --- /dev/null +++ b/run_ltx2_no_guidance_compare.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Manual parity runner: FastVideo vs official LTX2 one-stage with no CFG.""" + +from __future__ import annotations + +import argparse +import inspect +from pathlib import Path +import sys +from dataclasses import dataclass + +import torch + +from fastvideo import VideoGenerator + + +def _setup_ltx_paths(repo_root: Path) -> None: + ltx_core_path = repo_root / "LTX-2" / "packages" / "ltx-core" / "src" + ltx_pipelines_path = repo_root / "LTX-2" / "packages" / "ltx-pipelines" / "src" + if ltx_core_path.exists() and str(ltx_core_path) not in sys.path: + sys.path.insert(0, str(ltx_core_path)) + if ltx_pipelines_path.exists() and str(ltx_pipelines_path) not in sys.path: + sys.path.insert(0, str(ltx_pipelines_path)) + + +def _run_reference_pipeline( + ref_pipeline, + prompt: str, + negative_prompt: str, + seed: int, + height: int, + width: int, + num_frames: int, + fps: float, + steps: int, + guidance_scale: float, +): + from ltx_core.components.guiders import MultiModalGuiderParams + + sig = inspect.signature(ref_pipeline.__call__) + params = set(sig.parameters) + kwargs = { + "prompt": prompt, + "negative_prompt": negative_prompt, + "seed": seed, + "height": height, + "width": width, + "num_frames": num_frames, + "enhance_prompt": False, + } + + if "frame_rate" in params: + kwargs["frame_rate"] = fps + if "num_inference_steps" in params: + kwargs["num_inference_steps"] = steps + if "images" in params: + kwargs["images"] = [] + + if "video_guider_params" in params and "audio_guider_params" in params: + kwargs["video_guider_params"] = MultiModalGuiderParams( + cfg_scale=guidance_scale, + stg_scale=0.0, + rescale_scale=0.0, + modality_scale=1.0, + skip_step=0, + stg_blocks=[], + ) + kwargs["audio_guider_params"] = MultiModalGuiderParams( + cfg_scale=guidance_scale, + stg_scale=0.0, + rescale_scale=0.0, + modality_scale=1.0, + skip_step=0, + stg_blocks=[], + ) + elif "cfg_guidance_scale" in params: + kwargs["cfg_guidance_scale"] = guidance_scale + + with torch.no_grad(): + res = ref_pipeline(**kwargs) + return res + + +def _to_bcthw(video_iter) -> torch.Tensor: + chunks = list(video_iter) + ref_video = torch.cat( + [ + chunk if torch.is_tensor(chunk) else torch.from_numpy(chunk) + for chunk in chunks + ], + dim=0, + ) + ref_video = ref_video.to(torch.float32) / 255.0 + return ref_video.permute(3, 0, 1, 2).unsqueeze(0) + + +def _to_fhwc_uint8(video_iter) -> torch.Tensor: + chunks = list(video_iter) + video = torch.cat( + [ + chunk if torch.is_tensor(chunk) else torch.from_numpy(chunk) + for chunk in chunks + ], + dim=0, + ) + if video.dtype != torch.uint8: + video = video.to(torch.float32).clamp(0, 255).to(torch.uint8) + return video + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--diffusers-path", + default="converted_weights/ltx2-base") + parser.add_argument( + "--official-path", + default="official_ltx_weights/ltx-2-19b-dev.safetensors", + ) + parser.add_argument( + "--output-dir", + default="outputs_video/ltx2_no_guidance_compare", + help="Directory to save both generated mp4 files.", + ) + return parser.parse_args() + + +PROMPT = ( + "A warm sunny backyard. The camera starts in a tight cinematic close-up " + "of a woman and a man in their 30s, facing each other with serious " + "expressions. The woman, emotional and dramatic, says softly, \"That's " + "it... Dad's lost it. And we've lost Dad.\" The man exhales, slightly " + "annoyed: \"Stop being so dramatic, Jess.\" A beat. He glances aside, " + "then mutters defensively, \"He's just having fun.\" The camera slowly " + "pans right, revealing the grandfather in the garden wearing enormous " + "butterfly wings, waving his arms in the air like he's trying to take " + "off. He shouts, \"Wheeeew!\" as he flaps his wings with full commitment. " + "The woman covers her face, on the verge of tears. The tone is deadpan, " + "absurd, and quietly tragic.") + +official_negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) + + +@dataclass +class Setting: + height: int + width: int + guidance_scale: float + negative_prompt: str + output_name: str + fps: int = 24 + num_frames: int = 121 + steps: int = 40 + seed: int = 10 + prompt: str = PROMPT + + +SETTINGS = [ + Setting( + height=1088, + width=1920, + guidance_scale=1.0, + negative_prompt="", + output_name="ltx2_compare_1088_1920_1.0.mp4", + ), + Setting( + height=1088, + width=1920, + guidance_scale=3.0, + negative_prompt="blurry, low quality, distorted, noise", + output_name="ltx2_compare_1088_1920_3.0.mp4", + ), + Setting( + height=736, + width=1280, + guidance_scale=1.0, + negative_prompt="", + output_name="ltx2_compare_736_1280_1.0.mp4", + ), + Setting( + height=736, + width=1280, + guidance_scale=3.0, + negative_prompt="blurry, low quality, distorted, noise", + output_name="ltx2_compare_736_1280_3.0.mp4", + ), + Setting( + height=480, + width=640, + guidance_scale=1.0, + negative_prompt="", + output_name="ltx2_compare_480_640_1.0.mp4", + ), + Setting( + height=480, + width=640, + guidance_scale=3.0, + negative_prompt="blurry, low quality, distorted, noise", + output_name="ltx2_compare_480_640_3.0.mp4", + ), +] + + +def main() -> None: + args = parse_args() + if not torch.cuda.is_available(): + raise RuntimeError("CUDA is required.") + + # os.environ.setdefault("FASTVIDEO_ATTENTION_BACKEND", "FLASH_ATTN") + # torch.backends.cuda.enable_flash_sdp(False) + # torch.backends.cuda.enable_mem_efficient_sdp(False) + # torch.backends.cuda.enable_math_sdp(True) + + repo_root = Path(__file__).resolve().parent + _setup_ltx_paths(repo_root) + + from ltx_core.model.transformer import attention as ltx_attention + from ltx_pipelines.ti2vid_one_stage import TI2VidOneStagePipeline + from ltx_pipelines.utils.constants import AUDIO_SAMPLE_RATE + from ltx_pipelines.utils.media_io import encode_video + + ltx_attention.memory_efficient_attention = None + ltx_attention.flash_attn_interface = None + + diffusers_path = Path(args.diffusers_path).resolve() + official_path = Path(args.official_path).resolve() + gemma_root = diffusers_path / "text_encoder" / "gemma" + device = torch.device("cuda:0") + + if not diffusers_path.is_dir(): + raise FileNotFoundError(f"Missing diffusers path: {diffusers_path}") + if not official_path.is_file(): + raise FileNotFoundError(f"Missing official checkpoint: {official_path}") + if not gemma_root.is_dir(): + raise FileNotFoundError(f"Missing gemma root: {gemma_root}") + + output_dir = Path(args.output_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + # Setup generators + generator = VideoGenerator.from_pretrained( + str(diffusers_path), + num_gpus=1, + use_fsdp_inference=False, + dit_cpu_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=False, + pin_cpu_memory=False, + ltx2_vae_tiling=False, + ) + + ref_pipeline = TI2VidOneStagePipeline( + checkpoint_path=str(official_path), + gemma_root=str(gemma_root), + loras=[], + device=device, + fp8transformer=False, + ) + + # Generate videos + for setting in SETTINGS: + generator.generate_video( + prompt=setting.prompt, + negative_prompt=setting.negative_prompt, + output_path=str(output_dir / ("fv_" + setting.output_name)), + save_video=True, + height=setting.height, + width=setting.width, + num_frames=setting.num_frames, + fps=setting.fps, + num_inference_steps=setting.steps, + guidance_scale=setting.guidance_scale, + seed=setting.seed, + ) + generator.shutdown() + + with torch.no_grad(): + for setting in SETTINGS: + ref_video_iter, ref_audio = _run_reference_pipeline( + ref_pipeline=ref_pipeline, + prompt=setting.prompt, + negative_prompt=setting.negative_prompt, + seed=setting.seed, + height=setting.height, + width=setting.width, + num_frames=setting.num_frames, + fps=setting.fps, + steps=setting.steps, + guidance_scale=setting.guidance_scale, + ) + ref_video_u8 = _to_fhwc_uint8(ref_video_iter) + encode_video( + video=ref_video_u8, + fps=int(setting.fps), + audio=ref_audio, + audio_sample_rate=AUDIO_SAMPLE_RATE, + output_path=str(output_dir / + ("ref_" + setting.output_name)), + video_chunks_number=1, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/checkpoint_conversion/convert_ltx2_weights.sh b/scripts/checkpoint_conversion/convert_ltx2_weights.sh new file mode 100644 index 000000000..18bd2b16d --- /dev/null +++ b/scripts/checkpoint_conversion/convert_ltx2_weights.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Simple wrapper for convert_ltx2_weights.py +# Edit the values below directly + +python /home/d1su/codes/FastVideo/scripts/checkpoint_conversion/convert_ltx2_weights.py \ + --source "/Lightricks/LTX-2/ltx-2-19b-dev.safetensors" \ + --output "converted_weights/ltx2-base" \ + --class-name "LTX2Transformer3DModel" \ + --pipeline-class-name "LTX2Pipeline" \ + --diffusers-version "0.33.0.dev0" \ + --gemma-path "/google/gemma-3-12b-it"