diff --git a/examples/inference/basic/basic_ltx2_spatiotemporal_upsample.py b/examples/inference/basic/basic_ltx2_spatiotemporal_upsample.py new file mode 100644 index 000000000..50ca83864 --- /dev/null +++ b/examples/inference/basic/basic_ltx2_spatiotemporal_upsample.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: Apache-2.0 +from pathlib import Path + +import torch + +from fastvideo.configs.pipelines import PipelineConfig +from fastvideo.entrypoints.upsample import (_prepare_video, _read_video, + _write_video) +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.logger import init_logger +from fastvideo.models.loader.component_loader import UpsamplerLoader, VAELoader +from fastvideo.models.upsamplers import upsample_video +from fastvideo.utils import maybe_download_model + +logger = init_logger(__name__) + +# Input/output +INPUT_VIDEO = "outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4" +OUTPUT_VIDEO = "outputs_video/ltx2_upscale/ltx2_spatiotemporal_upscale_x2.mp4" + +# Diffusers-style LTX-2 repo with upsamplers included +MODEL_ID = "FastVideo/LTX2-Diffusers" + +# Controls +DOUBLE_FPS = True + + +def main() -> None: + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + precision_str = "bf16" if torch.cuda.is_available() else "fp32" + dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32 + + video, fps = _read_video(INPUT_VIDEO) + video = _prepare_video( + video, + trim_frames=False, + pad_frames=True, + crop_multiple=32, + ) + + model_root = maybe_download_model(MODEL_ID) + vae_path = str(Path(model_root) / "vae") + spatial_upsampler_path = str(Path(model_root) / "spatial_upsampler") + temporal_upsampler_path = str(Path(model_root) / "temporal_upsampler") + + args = FastVideoArgs( + model_path=vae_path, + pipeline_config=PipelineConfig(vae_precision=precision_str), + vae_cpu_offload=False, + ) + vae = VAELoader().load(vae_path, args).to(device=device, dtype=dtype) + spatial_upsampler = UpsamplerLoader().load( + spatial_upsampler_path, args).to(device=device, dtype=dtype) + temporal_upsampler = UpsamplerLoader().load( + temporal_upsampler_path, args).to(device=device, dtype=dtype) + + if hasattr(vae.decoder, "decode_noise_scale"): + vae.decoder.decode_noise_scale = 0.0 + + # [F, C, H, W] -> [B, C, F, H, W] + video = video.unsqueeze(0).permute(0, 2, 1, 3, 4).to( + device=device, dtype=dtype) + + with torch.no_grad(): + latents = vae.encoder(video) + up_latents = upsample_video(latents, vae.encoder, + getattr(spatial_upsampler, "model", + spatial_upsampler)) + up_latents = upsample_video(up_latents, vae.encoder, + getattr(temporal_upsampler, "model", + temporal_upsampler)) + + timestep_value = getattr(vae.decoder, "decode_timestep", 0.05) + timestep = torch.full((video.shape[0], ), + float(timestep_value), + device=device, + dtype=dtype) + decoded = vae.decoder(up_latents, timestep=timestep) + + # [B, C, F, H, W] -> [F, C, H, W] + decoded = decoded[0].permute(1, 0, 2, 3).detach().cpu() + + output_fps = fps * 2 if DOUBLE_FPS else fps + Path(OUTPUT_VIDEO).parent.mkdir(parents=True, exist_ok=True) + _write_video(decoded, OUTPUT_VIDEO, output_fps) + logger.info("Spatiotemporal upsampled video saved to %s", OUTPUT_VIDEO) + + +if __name__ == "__main__": + main() diff --git a/examples/inference/basic/basic_ltx2_temporal_upsample.py b/examples/inference/basic/basic_ltx2_temporal_upsample.py new file mode 100644 index 000000000..90b988d4b --- /dev/null +++ b/examples/inference/basic/basic_ltx2_temporal_upsample.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +from pathlib import Path + +import torch + +from fastvideo.configs.pipelines import PipelineConfig +from fastvideo.entrypoints.upsample import (_prepare_video, _read_video, + _write_video) +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.logger import init_logger +from fastvideo.models.loader.component_loader import UpsamplerLoader, VAELoader +from fastvideo.models.upsamplers import upsample_video +from fastvideo.utils import maybe_download_model + +logger = init_logger(__name__) + +# Input/output +INPUT_VIDEO = "outputs_video/ltx2_basic/output_ltx2_distilled_t2v.mp4" +OUTPUT_VIDEO = "outputs_video/ltx2_upscale/ltx2_temporal_upscale_x2.mp4" + +# Diffusers-style LTX-2 repo with upsamplers included +MODEL_ID = "FastVideo/LTX2-Diffusers" + +# Controls +DOUBLE_FPS = True + + +def main() -> None: + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + precision_str = "bf16" if torch.cuda.is_available() else "fp32" + dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32 + + video, fps = _read_video(INPUT_VIDEO) + video = _prepare_video( + video, + trim_frames=False, + pad_frames=True, + crop_multiple=32, + ) + + model_root = maybe_download_model(MODEL_ID) + vae_path = str(Path(model_root) / "vae") + temporal_upsampler_path = str(Path(model_root) / "temporal_upsampler") + + args = FastVideoArgs( + model_path=vae_path, + pipeline_config=PipelineConfig(vae_precision=precision_str), + vae_cpu_offload=False, + ) + vae = VAELoader().load(vae_path, args).to(device=device, dtype=dtype) + temporal_upsampler = UpsamplerLoader().load( + temporal_upsampler_path, args).to(device=device, dtype=dtype) + + if hasattr(vae.decoder, "decode_noise_scale"): + vae.decoder.decode_noise_scale = 0.0 + + # [F, C, H, W] -> [B, C, F, H, W] + video = video.unsqueeze(0).permute(0, 2, 1, 3, 4).to( + device=device, dtype=dtype) + + with torch.no_grad(): + latents = vae.encoder(video) + up_latents = upsample_video(latents, vae.encoder, + getattr(temporal_upsampler, "model", + temporal_upsampler)) + + timestep_value = getattr(vae.decoder, "decode_timestep", 0.05) + timestep = torch.full((video.shape[0], ), + float(timestep_value), + device=device, + dtype=dtype) + decoded = vae.decoder(up_latents, timestep=timestep) + + # [B, C, F, H, W] -> [F, C, H, W] + decoded = decoded[0].permute(1, 0, 2, 3).detach().cpu() + + output_fps = fps * 2 if DOUBLE_FPS else fps + Path(OUTPUT_VIDEO).parent.mkdir(parents=True, exist_ok=True) + _write_video(decoded, OUTPUT_VIDEO, output_fps) + logger.info("Temporal upsampled video saved to %s", OUTPUT_VIDEO) + + +if __name__ == "__main__": + main() diff --git a/examples/inference/basic/basic_ltx2_upscale.py b/examples/inference/basic/basic_ltx2_upscale.py index 7c442e0f9..ee1a9f9fb 100644 --- a/examples/inference/basic/basic_ltx2_upscale.py +++ b/examples/inference/basic/basic_ltx2_upscale.py @@ -15,19 +15,17 @@ PROMPT = ( "absurd, and quietly tragic." ) -# FastVideo-converted weights (align settings to the official distilled pipeline). -MODEL_PATH = "converted/ltx2_diffusers" -UPSAMPLER_PATH = "converted/ltx2_spatial_upscaler" +# HF model ID (downloaded automatically). Distilled repos include upsamplers + refine LoRA reference. +MODEL_ID = "FastVideo/LTX2-Distilled-Diffusers" -OUTPUT_PATH = "outputs_video/ltx2_upscale/ltx2_official_two_stage2222.mp4" +OUTPUT_PATH = "outputs_video/ltx2_upscale/ltx2_two_stage.mp4" def main() -> None: generator = VideoGenerator.from_pretrained( - MODEL_PATH, + MODEL_ID, num_gpus=1, ltx2_refine_enabled=True, - ltx2_refine_upsampler_path=UPSAMPLER_PATH, ltx2_refine_num_inference_steps=3, ltx2_refine_guidance_scale=1.0, ltx2_refine_add_noise=True, @@ -36,8 +34,8 @@ def main() -> None: generator.generate_video( prompt=PROMPT, output_path=OUTPUT_PATH, - height=960, - width=1664, + height=1080, + width=1920, num_frames=81, fps=24, seed=10, diff --git a/fastvideo/fastvideo_args.py b/fastvideo/fastvideo_args.py index fcb709028..1b62adc6f 100644 --- a/fastvideo/fastvideo_args.py +++ b/fastvideo/fastvideo_args.py @@ -172,6 +172,17 @@ class FastVideoArgs: ltx2_vae_temporal_tile_overlap_in_frames: int | None = None ltx2_initial_latent_path: str | None = None ltx2_audio_latent_path: str | None = None + # Generic stage-2 refine args (preferred API). These map to LTX-2 refine args + # for now, but keep the user-facing API model-agnostic. + refine_enabled: bool | None = None + refine_upsampler_path: str | None = None + refine_transformer_path: str | None = None + refine_lora_path: str | None = None + refine_num_inference_steps: int | None = None + refine_guidance_scale: float | None = None + refine_add_noise: bool | None = None + refine_noise_path: str | None = None + refine_audio_noise_path: str | None = None ltx2_refine_enabled: bool = False ltx2_refine_upsampler_path: str | None = None ltx2_refine_transformer_path: str | None = None @@ -233,6 +244,7 @@ class FastVideoArgs: self.moba_config_path, e) raise self._apply_ltx2_vae_overrides() + self._resolve_refine_args() self.check_fastvideo_args() def _apply_ltx2_vae_overrides(self) -> None: @@ -270,6 +282,27 @@ class FastVideoArgs: vae_config.ltx2_temporal_tile_overlap_in_frames = ( self.ltx2_vae_temporal_tile_overlap_in_frames) + def _resolve_refine_args(self) -> None: + """Map generic refine_* args to LTX-2-specific refine fields.""" + if self.refine_enabled is not None: + self.ltx2_refine_enabled = self.refine_enabled + if self.refine_upsampler_path is not None: + self.ltx2_refine_upsampler_path = self.refine_upsampler_path + if self.refine_transformer_path is not None: + self.ltx2_refine_transformer_path = self.refine_transformer_path + if self.refine_lora_path is not None: + self.ltx2_refine_lora_path = self.refine_lora_path + if self.refine_num_inference_steps is not None: + self.ltx2_refine_num_inference_steps = self.refine_num_inference_steps + if self.refine_guidance_scale is not None: + self.ltx2_refine_guidance_scale = self.refine_guidance_scale + if self.refine_add_noise is not None: + self.ltx2_refine_add_noise = self.refine_add_noise + if self.refine_noise_path is not None: + self.ltx2_refine_noise_path = self.refine_noise_path + if self.refine_audio_noise_path is not None: + self.ltx2_refine_audio_noise_path = self.refine_audio_noise_path + @staticmethod def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: # Model and path configuration diff --git a/fastvideo/pipelines/basic/ltx2/ltx2_pipeline.py b/fastvideo/pipelines/basic/ltx2/ltx2_pipeline.py index 0d5da0861..2c75a05ca 100644 --- a/fastvideo/pipelines/basic/ltx2/ltx2_pipeline.py +++ b/fastvideo/pipelines/basic/ltx2/ltx2_pipeline.py @@ -167,6 +167,51 @@ class LTX2Pipeline(LoRAPipeline): model_index = self._load_config(self.model_path) logger.info("Loading pipeline modules from config: %s", model_index) + # Apply optional FastVideo-specific refine defaults embedded in model_index.json. + def _resolve_refine_path(value: str | None) -> str | None: + if value is None: + return None + if os.path.isabs(value): + return value + candidate = os.path.join(self.model_path, value) + if os.path.exists(candidate): + return candidate + return value + + if model_index.get("fastvideo_refine_enabled") is True: + if fastvideo_args.refine_enabled is None: + fastvideo_args.ltx2_refine_enabled = True + if fastvideo_args.refine_upsampler_path is None and fastvideo_args.ltx2_refine_upsampler_path is None: + fastvideo_args.ltx2_refine_upsampler_path = _resolve_refine_path( + model_index.get("fastvideo_refine_upsampler_path")) + if fastvideo_args.ltx2_refine_upsampler_path is None and "spatial_upsampler" in model_index: + fastvideo_args.ltx2_refine_upsampler_path = _resolve_refine_path( + "spatial_upsampler") + if fastvideo_args.refine_transformer_path is None and fastvideo_args.ltx2_refine_transformer_path is None: + fastvideo_args.ltx2_refine_transformer_path = _resolve_refine_path( + model_index.get("fastvideo_refine_transformer_path")) + if fastvideo_args.refine_lora_path is None and fastvideo_args.ltx2_refine_lora_path is None: + fastvideo_args.ltx2_refine_lora_path = _resolve_refine_path( + model_index.get("fastvideo_refine_lora_path")) + if fastvideo_args.refine_num_inference_steps is None and model_index.get( + "fastvideo_refine_num_inference_steps") is not None: + fastvideo_args.ltx2_refine_num_inference_steps = int( + model_index["fastvideo_refine_num_inference_steps"]) + if fastvideo_args.refine_guidance_scale is None and model_index.get( + "fastvideo_refine_guidance_scale") is not None: + fastvideo_args.ltx2_refine_guidance_scale = float( + model_index["fastvideo_refine_guidance_scale"]) + if fastvideo_args.refine_add_noise is None and model_index.get( + "fastvideo_refine_add_noise") is not None: + fastvideo_args.ltx2_refine_add_noise = bool( + model_index["fastvideo_refine_add_noise"]) + if fastvideo_args.refine_noise_path is None and fastvideo_args.ltx2_refine_noise_path is None: + fastvideo_args.ltx2_refine_noise_path = _resolve_refine_path( + model_index.get("fastvideo_refine_noise_path")) + if fastvideo_args.refine_audio_noise_path is None and fastvideo_args.ltx2_refine_audio_noise_path is None: + fastvideo_args.ltx2_refine_audio_noise_path = _resolve_refine_path( + model_index.get("fastvideo_refine_audio_noise_path")) + model_index.pop("_class_name") model_index.pop("_diffusers_version") model_index.pop("workload_type", None) @@ -231,6 +276,15 @@ class LTX2Pipeline(LoRAPipeline): raise ValueError( "ltx2_refine_enabled is True but ltx2_refine_upsampler_path was not provided." ) + if not os.path.isdir(upsampler_path): + raise ValueError( + "ltx2_refine_upsampler_path must be a directory containing Diffusers-style " + f"upsampler weights; got {upsampler_path}") + config_path = os.path.join(upsampler_path, "config.json") + if not os.path.exists(config_path): + raise ValueError( + "ltx2_refine_upsampler_path must contain a Diffusers config.json; " + f"missing {config_path}") if loaded_modules is not None and "spatial_upsampler" in loaded_modules: modules["spatial_upsampler"] = loaded_modules[ "spatial_upsampler"] diff --git a/scripts/huggingface/upload_hf.py b/scripts/huggingface/upload_hf.py index 1e4ce88b7..b70cac506 100644 --- a/scripts/huggingface/upload_hf.py +++ b/scripts/huggingface/upload_hf.py @@ -2,8 +2,13 @@ from huggingface_hub import HfApi api = HfApi() +repo_id = "FastVideo/LTX2-Distilled-LoRA" + +# Create the repo if it doesn't exist +api.create_repo(repo_id=repo_id, repo_type="model", exist_ok=True) + api.upload_folder( - folder_path="Wan2.2-TI2V-5B-Diffusers", - repo_id="FastVideo/FastWan2.2-TI2V-5B-Diffusers", + folder_path="/mnt/user_storage/dev/FastVideo/ltx_distilled_lora", + repo_id=repo_id, repo_type="model", )