diff --git a/examples/inference/basic/basic.py b/examples/inference/basic/basic.py index 37f15b516..337b89550 100644 --- a/examples/inference/basic/basic.py +++ b/examples/inference/basic/basic.py @@ -11,11 +11,13 @@ def main(): generator = VideoGenerator.from_pretrained( "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", # FastVideo will automatically handle distributed setup - num_gpus=2, + num_gpus=1, use_fsdp_inference=True, dit_cpu_offload=False, vae_cpu_offload=False, - text_encoder_cpu_offload=False, + text_encoder_cpu_offload=True, + # Set pin_cpu_memory to false if CPU RAM is limited and there're no frequent CPU-GPU transfer + pin_cpu_memory=False, # image_encoder_cpu_offload=False, ) diff --git a/examples/inference/basic/basic_dmd.py b/examples/inference/basic/basic_dmd.py index ca806257c..ed30e6e0a 100644 --- a/examples/inference/basic/basic_dmd.py +++ b/examples/inference/basic/basic_dmd.py @@ -16,7 +16,8 @@ def main(): num_gpus=1, use_fsdp_inference=True, # Adjust these offload parameters if you have < 32GB of VRAM - text_encoder_cpu_offload=False, + text_encoder_cpu_offload=True, + pin_cpu_memory=False, dit_cpu_offload=False, vae_cpu_offload=False, VSA_sparsity=0.8, diff --git a/examples/inference/basic/basic_mps.py b/examples/inference/basic/basic_mps.py index ca53be02c..2300aa7c9 100644 --- a/examples/inference/basic/basic_mps.py +++ b/examples/inference/basic/basic_mps.py @@ -18,7 +18,7 @@ def main(): # Create sampling parameters with reduced number of frames sampling_param = SamplingParam.from_pretrained("Wan-AI/Wan2.1-T2V-1.3B-Diffusers") - sampling_param.num_frames = 3 # Reduce from default 81 to 25 frames bc we have to use the SDPA attn backend for mps + sampling_param.num_frames = 25 # Reduce from default 81 to 25 frames bc we have to use the SDPA attn backend for mps sampling_param.height = 256 sampling_param.width = 256 diff --git a/examples/inference/basic/default_args.py b/examples/inference/basic/default_args.py deleted file mode 100644 index 00d877255..000000000 --- a/examples/inference/basic/default_args.py +++ /dev/null @@ -1,62 +0,0 @@ -from fastvideo import VideoGenerator -from fastvideo.configs.pipelines.base import PipelineConfig - -def main(): - - # This is the config class for the model initialization - config = PipelineConfig.from_pretrained("FastVideo/FastHunyuan-Diffusers") - # can be used to dump the config to a yaml file - config.dump_to_yaml("config.yaml") - print(config) - # { - # 'vae_config': { - # 'scale_factor': 8, - # 'sp': True, - # 'tiling': True, - # 'precision': 'fp16' - # }, - # 'text_encoder_config': { - # 'precision': 'fp16' - # }, - # 'dit_config': { - # 'precision': 'fp16' - # }, - # 'inference_args': { - # 'guidance_scale': 7.5, - # 'num_inference_steps': 5, - # 'seed': 1024, - # 'guidance_rescale': 0.0, - # 'flow_shift': 17, - # 'num_inference_steps': 5, - # } - # } - - config.vae_config.scale_factor = 16 - - # FastVideo will automatically used the optimal default arguments for the model - # If a local path is provided, FastVideo will make a best effort attempt to - # identify the optimal arguments. - generator = VideoGenerator.from_pretrained( - "FastVideo/FastHunyuan-Diffusers", - num_gpus=4, - config=config, - # or - config_path="config.yaml", - ) - - sampling_param = SamplingParam.from_pretrained( - "FastVideo/FastHunyuan-Diffusers") - sampling_param.num_inference_steps = 5 - - # Generate videos with the same simple API, regardless of GPU count - prompt = "A beautiful woman in a red dress walking down a street" - video = generator.generate_video(prompt, - sampling_param=sampling_param, - num_inference_steps=6) - - video2 = generator.generate_video(prompt2) - prompt2 = "A beautiful woman in a blue dress walking down a street" - - -if __name__ == "__main__": - main() diff --git a/examples/inference/basic/test.py b/examples/inference/basic/test.py deleted file mode 100644 index 050da2584..000000000 --- a/examples/inference/basic/test.py +++ /dev/null @@ -1,39 +0,0 @@ -from fastvideo import VideoGenerator, PipelineConfig, SamplingParam - -# from fastvideo.configs.sample import SamplingParam - -OUTPUT_PATH = "video_samples_fp16" -def main(): - # FastVideo will automatically use the optimal default arguments for the - # model. - # If a local path is provided, FastVideo will make a best effort - # attempt to identify the optimal arguments. - model = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" - pipeline_config = PipelineConfig.from_pretrained(model) - pipeline_config.text_encoder_precisions = ("bf16", ) - generator = VideoGenerator.from_pretrained( - model, - # if num_gpus > 1, FastVideo will automatically handle distributed setup - pipeline_config=pipeline_config, - use_fsdp_inference=False, # Disable FSDP for MPS - dit_cpu_offload=True, - text_encoder_cpu_offload=True, - pin_cpu_memory=True, - disable_autocast=False, - num_gpus=1, - ) - - sampling_param = SamplingParam.from_pretrained(model) - sampling_param.num_frames = 30 - # sampling_param.image_path = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg" - # Generate videos with the same simple API, regardless of GPU count - prompt = "Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of a bustling street food market. The scene captures a mix of humor and authenticity. Mid-shot framing, vibrant lighting." - video = generator.generate_video(prompt, output_path=OUTPUT_PATH, save_video=True, sampling_param=sampling_param) - # video = generator.generate_video(prompt, sampling_param=sampling_param, output_path="wan_t2v_videos/") - - # Generate another video with a different prompt, without reloading the - # model! - - -if __name__ == "__main__": - main() diff --git a/examples/inference/lora/wan_lora_inference.py b/examples/inference/lora/wan_lora_inference.py index 74fa73328..5471a8980 100644 --- a/examples/inference/lora/wan_lora_inference.py +++ b/examples/inference/lora/wan_lora_inference.py @@ -7,6 +7,10 @@ def main(): generator = VideoGenerator.from_pretrained( "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", num_gpus=1, + dit_cpu_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=True, + pin_cpu_memory=False, lora_path="benjamin-paine/steamboat-willie-1.3b", lora_nickname="steamboat" ) diff --git a/examples/inference/lora/wan_lora_inference_from_ckpt.py b/examples/inference/lora/wan_lora_inference_from_ckpt.py index 2972d11da..444fda6e2 100644 --- a/examples/inference/lora/wan_lora_inference_from_ckpt.py +++ b/examples/inference/lora/wan_lora_inference_from_ckpt.py @@ -10,6 +10,10 @@ def main(): generator = VideoGenerator.from_pretrained( "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", num_gpus=1, + dit_cpu_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=True, + pin_cpu_memory=False, lora_path="checkpoints/wan_t2v_finetune_lora/checkpoint-1250/transformer", lora_nickname="crush_smol" ) diff --git a/examples/inference/optimizations/attention_example.py b/examples/inference/optimizations/attention_example.py index ddb1868f6..ce727212f 100644 --- a/examples/inference/optimizations/attention_example.py +++ b/examples/inference/optimizations/attention_example.py @@ -11,6 +11,10 @@ def main(): gen = VideoGenerator.from_pretrained( model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", num_gpus=1, + dit_cpu_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=True, + pin_cpu_memory=False, ) load_time = time.perf_counter() - start_time print(f"Model loading time: {load_time:.2f} seconds") diff --git a/examples/inference/optimizations/teacache_example.py b/examples/inference/optimizations/teacache_example.py index 1dab0f92e..ba2e353ea 100644 --- a/examples/inference/optimizations/teacache_example.py +++ b/examples/inference/optimizations/teacache_example.py @@ -10,6 +10,9 @@ def main(): model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", num_gpus=1, dit_cpu_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=True, + pin_cpu_memory=False, ) load_time = time.perf_counter() - start_time print(f"Model loading time: {load_time:.2f} seconds") diff --git a/fastvideo/entrypoints/video_generator.py b/fastvideo/entrypoints/video_generator.py index 2933b2f05..2e6f63696 100644 --- a/fastvideo/entrypoints/video_generator.py +++ b/fastvideo/entrypoints/video_generator.py @@ -288,10 +288,6 @@ class VideoGenerator: """ # type: ignore[attr-defined] logger.info(debug_str) - # Use prompt[:100] for video name - if sampling_param.output_video_name is None: - sampling_param.output_video_name = prompt[:100] - # Prepare batch batch = ForwardBatch( **shallow_asdict(sampling_param), @@ -301,6 +297,10 @@ class VideoGenerator: extra={}, ) + # Use prompt[:100] for video name + if batch.output_video_name is None: + batch.output_video_name = prompt[:100] + # Run inference start_time = time.perf_counter() output_batch = self.executor.execute_forward(batch, fastvideo_args) diff --git a/scripts/inference/v1_inference_wan.sh b/scripts/inference/v1_inference_wan.sh index 60f96843e..42d030164 100755 --- a/scripts/inference/v1_inference_wan.sh +++ b/scripts/inference/v1_inference_wan.sh @@ -10,6 +10,10 @@ fastvideo generate \ --sp-size $num_gpus \ --tp-size 1 \ --num-gpus $num_gpus \ + --dit-cpu-offload False \ + --vae-cpu-offload False \ + --text-encoder-cpu-offload True \ + --pin-cpu-memory False \ --height 480 \ --width 832 \ --num-frames 77 \ diff --git a/scripts/inference/v1_inference_wan_VSA.sh b/scripts/inference/v1_inference_wan_VSA.sh index e75d2c18e..721fb717e 100755 --- a/scripts/inference/v1_inference_wan_VSA.sh +++ b/scripts/inference/v1_inference_wan_VSA.sh @@ -10,6 +10,10 @@ fastvideo generate \ --sp-size $num_gpus \ --tp-size 1 \ --num-gpus $num_gpus \ + --dit-cpu-offload False \ + --vae-cpu-offload False \ + --text-encoder-cpu-offload True \ + --pin-cpu-memory False \ --height 448 \ --width 832 \ --num-frames 77 \ diff --git a/scripts/inference/v1_inference_wan_dmd.sh b/scripts/inference/v1_inference_wan_dmd.sh index 59b5048f2..156f31cb0 100755 --- a/scripts/inference/v1_inference_wan_dmd.sh +++ b/scripts/inference/v1_inference_wan_dmd.sh @@ -10,6 +10,10 @@ fastvideo generate \ --sp-size $num_gpus \ --tp-size 1 \ --num-gpus $num_gpus \ + --dit-cpu-offload False \ + --vae-cpu-offload False \ + --text-encoder-cpu-offload True \ + --pin-cpu-memory False \ --height 480 \ --width 848 \ --num-frames 81 \