From d8b9d7c6abfdd8b4e6df67e50bf9fcac9e24253b Mon Sep 17 00:00:00 2001 From: XOR-op <17672363+XOR-op@users.noreply.github.com> Date: Tue, 13 Jan 2026 17:20:05 -0500 Subject: [PATCH] [feat!] Disable FSDP inference by default (#1001) --- docs/inference/inference_quick_start.md | 2 ++ examples/inference/basic/basic.py | 2 +- examples/inference/basic/basic_dmd.py | 2 +- examples/inference/basic/basic_hy15.py | 2 +- examples/inference/basic/basic_longcat_i2v.py | 2 +- examples/inference/basic/basic_longcat_t2v.py | 2 +- examples/inference/basic/basic_longcat_vc.py | 2 +- examples/inference/basic/basic_matrixgame.py | 2 +- examples/inference/basic/basic_matrixgame_streaming.py | 2 +- examples/inference/basic/basic_self_forcing_causal.py | 2 +- .../inference/basic/basic_self_forcing_causal_wan2_2_i2v.py | 2 +- .../inference/basic/basic_self_forcing_causal_wan2_2_t2v.py | 2 +- examples/inference/basic/basic_turbodiffusion.py | 1 + examples/inference/basic/basic_wan2_2.py | 2 +- examples/inference/basic/basic_wan2_2_Fun.py | 2 +- examples/inference/basic/basic_wan2_2_i2v.py | 2 +- examples/inference/basic/basic_wan2_2_ti2v.py | 2 +- fastvideo/fastvideo_args.py | 6 ++++-- 18 files changed, 22 insertions(+), 17 deletions(-) diff --git a/docs/inference/inference_quick_start.md b/docs/inference/inference_quick_start.md index ba2712901..a55c85903 100644 --- a/docs/inference/inference_quick_start.md +++ b/docs/inference/inference_quick_start.md @@ -106,6 +106,8 @@ If you encounter CUDA out of memory errors: - Enable memory optimization with `enable_model_cpu_offload` - Try a smaller model or use distilled versions - Use `num_gpus` > 1 if multiple GPUs are available +- Try enabling FSDP inference with `use_fsdp_inference=True` (may slow down generation) +- Try enabling DiT layerwise offload with `dit_layerwise_offload=True` (now only a few models support this, but may introduce less overhead than FSDP) ### Slow Generation diff --git a/examples/inference/basic/basic.py b/examples/inference/basic/basic.py index f12a4969e..3f00832b9 100644 --- a/examples/inference/basic/basic.py +++ b/examples/inference/basic/basic.py @@ -12,7 +12,7 @@ def main(): "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=False, vae_cpu_offload=False, text_encoder_cpu_offload=True, diff --git a/examples/inference/basic/basic_dmd.py b/examples/inference/basic/basic_dmd.py index 48b2965ce..b3b63b199 100644 --- a/examples/inference/basic/basic_dmd.py +++ b/examples/inference/basic/basic_dmd.py @@ -14,7 +14,7 @@ def main(): model_name, # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory # Adjust these offload parameters if you have < 32GB of VRAM text_encoder_cpu_offload=True, pin_cpu_memory=True, # set to false if low CPU RAM or hit obscure "CUDA error: Invalid argument" diff --git a/examples/inference/basic/basic_hy15.py b/examples/inference/basic/basic_hy15.py index 71e491a41..39d8772d3 100644 --- a/examples/inference/basic/basic_hy15.py +++ b/examples/inference/basic/basic_hy15.py @@ -12,7 +12,7 @@ def main(): "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=True, vae_cpu_offload=True, text_encoder_cpu_offload=True, diff --git a/examples/inference/basic/basic_longcat_i2v.py b/examples/inference/basic/basic_longcat_i2v.py index 9e13c66ef..f170dc4a3 100644 --- a/examples/inference/basic/basic_longcat_i2v.py +++ b/examples/inference/basic/basic_longcat_i2v.py @@ -55,7 +55,7 @@ def basic_generation(): generator = VideoGenerator.from_pretrained( "FastVideo/LongCat-Video-I2V-Diffusers", num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=False, vae_cpu_offload=True, text_encoder_cpu_offload=True, diff --git a/examples/inference/basic/basic_longcat_t2v.py b/examples/inference/basic/basic_longcat_t2v.py index aee8ab748..354228de2 100644 --- a/examples/inference/basic/basic_longcat_t2v.py +++ b/examples/inference/basic/basic_longcat_t2v.py @@ -48,7 +48,7 @@ def basic_generation(): generator = VideoGenerator.from_pretrained( "FastVideo/LongCat-Video-T2V-Diffusers", num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=False, vae_cpu_offload=True, text_encoder_cpu_offload=True, diff --git a/examples/inference/basic/basic_longcat_vc.py b/examples/inference/basic/basic_longcat_vc.py index 500a02091..313cbb72c 100644 --- a/examples/inference/basic/basic_longcat_vc.py +++ b/examples/inference/basic/basic_longcat_vc.py @@ -66,7 +66,7 @@ def basic_generation(): generator = VideoGenerator.from_pretrained( "FastVideo/LongCat-Video-VC-Diffusers", num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=False, vae_cpu_offload=True, text_encoder_cpu_offload=True, diff --git a/examples/inference/basic/basic_matrixgame.py b/examples/inference/basic/basic_matrixgame.py index bae6fba3f..b88fb5fcc 100644 --- a/examples/inference/basic/basic_matrixgame.py +++ b/examples/inference/basic/basic_matrixgame.py @@ -43,7 +43,7 @@ def main(): config["model_path"], # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=True, # DiT need to be offloaded for MoE vae_cpu_offload=False, text_encoder_cpu_offload=True, diff --git a/examples/inference/basic/basic_matrixgame_streaming.py b/examples/inference/basic/basic_matrixgame_streaming.py index 2454cda4a..ea3f2abd4 100644 --- a/examples/inference/basic/basic_matrixgame_streaming.py +++ b/examples/inference/basic/basic_matrixgame_streaming.py @@ -46,7 +46,7 @@ async def main(): config["model_path"], # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=True, # DiT need to be offloaded for MoE vae_cpu_offload=False, text_encoder_cpu_offload=True, diff --git a/examples/inference/basic/basic_self_forcing_causal.py b/examples/inference/basic/basic_self_forcing_causal.py index e2241a94e..0e6d46e9e 100644 --- a/examples/inference/basic/basic_self_forcing_causal.py +++ b/examples/inference/basic/basic_self_forcing_causal.py @@ -13,7 +13,7 @@ def main(): model_name, # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory text_encoder_cpu_offload=False, dit_cpu_offload=False, ) diff --git a/examples/inference/basic/basic_self_forcing_causal_wan2_2_i2v.py b/examples/inference/basic/basic_self_forcing_causal_wan2_2_i2v.py index 5579ababe..bec8575c4 100644 --- a/examples/inference/basic/basic_self_forcing_causal_wan2_2_i2v.py +++ b/examples/inference/basic/basic_self_forcing_causal_wan2_2_i2v.py @@ -14,7 +14,7 @@ def main(): "FastVideo/SFWan2.2-I2V-A14B-Preview-Diffusers", # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=True, # DiT need to be offloaded for MoE dit_precision="fp32", vae_cpu_offload=False, diff --git a/examples/inference/basic/basic_self_forcing_causal_wan2_2_t2v.py b/examples/inference/basic/basic_self_forcing_causal_wan2_2_t2v.py index 6c11689ec..2d6032c5d 100644 --- a/examples/inference/basic/basic_self_forcing_causal_wan2_2_t2v.py +++ b/examples/inference/basic/basic_self_forcing_causal_wan2_2_t2v.py @@ -14,7 +14,7 @@ def main(): "rand0nmr/SFWan2.2-T2V-A14B-Diffusers", # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=True, # DiT need to be offloaded for MoE vae_cpu_offload=False, text_encoder_cpu_offload=True, diff --git a/examples/inference/basic/basic_turbodiffusion.py b/examples/inference/basic/basic_turbodiffusion.py index 355db0c24..b3e314869 100644 --- a/examples/inference/basic/basic_turbodiffusion.py +++ b/examples/inference/basic/basic_turbodiffusion.py @@ -15,6 +15,7 @@ def main() -> None: "loayrashid/TurboWan2.1-T2V-1.3B-Diffusers", # FastVideo will automatically handle distributed setup num_gpus=1, + use_fsdp_inference=False, # set to True if GPU is out of memory # set to false if using RTX 4090 # pin_cpu_memory=False, diff --git a/examples/inference/basic/basic_wan2_2.py b/examples/inference/basic/basic_wan2_2.py index b1420b7da..f1b324c73 100644 --- a/examples/inference/basic/basic_wan2_2.py +++ b/examples/inference/basic/basic_wan2_2.py @@ -12,7 +12,7 @@ def main(): "Wan-AI/Wan2.2-T2V-A14B-Diffusers", # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=True, # DiT need to be offloaded for MoE vae_cpu_offload=False, text_encoder_cpu_offload=True, diff --git a/examples/inference/basic/basic_wan2_2_Fun.py b/examples/inference/basic/basic_wan2_2_Fun.py index ca779941b..61c216074 100644 --- a/examples/inference/basic/basic_wan2_2_Fun.py +++ b/examples/inference/basic/basic_wan2_2_Fun.py @@ -14,7 +14,7 @@ def main(): # "alibaba-pai/Wan2.2-Fun-A14B-Control", # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=True, # DiT need to be offloaded for MoE vae_cpu_offload=False, text_encoder_cpu_offload=True, diff --git a/examples/inference/basic/basic_wan2_2_i2v.py b/examples/inference/basic/basic_wan2_2_i2v.py index 9c55694e3..08eb2c21a 100644 --- a/examples/inference/basic/basic_wan2_2_i2v.py +++ b/examples/inference/basic/basic_wan2_2_i2v.py @@ -12,7 +12,7 @@ def main(): "Wan-AI/Wan2.2-I2V-A14B-Diffusers", # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=True, # DiT need to be offloaded for MoE vae_cpu_offload=False, text_encoder_cpu_offload=True, diff --git a/examples/inference/basic/basic_wan2_2_ti2v.py b/examples/inference/basic/basic_wan2_2_ti2v.py index 0b16bb4d0..df0c18848 100644 --- a/examples/inference/basic/basic_wan2_2_ti2v.py +++ b/examples/inference/basic/basic_wan2_2_ti2v.py @@ -11,7 +11,7 @@ def main(): model_name, # FastVideo will automatically handle distributed setup num_gpus=1, - use_fsdp_inference=True, + use_fsdp_inference=False, # set to True if GPU is out of memory dit_cpu_offload=True, vae_cpu_offload=False, text_encoder_cpu_offload=True, diff --git a/fastvideo/fastvideo_args.py b/fastvideo/fastvideo_args.py index 5845449a9..2a41e6d67 100644 --- a/fastvideo/fastvideo_args.py +++ b/fastvideo/fastvideo_args.py @@ -132,7 +132,7 @@ class FastVideoArgs: # CPU offload parameters dit_cpu_offload: bool = True - use_fsdp_inference: bool = True + use_fsdp_inference: bool = False dit_layerwise_offload: bool = False text_encoder_cpu_offload: bool = True image_encoder_cpu_offload: bool = True @@ -431,7 +431,9 @@ class FastVideoArgs: "--use-fsdp-inference", action=StoreBoolean, help= - "Use FSDP for inference by sharding the model weights. Latency is very low due to prefetch--enable if run out of memory.", + "Use FSDP for inference by sharding the model weights. FSDP helps reduce GPU memory usage but may introduce" + + + " weight transfer overhead depending on the specific setup. Enable if run out of memory.", ) parser.add_argument( "--text-encoder-cpu-offload",