update

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

docs/inference/inference_quick_start.md

@@ -106,6 +106,8 @@ If you encounter CUDA out of memory errors:
 - Enable memory optimization with `enable_model_cpu_offload`
 - Try a smaller model or use distilled versions
 - Use `num_gpus` > 1 if multiple GPUs are available
+- Try enabling FSDP inference with `use_fsdp_inference=True` (may slow down generation)
+- Try enabling DiT layerwise offload with `dit_layerwise_offload=True` (now only a few models support this, but may introduce less overhead than FSDP)
 
 ### Slow Generation
 

examples/inference/basic/basic.py

@@ -12,7 +12,7 @@ def main():
         "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=False,
         vae_cpu_offload=False,
         text_encoder_cpu_offload=True,

examples/inference/basic/basic_dmd.py

@@ -14,7 +14,7 @@ def main():
         model_name,
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         # Adjust these offload parameters if you have < 32GB of VRAM
         text_encoder_cpu_offload=True,
         pin_cpu_memory=True, # set to false if low CPU RAM or hit obscure "CUDA error: Invalid argument"

examples/inference/basic/basic_hy15.py

@@ -12,7 +12,7 @@ def main():
         "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=True,
         vae_cpu_offload=True,
         text_encoder_cpu_offload=True,

examples/inference/basic/basic_longcat_i2v.py

@@ -55,7 +55,7 @@ def basic_generation():
     generator = VideoGenerator.from_pretrained(
         "FastVideo/LongCat-Video-I2V-Diffusers",
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=False,
         vae_cpu_offload=True,
         text_encoder_cpu_offload=True,

examples/inference/basic/basic_longcat_t2v.py

@@ -48,7 +48,7 @@ def basic_generation():
     generator = VideoGenerator.from_pretrained(
         "FastVideo/LongCat-Video-T2V-Diffusers",
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=False,
         vae_cpu_offload=True,
         text_encoder_cpu_offload=True,

examples/inference/basic/basic_longcat_vc.py

@@ -66,7 +66,7 @@ def basic_generation():
     generator = VideoGenerator.from_pretrained(
         "FastVideo/LongCat-Video-VC-Diffusers",
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=False,
         vae_cpu_offload=True,
         text_encoder_cpu_offload=True,

examples/inference/basic/basic_matrixgame.py

@@ -43,7 +43,7 @@ def main():
         config["model_path"],
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=True, # DiT need to be offloaded for MoE
         vae_cpu_offload=False,
         text_encoder_cpu_offload=True,

examples/inference/basic/basic_matrixgame_streaming.py

@@ -46,7 +46,7 @@ async def main():
         config["model_path"],
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=True, # DiT need to be offloaded for MoE
         vae_cpu_offload=False,
         text_encoder_cpu_offload=True,

examples/inference/basic/basic_self_forcing_causal.py

@@ -13,7 +13,7 @@ def main():
         model_name,
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         text_encoder_cpu_offload=False,
         dit_cpu_offload=False,
     )

examples/inference/basic/basic_self_forcing_causal_wan2_2_i2v.py

@@ -14,7 +14,7 @@ def main():
         "FastVideo/SFWan2.2-I2V-A14B-Preview-Diffusers",
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=True, # DiT need to be offloaded for MoE
         dit_precision="fp32",
         vae_cpu_offload=False,

examples/inference/basic/basic_self_forcing_causal_wan2_2_t2v.py

@@ -14,7 +14,7 @@ def main():
         "rand0nmr/SFWan2.2-T2V-A14B-Diffusers",
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=True, # DiT need to be offloaded for MoE
         vae_cpu_offload=False,
         text_encoder_cpu_offload=True,

examples/inference/basic/basic_turbodiffusion.py

@@ -15,6 +15,7 @@ def main() -> None:
         "loayrashid/TurboWan2.1-T2V-1.3B-Diffusers",
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
 
         # set to false if using RTX 4090 
         # pin_cpu_memory=False,

examples/inference/basic/basic_wan2_2.py

@@ -12,7 +12,7 @@ def main():
         "Wan-AI/Wan2.2-T2V-A14B-Diffusers",
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory 
         dit_cpu_offload=True, # DiT need to be offloaded for MoE
         vae_cpu_offload=False,
         text_encoder_cpu_offload=True,

examples/inference/basic/basic_wan2_2_Fun.py

@@ -14,7 +14,7 @@ def main():
         # "alibaba-pai/Wan2.2-Fun-A14B-Control",
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=True, # DiT need to be offloaded for MoE
         vae_cpu_offload=False,
         text_encoder_cpu_offload=True,

examples/inference/basic/basic_wan2_2_i2v.py

@@ -12,7 +12,7 @@ def main():
         "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=True, # DiT need to be offloaded for MoE
         vae_cpu_offload=False,
         text_encoder_cpu_offload=True,

examples/inference/basic/basic_wan2_2_ti2v.py

@@ -11,7 +11,7 @@ def main():
         model_name,
         # FastVideo will automatically handle distributed setup
         num_gpus=1,
-        use_fsdp_inference=True,
+        use_fsdp_inference=False, # set to True if GPU is out of memory
         dit_cpu_offload=True,
         vae_cpu_offload=False,
         text_encoder_cpu_offload=True,

fastvideo/fastvideo_args.py

@@ -132,7 +132,7 @@ class FastVideoArgs:
 
     # CPU offload parameters
     dit_cpu_offload: bool = True
-    use_fsdp_inference: bool = True
+    use_fsdp_inference: bool = False
     dit_layerwise_offload: bool = False
     text_encoder_cpu_offload: bool = True
     image_encoder_cpu_offload: bool = True
@@ -431,7 +431,9 @@ class FastVideoArgs:
             "--use-fsdp-inference",
             action=StoreBoolean,
             help=
-            "Use FSDP for inference by sharding the model weights. Latency is very low due to prefetch--enable if run out of memory.",
+            "Use FSDP for inference by sharding the model weights. FSDP helps reduce GPU memory usage but may introduce"
+            +
+            " weight transfer overhead depending on the specific setup. Enable if run out of memory.",
         )
         parser.add_argument(
             "--text-encoder-cpu-offload",

fastvideo/tests/ssim/test_turbodiffusion_similarity.py

@@ -161,7 +161,7 @@ def test_turbodiffusion_inference_similarity(prompt, model_id):
 
 # TurboDiffusion I2V parameters (dual-model with RCM scheduler + SLA attention)
 TURBODIFFUSION_I2V_PARAMS = {
-    "num_gpus": 2,
+    "num_gpus": 4,
     "model_path": "loayrashid/TurboWan2.2-I2V-A14B-Diffusers",
     "height": 480,
     "width": 832,
@@ -169,7 +169,7 @@ TURBODIFFUSION_I2V_PARAMS = {
     "num_inference_steps": 4,  # TurboDiffusion uses 1-4 steps
     "guidance_scale": 1.0,  # No CFG for TurboDiffusion
     "seed": 42,
-    "sp_size": 2,
+    "sp_size": 4,
     "tp_size": 1,
     "fps": 24,
 }
@@ -218,6 +218,8 @@ def test_turbodiffusion_i2v_inference_similarity(prompt, model_id):
         "sp_size": BASE_PARAMS["sp_size"],
         "tp_size": BASE_PARAMS["tp_size"],
         "override_pipeline_cls_name": "TurboDiffusionI2VPipeline",
+        # Keep both transformers in VRAM - avoids CPU RAM bottleneck
+        "dit_cpu_offload": False,
     }
 
     generation_kwargs = {

pyproject.toml

@@ -111,7 +111,7 @@ lint = [
 ]
 
 test = [
-    "av==14.3.0",
+    "av",
     "pytorch-msssim==1.0.0",
     "pytest",
 ]

pyproject_other.toml

@@ -90,7 +90,7 @@ lint = [
 ]
 
 test = [
-    "av==14.3.0",
+    "av",
     "pytorch-msssim==1.0.0",
     "pytest",
 ]