diff --git a/fastvideo/tests/inference/vmoba/test_vmoba_inference.py b/fastvideo/tests/inference/vmoba/test_vmoba_inference.py index 0cf06f35e..0b0b80021 100644 --- a/fastvideo/tests/inference/vmoba/test_vmoba_inference.py +++ b/fastvideo/tests/inference/vmoba/test_vmoba_inference.py @@ -27,7 +27,7 @@ def test_inference_vmoba(): "--height", "480", "--width", "832", "--num-frames", "77", - "--num-inference-steps", "50", + "--num-inference-steps", "10", "--moba-config-path", moba_config, "--fps", "16", "--guidance-scale", "6.0", diff --git a/fastvideo/tests/ssim/L40S_reference_videos/FastHunyuan-diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 b/fastvideo/tests/ssim/L40S_reference_videos/FastHunyuan-diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 index 5cfa926b0..378db8036 100644 Binary files a/fastvideo/tests/ssim/L40S_reference_videos/FastHunyuan-diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 and b/fastvideo/tests/ssim/L40S_reference_videos/FastHunyuan-diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 differ diff --git a/fastvideo/tests/ssim/L40S_reference_videos/FastHunyuan-diffusers/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 b/fastvideo/tests/ssim/L40S_reference_videos/FastHunyuan-diffusers/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 index 797ae1d61..cae4ff6a6 100644 Binary files a/fastvideo/tests/ssim/L40S_reference_videos/FastHunyuan-diffusers/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 and b/fastvideo/tests/ssim/L40S_reference_videos/FastHunyuan-diffusers/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 differ diff --git a/fastvideo/tests/ssim/L40S_reference_videos/SFWan2.1-T2V-1.3B-Diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 b/fastvideo/tests/ssim/L40S_reference_videos/SFWan2.1-T2V-1.3B-Diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 index 3f60bf983..f650d4862 100644 Binary files a/fastvideo/tests/ssim/L40S_reference_videos/SFWan2.1-T2V-1.3B-Diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 and b/fastvideo/tests/ssim/L40S_reference_videos/SFWan2.1-T2V-1.3B-Diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 differ diff --git a/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-I2V-14B-480P-Diffusers/FLASH_ATTN/An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space reali.mp4 b/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-I2V-14B-480P-Diffusers/FLASH_ATTN/An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space reali.mp4 index 49dac8f7f..caf004b66 100644 Binary files a/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-I2V-14B-480P-Diffusers/FLASH_ATTN/An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space reali.mp4 and b/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-I2V-14B-480P-Diffusers/FLASH_ATTN/An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space reali.mp4 differ diff --git a/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-T2V-1.3B-Diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 b/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-T2V-1.3B-Diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 index a454aee9e..64e827f89 100644 Binary files a/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-T2V-1.3B-Diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 and b/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-T2V-1.3B-Diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 differ diff --git a/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-T2V-1.3B-Diffusers/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 b/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-T2V-1.3B-Diffusers/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 index b0da7496d..4cf3146cd 100644 Binary files a/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-T2V-1.3B-Diffusers/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 and b/fastvideo/tests/ssim/L40S_reference_videos/Wan2.1-T2V-1.3B-Diffusers/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of.mp4 differ diff --git a/fastvideo/tests/training/VSA/l40s_reference_wandb_summary_VSA.json b/fastvideo/tests/training/VSA/h100_reference_wandb_summary_VSA.json similarity index 73% rename from fastvideo/tests/training/VSA/l40s_reference_wandb_summary_VSA.json rename to fastvideo/tests/training/VSA/h100_reference_wandb_summary_VSA.json index ac67d779b..97e793806 100644 --- a/fastvideo/tests/training/VSA/l40s_reference_wandb_summary_VSA.json +++ b/fastvideo/tests/training/VSA/h100_reference_wandb_summary_VSA.json @@ -1,10 +1,10 @@ { "step_time": 0.6983645600266755, - "grad_norm": 0.245593056678772, + "grad_norm": 1.278342604637146, "avg_step_time": 1.002151239803061, "_timestamp": 1751181952.70901, "vsa_sparsity": 0.05, "learning_rate": 1e-05, - "train_loss": 0.2530866410434246, + "train_loss": 0.3085433095693588, "_runtime": 107.325113071 } diff --git a/fastvideo/tests/training/VSA/test_training_loss_VSA.py b/fastvideo/tests/training/VSA/test_training_loss_VSA.py index e0abe0907..0a3e6743d 100644 --- a/fastvideo/tests/training/VSA/test_training_loss_VSA.py +++ b/fastvideo/tests/training/VSA/test_training_loss_VSA.py @@ -13,7 +13,7 @@ from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs from fastvideo.utils import FlexibleArgumentParser wandb_name = "test_training_loss_VSA" -l40s_reference_wandb_summary_file = "fastvideo/tests/training/VSA/reference_wandb_summary_VSA.json" +h100_reference_wandb_summary_file = "fastvideo/tests/training/VSA/h100_reference_wandb_summary_VSA.json" h200_reference_wandb_summary_file = "fastvideo/tests/training/VSA/h200_reference_wandb_summary_VSA.json" NUM_NODES = "1" @@ -107,8 +107,8 @@ def test_distributed_training(): summary_file = 'data/wan_finetune_test_VSA/tracker/wandb/latest-run/files/wandb-summary.json' device_name = torch.cuda.get_device_name() - if "L40S" in device_name: - reference_wandb_summary_file = l40s_reference_wandb_summary_file + if "H100" in device_name: + reference_wandb_summary_file = h100_reference_wandb_summary_file elif "H200" in device_name: reference_wandb_summary_file = h200_reference_wandb_summary_file else: @@ -121,7 +121,7 @@ def test_distributed_training(): 'avg_step_time': 1.0, 'grad_norm': 0.1, 'step_time': 1.0, - 'train_loss': 0.005 + 'train_loss': 0.02 } failures = [] diff --git a/fastvideo/tests/training/Vanilla/l40s_reference_wandb_summary.json b/fastvideo/tests/training/Vanilla/l40s_reference_wandb_summary.json index e9b3fe87d..e55facb6c 100644 --- a/fastvideo/tests/training/Vanilla/l40s_reference_wandb_summary.json +++ b/fastvideo/tests/training/Vanilla/l40s_reference_wandb_summary.json @@ -1,5 +1,5 @@ { - "train_loss": 0.1021774671971798, + "train_loss": 0.13823603838682175, "_step": 5, "step_time": 2.2575189135968685, "grad_norm": 0.11582941561937332, diff --git a/fastvideo/tests/transformers/test_hunyuanvideo.py b/fastvideo/tests/transformers/test_hunyuanvideo.py index def599a52..8fda5c49b 100644 --- a/fastvideo/tests/transformers/test_hunyuanvideo.py +++ b/fastvideo/tests/transformers/test_hunyuanvideo.py @@ -1,108 +1,84 @@ # SPDX-License-Identifier: Apache-2.0 +import glob +import json import os -from itertools import chain - import pytest import torch -import torch.nn as nn +from fastvideo.configs.pipelines.base import PipelineConfig from fastvideo.distributed.parallel_state import ( get_sp_parallel_rank, get_sp_world_size) from fastvideo.logger import init_logger -from fastvideo.models.dits.hunyuanvideo import ( - HunyuanVideoTransformer3DModel as HunyuanVideoDit) -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch - -from fastvideo.models.loader.fsdp_load import shard_model +from fastvideo.models.loader.component_loader import TransformerLoader +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.utils import maybe_download_model from fastvideo.forward_context import set_forward_context - +from fastvideo.configs.models.dits import HunyuanVideoConfig +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch logger = init_logger(__name__) os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = "29503" +BASE_MODEL_PATH = "hunyuanvideo-community/HunyuanVideo" +MODEL_PATH = maybe_download_model(BASE_MODEL_PATH, + local_dir=os.path.join("data", BASE_MODEL_PATH) # store in the large /workspace disk on Runpod + ) +TRANSFORMER_PATH = os.path.join(MODEL_PATH, "transformer") +CONFIG_PATH = os.path.join(TRANSFORMER_PATH, "config.json") + +LOCAL_RANK = 0 +RANK = 0 +WORLD_SIZE = 1 + # Latent generated on commit c021e8a27cf437ac22827f2bc58b7f006561317f with 1 x L40S -REFERENCE_LATENT = 1472.079828262329 +REFERENCE_LATENT = 93.23263549804688 -def initialize_identical_weights(model, seed=42): - """Initialize both models with identical weights using a fixed seed for reproducibility.""" - # Get all parameters from both models - params1 = dict(model.named_parameters()) - - # Initialize each layer with identical values - with torch.no_grad(): - # Initialize weights - for name1, param1 in params1.items(): - if 'weight' in name1: - # Set seed before each weight initialization - torch.manual_seed(seed) - nn.init.normal_(param1, mean=0.0, std=0.05) - - # Initialize biases - for name1, param1 in params1.items(): - if 'bias' in name1: - torch.manual_seed(seed) - nn.init.normal_(param1, mean=0.0, std=0.05) - param1.data = param1.data.to(torch.bfloat16) - - logger.info("Model initialized with identical weights in bfloat16") - return model - - -@pytest.mark.skip(reason="Incompatible with the new config") +@pytest.mark.usefixtures("distributed_setup") def test_hunyuanvideo_distributed(): + logger.info( + f"Initializing process: rank={RANK}, local_rank={LOCAL_RANK}, world_size={WORLD_SIZE}" + ) + + torch.cuda.set_device(f"cuda:{LOCAL_RANK}") + # Get tensor parallel info sp_rank = get_sp_parallel_rank() sp_world_size = get_sp_world_size() - # Small model parameters for testing - hidden_size = 128 - heads_num = 4 - mm_double_blocks_depth = 2 - mm_single_blocks_depth = 2 - torch.cuda.set_device("cuda:0") - # Initialize the two model implementations - model = HunyuanVideoDit( - patch_size=2, - patch_size_t=1, - in_channels=4, - out_channels=4, - attention_head_dim=hidden_size // heads_num, - num_attention_heads=heads_num, - num_layers=mm_double_blocks_depth, - num_single_layers=mm_single_blocks_depth, - rope_axes_dim=[8, 16, 8], # sum = hidden_size // heads_num = 32 - dtype=torch.bfloat16).to(torch.bfloat16) + logger.info( + f"Process rank {RANK} initialized with SP rank {sp_rank} in SP world size {sp_world_size}" + ) - # Initialize with identical weights - model = initialize_identical_weights(model, seed=42) - shard_model(model, cpu_offload=True, - reshard_after_forward=True, - fsdp_shard_conditions=model._fsdp_shard_conditions - ) - for n, p in chain(model.named_parameters(), model.named_buffers()): - if p.is_meta: - raise RuntimeError( - f"Unexpected param or buffer {n} on meta device.") - for p in model.parameters(): - p.requires_grad = False + config = json.load(open(CONFIG_PATH)) + # remove "_class_name": "HunyuanVideoTransformer3DModel", "_diffusers_version": "0.32.0.dev0", + # TODO: write normalize config function + config.pop("_class_name") + config.pop("_diffusers_version") + + precision_str = "bf16" + args = FastVideoArgs(model_path=TRANSFORMER_PATH, + dit_cpu_offload=True, + pipeline_config=PipelineConfig(dit_config=HunyuanVideoConfig(), dit_precision=precision_str)) + args.device = torch.device(f"cuda:{LOCAL_RANK}") + + loader = TransformerLoader() + model = loader.load(TRANSFORMER_PATH, args) model.eval() - # Move to GPU based on local rank (0 or 1 for 2 GPUs) - device = torch.device(f"cuda:0") - model = model - + # Create random inputs for testing batch_size = 1 seq_len = 3 + device = torch.device(f"cuda:{LOCAL_RANK}") # Video latents [B, C, T, H, W] hidden_states = torch.randn(batch_size, - 4, + 16, 8, 16, 16, @@ -121,19 +97,20 @@ def test_hunyuanvideo_distributed(): # Timestep timestep = torch.tensor([500], device=device, dtype=torch.bfloat16) - forward_batch = ForwardBatch( data_type="dummy", ) # Disable gradients for inference with torch.no_grad(): - with set_forward_context(current_timestep=0, attn_metadata=None, forward_batch=forward_batch): - output = model( - hidden_states=hidden_states, - encoder_hidden_states=encoder_hidden_states, - timestep=timestep, - ) + # Run inference on model + with torch.amp.autocast(device_type="cuda", dtype=torch.bfloat16): + with set_forward_context(current_timestep=0, attn_metadata=None, forward_batch=forward_batch): + output = model( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + ) latent = output.double().sum().item() diff --git a/fastvideo/tests/transformers/test_hunyuanvideo_load.py b/fastvideo/tests/transformers/test_hunyuanvideo_load.py deleted file mode 100644 index b223dcc52..000000000 --- a/fastvideo/tests/transformers/test_hunyuanvideo_load.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -import glob -import json -import os -import pytest -import torch - -from fastvideo.configs.pipelines.base import PipelineConfig -from fastvideo.distributed.parallel_state import ( - get_sp_parallel_rank, - get_sp_world_size) -from fastvideo.logger import init_logger -from fastvideo.models.loader.component_loader import TransformerLoader -from fastvideo.fastvideo_args import FastVideoArgs -from fastvideo.utils import maybe_download_model -from fastvideo.forward_context import set_forward_context -from fastvideo.configs.models.dits import HunyuanVideoConfig -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch - -logger = init_logger(__name__) - -os.environ["MASTER_ADDR"] = "localhost" -os.environ["MASTER_PORT"] = "29503" - -BASE_MODEL_PATH = "hunyuanvideo-community/HunyuanVideo" -MODEL_PATH = maybe_download_model(BASE_MODEL_PATH, - local_dir=os.path.join("data", BASE_MODEL_PATH) # store in the large /workspace disk on Runpod - ) -TRANSFORMER_PATH = os.path.join(MODEL_PATH, "transformer") -CONFIG_PATH = os.path.join(TRANSFORMER_PATH, "config.json") - -LOCAL_RANK = 0 -RANK = 0 -WORLD_SIZE = 1 - -# Latent generated on commit c021e8a27cf437ac22827f2bc58b7f006561317f with 1 x L40S -REFERENCE_LATENT = 89.7002067565918 - - -@pytest.mark.usefixtures("distributed_setup") -def test_hunyuanvideo_distributed(): - logger.info( - f"Initializing process: rank={RANK}, local_rank={LOCAL_RANK}, world_size={WORLD_SIZE}" - ) - - torch.cuda.set_device(f"cuda:{LOCAL_RANK}") - - # Get tensor parallel info - sp_rank = get_sp_parallel_rank() - sp_world_size = get_sp_world_size() - - logger.info( - f"Process rank {RANK} initialized with SP rank {sp_rank} in SP world size {sp_world_size}" - ) - - config = json.load(open(CONFIG_PATH)) - # remove "_class_name": "HunyuanVideoTransformer3DModel", "_diffusers_version": "0.32.0.dev0", - # TODO: write normalize config function - config.pop("_class_name") - config.pop("_diffusers_version") - - precision_str = "bf16" - args = FastVideoArgs(model_path=TRANSFORMER_PATH, - dit_cpu_offload=True, - pipeline_config=PipelineConfig(dit_config=HunyuanVideoConfig(), dit_precision=precision_str)) - args.device = torch.device(f"cuda:{LOCAL_RANK}") - - loader = TransformerLoader() - model = loader.load(TRANSFORMER_PATH, args) - - model.eval() - - # Create random inputs for testing - batch_size = 1 - seq_len = 3 - device = torch.device(f"cuda:{LOCAL_RANK}") - - # Video latents [B, C, T, H, W] - hidden_states = torch.randn(batch_size, - 16, - 8, - 16, - 16, - device=device, - dtype=torch.bfloat16) - chunk_per_rank = hidden_states.shape[2] // sp_world_size - hidden_states = hidden_states[:, :, sp_rank * chunk_per_rank:(sp_rank + 1) * - chunk_per_rank] - - # Text embeddings [B, L, D] (including global token) - encoder_hidden_states = torch.randn(batch_size, - seq_len + 1, - 4096, - device=device, - dtype=torch.bfloat16) - - # Timestep - timestep = torch.tensor([500], device=device, dtype=torch.bfloat16) - forward_batch = ForwardBatch( - data_type="dummy", - ) - - # Disable gradients for inference - with torch.no_grad(): - # Run inference on model - with torch.amp.autocast(device_type="cuda", dtype=torch.bfloat16): - with set_forward_context(current_timestep=0, attn_metadata=None, forward_batch=forward_batch): - output = model( - hidden_states=hidden_states, - encoder_hidden_states=encoder_hidden_states, - timestep=timestep, - ) - - latent = output.double().sum().item() - - # Check if latents are similar - diff_output_latents = abs(REFERENCE_LATENT - latent) - logger.info( - f"Reference latent: {REFERENCE_LATENT}, Current latent: {latent}" - ) - assert diff_output_latents < 1e-4, f"Output latents differ significantly: max diff = {diff_output_latents}" - - diff --git a/fastvideo/tests/vaes/test_hunyuan_vae.py b/fastvideo/tests/vaes/test_hunyuan_vae.py index f37b78f83..ed628d1fc 100644 --- a/fastvideo/tests/vaes/test_hunyuan_vae.py +++ b/fastvideo/tests/vaes/test_hunyuan_vae.py @@ -33,7 +33,7 @@ CONFIG_PATH = os.path.join(VAE_PATH, "config.json") A40_REFERENCE_LATENT = -106.22467041015625 # Latent generated on commit 2b54068960c41d42221e8b8719a374b499855029 with 1 x L40S -L40S_REFERENCE_LATENT = -158.32318115234375 +L40S_REFERENCE_LATENT = -157.09130859375 @pytest.mark.usefixtures("distributed_setup") diff --git a/fastvideo/training/self_forcing_distillation_pipeline.py b/fastvideo/training/self_forcing_distillation_pipeline.py index e23b97a8c..fc68e64ce 100644 --- a/fastvideo/training/self_forcing_distillation_pipeline.py +++ b/fastvideo/training/self_forcing_distillation_pipeline.py @@ -310,14 +310,8 @@ class SelfForcingDistillationPipeline(DistillationPipeline): if self.boundary_timestep is not None and current_timestep < self.boundary_timestep and self.transformer_2 is not None: current_model = self.transformer_2 - self._enable_training(self.transformer_2, self.optimizer_2) - self._disable_training(self.transformer, self.optimizer) else: current_model = self.transformer - self._enable_training(self.transformer, self.optimizer) - if self.boundary_timestep is not None and self.transformer_2 is not None: - self._disable_training(self.transformer_2, - self.optimizer_2) if not exit_flag: with torch.no_grad(): diff --git a/pyproject.toml b/pyproject.toml index 371a441f0..314f174f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "requests>=2.32.2", # Machine Learning & Transformers - "transformers>=4.46.1", + "transformers==4.57.3", "tokenizers>=0.20.1", "sentencepiece==0.2.0", "timm==1.0.11",