From 8675800db8a002621439ff2addedb16639ff0a32 Mon Sep 17 00:00:00 2001 From: JerryZhou54 Date: Sat, 28 Jun 2025 00:20:31 +0000 Subject: [PATCH] Update --- fastvideo/v1/configs/models/dits/wanvideo.py | 67 +++++++++++++++++++ fastvideo/v1/models/dits/wanvideo.py | 3 +- .../v1/models/loader/component_loader.py | 13 +++- fastvideo/v1/models/loader/fsdp_load.py | 2 +- scripts/inference/v1_inference_wan_i2v.sh | 10 +-- 5 files changed, 87 insertions(+), 8 deletions(-) diff --git a/fastvideo/v1/configs/models/dits/wanvideo.py b/fastvideo/v1/configs/models/dits/wanvideo.py index ea1048c1b..a32c34441 100644 --- a/fastvideo/v1/configs/models/dits/wanvideo.py +++ b/fastvideo/v1/configs/models/dits/wanvideo.py @@ -13,6 +13,73 @@ def is_blocks(n: str, m) -> bool: class WanVideoArchConfig(DiTArchConfig): _fsdp_shard_conditions: list = field(default_factory=lambda: [is_blocks]) + _videox_param_names_mapping: dict = field( + default_factory=lambda: { + r"^patch_embedding\.(.*)$": + r"patch_embedding.proj.\1", + r"^text_embedding\.0\.(.*)$": + r"condition_embedder.text_embedder.fc_in.\1", + r"^text_embedding\.2\.(.*)$": + r"condition_embedder.text_embedder.fc_out.\1", + r"^time_embedding\.0\.(.*)$": + r"condition_embedder.time_embedder.mlp.fc_in.\1", + r"^time_embedding\.2\.(.*)$": + r"condition_embedder.time_embedder.mlp.fc_out.\1", + r"^time_projection\.1\.(.*)$": + r"condition_embedder.time_modulation.linear.\1", + r"^img_emb\.proj\.0\.(.*)$": + r"condition_embedder.image_embedder.norm1.\1", + r"^img_emb\.proj\.1\.(.*)$": + r"condition_embedder.image_embedder.ff.fc_in.\1", + r"^img_emb\.proj\.3\.(.*)$": + r"condition_embedder.image_embedder.ff.fc_out.\1", + r"^img_emb\.proj\.4\.(.*)$": + r"condition_embedder.image_embedder.norm2.\1", + r"^head\.modulation": + r"scale_shift_table", + r"^head\.head\.(.*)$": + r"proj_out.\1", + r"^blocks\.(\d+)\.self_attn\.q\.(.*)$": + r"blocks.\1.to_q.\2", + r"^blocks\.(\d+)\.self_attn\.k\.(.*)$": + r"blocks.\1.to_k.\2", + r"^blocks\.(\d+)\.self_attn\.v\.(.*)$": + r"blocks.\1.to_v.\2", + r"^blocks\.(\d+)\.self_attn\.o\.(.*)$": + r"blocks.\1.to_out.\2", + r"^blocks\.(\d+)\.self_attn\.norm_q\.(.*)$": + r"blocks.\1.norm_q.\2", + r"^blocks\.(\d+)\.self_attn\.norm_k\.(.*)$": + r"blocks.\1.norm_k.\2", + r"^blocks\.(\d+)\.cross_attn\.q\.(.*)$": + r"blocks.\1.attn2.to_q.\2", + r"^blocks\.(\d+)\.cross_attn\.k\.(.*)$": + r"blocks.\1.attn2.to_k.\2", + r"^blocks\.(\d+)\.cross_attn\.k_img\.(.*)$": + r"blocks.\1.attn2.add_k_proj.\2", + r"^blocks\.(\d+)\.cross_attn\.v\.(.*)$": + r"blocks.\1.attn2.to_v.\2", + r"^blocks\.(\d+)\.cross_attn\.v_img\.(.*)$": + r"blocks.\1.attn2.add_v_proj.\2", + r"^blocks\.(\d+)\.cross_attn\.o\.(.*)$": + r"blocks.\1.attn2.to_out.\2", + r"^blocks\.(\d+)\.cross_attn\.norm_q\.(.*)$": + r"blocks.\1.attn2.norm_q.\2", + r"^blocks\.(\d+)\.cross_attn\.norm_k\.(.*)$": + r"blocks.\1.attn2.norm_k.\2", + r"^blocks\.(\d+)\.cross_attn\.norm_k_img\.(.*)$": + r"blocks.\1.attn2.norm_added_k.\2", + r"^blocks\.(\d+)\.ffn\.0\.(.*)$": + r"blocks.\1.ffn.fc_in.\2", + r"^blocks\.(\d+)\.ffn\.2\.(.*)$": + r"blocks.\1.ffn.fc_out.\2", + r"^blocks\.(\d+)\.modulation": + r"blocks.\1.scale_shift_table", + r"^blocks\.(\d+)\.norm3\.(.*)$": + r"blocks.\1.self_attn_residual_norm.norm.\2", + } + ) + _param_names_mapping: dict = field( default_factory=lambda: { r"^patch_embedding\.(.*)$": diff --git a/fastvideo/v1/models/dits/wanvideo.py b/fastvideo/v1/models/dits/wanvideo.py index 62d542c18..0cdbaddb9 100644 --- a/fastvideo/v1/models/dits/wanvideo.py +++ b/fastvideo/v1/models/dits/wanvideo.py @@ -183,7 +183,7 @@ class WanI2VCrossAttention(WanSelfAttention): self.add_k_proj = ReplicatedLinear(dim, dim) self.add_v_proj = ReplicatedLinear(dim, dim) self.norm_added_k = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() - self.norm_added_q = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + # self.norm_added_q = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() def forward(self, x, context, context_lens): r""" @@ -518,6 +518,7 @@ class WanTransformer3DModel(CachableDiT): _supported_attention_backends = WanVideoConfig( )._supported_attention_backends _param_names_mapping = WanVideoConfig()._param_names_mapping + _videox_param_names_mapping = WanVideoConfig()._videox_param_names_mapping _reverse_param_names_mapping = WanVideoConfig()._reverse_param_names_mapping _lora_param_names_mapping = WanVideoConfig()._lora_param_names_mapping diff --git a/fastvideo/v1/models/loader/component_loader.py b/fastvideo/v1/models/loader/component_loader.py index 270bc8387..786cee71e 100644 --- a/fastvideo/v1/models/loader/component_loader.py +++ b/fastvideo/v1/models/loader/component_loader.py @@ -378,11 +378,20 @@ class TransformerLoader(ComponentLoader): # Config from Diffusers supersedes fastvideo's model config dit_config = fastvideo_args.pipeline_config.dit_config - dit_config.update_model_arch(config) + # dit_config.update_model_arch(config) + dit_config.update_model_arch({ + "num_attention_heads": 12, + "ffn_dim": 8960, + "in_channels": 36, + "num_layers": 30, + "image_dim": 1280, + "added_kv_proj_dim": 0 + }) model_cls, _ = ModelRegistry.resolve_model_cls(cls_name) # Find all safetensors files + model_path="/mnt/weka/home/hao.zhang/wei/FastVideo" safetensors_list = glob.glob( os.path.join(str(model_path), "*.safetensors")) if not safetensors_list: @@ -458,6 +467,8 @@ class SchedulerLoader(ComponentLoader): assert class_name is not None, "Model config does not contain a _class_name attribute. Only diffusers format is supported." scheduler_cls, _ = ModelRegistry.resolve_model_cls(class_name) + from fastvideo.v1.models.schedulers.scheduling_flow_match_euler_discrete import FlowMatchDiscreteScheduler + scheduler_cls = FlowMatchDiscreteScheduler scheduler = scheduler_cls(**config) if fastvideo_args.pipeline_config.flow_shift is not None: diff --git a/fastvideo/v1/models/loader/fsdp_load.py b/fastvideo/v1/models/loader/fsdp_load.py index a9c890f69..0d811bbb8 100644 --- a/fastvideo/v1/models/loader/fsdp_load.py +++ b/fastvideo/v1/models/loader/fsdp_load.py @@ -104,7 +104,7 @@ def maybe_load_fsdp_model( mesh=device_mesh) weight_iterator = safetensors_weights_iterator(weight_dir_list) - param_names_mapping_fn = get_param_names_mapping(model._param_names_mapping) + param_names_mapping_fn = get_param_names_mapping(model._videox_param_names_mapping) load_model_from_full_model_state_dict( model, weight_iterator, diff --git a/scripts/inference/v1_inference_wan_i2v.sh b/scripts/inference/v1_inference_wan_i2v.sh index 3601ab1fe..4ed1b433a 100644 --- a/scripts/inference/v1_inference_wan_i2v.sh +++ b/scripts/inference/v1_inference_wan_i2v.sh @@ -1,5 +1,5 @@ #!/bin/bash - +export HOME=/mnt/weka/home/hao.zhang/wei num_gpus=2 export FASTVIDEO_ATTENTION_BACKEND= export MODEL_BASE=Wan-AI/Wan2.1-I2V-14B-480P-Diffusers @@ -14,13 +14,13 @@ fastvideo generate \ --num-gpus $num_gpus \ --height 480 \ --width 832 \ - --num-frames 77 \ + --num-frames 29 \ --num-inference-steps 40 \ --fps 16 \ --flow-shift 3.0 \ --guidance-scale 5.0 \ - --image-path "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg" \ - --prompt "An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space realised in the background. High quality, ultrarealistic detail and breath-taking movie-like camera shot." \ + --image-path "assets/dog.png" \ + --prompt "A brown dog is shaking its head while sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on the shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." \ --negative-prompt "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" \ --seed 1024 \ - --output-path outputs_i2v/ \ No newline at end of file + --output-path outputs_videox_i2v/ \ No newline at end of file