nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
JerryZhou54
2025-06-28 00:20:31 +00:00
parent ab01dc4ba5
commit 8675800db8
5 changed files with 87 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastvideo/v1/configs/models/dits/wanvideo.py
+67
View File
@@ -13,6 +13,73 @@ def is_blocks(n: str, m) -> bool:
class WanVideoArchConfig(DiTArchConfig):
_fsdp_shard_conditions: list = field(default_factory=lambda: [is_blocks])
_videox_param_names_mapping: dict = field(
default_factory=lambda: {
r"^patch_embedding\.(.*)$":
r"patch_embedding.proj.\1",
r"^text_embedding\.0\.(.*)$":
r"condition_embedder.text_embedder.fc_in.\1",
r"^text_embedding\.2\.(.*)$":
r"condition_embedder.text_embedder.fc_out.\1",
r"^time_embedding\.0\.(.*)$":
r"condition_embedder.time_embedder.mlp.fc_in.\1",
r"^time_embedding\.2\.(.*)$":
r"condition_embedder.time_embedder.mlp.fc_out.\1",
r"^time_projection\.1\.(.*)$":
r"condition_embedder.time_modulation.linear.\1",
r"^img_emb\.proj\.0\.(.*)$":
r"condition_embedder.image_embedder.norm1.\1",
r"^img_emb\.proj\.1\.(.*)$":
r"condition_embedder.image_embedder.ff.fc_in.\1",
r"^img_emb\.proj\.3\.(.*)$":
r"condition_embedder.image_embedder.ff.fc_out.\1",
r"^img_emb\.proj\.4\.(.*)$":
r"condition_embedder.image_embedder.norm2.\1",
r"^head\.modulation":
r"scale_shift_table",
r"^head\.head\.(.*)$":
r"proj_out.\1",
r"^blocks\.(\d+)\.self_attn\.q\.(.*)$":
r"blocks.\1.to_q.\2",
r"^blocks\.(\d+)\.self_attn\.k\.(.*)$":
r"blocks.\1.to_k.\2",
r"^blocks\.(\d+)\.self_attn\.v\.(.*)$":
r"blocks.\1.to_v.\2",
r"^blocks\.(\d+)\.self_attn\.o\.(.*)$":
r"blocks.\1.to_out.\2",
r"^blocks\.(\d+)\.self_attn\.norm_q\.(.*)$":
r"blocks.\1.norm_q.\2",
r"^blocks\.(\d+)\.self_attn\.norm_k\.(.*)$":
r"blocks.\1.norm_k.\2",
r"^blocks\.(\d+)\.cross_attn\.q\.(.*)$":
r"blocks.\1.attn2.to_q.\2",
r"^blocks\.(\d+)\.cross_attn\.k\.(.*)$":
r"blocks.\1.attn2.to_k.\2",
r"^blocks\.(\d+)\.cross_attn\.k_img\.(.*)$":
r"blocks.\1.attn2.add_k_proj.\2",
r"^blocks\.(\d+)\.cross_attn\.v\.(.*)$":
r"blocks.\1.attn2.to_v.\2",
r"^blocks\.(\d+)\.cross_attn\.v_img\.(.*)$":
r"blocks.\1.attn2.add_v_proj.\2",
r"^blocks\.(\d+)\.cross_attn\.o\.(.*)$":
r"blocks.\1.attn2.to_out.\2",
r"^blocks\.(\d+)\.cross_attn\.norm_q\.(.*)$":
r"blocks.\1.attn2.norm_q.\2",
r"^blocks\.(\d+)\.cross_attn\.norm_k\.(.*)$":
r"blocks.\1.attn2.norm_k.\2",
r"^blocks\.(\d+)\.cross_attn\.norm_k_img\.(.*)$":
r"blocks.\1.attn2.norm_added_k.\2",
r"^blocks\.(\d+)\.ffn\.0\.(.*)$":
r"blocks.\1.ffn.fc_in.\2",
r"^blocks\.(\d+)\.ffn\.2\.(.*)$":
r"blocks.\1.ffn.fc_out.\2",
r"^blocks\.(\d+)\.modulation":
r"blocks.\1.scale_shift_table",
r"^blocks\.(\d+)\.norm3\.(.*)$":
r"blocks.\1.self_attn_residual_norm.norm.\2",
}
)
_param_names_mapping: dict = field(
default_factory=lambda: {
r"^patch_embedding\.(.*)$":
fastvideo/v1/models/dits/wanvideo.py
+2 -1
View File
@@ -183,7 +183,7 @@ class WanI2VCrossAttention(WanSelfAttention):
self.add_k_proj = ReplicatedLinear(dim, dim)
self.add_v_proj = ReplicatedLinear(dim, dim)
self.norm_added_k = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity()
self.norm_added_q = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity()
# self.norm_added_q = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity()
def forward(self, x, context, context_lens):
r"""
@@ -518,6 +518,7 @@ class WanTransformer3DModel(CachableDiT):
_supported_attention_backends = WanVideoConfig(
)._supported_attention_backends
_param_names_mapping = WanVideoConfig()._param_names_mapping
_videox_param_names_mapping = WanVideoConfig()._videox_param_names_mapping
_reverse_param_names_mapping = WanVideoConfig()._reverse_param_names_mapping
_lora_param_names_mapping = WanVideoConfig()._lora_param_names_mapping
fastvideo/v1/models/loader/component_loader.py
+12 -1
View File
@@ -378,11 +378,20 @@ class TransformerLoader(ComponentLoader):
# Config from Diffusers supersedes fastvideo's model config
dit_config = fastvideo_args.pipeline_config.dit_config
dit_config.update_model_arch(config)
# dit_config.update_model_arch(config)
dit_config.update_model_arch({
"num_attention_heads": 12,
"ffn_dim": 8960,
"in_channels": 36,
"num_layers": 30,
"image_dim": 1280,
"added_kv_proj_dim": 0
})
model_cls, _ = ModelRegistry.resolve_model_cls(cls_name)
# Find all safetensors files
model_path="/mnt/weka/home/hao.zhang/wei/FastVideo"
safetensors_list = glob.glob(
os.path.join(str(model_path), "*.safetensors"))
if not safetensors_list:
@@ -458,6 +467,8 @@ class SchedulerLoader(ComponentLoader):
assert class_name is not None, "Model config does not contain a _class_name attribute. Only diffusers format is supported."
scheduler_cls, _ = ModelRegistry.resolve_model_cls(class_name)
from fastvideo.v1.models.schedulers.scheduling_flow_match_euler_discrete import FlowMatchDiscreteScheduler
scheduler_cls = FlowMatchDiscreteScheduler
scheduler = scheduler_cls(**config)
if fastvideo_args.pipeline_config.flow_shift is not None:
fastvideo/v1/models/loader/fsdp_load.py
+1 -1
View File
@@ -104,7 +104,7 @@ def maybe_load_fsdp_model(
mesh=device_mesh)
weight_iterator = safetensors_weights_iterator(weight_dir_list)
param_names_mapping_fn = get_param_names_mapping(model._param_names_mapping)
param_names_mapping_fn = get_param_names_mapping(model._videox_param_names_mapping)
load_model_from_full_model_state_dict(
model,
weight_iterator,
scripts/inference/v1_inference_wan_i2v.sh
+5 -5
View File
@@ -1,5 +1,5 @@
#!/bin/bash
export HOME=/mnt/weka/home/hao.zhang/wei
num_gpus=2
export FASTVIDEO_ATTENTION_BACKEND=
export MODEL_BASE=Wan-AI/Wan2.1-I2V-14B-480P-Diffusers
@@ -14,13 +14,13 @@ fastvideo generate \
--num-gpus $num_gpus \
--height 480 \
--width 832 \
--num-frames 77 \
--num-frames 29 \
--num-inference-steps 40 \
--fps 16 \
--flow-shift 3.0 \
--guidance-scale 5.0 \
--image-path "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg" \
--prompt "An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space realised in the background. High quality, ultrarealistic detail and breath-taking movie-like camera shot." \
--image-path "assets/dog.png" \
--prompt "A brown dog is shaking its head while sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on the shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." \
--negative-prompt "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" \
--seed 1024 \
--output-path outputs_i2v/
--output-path outputs_videox_i2v/