diff --git a/examples/phantom/predict_s2v.py b/examples/phantom/predict_s2v.py index 0d7e8f4..28314f1 100644 --- a/examples/phantom/predict_s2v.py +++ b/examples/phantom/predict_s2v.py @@ -18,6 +18,8 @@ from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, WanT5EncoderModel, WanTransformer3DModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import WanFunPhantomPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) @@ -27,7 +29,7 @@ from videox_fun.utils.utils import (filter_kwargs, get_image_latent, from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -38,6 +40,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -217,6 +222,9 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/turbodiffusion/predict_i2v_wan2.2.py b/examples/turbodiffusion/predict_i2v_wan2.2.py index ee09f84..76cc736 100644 --- a/examples/turbodiffusion/predict_i2v_wan2.2.py +++ b/examples/turbodiffusion/predict_i2v_wan2.2.py @@ -20,12 +20,14 @@ from videox_fun.pipeline import Wan2_2I2VPipeline from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, convert_weight_dtype_wrapper) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, save_videos_grid) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -36,6 +38,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -238,6 +243,10 @@ if GPU_memory_mode == "sequential_cpu_offload": transformer.freqs = transformer.freqs.to(device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(transformer) + register_auto_device_hook(transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) diff --git a/examples/turbodiffusion/predict_t2v_wan2.1.py b/examples/turbodiffusion/predict_t2v_wan2.1.py index ce28bae..74c915f 100644 --- a/examples/turbodiffusion/predict_t2v_wan2.1.py +++ b/examples/turbodiffusion/predict_t2v_wan2.1.py @@ -23,10 +23,12 @@ from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -37,6 +39,9 @@ from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -192,7 +197,11 @@ if compile_dit: if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.1/predict_i2v.py b/examples/wan2.1/predict_i2v.py index c2de018..2f56563 100755 --- a/examples/wan2.1/predict_i2v.py +++ b/examples/wan2.1/predict_i2v.py @@ -15,18 +15,21 @@ for project_root in project_roots: from videox_fun.dist import set_multi_gpus_devices, shard_model from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, - WanT5EncoderModel, WanTransformer3DModel) + WanT5EncoderModel, WanTransformer3DModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import WanI2VPipeline -from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, - convert_weight_dtype_wrapper) -from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, - save_videos_grid) +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -37,6 +40,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -218,6 +224,9 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.1/predict_t2v.py b/examples/wan2.1/predict_t2v.py index 2e96d13..c50db5f 100755 --- a/examples/wan2.1/predict_t2v.py +++ b/examples/wan2.1/predict_t2v.py @@ -13,19 +13,22 @@ for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, WanT5EncoderModel, AutoTokenizer, - WanTransformer3DModel) +from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, + WanT5EncoderModel, WanTransformer3DModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import WanPipeline -from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, - convert_weight_dtype_wrapper) -from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, - save_videos_grid) +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -36,6 +39,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -205,6 +211,9 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.1_fun/predict_i2v.py b/examples/wan2.1_fun/predict_i2v.py index dc86a68..fb36ce5 100755 --- a/examples/wan2.1_fun/predict_i2v.py +++ b/examples/wan2.1_fun/predict_i2v.py @@ -15,18 +15,21 @@ for project_root in project_roots: from videox_fun.dist import set_multi_gpus_devices, shard_model from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, - WanTransformer3DModel) + WanTransformer3DModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import WanFunInpaintPipeline -from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, - convert_weight_dtype_wrapper) -from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, - save_videos_grid) +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -37,6 +40,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -219,6 +225,9 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.1_fun/predict_t2v.py b/examples/wan2.1_fun/predict_t2v.py index bfb2759..dd1c07c 100755 --- a/examples/wan2.1_fun/predict_t2v.py +++ b/examples/wan2.1_fun/predict_t2v.py @@ -15,18 +15,21 @@ for project_root in project_roots: from videox_fun.dist import set_multi_gpus_devices, shard_model from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, - WanTransformer3DModel) + WanTransformer3DModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import WanFunInpaintPipeline, WanFunPipeline -from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, - convert_weight_dtype_wrapper) -from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, - save_videos_grid) +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -37,6 +40,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -226,6 +232,9 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.1_fun/predict_v2v_control.py b/examples/wan2.1_fun/predict_v2v_control.py index 4cf0306..6d5514d 100755 --- a/examples/wan2.1_fun/predict_v2v_control.py +++ b/examples/wan2.1_fun/predict_v2v_control.py @@ -13,12 +13,16 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data import process_pose_file from videox_fun.dist import set_multi_gpus_devices, shard_model from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, WanT5EncoderModel, WanTransformer3DModel) -from videox_fun.data import process_pose_file from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import WanFunControlPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) @@ -26,10 +30,8 @@ from videox_fun.utils.lora_utils import merge_lora, unmerge_lora from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +42,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -229,6 +234,9 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.1_fun/predict_v2v_control_camera.py b/examples/wan2.1_fun/predict_v2v_control_camera.py index 4a39ca1..39ed5cb 100755 --- a/examples/wan2.1_fun/predict_v2v_control_camera.py +++ b/examples/wan2.1_fun/predict_v2v_control_camera.py @@ -13,23 +13,26 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data import process_pose_file from videox_fun.dist import set_multi_gpus_devices, shard_model from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, WanT5EncoderModel, WanTransformer3DModel) -from videox_fun.data import process_pose_file from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import WanFunControlPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +43,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -229,6 +235,9 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.1_fun/predict_v2v_control_ref.py b/examples/wan2.1_fun/predict_v2v_control_ref.py index ac1603c..7acbef4 100755 --- a/examples/wan2.1_fun/predict_v2v_control_ref.py +++ b/examples/wan2.1_fun/predict_v2v_control_ref.py @@ -13,23 +13,26 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data import process_pose_file from videox_fun.dist import set_multi_gpus_devices, shard_model from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, WanT5EncoderModel, WanTransformer3DModel) -from videox_fun.data import process_pose_file from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import WanFunControlPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +43,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -229,6 +235,9 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.1_vace/predict_i2v.py b/examples/wan2.1_vace/predict_i2v.py index 5774d2f..f121e8f 100644 --- a/examples/wan2.1_vace/predict_i2v.py +++ b/examples/wan2.1_vace/predict_i2v.py @@ -13,23 +13,26 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data import process_pose_file from videox_fun.dist import set_multi_gpus_devices, shard_model from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, - WanT5EncoderModel, VaceWanTransformer3DModel) -from videox_fun.data import process_pose_file + VaceWanTransformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients -from videox_fun.pipeline import WanVacePipeline, WanPipeline +from videox_fun.pipeline import WanPipeline, WanVacePipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +43,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -220,6 +226,9 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.1_vace/predict_s2v.py b/examples/wan2.1_vace/predict_s2v.py index 3be03ba..6cfaa76 100644 --- a/examples/wan2.1_vace/predict_s2v.py +++ b/examples/wan2.1_vace/predict_s2v.py @@ -13,23 +13,26 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data import process_pose_file from videox_fun.dist import set_multi_gpus_devices, shard_model from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, - WanT5EncoderModel, VaceWanTransformer3DModel) -from videox_fun.data import process_pose_file + VaceWanTransformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients -from videox_fun.pipeline import WanVacePipeline, WanPipeline +from videox_fun.pipeline import WanPipeline, WanVacePipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +43,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -220,6 +226,9 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.1_vace/predict_v2v_control.py b/examples/wan2.1_vace/predict_v2v_control.py index 774cd9c..09c2b9a 100644 --- a/examples/wan2.1_vace/predict_v2v_control.py +++ b/examples/wan2.1_vace/predict_v2v_control.py @@ -13,23 +13,26 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data import process_pose_file from videox_fun.dist import set_multi_gpus_devices, shard_model from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, - WanT5EncoderModel, VaceWanTransformer3DModel) -from videox_fun.data import process_pose_file + VaceWanTransformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients -from videox_fun.pipeline import WanVacePipeline, WanPipeline +from videox_fun.pipeline import WanPipeline, WanVacePipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +43,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -220,6 +226,9 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer, ["modulation",], device=device) transformer.freqs = transformer.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2/predict_animate.py b/examples/wan2.2/predict_animate.py index d854426..3d24f8d 100644 --- a/examples/wan2.2/predict_animate.py +++ b/examples/wan2.2/predict_animate.py @@ -19,6 +19,8 @@ from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2AnimatePipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, @@ -30,7 +32,7 @@ from videox_fun.utils.utils import (filter_kwargs, get_image, get_video_to_video_latent, save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -41,6 +43,9 @@ from videox_fun.utils.utils import (filter_kwargs, get_image, # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -266,6 +271,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2/predict_i2v.py b/examples/wan2.2/predict_i2v.py index 52c93c2..6fd2a07 100644 --- a/examples/wan2.2/predict_i2v.py +++ b/examples/wan2.2/predict_i2v.py @@ -13,19 +13,23 @@ for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2I2VPipeline -from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, - convert_weight_dtype_wrapper) -from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, - save_videos_grid) +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -36,6 +40,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -245,6 +252,10 @@ if GPU_memory_mode == "sequential_cpu_offload": transformer.freqs = transformer.freqs.to(device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) diff --git a/examples/wan2.2/predict_s2v.py b/examples/wan2.2/predict_s2v.py index f38296a..45c8209 100644 --- a/examples/wan2.2/predict_s2v.py +++ b/examples/wan2.2/predict_s2v.py @@ -19,6 +19,8 @@ from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2S2VPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, @@ -30,7 +32,7 @@ from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_video_to_video_latent, merge_video_audio, save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -41,6 +43,9 @@ from videox_fun.utils.utils import (filter_kwargs, get_image_latent, # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -270,6 +275,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2/predict_t2v.py b/examples/wan2.2/predict_t2v.py index 28d56ce..c64d73a 100755 --- a/examples/wan2.2/predict_t2v.py +++ b/examples/wan2.2/predict_t2v.py @@ -13,19 +13,23 @@ for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, WanT5EncoderModel, AutoTokenizer, - Wan2_2Transformer3DModel) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, Wan2_2Transformer3DModel, + WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2Pipeline -from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, - convert_weight_dtype_wrapper) -from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, - save_videos_grid) +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -36,6 +40,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -240,6 +247,10 @@ if GPU_memory_mode == "sequential_cpu_offload": transformer.freqs = transformer.freqs.to(device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) diff --git a/examples/wan2.2/predict_ti2v.py b/examples/wan2.2/predict_ti2v.py index d039210..36bdbfd 100755 --- a/examples/wan2.2/predict_ti2v.py +++ b/examples/wan2.2/predict_ti2v.py @@ -13,19 +13,23 @@ for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan3_8, AutoencoderKLWan, WanT5EncoderModel, AutoTokenizer, - Wan2_2Transformer3DModel) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, Wan2_2Transformer3DModel, + WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2TI2VPipeline -from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, - convert_weight_dtype_wrapper) -from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, - save_videos_grid) +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -36,6 +40,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -253,6 +260,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_fun/predict_i2v.py b/examples/wan2.2_fun/predict_i2v.py index 79181b4..161e083 100644 --- a/examples/wan2.2_fun/predict_i2v.py +++ b/examples/wan2.2_fun/predict_i2v.py @@ -13,19 +13,23 @@ for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2FunInpaintPipeline -from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, - convert_weight_dtype_wrapper) -from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, - save_videos_grid) +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -36,6 +40,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -96,7 +103,7 @@ vae_path = None # Load lora model if need # The lora_path is used for low noise model, the lora_high_path is used for high noise model. lora_path = None -lora_high_path = None +lora_high_path = None # Other params sample_size = [480, 832] @@ -255,6 +262,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_fun/predict_i2v_5b.py b/examples/wan2.2_fun/predict_i2v_5b.py index 1bbccaa..339565f 100644 --- a/examples/wan2.2_fun/predict_i2v_5b.py +++ b/examples/wan2.2_fun/predict_i2v_5b.py @@ -13,19 +13,23 @@ for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, AutoencoderKLWan3_8, - WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2FunInpaintPipeline -from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, - convert_weight_dtype_wrapper) -from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, - save_videos_grid) +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -36,6 +40,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload means that the internal layer groups will be transferred between CPU and CUDA, +# balancing memory efficiency and speed. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -257,6 +264,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_fun/predict_t2v.py b/examples/wan2.2_fun/predict_t2v.py index 347128a..9f5efeb 100644 --- a/examples/wan2.2_fun/predict_t2v.py +++ b/examples/wan2.2_fun/predict_t2v.py @@ -13,19 +13,23 @@ for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, AutoencoderKLWan3_8, - WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2FunInpaintPipeline -from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, - convert_weight_dtype_wrapper) -from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, - save_videos_grid) +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -36,6 +40,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload means that the internal layer groups will be transferred between CPU and CUDA, +# balancing memory efficiency and speed. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -244,6 +251,10 @@ if GPU_memory_mode == "sequential_cpu_offload": transformer.freqs = transformer.freqs.to(device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) diff --git a/examples/wan2.2_fun/predict_t2v_5b.py b/examples/wan2.2_fun/predict_t2v_5b.py index 49c5bb2..6f7290b 100644 --- a/examples/wan2.2_fun/predict_t2v_5b.py +++ b/examples/wan2.2_fun/predict_t2v_5b.py @@ -13,19 +13,23 @@ for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2FunInpaintPipeline -from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, - convert_weight_dtype_wrapper) -from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, - save_videos_grid) +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -36,6 +40,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload means that the internal layer groups will be transferred between CPU and CUDA, +# balancing memory efficiency and speed. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -251,6 +258,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_fun/predict_v2v_control.py b/examples/wan2.2_fun/predict_v2v_control.py index 8680827..63cd841 100644 --- a/examples/wan2.2_fun/predict_v2v_control.py +++ b/examples/wan2.2_fun/predict_v2v_control.py @@ -13,23 +13,27 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None -from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, Wan2_2Transformer3DModel) from videox_fun.data import process_pose_file +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +44,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload means that the internal layer groups will be transferred between CPU and CUDA, +# balancing memory efficiency and speed. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -268,6 +275,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_fun/predict_v2v_control_5b.py b/examples/wan2.2_fun/predict_v2v_control_5b.py index 2bde229..8cf222f 100644 --- a/examples/wan2.2_fun/predict_v2v_control_5b.py +++ b/examples/wan2.2_fun/predict_v2v_control_5b.py @@ -13,23 +13,27 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None -from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, Wan2_2Transformer3DModel) from videox_fun.data import process_pose_file +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +44,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload means that the internal layer groups will be transferred between CPU and CUDA, +# balancing memory efficiency and speed. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -268,6 +275,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_fun/predict_v2v_control_camera.py b/examples/wan2.2_fun/predict_v2v_control_camera.py index 7b00010..666e0a4 100644 --- a/examples/wan2.2_fun/predict_v2v_control_camera.py +++ b/examples/wan2.2_fun/predict_v2v_control_camera.py @@ -13,23 +13,27 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None -from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, Wan2_2Transformer3DModel) from videox_fun.data import process_pose_file +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +44,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload means that the internal layer groups will be transferred between CPU and CUDA, +# balancing memory efficiency and speed. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -268,6 +275,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_fun/predict_v2v_control_camera_5b.py b/examples/wan2.2_fun/predict_v2v_control_camera_5b.py index 1369656..a50173b 100644 --- a/examples/wan2.2_fun/predict_v2v_control_camera_5b.py +++ b/examples/wan2.2_fun/predict_v2v_control_camera_5b.py @@ -13,23 +13,27 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None -from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, Wan2_2Transformer3DModel) from videox_fun.data import process_pose_file +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +44,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload means that the internal layer groups will be transferred between CPU and CUDA, +# balancing memory efficiency and speed. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -268,6 +275,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_fun/predict_v2v_control_ref.py b/examples/wan2.2_fun/predict_v2v_control_ref.py index e93162b..eaf088d 100644 --- a/examples/wan2.2_fun/predict_v2v_control_ref.py +++ b/examples/wan2.2_fun/predict_v2v_control_ref.py @@ -13,23 +13,27 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None -from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, Wan2_2Transformer3DModel) from videox_fun.data import process_pose_file +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +44,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload means that the internal layer groups will be transferred between CPU and CUDA, +# balancing memory efficiency and speed. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -268,6 +275,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_fun/predict_v2v_control_ref_5b.py b/examples/wan2.2_fun/predict_v2v_control_ref_5b.py index 880906f..5ecd1c7 100644 --- a/examples/wan2.2_fun/predict_v2v_control_ref_5b.py +++ b/examples/wan2.2_fun/predict_v2v_control_ref_5b.py @@ -13,23 +13,27 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None -from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, Wan2_2Transformer3DModel) from videox_fun.data import process_pose_file +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +44,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload means that the internal layer groups will be transferred between CPU and CUDA, +# balancing memory efficiency and speed. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -268,6 +275,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_vace_fun/predict_i2v.py b/examples/wan2.2_vace_fun/predict_i2v.py index 6f54ff4..1abf301 100644 --- a/examples/wan2.2_vace_fun/predict_i2v.py +++ b/examples/wan2.2_vace_fun/predict_i2v.py @@ -13,23 +13,27 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None -from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, VaceWanTransformer3DModel) from videox_fun.data import process_pose_file +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + VaceWanTransformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2VaceFunPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +44,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -272,6 +279,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_vace_fun/predict_s2v.py b/examples/wan2.2_vace_fun/predict_s2v.py index 912afbd..5d90aa3 100644 --- a/examples/wan2.2_vace_fun/predict_s2v.py +++ b/examples/wan2.2_vace_fun/predict_s2v.py @@ -13,23 +13,27 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None -from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, VaceWanTransformer3DModel) from videox_fun.data import process_pose_file +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + VaceWanTransformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2VaceFunPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +44,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -272,6 +279,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_vace_fun/predict_v2v_control.py b/examples/wan2.2_vace_fun/predict_v2v_control.py index c9a12cf..a063e4f 100644 --- a/examples/wan2.2_vace_fun/predict_v2v_control.py +++ b/examples/wan2.2_vace_fun/predict_v2v_control.py @@ -13,23 +13,27 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None -from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, VaceWanTransformer3DModel) from videox_fun.data import process_pose_file +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + VaceWanTransformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2VaceFunPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +44,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -272,6 +279,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_vace_fun/predict_v2v_control_ref.py b/examples/wan2.2_vace_fun/predict_v2v_control_ref.py index ae9529a..b57099b 100644 --- a/examples/wan2.2_vace_fun/predict_v2v_control_ref.py +++ b/examples/wan2.2_vace_fun/predict_v2v_control_ref.py @@ -13,23 +13,27 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None -from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, VaceWanTransformer3DModel) from videox_fun.data import process_pose_file +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + VaceWanTransformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2VaceFunPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +44,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -272,6 +279,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/examples/wan2.2_vace_fun/predict_v2v_mask.py b/examples/wan2.2_vace_fun/predict_v2v_mask.py index 44e45f6..dd28d98 100644 --- a/examples/wan2.2_vace_fun/predict_v2v_mask.py +++ b/examples/wan2.2_vace_fun/predict_v2v_mask.py @@ -13,23 +13,27 @@ project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dir for project_root in project_roots: sys.path.insert(0, project_root) if project_root not in sys.path else None -from videox_fun.dist import set_multi_gpus_devices, shard_model -from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, - WanT5EncoderModel, VaceWanTransformer3DModel) from videox_fun.data import process_pose_file +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + VaceWanTransformer3DModel, WanT5EncoderModel) from videox_fun.models.cache_utils import get_teacache_coefficients from videox_fun.pipeline import Wan2_2VaceFunPipeline, WanPipeline +from videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) from videox_fun.utils.lora_utils import merge_lora, unmerge_lora -from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, get_video_to_video_latent, save_videos_grid) -from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler -from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler -# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, model_group_offload, sequential_cpu_offload]. # model_full_load means that the entire model will be moved to the GPU. # # model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, @@ -40,6 +44,9 @@ from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, # and the transformer model has been quantized to float8, which can save more GPU memory. # +# model_group_offload transfers internal layer groups between CPU/CUDA, +# balancing memory efficiency and speed between full-module and leaf-level offloading methods. +# # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, # resulting in slower speeds but saving a large amount of GPU memory. GPU_memory_mode = "sequential_cpu_offload" @@ -272,6 +279,11 @@ if GPU_memory_mode == "sequential_cpu_offload": replace_parameters_by_name(transformer_2, ["modulation",], device=device) transformer_2.freqs = transformer_2.freqs.to(device=device) pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + if transformer_2 is not None: + register_auto_device_hook(pipeline.transformer_2) + safe_enable_group_offload(pipeline, onload_device=device, offload_device="cpu", offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) diff --git a/scripts/cogvideox_fun/README_TRAIN.md b/scripts/cogvideox_fun/README_TRAIN.md index b58198b..b995226 100755 --- a/scripts/cogvideox_fun/README_TRAIN.md +++ b/scripts/cogvideox_fun/README_TRAIN.md @@ -1,47 +1,189 @@ -## Training Code +# CogVideoX-Fun Full Parameter Training Guide -The default training commands for the different versions are as follows: +This document provides a complete workflow for full parameter training of CogVideoX-Fun Diffusion Transformer, including environment configuration, data preparation, distributed training, and inference testing. -We can choose whether to use deepspeed and fsdp in CogVideoX-Fun, which can save a lot of video memory. +> **Note**: CogVideoX-Fun is a video generation model that supports Text-to-Video (T2V), Image-to-Video (I2V), and Video-to-Video (V2V). This document covers the training workflow for general video generation tasks. -Some parameters in the sh file can be confusing, and they are explained in this document: +--- -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since CogVideoX-Fun uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +## Table of Contents +- [1. Environment Configuration](#1-environment-configuration) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative vs Absolute Path Usage](#24-relative-vs-absolute-path-usage) +- [3. Full Parameter Training](#3-full-parameter-training) + - [3.1 Download Pretrained Model](#31-download-pretrained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 Common Training Parameters](#33-common-training-parameters) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Training without DeepSpeed and FSDP](#36-training-without-deepspeed-and-fsdp) + - [3.7 Multi-Machine Distributed Training](#37-multi-machine-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameters](#41-inference-parameters) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Video-to-Video (V2V) Inference](#44-video-to-video-v2v-inference) + - [4.5 Multi-GPU Parallel Inference](#45-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -CogVideoX-Fun without deepspeed: -```sh -export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +--- + +## 1. Environment Configuration + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**Method 2: Manual Dependency Installation** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**Method 3: Using Docker** + +When using Docker, please ensure that the GPU driver and CUDA environment are correctly installed on your machine, then execute the following commands: + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official demo dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video dimensions (**recommended** to provide for bucket training. If not provided, they will be automatically read during training, which may slow down training when data is stored on slow systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height fields for JSON files without them, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, use absolute paths. + +--- + +## 3. Full Parameter Training + +### 3.1 Download Pretrained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download CogVideoX-Fun official weights +modelscope download --model PAI/CogVideoX-Fun-2b-InP --local_dir models/Diffusion_Transformer/CogVideoX-Fun-2b-InP +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After downloading the data as per **2.1 Quick Test Dataset** and the weights as per **3.1 Download Pretrained Model**, you can directly copy and run the quick start command. + +We recommend using DeepSpeed-Zero-2 or FSDP for training. Here we use DeepSpeed-Zero-2 as an example. + +The difference between DeepSpeed-Zero-2 and FSDP lies in whether model weights are sharded. **If you run out of GPU memory with multiple GPUs using DeepSpeed-Zero-2**, you can switch to FSDP. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train.py \ +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/cogvideox_fun/train.py \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -60,7 +202,7 @@ accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train.py \ --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_cogvideox_fun" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -74,56 +216,100 @@ accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train.py \ --trainable_modules "." ``` -CogVideoX-Fun with Deepspeed Zero-2: -```sh -export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +### 3.3 Common Training Parameters -accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/cogvideox_fun/train.py \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=512 \ - --video_sample_size=512 \ - --token_sample_size=512 \ - --video_sample_stride=3 \ - --video_sample_n_frames=49 \ - --train_batch_size=4 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-05 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --use_deepspeed \ - --train_mode="inpaint" \ - --trainable_modules "." +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/CogVideoX-Fun-2b-InP` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Batch size per GPU | 1 | +| `--image_sample_size` | Maximum image training resolution | 512 | +| `--video_sample_size` | Maximum video training resolution | 512 | +| `--token_sample_size` | Token sample size | 512 | +| `--video_sample_stride` | Video sampling stride | 3 | +| `--video_sample_n_frames` | Video sampling frames | 49 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch) | 1 | +| `--dataloader_num_workers` | DataLoader worker processes | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate | 2e-05 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_cogvideox_fun` | +| `--gradient_checkpointing` | Enable gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon | 1e-10 | +| `--vae_mini_batch` | VAE encoding mini-batch size | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping, group by resolution | - | +| `--random_hw_adapt` | Auto-scale images/videos to random size in `[min_size, max_size]` range | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--train_mode` | Training mode: `inpaint` (I2V/V2V) or `normal` (T2V) | `inpaint` | +| `--resume_from_checkpoint` | Resume training path, use `"latest"` to auto-select latest checkpoint | None | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for video generation validation | `"A dog shaking head..."` | +| `--trainable_modules` | Trainable modules (`"."` means all modules) | `"."` | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Guide**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: a video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +**Training Mode Guide**: +- `train_mode="inpaint"`: Default mode for CogVideoX-Fun, uses inpaint model to achieve image-to-video and video-to-video generation. +- `train_mode="normal"`: Standard text-to-video mode. Remove this parameter or set to `normal` if you only want text-to-video generation. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|------|------|--------| +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for video generation validation, space-separated for multiple prompts | Multiple space-separated prompts | + +**Example**: + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." ``` -With FSDP: +**Notes**: +- Validation videos are saved to the `output_dir` directory +- Multiple prompts format: `--validation_prompts "prompt1" "prompt2" "prompt3"` + +### 3.5 Training with FSDP + +**If you run out of GPU memory with multiple GPUs using DeepSpeed-Zero-2**, you can switch to FSDP. ```sh export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -148,7 +334,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_cogvideox_fun" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -161,3 +347,300 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --train_mode="inpaint" \ --trainable_modules "." ``` + +### 3.6 Training without DeepSpeed and FSDP + +**This approach is not recommended due to lack of memory-saving backends, which may easily cause out-of-memory errors**. Provided here only for reference. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.7 Multi-Machine Distributed Training + +**Suitable for**: Ultra-large datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/cogvideox_fun/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Machine Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameters + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|------|------|-------| +| `GPU_memory_mode` | GPU memory mode, see table below for options | `model_cpu_offload_and_qfloat8` | +| `ulysses_degree` | Ulysses parallelism degree for multi-GPU inference | 1 | +| `ring_degree` | Ring parallelism degree for multi-GPU inference | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save memory | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective at fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP` | +| `sampler_name` | Sampler type: `Euler`, `Euler A`, `DPM++`, `PNDM`, `DDIM_Cog`, `DDIM_Origin` | `DDIM_Origin` | +| `transformer_path` | Path to trained Transformer weights | `None` | +| `vae_path` | Path to trained VAE weights | `None` | +| `lora_path` | LoRA weights path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[384, 672]` | +| `video_length` | Number of generated frames (V1.0/V1.1: up to 49, V1.5: up to 85) | `49` | +| `fps` | Frames per second | `8` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs without bf16 support | `torch.bfloat16` | +| `validation_image_start` | Reference image path for I2V mode | `"asset/1.png"` | +| `validation_video` | Reference video path for V2V mode | `"asset/1.mp4"` | +| `prompt` | Positive prompt describing generated content | `"The dog is shaking head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"Low quality, low resolution..."` | +| `guidance_scale` | Guidance strength | 6.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 50 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated videos | `samples/cogvideox-fun-videos-i2v` | + +**GPU Memory Mode Descriptions**: + +| Mode | Description | Memory Usage | +|------|------|---------| +| `model_full_load` | Load entire model to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups offloaded between CPU/CUDA | Low | +| `sequential_cpu_offload` | Sequential offload layer by layer (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run single-GPU inference: + +```bash +python examples/cogvideox_fun/predict_t2v.py +``` + +Edit `examples/cogvideox_fun/predict_t2v.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Your actual model path +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# Path to trained weights, e.g., "output_dir_cogvideox_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Write based on your generation content +prompt = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run single-GPU inference: + +```bash +python examples/cogvideox_fun/predict_i2v.py +``` + +Edit `examples/cogvideox_fun/predict_i2v.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Your actual model path +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# Path to trained weights, e.g., "output_dir_cogvideox_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Reference image path +validation_image_start = "asset/1.png" +validation_image_end = None +# Write based on your generation content +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 Video-to-Video (V2V) Inference + +Run single-GPU inference: + +```bash +python examples/cogvideox_fun/predict_v2v.py +``` + +Edit `examples/cogvideox_fun/predict_v2v.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Your actual model path +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# Path to trained weights, e.g., "output_dir_cogvideox_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Reference video path +validation_video = "asset/1.mp4" +validation_video_mask = None # Set to mask path for partial video redraw +denoise_strength = 0.70 # Use 1.00 when using validation_video_mask +# Write based on your generation content +prompt = "A cute cat is playing the guitar." +# ... +``` + +### 4.5 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/cogvideox_fun/predict_t2v.py`, `examples/cogvideox_fun/predict_i2v.py`, or `examples/cogvideox_fun/predict_v2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must evenly divide the model's head count +- `ring_degree` splits on the sequence dimension, which affects communication overhead. Try to avoid using it when heads can be evenly divided. + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/cogvideox_fun/predict_t2v.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/cogvideox_fun/README_TRAIN_LORA.md b/scripts/cogvideox_fun/README_TRAIN_LORA.md index 187401f..e4a6c5f 100755 --- a/scripts/cogvideox_fun/README_TRAIN_LORA.md +++ b/scripts/cogvideox_fun/README_TRAIN_LORA.md @@ -1,89 +1,183 @@ -## Lora Training Code +# CogVideoX-Fun LoRA Fine-tuning Training Guide -We can choose whether to use deepspeed and fsdp in CogVideoX-Fun, which can save a lot of video memory. +This document provides a complete guide for CogVideoX-Fun LoRA fine-tuning training, including environment configuration, data preparation, multiple distributed training strategies, and inference testing. -Some parameters in the sh file can be confusing, and they are explained in this document: +> **Note**: CogVideoX-Fun is a video generation model that supports Text-to-Video (T2V), Image-to-Video (I2V), and Video-to-Video (V2V). This guide covers the LoRA fine-tuning training process, suitable for custom dataset fine-tuning scenarios. -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since CogVideoX-Fun uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. -- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. -- `rank` means the dimension of the LoRA update matrices. -- `network_alpha` means the scale of the LoRA update matrices. +--- -CogVideoX-Fun without deepspeed: +## Table of Contents +- [1. Environment Configuration](#1-environment-configuration) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative Path vs Absolute Path Usage](#24-relative-path-vs-absolute-path-usage) +- [3. LoRA Training](#3-lora-training) + - [3.1 Download Pre-trained Model](#31-download-pre-trained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 LoRA-Specific Parameter Explanation](#33-lora-specific-parameter-explanation) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Training without DeepSpeed and FSDP](#36-training-without-deepspeed-and-fsdp) + - [3.7 Multi-Machine Distributed Training](#37-multi-machine-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameter Explanation](#41-inference-parameter-explanation) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Video-to-Video (V2V) Inference](#44-video-to-video-v2v-inference) + - [4.5 Multi-GPU Parallel Inference](#45-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -```sh -export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +--- -accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train_lora.py \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=512 \ - --video_sample_size=512 \ - --token_sample_size=512 \ - --video_sample_stride=3 \ - --video_sample_n_frames=49 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-04 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --rank=64 \ - --network_alpha=32 \ - --target_name="to_q,to_k,to_v,ff.0,ff.2" \ - --use_peft_lora \ - --low_vram \ - --train_mode="inpaint" +## 1. Environment Configuration + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -CogVideoX-Fun with Deepspeed Zero-2: -```sh -export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +**Method 2: Manual Installation** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**Method 3: Using Docker** + +When using Docker, please ensure that the GPU driver and CUDA environment are correctly installed on your machine, then execute the following commands: + +```bash +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official example dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video width and height (**recommended to provide**, used for bucket training; if not provided, they will be automatically read during training, which may affect training speed when data is stored on slow systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height from JSON files without these fields, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative Path vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, relative paths are recommended. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, absolute paths are recommended. + +--- + +## 3. LoRA Training + +### 3.1 Download Pre-trained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download CogVideoX-Fun official weights +modelscope download --model PAI/CogVideoX-Fun-2b-InP --local_dir models/Diffusion_Transformer/CogVideoX-Fun-2b-InP +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +If you have downloaded the data following **Section 2.1 Quick Test Dataset** and the weights following **Section 3.1 Download Pre-trained Model**, you can directly copy the quick start instructions to launch training. + +DeepSpeed-Zero-2 and FSDP are recommended for training. Here we use DeepSpeed-Zero-2 as an example to configure the shell file. + +The difference between DeepSpeed-Zero-2 and FSDP lies in whether model weights are sharded. **If you run out of VRAM when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -106,7 +200,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_cogvideox_fun_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -121,21 +215,113 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --target_name="to_q,to_k,to_v,ff.0,ff.2" \ --use_peft_lora \ --low_vram \ - --train_mode="inpaint" + --train_mode="inpaint" ``` -With FSDP: +### 3.3 LoRA-Specific Parameter Explanation + +**Key LoRA Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `--pretrained_model_name_or_path` | Pre-trained model path | `models/Diffusion_Transformer/CogVideoX-Fun-2b-InP` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Batch size | 1 | +| `--image_sample_size` | Maximum training resolution for images | 512 | +| `--video_sample_size` | Maximum training resolution for videos | 512 | +| `--token_sample_size` | Token sampling size | 512 | +| `--video_sample_stride` | Video sampling stride | 3 | +| `--video_sample_n_frames` | Number of video frames to sample | 49 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader worker processes | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (recommended for LoRA) | 1e-04 | +| `--lr_scheduler` | Learning rate scheduler | `constant` | +| `--lr_warmup_steps` | Learning rate warmup steps | 500 | +| `--seed` | Random seed (for reproducibility) | 42 | +| `--output_dir` | Output directory | `output_dir_cogvideox_fun_lora` | +| `--gradient_checkpointing` | Enable gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini batch size for VAE encoding | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training; trains without cropping images/videos, groups by resolution | - | +| `--random_hw_adapt` | Automatically scale images/videos to random sizes within `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `inpaint` (I2V/V2V) or `normal` (T2V) | `inpaint` | +| `--resume_from_checkpoint` | Resume training from checkpoint path; use `"latest"` to auto-select the latest checkpoint | None | +| `--rank` | Dimension of LoRA update matrices (higher rank = stronger expressiveness, but more VRAM) | 128 | +| `--network_alpha` | Scaling factor for LoRA update matrices (usually set to half of rank or same) | 64 | +| `--target_name` | Components/modules to apply LoRA, separated by commas | `to_q,to_k,to_v,ff.0,ff.2` | +| `--use_peft_lora` | Use PEFT module to add LoRA (more memory-efficient) | - | +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for validation video generation | `"A young woman..."` | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: A video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +**Training Mode Guide**: +- `train_mode="inpaint"`: Default mode for CogVideoX-Fun, uses inpaint model to achieve image-to-video and video-to-video generation. +- `train_mode="normal"`: Standard text-to-video mode. Remove this parameter or set to `normal` if you only want text-to-video generation. + +### 3.4 Training Validation + +You can configure validation parameters to regularly generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|-----------|-------------|-------------------| +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for validation video generation, can separate multiple prompts with spaces | Multiple space-separated prompts | + +**Example**: + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +``` + +**Notes**: +- Validation videos will be saved in the `output_dir` directory. +- Multiple prompts validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` + +### 3.5 Training with FSDP + +**If you run out of VRAM when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. ```sh export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap CogVideoXBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/cogvideox_fun/train_lora.py \ +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=CogVideoXBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/cogvideox_fun/train_lora.py \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -152,7 +338,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_cogvideox_fun_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -167,5 +353,312 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --target_name="to_q,to_k,to_v,ff.0,ff.2" \ --use_peft_lora \ --low_vram \ - --train_mode="inpaint" -``` \ No newline at end of file + --train_mode="inpaint" +``` + +### 3.6 Training without DeepSpeed and FSDP + +**This approach is NOT recommended, as it lacks memory-saving backends and can easily cause VRAM issues**. It is only provided here as a reference. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +### 3.7 Multi-Machine Distributed Training + +**Suitable for**: Ultra-large datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/cogvideox_fun/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Machine Training Considerations + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data path (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameter Explanation + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `GPU_memory_mode` | VRAM management mode, see table below for options | `model_cpu_offload_and_qfloat8` | +| `ulysses_degree` | Head dimension parallelism degree, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism degree, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save VRAM | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolutions) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP` | +| `sampler_name` | Sampler type: `Euler`, `Euler A`, `DPM++`, `PNDM`, `DDIM_Cog`, `DDIM_Origin` | `DDIM_Origin` | +| `transformer_path` | Path to load trained Transformer weights | `None` | +| `vae_path` | Path to load trained VAE weights | `None` | +| `lora_path` | LoRA weights path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[384, 672]` | +| `video_length` | Number of generated video frames (V1.0/V1.1: up to 49, V1.5: up to 85) | `49` | +| `fps` | Frames per second | `8` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs not supporting bf16 | `torch.bfloat16` | +| `validation_image_start` | Reference image path for Image-to-Video (I2V mode) | `"asset/1.png"` | +| `validation_video` | Reference video path for Video-to-Video (V2V mode) | `"asset/1.mp4"` | +| `prompt` | Positive prompt describing generated content | `"The dog is shaking head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"lowres, low quality..."` | +| `guidance_scale` | Guidance strength | 6.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 50 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated video | `samples/cogvideox-fun-videos-i2v` or `samples/cogvideox-fun-videos-t2v` | + +**VRAM Management Mode Descriptions**: + +| Mode | Description | VRAM Usage | +|------|-------------|------------| +| `model_full_load` | Load entire model to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/cogvideox_fun/predict_t2v.py +``` + +Modify `examples/cogvideox_fun/predict_t2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameter explanation above. + +```python +# Choose based on GPU VRAM +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Based on actual model path +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# Path to trained weights, e.g., "output_dir_cogvideox_fun_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA weight strength +lora_weight = 0.55 +# Write based on generated content +prompt = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/cogvideox_fun/predict_i2v.py +``` + +Modify `examples/cogvideox_fun/predict_i2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameter explanation above. + +```python +# Choose based on GPU VRAM +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Based on actual model path +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# LoRA weights path, e.g., "output_dir_cogvideox_fun_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA weight strength +lora_weight = 0.55 +# Starting image for Image-to-Video +validation_image_start = "asset/1.png" +validation_image_end = None +# Write based on generated content +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 Video-to-Video (V2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/cogvideox_fun/predict_v2v.py +``` + +Modify `examples/cogvideox_fun/predict_v2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameter explanation above. + +```python +# Choose based on GPU VRAM +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Based on actual model path +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# LoRA weights path, e.g., "output_dir_cogvideox_fun_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA weight strength +lora_weight = 0.55 +# Reference video for Video-to-Video +validation_video = "asset/1.mp4" +validation_video_mask = None # Set to mask path for partial video redraw +denoise_strength = 0.70 # Use 1.00 when using validation_video_mask +# Write based on generated content +prompt = "A cute cat is playing the guitar." +# ... +``` + +### 4.5 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/cogvideox_fun/predict_t2v.py`, `examples/cogvideox_fun/predict_i2v.py`, or `examples/cogvideox_fun/predict_v2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be evenly divisible by the model's number of heads +- `ring_degree` splits along the sequence dimension and affects communication overhead; try to avoid it when heads are evenly divisible + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|-----------|---------------|-------------|-------------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/cogvideox_fun/predict_t2v.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/cogvideox_fun/README_TRAIN_LORA_zh-CN.md b/scripts/cogvideox_fun/README_TRAIN_LORA_zh-CN.md new file mode 100644 index 0000000..3d6f772 --- /dev/null +++ b/scripts/cogvideox_fun/README_TRAIN_LORA_zh-CN.md @@ -0,0 +1,664 @@ +# CogVideoX-Fun LoRA 微调训练指南 + +本文档提供 CogVideoX-Fun LoRA 微调训练的完整流程，包括环境配置、数据准备、多种分布式训练策略和推理测试。 + +> **说明**：CogVideoX-Fun 是支持文生视频（T2V）、图生视频（I2V）和视频生视频（V2V）的视频生成模型。本指南涵盖 LoRA 微调训练流程，适用于自定义数据集的微调场景。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、LoRA 训练](#三lora-训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 LoRA 专用参数解析](#33-lora-专用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 不使用 DeepSpeed 与 FSDP 训练](#36-不使用-deepspeed-与-fsdp-训练) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 视频生视频（V2V）推理](#44-视频生视频v2v推理) + - [4.5 多卡并行推理](#45-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、LoRA 训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 CogVideoX-Fun 官方权重 +modelscope download --model PAI/CogVideoX-Fun-2b-InP --local_dir models/Diffusion_Transformer/CogVideoX-Fun-2b-InP +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用 DeepSpeed-Zero-2 与 FSDP 方案进行训练。这里以 DeepSpeed-Zero-2 为例配置 shell 文件。 + +本文中 DeepSpeed-Zero-2 与 FSDP 的差别在于是否对模型权重进行分片，**如果使用多卡且使用 DeepSpeed-Zero-2 的情况下显存不足**，可以切换使用 FSDP 进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/cogvideox_fun/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +### 3.3 LoRA 专用参数解析 + +**LoRA 关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/CogVideoX-Fun-2b-InP` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 512 | +| `--video_sample_size` | 视频最大训练分辨率 | 512 | +| `--token_sample_size` | Token 采样尺寸 | 512 | +| `--video_sample_stride` | 视频采样步幅 | 3 | +| `--video_sample_n_frames` | 视频采样帧数 | 49 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率（LoRA 推荐值） | 1e-04 | +| `--lr_scheduler` | 学习率调度器 | `constant` | +| `--lr_warmup_steps` | 学习率预热步数 | 500 | +| `--seed` | 随机种子（可复现训练） | 42 | +| `--output_dir` | 输出目录 | `output_dir_cogvideox_fun_lora` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式：`inpaint`（图生视频/视频生视频）或 `normal`（文生视频） | `inpaint` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--rank` | LoRA 更新矩阵的维度（rank 越大表达能力越强，但显存占用越高） | 128 | +| `--network_alpha` | LoRA 更新矩阵的缩放系数（通常设置为 rank 的一半或相同） | 64 | +| `--target_name` | 应用 LoRA 的组件/模块，用逗号分隔 | `to_q,to_k,to_v,ff.0,ff.2` | +| `--use_peft_lora` | 使用 PEFT 模块添加 LoRA（更节省显存） | - | +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | `"A dog shaking head..."` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +**训练模式说明**： +- `train_mode="inpaint"`：CogVideoX-Fun 的默认模式，使用 inpaint 模型实现图生视频和视频生视频功能。 +- `train_mode="normal"`：标准文生视频模式。如果只需要文生视频功能，移除此参数或设置为 `normal`。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词,可用空格分隔多个提示词 | 多个空格分隔的提示词 | + +**示例**： + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用 DeepSpeed-Zero-2 的情况下显存不足**，可以切换使用 FSDP 进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=CogVideoXBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/cogvideox_fun/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +### 3.6 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练 Shell 用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/cogvideox_fun/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_cpu_offload_and_qfloat8` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP` | +| `sampler_name` | 采样器类型：`Euler`、`Euler A`、`DPM++`、`PNDM`、`DDIM_Cog`、`DDIM_Origin` | `DDIM_Origin` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[384, 672]` | +| `video_length` | 生成视频帧数（V1.0/V1.1：最多49帧，V1.5：最多85帧） | `49` | +| `fps` | 每秒帧数 | `8` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V 模式） | `"asset/1.png"` | +| `validation_video` | 视频生视频的参考视频路径（V2V 模式） | `"asset/1.mp4"` | +| `prompt` | 正向提示词，描述生成内容 | `"The dog is shaking head..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"Low quality, low resolution..."` | +| `guidance_scale` | 引导强度 | 6.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 50 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/cogvideox-fun-videos-i2v` 或 `samples/cogvideox-fun-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/cogvideox_fun/predict_t2v.py +``` + +根据需求修改编辑 `examples/cogvideox_fun/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# 训练好的权重路径，如 "output_dir_cogvideox_fun_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA 权重强度 +lora_weight = 0.55 +# 根据生成内容编写 +prompt = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/cogvideox_fun/predict_i2v.py +``` + +根据需求修改编辑 `examples/cogvideox_fun/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# LoRA 权重路径，如 "output_dir_cogvideox_fun_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA 权重强度 +lora_weight = 0.55 +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +validation_image_end = None +# 根据生成内容编写 +prompt = "A dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 视频生视频（V2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/cogvideox_fun/predict_v2v.py +``` + +根据需求修改编辑 `examples/cogvideox_fun/predict_v2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# LoRA 权重路径，如 "output_dir_cogvideox_fun_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA 权重强度 +lora_weight = 0.55 +# 视频生视频的参考视频 +validation_video = "asset/1.mp4" +validation_video_mask = None # 设置为掩码路径可进行部分视频重绘 +denoise_strength = 0.70 # 使用 validation_video_mask 时使用 1.00 +# 根据生成内容编写 +prompt = "A cute cat is playing the guitar." +# ... +``` + +### 4.5 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/cogvideox_fun/predict_t2v.py`、`examples/cogvideox_fun/predict_i2v.py` 或 `examples/cogvideox_fun/predict_v2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = GPU 数量 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/cogvideox_fun/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/cogvideox_fun/README_TRAIN_zh-CN.md b/scripts/cogvideox_fun/README_TRAIN_zh-CN.md new file mode 100644 index 0000000..6855d4f --- /dev/null +++ b/scripts/cogvideox_fun/README_TRAIN_zh-CN.md @@ -0,0 +1,646 @@ +# CogVideoX-Fun 全参数训练指南 + +本文档提供 CogVideoX-Fun Diffusion Transformer 全参数训练的完整流程,包括环境配置、数据准备、分布式训练和推理测试。 + +> **说明**：CogVideoX-Fun 是一个支持文生视频（T2V）、图生视频（I2V）和视频生视频（V2V）的视频生成模型。本文档涵盖普通视频生成任务的训练流程。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、全量参数训练](#三全量参数训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 训练常用参数解析](#33-训练常用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 不使用 DeepSpeed 与 FSDP 训练](#36-不使用-deepspeed-与-fsdp-训练) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 视频生视频（V2V）推理](#44-视频生视频v2v推理) + - [4.5 多卡并行推理](#45-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、全量参数训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 CogVideoX-Fun 官方权重 +modelscope download --model PAI/CogVideoX-Fun-2b-InP --local_dir models/Diffusion_Transformer/CogVideoX-Fun-2b-InP +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用DeepSpeed-Zero-2与FSDP方案进行训练。这里使用DeepSpeed-Zero-2为例配置shell文件。 + +本文中DeepSpeed-Zero-2与FSDP的差别在于是否对模型权重进行分片，**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/cogvideox_fun/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.3 训练常用参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/CogVideoX-Fun-2b-InP` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 512 | +| `--video_sample_size` | 视频最大训练分辨率 | 512 | +| `--token_sample_size` | Token 采样尺寸 | 512 | +| `--video_sample_stride` | 视频采样步幅 | 3 | +| `--video_sample_n_frames` | 视频采样帧数 | 49 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率 | 2e-05 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_cogvideox_fun` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--train_mode` | 训练模式：`inpaint`（图生视频/视频生视频）或 `normal`（文生视频） | `inpaint` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词 | `"A dog shaking head..."` | +| `--trainable_modules` | 可训练模块（`"."` 表示所有模块） | `"."` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +**训练模式说明**： +- `train_mode="inpaint"`：CogVideoX-Fun 的默认模式，使用 inpaint 模型实现图生视频和视频生视频功能。 +- `train_mode="normal"`：标准文生视频模式。如果只需要文生视频功能，可以移除此参数或设置为 `normal`。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词,可用空格分隔多个提示词 | 多个空格分隔的提示词 | + +**示例**： + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap CogVideoXBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/cogvideox_fun/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.6 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/cogvideox_fun/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_cogvideox_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_cpu_offload_and_qfloat8` | +| `ulysses_degree` | Ulysses 并行度，用于多卡推理 | 1 | +| `ring_degree` | Ring 并行度，用于多卡推理 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP` | +| `sampler_name` | 采样器类型：`Euler`、`Euler A`、`DPM++`、`PNDM`、`DDIM_Cog`、`DDIM_Origin` | `DDIM_Origin` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[384, 672]` | +| `video_length` | 生成视频帧数（V1.0/V1.1：最多49帧，V1.5：最多85帧） | `49` | +| `fps` | 每秒帧数 | `8` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V 模式） | `"asset/1.png"` | +| `validation_video` | 视频生视频的参考视频路径（V2V 模式） | `"asset/1.mp4"` | +| `prompt` | 正向提示词，描述生成内容 | `"The dog is shaking head..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"Low quality, low resolution..."` | +| `guidance_scale` | 引导强度 | 6.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 50 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/cogvideox-fun-videos-i2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/cogvideox_fun/predict_t2v.py +``` + +根据需求修改编辑 `examples/cogvideox_fun/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# 训练好的权重路径，如 "output_dir_cogvideox_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 根据生成内容编写 +prompt = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/cogvideox_fun/predict_i2v.py +``` + +根据需求修改编辑 `examples/cogvideox_fun/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# 训练好的权重路径，如 "output_dir_cogvideox_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +validation_image_end = None +# 根据生成内容编写 +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 视频生视频（V2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/cogvideox_fun/predict_v2v.py +``` + +根据需求修改编辑 `examples/cogvideox_fun/predict_v2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +# 训练好的权重路径，如 "output_dir_cogvideox_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 参考视频路径 +validation_video = "asset/1.mp4" +validation_video_mask = None # 设置为掩码路径可进行部分视频重绘 +denoise_strength = 0.70 # 使用 validation_video_mask 时使用 1.00 +# 根据生成内容编写 +prompt = "A cute cat is playing the guitar." +# ... +``` + +### 4.5 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/cogvideox_fun/predict_t2v.py`、`examples/cogvideox_fun/predict_i2v.py` 或 `examples/cogvideox_fun/predict_v2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/cogvideox_fun/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/cogvideox_fun/train.sh b/scripts/cogvideox_fun/train.sh index 889b0a3..4949289 100755 --- a/scripts/cogvideox_fun/train.sh +++ b/scripts/cogvideox_fun/train.sh @@ -25,7 +25,7 @@ accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train.py \ --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir_cog" \ + --output_dir="output_dir_cogvideox_fun" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -65,7 +65,7 @@ accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train.py \ # --lr_scheduler="constant_with_warmup" \ # --lr_warmup_steps=100 \ # --seed=42 \ -# --output_dir="output_dir_cog" \ +# --output_dir="output_dir_cogvideox_fun" \ # --gradient_checkpointing \ # --mixed_precision="bf16" \ # --adam_weight_decay=3e-2 \ diff --git a/scripts/cogvideox_fun/train_lora.py b/scripts/cogvideox_fun/train_lora.py index a5e5c6d..19ecfb3 100755 --- a/scripts/cogvideox_fun/train_lora.py +++ b/scripts/cogvideox_fun/train_lora.py @@ -588,12 +588,6 @@ def parse_args(): parser.add_argument( "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", ) - parser.add_argument( - "--noise_share_in_frames", action="store_true", help="Whether enable noise share in frames." - ) - parser.add_argument( - "--noise_share_in_frames_ratio", type=float, default=0.5, help="Noise share ratio.", - ) parser.add_argument( "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." ) @@ -647,12 +641,6 @@ def parse_args(): default=0, help="Num of repeat video.", ) - parser.add_argument( - "--image_repeat_in_forward", - type=int, - default=0, - help="Num of repeat image in forward.", - ) parser.add_argument( "--transformer_path", type=str, diff --git a/scripts/cogvideox_fun/train_lora.sh b/scripts/cogvideox_fun/train_lora.sh index de54bda..482f07e 100755 --- a/scripts/cogvideox_fun/train_lora.sh +++ b/scripts/cogvideox_fun/train_lora.sh @@ -23,7 +23,7 @@ accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train_lora.py \ --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir_cog_lora" \ + --output_dir="output_dir_cogvideox_fun_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -65,7 +65,7 @@ accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train_lora.py \ # --checkpointing_steps=50 \ # --learning_rate=1e-04 \ # --seed=42 \ -# --output_dir="output_dir_cog_lora" \ +# --output_dir="output_dir_cogvideox_fun_lora" \ # --gradient_checkpointing \ # --mixed_precision="bf16" \ # --adam_weight_decay=3e-2 \ diff --git a/scripts/fantasytalking/README_TRAIN.md b/scripts/fantasytalking/README_TRAIN.md index 1d705bf..8552f96 100755 --- a/scripts/fantasytalking/README_TRAIN.md +++ b/scripts/fantasytalking/README_TRAIN.md @@ -198,8 +198,8 @@ The difference between DeepSpeed-Zero-2 and FSDP is whether model weights are sh ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # Uncomment the following two lines for multi-node training without RDMA # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -343,8 +343,8 @@ You can configure validation parameters to periodically generate test videos dur ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=AudioAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/fantasytalking/train.py \ @@ -401,8 +401,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" scripts/fantasytalking/train.py \ @@ -451,9 +451,9 @@ Assuming 2 machines with 8 GPUs each: **Machine 0 (Master)**: ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" -export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" # If None, will use $MODEL_NAME/audio_encoder -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Master machine IP export MASTER_PORT=10086 export WORLD_SIZE=2 # Total number of machines @@ -503,8 +503,8 @@ accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Same as Master export MASTER_PORT=10086 export WORLD_SIZE=2 diff --git a/scripts/fantasytalking/README_TRAIN_zh-CN.md b/scripts/fantasytalking/README_TRAIN_zh-CN.md index c37d5e6..9afb731 100755 --- a/scripts/fantasytalking/README_TRAIN_zh-CN.md +++ b/scripts/fantasytalking/README_TRAIN_zh-CN.md @@ -198,8 +198,8 @@ DeepSpeed-Zero-2 与 FSDP 的区别在于模型权重是否分片。**如果多 ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # 如果没有 RDMA 的多节点训练，取消注释以下两行 # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -343,8 +343,8 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=AudioAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/fantasytalking/train.py \ @@ -401,8 +401,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" scripts/fantasytalking/train.py \ @@ -451,9 +451,9 @@ accelerate launch --mixed_precision="bf16" scripts/fantasytalking/train.py \ **机器 0（Master）**： ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" -export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" # 如果为 None，将使用 $MODEL_NAME/audio_encoder -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Master 机器 IP export MASTER_PORT=10086 export WORLD_SIZE=2 # 总机器数 @@ -503,8 +503,8 @@ accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" export MODEL_NAME_AUDIO="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # 与 Master 相同 export MASTER_PORT=10086 export WORLD_SIZE=2 diff --git a/scripts/flashhead/README_TRAIN_S2V.md b/scripts/flashhead/README_TRAIN_S2V.md index 4454045..ab8cb60 100644 --- a/scripts/flashhead/README_TRAIN_S2V.md +++ b/scripts/flashhead/README_TRAIN_S2V.md @@ -315,8 +315,8 @@ You can configure validation parameters to periodically generate test videos dur ```bash export MODEL_NAME="models/Diffusion_Transformer/SoulX-FlashHead-1_3B" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -363,8 +363,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```bash export MODEL_NAME="models/Diffusion_Transformer/SoulX-FlashHead-1_3B" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 diff --git a/scripts/flashhead/README_TRAIN_S2V_zh-CN.md b/scripts/flashhead/README_TRAIN_S2V_zh-CN.md index 112592c..b880a64 100644 --- a/scripts/flashhead/README_TRAIN_S2V_zh-CN.md +++ b/scripts/flashhead/README_TRAIN_S2V_zh-CN.md @@ -308,8 +308,8 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con ```bash export MODEL_NAME="models/Diffusion_Transformer/SoulX-FlashHead-1_3B" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 和 NCCL_P2P_DISABLE=1 用于无 RDMA 的多机环境 # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -356,8 +356,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```bash export MODEL_NAME="models/Diffusion_Transformer/SoulX-FlashHead-1_3B" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/wav2vec2-base-960h" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 和 NCCL_P2P_DISABLE=1 用于无 RDMA 的多机环境 # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 diff --git a/scripts/hunyuanvideo/README_TRAIN.md b/scripts/hunyuanvideo/README_TRAIN.md index efd9e5f..e31445e 100644 --- a/scripts/hunyuanvideo/README_TRAIN.md +++ b/scripts/hunyuanvideo/README_TRAIN.md @@ -1,99 +1,182 @@ -## Training Code +# HunyuanVideo Full Parameter Training Guide -The default training commands for the different versions are as follows: +This document provides a complete workflow for HunyuanVideo Diffusion Transformer full parameter training, including environment setup, data preparation, distributed training, and inference testing. -We can choose whether to use DeepSpeed and FSDP in HunyuanVideo, which can save a lot of video memory. +> **Note**: HunyuanVideo is a video generation model that supports text-to-video (T2V) and image-to-video (I2V). This document covers the training workflow for normal video generation tasks. -Some parameters in the sh file can be confusing, and they are explained in this document: +--- -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since HunyuanVideo uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +## Table of Contents +- [I. Environment Setup](#i-environment-setup) +- [II. Data Preparation](#ii-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative Path vs Absolute Path Usage](#24-relative-path-vs-absolute-path-usage) +- [III. Full Parameter Training](#iii-full-parameter-training) + - [3.1 Download Pretrained Model](#31-download-pretrained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 Training Parameters Reference](#33-training-parameters-reference) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Training without DeepSpeed and FSDP](#36-training-without-deepspeed-and-fsdp) + - [3.7 Multi-Node Distributed Training](#37-multi-node-distributed-training) +- [IV. Inference Testing](#iv-inference-testing) + - [4.1 Inference Parameters Reference](#41-inference-parameters-reference) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [V. Additional Resources](#v-additional-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +## I. Environment Setup + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -HunyuanVideo T2V without deepspeed: +**Method 2: Manual Installation** -```sh -export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/hunyuanvideo/train.py \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-05 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir2" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --low_vram \ - --train_mode="normal" \ - --trainable_modules "." +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -Hun yuan T2V with Deepspeed Zero-2: +**Method 3: Using Docker** -```sh -export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +When using Docker, please ensure that the graphics card driver and CUDA environment are correctly installed, then execute the following commands: + +```bash +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## II. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official example dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video width and height (**recommended to provide**, used for bucket training. If not provided, they will be automatically read during training, which may affect training speed when data is stored on slow systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height fields from JSON without these fields, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative Path vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, relative paths are recommended. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, absolute paths are recommended. + +--- + +## III. Full Parameter Training + +### 3.1 Download Pretrained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download HunyuanVideo official weights +hf download hunyuanvideo-community/HunyuanVideo --local-dir models/Diffusion_Transformer/HunyuanVideo +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After downloading data as per **2.1 Quick Test Dataset** and weights as per **3.1 Download Pretrained Model**, you can directly copy and run the quick start command. + +We recommend using DeepSpeed-Zero-2 or FSDP for training. Here we use DeepSpeed-Zero-2 as an example. + +The difference between DeepSpeed-Zero-2 and FSDP lies in whether model weights are sharded. **If you run out of GPU memory with multiple GPUs using DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -118,7 +201,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir2" \ + --output_dir="output_dir_hunyuanvideo" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -134,12 +217,98 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --trainable_modules "." ``` -HunyuanVideo T2V with FSDP: +### 3.3 Training Parameters Reference + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/HunyuanVideo` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Number of samples per batch | 16 | +| `--image_sample_size` | Maximum training resolution for images | 512 | +| `--video_sample_size` | Maximum training resolution for videos | 512 | +| `--token_sample_size` | Token sampling size | 512 | +| `--video_sample_stride` | Video sampling stride | 4 | +| `--video_sample_n_frames` | Number of video frames to sample | 17 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader subprocesses | 0 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 500 | +| `--learning_rate` | Initial learning rate | 1e-4 | +| `--lr_scheduler` | Learning rate scheduler | `constant` | +| `--lr_warmup_steps` | Learning rate warmup steps | 500 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_hunyuanvideo` | +| `--gradient_checkpointing` | Enable activation recomputation | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 1e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-08 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 32 | +| `--max_grad_norm` | Gradient clipping threshold | 1.0 | +| `--enable_bucket` | Enable bucket training without cropping, groups by resolution | - | +| `--random_hw_adapt` | Automatically scale images/videos to random sizes within `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `normal` (standard) or `i2v` (image-to-video, note: validation phase not yet implemented) | `normal` | +| `--resume_from_checkpoint` | Resume training from checkpoint, use `"latest"` to auto-select | None | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for validation video generation | `"A young woman..."` | +| `--trainable_modules` | Trainable modules (`"."` means all modules) | `"."` | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: A video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|------|------|--------| +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for validation video generation, space-separated for multiple prompts | Multiple space-separated prompts | + +**Example**: + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A dog is shaking its head, high video quality, very clear view. High quality, masterpiece, best quality, high resolution, ultra-detailed, fantastic." +``` + +**Notes**: +- Validation videos are saved to the `output_dir` directory +- Multi-prompt validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` + +### 3.5 Training with FSDP + +**If you run out of GPU memory with multiple GPUs using DeepSpeed-Zero-2**, you can switch to FSDP for training. ```sh export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -164,7 +333,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir2" \ + --output_dir="output_dir_hunyuanvideo" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -180,18 +349,20 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --trainable_modules "." ``` -HunyuanVideo I2V with FSDP: +### 3.6 Training without DeepSpeed and FSDP + +**This approach is not recommended as it lacks memory-saving backends and may easily cause out-of-memory errors**. We only provide the training shell for reference. ```sh -export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo-I2V" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=HunyuanVideoTokenReplaceTransformerBlock,HunyuanVideoTokenReplaceSingleTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/hunyuanvideo/train.py \ +accelerate launch --mixed_precision="bf16" scripts/hunyuanvideo/train.py \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -210,7 +381,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir2" \ + --output_dir="output_dir_hunyuanvideo" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -222,6 +393,231 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --enable_bucket \ --uniform_sampling \ --low_vram \ - --train_mode="i2v" \ + --train_mode="normal" \ --trainable_modules "." -``` \ No newline at end of file +``` + +### 3.7 Multi-Node Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/hunyuanvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_hunyuanvideo" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Node Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data paths (NFS/shared storage) + +--- + +## IV. Inference Testing + +### 4.1 Inference Parameters Reference + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|------|------|-------| +| `GPU_memory_mode` | VRAM management mode, see table below for options | `sequential_cpu_offload` | +| `ulysses_degree` | Head dimension parallelism, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save VRAM | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolutions) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/HunyuanVideo` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow` | +| `transformer_path` | Path to trained Transformer weights | `None` | +| `vae_path` | Path to trained VAE weights | `None` | +| `lora_path` | LoRA weights path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[832, 480]` (T2V) or `[480, 832]` (I2V) | +| `video_length` | Number of generated video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` if GPU doesn't support bf16 | `torch.bfloat16` | +| `validation_image_start` | Reference image path for image-to-video (I2V mode) | `"asset/1.png"` | +| `prompt` | Positive prompt describing content to generate | `"The dog is shaking head..."` | +| `negative_prompt` | Negative prompt for content to avoid | `"Low resolution, low quality..."` | +| `guidance_scale` | Guidance strength | 1.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 40 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated videos | `samples/hunyuanvideo-videos-i2v` or `samples/hunyuanvideo-videos-t2v` | + +**GPU Memory Mode Descriptions**: + +| Mode | Description | VRAM Usage | +|------|------|---------| +| `model_full_load` | Entire model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Model offloaded to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switched between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offloading (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/hunyuanvideo/predict_t2v.py +``` + +Modify `examples/hunyuanvideo/predict_t2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameters reference above. + +```python +# Choose based on GPU VRAM +GPU_memory_mode = "model_group_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/HunyuanVideo" +# Path to trained weights, e.g., "output_dir_hunyuanvideo/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Write based on content to generate +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/hunyuanvideo/predict_i2v.py +``` + +Modify `examples/hunyuanvideo/predict_i2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameters reference above. + +```python +# Choose based on GPU VRAM +GPU_memory_mode = "model_group_offload" +# Based on actual model path (I2V uses HunyuanVideo-I2V model) +model_name = "models/Diffusion_Transformer/HunyuanVideo-I2V" +# Path to trained weights, e.g., "output_dir_hunyuanvideo/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Starting image for image-to-video +validation_image_start = "asset/1.png" +# Write based on content to generate +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/hunyuanvideo/predict_t2v.py` or `examples/hunyuanvideo/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, if using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be evenly divisible by the model's head count +- `ring_degree` splits along the sequence dimension and affects communication overhead; avoid using it if heads are evenly divisible + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/hunyuanvideo/predict_t2v.py +``` + +--- + +## V. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/hunyuanvideo/README_TRAIN_LORA.md b/scripts/hunyuanvideo/README_TRAIN_LORA.md index c969fdf..c375c7e 100644 --- a/scripts/hunyuanvideo/README_TRAIN_LORA.md +++ b/scripts/hunyuanvideo/README_TRAIN_LORA.md @@ -1,104 +1,182 @@ -## Training Code +# HunyuanVideo LoRA Fine-tuning Training Guide -The default training commands for the different versions are as follows: +This document provides a complete workflow for HunyuanVideo LoRA fine-tuning training, including environment configuration, data preparation, multiple distributed training strategies, and inference testing. -We can choose whether to use DeepSpeed and FSDP in HunyuanVideo, which can save a lot of video memory. +> **Note**: HunyuanVideo is a video generation model that supports text-to-video (T2V) and image-to-video (I2V). This guide covers the LoRA fine-tuning training workflow, suitable for custom dataset fine-tuning scenarios. -Some parameters in the sh file can be confusing, and they are explained in this document: +--- -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since HunyuanVideo uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. -- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. -- `rank` means the dimension of the LoRA update matrices. -- `network_alpha` means the scale of the LoRA update matrices. +## Table of Contents +- [1. Environment Configuration](#1-environment-configuration) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative and Absolute Path Usage](#24-relative-and-absolute-path-usage) +- [3. LoRA Training](#3-lora-training) + - [3.1 Download Pre-trained Model](#31-download-pre-trained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 LoRA-Specific Parameters](#33-lora-specific-parameters) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Training without DeepSpeed and FSDP](#36-training-without-deepspeed-and-fsdp) + - [3.7 Multi-machine Distributed Training](#37-multi-machine-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameters](#41-inference-parameters) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +## 1. Environment Configuration + +**Option 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -HunyuanVideo T2V without deepspeed: +**Option 2: Manual dependency installation** -```sh -export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" -export DATASET_NAME="datasets/internaldatasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/hunyuanvideo/train_lora.py \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-04 \ - --seed=42 \ - --output_dir="output_dir_lora" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --rank=64 \ - --network_alpha=32 \ - --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ - --use_peft_lora \ - --low_vram \ - --train_mode="normal" +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -Hun yuan T2V with Deepspeed Zero-2: +**Option 3: Using Docker** -```sh -export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +When using Docker, please ensure that the GPU driver and CUDA environment are correctly installed, then execute the following commands: + +```bash +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset that contains several training samples. + +```bash +# Download official example dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative path format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute path format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key field descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video width and height (**recommended to provide**, used for bucket training. If not provided, they will be automatically read during training, which may affect training speed when data is stored on slow systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height fields from JSON files without these fields, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative and Absolute Path Usage + +**Relative paths**: + +If your data uses relative paths, set in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute paths**: + +If your data uses absolute paths, set in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, relative paths are recommended. If the dataset is stored on external storage (such as NAS, OSS) or shared across multiple machines, absolute paths are recommended. + +--- + +## 3. LoRA Training + +### 3.1 Download Pre-trained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download HunyuanVideo official weights +hf download hunyuanvideo-community/HunyuanVideo --local-dir models/Diffusion_Transformer/HunyuanVideo +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After downloading data according to **2.1 Quick Test Dataset** and weights according to **3.1 Download Pre-trained Model**, you can directly copy and run the quick start command. + +DeepSpeed-Zero-2 and FSDP are recommended for training. Here we use DeepSpeed-Zero-2 as an example to configure the shell file. + +The difference between DeepSpeed-Zero-2 and FSDP is whether to shard model weights. **If GPU memory is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -121,7 +199,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir_lora" \ + --output_dir="output_dir_hunyuanvideo_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -136,22 +214,110 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --network_alpha=32 \ --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ --use_peft_lora \ - --low_vram \ - --train_mode="normal" + --low_vram ``` -HunyuanVideo T2V with FSDP: +### 3.3 LoRA-Specific Parameters + +**Key LoRA Parameters**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `--pretrained_model_name_or_path` | Pre-trained model path | `models/Diffusion_Transformer/HunyuanVideo` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Number of samples per batch | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sampling size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Number of video frames to sample | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader workers | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (recommended for LoRA) | 1e-04 | +| `--lr_scheduler` | Learning rate scheduler | `constant` | +| `--lr_warmup_steps` | Learning rate warmup steps | 500 | +| `--seed` | Random seed (for reproducible training) | 42 | +| `--output_dir` | Output directory | `output_dir_hunyuanvideo_lora` | +| `--gradient_checkpointing` | Enable gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping, group by resolution | - | +| `--random_hw_adapt` | Auto-resize images/videos to random sizes within `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train by token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling (recommended) | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `normal` (standard) or `i2v` (image-to-video) | `normal` | +| `--resume_from_checkpoint` | Resume training from checkpoint, use `"latest"` to auto-select | None | +| `--rank` | LoRA update matrix dimension (higher rank = more expressive, more memory) | 128 | +| `--network_alpha` | LoRA update matrix scaling factor (usually half of rank or same) | 64 | +| `--target_name` | Components/modules to apply LoRA, comma-separated | `to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2` | +| `--use_peft_lora` | Use PEFT module for LoRA (more memory-efficient) | - | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for validation video generation | `"A young woman..."` | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Notes**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: a video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameters**: + +| Parameter | Description | Recommended Value | +|-----------|-------------|-------------------| +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for validation video generation, space-separated for multiple prompts | Space-separated prompts | + +**Example**: + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A young woman standing on a sunny coastline, white dress gently blowing in the sea breeze." +``` + +**Notes**: +- Validation videos are saved to the `output_dir` directory +- Multi-prompt validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` + +### 3.5 Training with FSDP + +**If GPU memory is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. ```sh export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=HunyuanVideoTransformerBlock,HunyuanVideoSingleTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/hunyuanvideo/train_lora.py \ +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=HunyuanVideoTransformer3DModel --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/hunyuanvideo/train_lora.py \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -168,7 +334,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir_lora" \ + --output_dir="output_dir_hunyuanvideo_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -183,22 +349,23 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --network_alpha=32 \ --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ --use_peft_lora \ - --low_vram \ - --train_mode="normal" + --low_vram ``` -HunyuanVideo I2V with FSDP: +### 3.6 Training without DeepSpeed and FSDP + +**This approach is not recommended because it lacks memory-saving backends and may easily cause out-of-memory errors**. It is provided here for reference only. ```sh -export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo-I2V" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=HunyuanVideoTokenReplaceTransformerBlock,HunyuanVideoTokenReplaceSingleTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/hunyuanvideo/train_lora.py \ +accelerate launch --mixed_precision="bf16" scripts/hunyuanvideo/train_lora.py \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -215,7 +382,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir_lora" \ + --output_dir="output_dir_hunyuanvideo_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -230,6 +397,234 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --network_alpha=32 \ --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ --use_peft_lora \ - --low_vram \ - --train_mode="i2v" -``` \ No newline at end of file + --low_vram +``` + +### 3.7 Multi-machine Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines with 8 GPUs each: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/hunyuanvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_hunyuanvideo_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --low_vram +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-machine Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameters + +**Key Parameters**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `GPU_memory_mode` | Memory management mode, see table below for options | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save memory | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolutions) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/HunyuanVideo` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow` | +| `transformer_path` | Path to load trained Transformer weights | `None` | +| `vae_path` | Path to load trained VAE weights | `None` | +| `lora_path` | LoRA weights path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs without bf16 support | `torch.bfloat16` | +| `validation_image_start` | Reference image path for I2V mode | `"asset/1.png"` | +| `prompt` | Positive prompt, describes generated content | `"The dog is shaking head..."` | +| `negative_prompt` | Negative prompt, content to avoid | `"Low resolution, low quality..."` | +| `guidance_scale` | Guidance strength (HunyuanVideo uses true_cfg_scale) | 1.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 40 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Generated video save path | `samples/hunyuanvideo-videos-i2v` or `samples/hunyuanvideo-videos-t2v` | + +**Memory Management Modes**: + +| Mode | Description | Memory Usage | +|------|-------------|--------------| +| `model_full_load` | Entire model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Model offloaded to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/hunyuanvideo/predict_t2v.py +``` + +Edit `examples/hunyuanvideo/predict_t2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, refer to the inference parameter analysis above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "model_group_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/HunyuanVideo" +# Path to trained weights, e.g., "output_dir_hunyuanvideo_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA weight strength +lora_weight = 0.55 +# Write based on generated content +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/hunyuanvideo/predict_i2v.py +``` + +Edit `examples/hunyuanvideo/predict_i2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, refer to the inference parameter analysis above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "model_group_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/HunyuanVideo" +# LoRA weights path, e.g., "output_dir_hunyuanvideo_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA weight strength +lora_weight = 0.55 +# Starting image for image-to-video +validation_image_start = "asset/1.png" +# Write based on generated content +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, faster inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/hunyuanvideo/predict_t2v.py` or `examples/hunyuanvideo/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be divisible by the model's head count +- `ring_degree` splits along the sequence dimension and affects communication overhead; avoid using it if heads are evenly divisible + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|-----------|---------------|-------------|-------------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/hunyuanvideo/predict_t2v.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/hunyuanvideo/README_TRAIN_LORA_zh-CN.md b/scripts/hunyuanvideo/README_TRAIN_LORA_zh-CN.md new file mode 100644 index 0000000..47393fa --- /dev/null +++ b/scripts/hunyuanvideo/README_TRAIN_LORA_zh-CN.md @@ -0,0 +1,630 @@ +# HunyuanVideo LoRA 微调训练指南 + +本文档提供 HunyuanVideo LoRA 微调训练的完整流程，包括环境配置、数据准备、多种分布式训练策略和推理测试。 + +> **说明**：HunyuanVideo 是支持文生视频（T2V）和图生视频（I2V）的视频生成模型。本指南涵盖 LoRA 微调训练流程，适用于自定义数据集的微调场景。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、LoRA 训练](#三lora-训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 LoRA 专用参数解析](#33-lora-专用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 不使用 DeepSpeed 与 FSDP 训练](#36-不使用-deepspeed-与-fsdp-训练) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、LoRA 训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 HunyuanVideo 官方权重 +hf download hunyuanvideo-community/HunyuanVideo --local-dir models/Diffusion_Transformer/HunyuanVideo +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用 DeepSpeed-Zero-2 与 FSDP 方案进行训练。这里以 DeepSpeed-Zero-2 为例配置 shell 文件。 + +本文中 DeepSpeed-Zero-2 与 FSDP 的差别在于是否对模型权重进行分片，**如果使用多卡且使用 DeepSpeed-Zero-2 的情况下显存不足**，可以切换使用 FSDP 进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/hunyuanvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_hunyuanvideo_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --low_vram +``` + +### 3.3 LoRA 专用参数解析 + +**LoRA 关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/HunyuanVideo` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率（LoRA 推荐值） | 1e-04 | +| `--lr_scheduler` | 学习率调度器 | `constant` | +| `--lr_warmup_steps` | 学习率预热步数 | 500 | +| `--seed` | 随机种子（可复现训练） | 42 | +| `--output_dir` | 输出目录 | `output_dir_hunyuanvideo_lora` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep（推荐启用） | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式：`normal`（普通）或 `i2v`（图生视频） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--rank` | LoRA 更新矩阵的维度（rank 越大表达能力越强，但显存占用越高） | 128 | +| `--network_alpha` | LoRA 更新矩阵的缩放系数（通常设置为 rank 的一半或相同） | 64 | +| `--target_name` | 应用 LoRA 的组件/模块，用逗号分隔 | `to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2` | +| `--use_peft_lora` | 使用 PEFT 模块添加 LoRA（更节省显存） | - | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一位年轻女子..."` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词,可用空格分隔多个提示词 | 多个空格分隔的提示词 | + +**示例**： + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A young woman standing on a sunny coastline, her white dress gently swaying in the sea breeze." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用 DeepSpeed-Zero-2 的情况下显存不足**，可以切换使用 FSDP 进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=HunyuanVideoTransformer3DModel --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/hunyuanvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_hunyuanvideo_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --low_vram +``` + +### 3.6 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练 Shell 用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/hunyuanvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_hunyuanvideo_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --low_vram +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/hunyuanvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_hunyuanvideo_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --low_vram +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/HunyuanVideo` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V 模式） | `"asset/1.png"` | +| `prompt` | 正向提示词，描述生成内容 | `"The dog is shaking head..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"Low resolution, low quality..."` | +| `guidance_scale` | 引导强度（HunyuanVideo使用true_cfg_scale） | 1.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 40 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/hunyuanvideo-videos-i2v` 或 `samples/hunyuanvideo-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/hunyuanvideo/predict_t2v.py +``` + +根据需求修改编辑 `examples/hunyuanvideo/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_group_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/HunyuanVideo" +# 训练好的权重路径，如 "output_dir_hunyuanvideo_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA 权重强度 +lora_weight = 0.55 +# 根据生成内容编写 +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/hunyuanvideo/predict_i2v.py +``` + +根据需求修改编辑 `examples/hunyuanvideo/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_group_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/HunyuanVideo" +# LoRA 权重路径，如 "output_dir_hunyuanvideo_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA 权重强度 +lora_weight = 0.55 +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/hunyuanvideo/predict_t2v.py` 或 `examples/hunyuanvideo/predict_i2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = GPU 数量 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/hunyuanvideo/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/hunyuanvideo/README_TRAIN_zh-CN.md b/scripts/hunyuanvideo/README_TRAIN_zh-CN.md new file mode 100644 index 0000000..46b7ec0 --- /dev/null +++ b/scripts/hunyuanvideo/README_TRAIN_zh-CN.md @@ -0,0 +1,623 @@ +# HunyuanVideo 全参数训练指南 + +本文档提供 HunyuanVideo Diffusion Transformer 全参数训练的完整流程，包括环境配置、数据准备、分布式训练和推理测试。 + +> **说明**：HunyuanVideo 是一个支持文生视频（T2V）和图生视频（I2V）的视频生成模型。本文档涵盖普通视频生成任务的训练流程。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、全量参数训练](#三全量参数训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 训练常用参数解析](#33-训练常用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 不使用 DeepSpeed 与 FSDP 训练](#36-不使用-deepspeed-与-fsdp-训练) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、全量参数训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 HunyuanVideo 官方权重 +hf download hunyuanvideo-community/HunyuanVideo --local-dir models/Diffusion_Transformer/HunyuanVideo +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用DeepSpeed-Zero-2与FSDP方案进行训练。这里使用DeepSpeed-Zero-2为例配置shell文件。 + +本文中DeepSpeed-Zero-2与FSDP的差别在于是否对模型权重进行分片，**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/hunyuanvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_hunyuanvideo" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.3 训练常用参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/HunyuanVideo` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 16 | +| `--image_sample_size` | 图像最大训练分辨率 | 512 | +| `--video_sample_size` | 视频最大训练分辨率 | 512 | +| `--token_sample_size` | Token 采样尺寸 | 512 | +| `--video_sample_stride` | 视频采样步幅 | 4 | +| `--video_sample_n_frames` | 视频采样帧数 | 17 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 0 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 500 | +| `--learning_rate` | 初始学习率 | 1e-4 | +| `--lr_scheduler` | 学习率调度器 | `constant` | +| `--lr_warmup_steps` | 学习率预热步数 | 500 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_hunyuanvideo` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 1e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-08 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 32 | +| `--max_grad_norm` | 梯度裁剪阈值 | 1.0 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式：`normal`（普通）或 `i2v`（图生视频，注：验证阶段尚未实现） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一位年轻女子..."` | +| `--trainable_modules` | 可训练模块（`"."` 表示所有模块） | `"."` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词,可用空格分隔多个提示词 | 多个空格分隔的提示词 | + +**示例**： + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=HunyuanVideoTransformerBlock,HunyuanVideoSingleTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/hunyuanvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_hunyuanvideo" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.6 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/hunyuanvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_hunyuanvideo" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/hunyuanvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_hunyuanvideo" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `sequential_cpu_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/HunyuanVideo` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[832, 480]`（T2V）或 `[480, 832]`（I2V） | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V 模式） | `"asset/1.png"` | +| `prompt` | 正向提示词，描述生成内容 | `"The dog is shaking head..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"Low resolution, low quality..."` | +| `guidance_scale` | 引导强度 | 1.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 40 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/hunyuanvideo-videos-i2v` 或 `samples/hunyuanvideo-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/hunyuanvideo/predict_t2v.py +``` + +根据需求修改编辑 `examples/hunyuanvideo/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_group_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/HunyuanVideo" +# 训练好的权重路径，如 "output_dir_hunyuanvideo/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 根据生成内容编写 +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/hunyuanvideo/predict_i2v.py +``` + +根据需求修改编辑 `examples/hunyuanvideo/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_group_offload" +# 根据实际模型路径（I2V 使用 HunyuanVideo-I2V 模型） +model_name = "models/Diffusion_Transformer/HunyuanVideo-I2V" +# 训练好的权重路径，如 "output_dir_hunyuanvideo/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/hunyuanvideo/predict_t2v.py` 或 `examples/hunyuanvideo/predict_i2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/hunyuanvideo/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/infinitetalk/README_TRAIN_S2V.md b/scripts/infinitetalk/README_TRAIN_S2V.md index a9fa2cd..3eaf964 100644 --- a/scripts/infinitetalk/README_TRAIN_S2V.md +++ b/scripts/infinitetalk/README_TRAIN_S2V.md @@ -155,7 +155,7 @@ If your data uses relative paths, configure the training script as follows: ```bash export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" ``` **Absolute Paths**: @@ -164,7 +164,7 @@ If your data uses absolute paths, configure the training script as follows: ```bash export DATASET_NAME="" -export DATASET_META_NAME="/mnt/data/metadata_control.json" +export DATASET_META_NAME="/mnt/data/metadata.json" ``` > 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, use absolute paths. @@ -201,8 +201,8 @@ The difference between DeepSpeed-Zero-2 and FSDP lies in whether the model weigh ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/chinese-wav2vec2-base/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -254,7 +254,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con | `--pretrained_model_name_or_path` | Path to pretrained model | `models/Diffusion_Transformer/Wan2.1-I2V-14B-480P` | | `--audio_encoder_path` | Audio encoder path (**InfiniteTalk-S2V specific**) | `models/Diffusion_Transformer/chinese-wav2vec2-base/` | | `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | -| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata_control.json` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | | `--train_batch_size` | Samples per batch | 1 | | `--video_sample_size` | Maximum video resolution for training | 640 | | `--token_sample_size` | Token length sampling size | 640 | @@ -323,8 +323,8 @@ You can configure validation parameters to periodically generate test videos dur ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/chinese-wav2vec2-base/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -373,8 +373,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/chinese-wav2vec2-base/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -428,8 +428,8 @@ Assuming 2 machines with 8 GPUs each: ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/chinese-wav2vec2-base/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Master machine IP export MASTER_PORT=10086 export WORLD_SIZE=2 # Total number of machines @@ -480,8 +480,8 @@ accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/chinese-wav2vec2-base/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Same as Master export MASTER_PORT=10086 export WORLD_SIZE=2 diff --git a/scripts/infinitetalk/README_TRAIN_S2V_zh-CN.md b/scripts/infinitetalk/README_TRAIN_S2V_zh-CN.md index cf3a75e..6d8e04a 100644 --- a/scripts/infinitetalk/README_TRAIN_S2V_zh-CN.md +++ b/scripts/infinitetalk/README_TRAIN_S2V_zh-CN.md @@ -196,8 +196,8 @@ DeepSpeed-Zero-2 与 FSDP 的区别在于模型权重是否分片。**如果多 ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/chinese-wav2vec2-base/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # 如果没有 RDMA 的多节点训练，取消注释以下两行 # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -249,7 +249,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con | `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.1-I2V-14B-480P` | | `--audio_encoder_path` | 音频编码器路径（**InfiniteTalk 特有**） | `models/Diffusion_Transformer/chinese-wav2vec2-base/` | | `--train_data_dir` | 训练数据集目录 | `datasets/internal_datasets/` | -| `--train_data_meta` | 训练数据集元数据文件 | `datasets/internal_datasets/metadata_control.json` | +| `--train_data_meta` | 训练数据集元数据文件 | `datasets/internal_datasets/metadata.json` | | `--video_sample_size` | 视频采样尺寸（最大分辨率） | `640` | | `--token_sample_size` | Token 采样尺寸 | `640` | | `--video_sample_stride` | 视频采样步幅 | `1` | @@ -307,8 +307,8 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/chinese-wav2vec2-base/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/infinitetalk/train_s2v.py \ @@ -354,8 +354,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/chinese-wav2vec2-base/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" scripts/infinitetalk/train_s2v.py \ @@ -406,8 +406,8 @@ accelerate launch --mixed_precision="bf16" scripts/infinitetalk/train_s2v.py \ ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/chinese-wav2vec2-base/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Master 机器 IP export MASTER_PORT=10086 export WORLD_SIZE=2 # 总机器数 @@ -458,8 +458,8 @@ accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" export AUDIO_MODEL_NAME="models/Diffusion_Transformer/chinese-wav2vec2-base/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # 与 Master 相同 export MASTER_PORT=10086 export WORLD_SIZE=2 diff --git a/scripts/longcatvideo/README_TRAIN.md b/scripts/longcatvideo/README_TRAIN.md index 1b2672e..cf1a7a3 100644 --- a/scripts/longcatvideo/README_TRAIN.md +++ b/scripts/longcatvideo/README_TRAIN.md @@ -1,100 +1,182 @@ -## Training Code +# LongCat-Video Full Parameter Training Guide -The default training commands for the different versions are as follows: +This document provides a complete workflow for full parameter training of LongCat-Video Diffusion Transformer, including environment configuration, data preparation, distributed training, and inference testing. -We can choose whether to use DeepSpeed and FSDP in LongCatVideo, which can save a lot of video memory. +> **Note**: LongCat-Video is a long video generation model that supports Text-to-Video (T2V) and Image-to-Video (I2V). This document covers the training workflow for general video generation tasks. -Some parameters in the sh file can be confusing, and they are explained in this document: +--- -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +## Table of Contents +- [1. Environment Configuration](#1-environment-configuration) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative vs Absolute Path Usage](#24-relative-vs-absolute-path-usage) +- [3. Full Parameter Training](#3-full-parameter-training) + - [3.1 Download Pretrained Model](#31-download-pretrained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 Common Training Parameters](#33-common-training-parameters) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Training without DeepSpeed and FSDP](#36-training-without-deepspeed-and-fsdp) + - [3.7 Multi-Machine Distributed Training](#37-multi-machine-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameters](#41-inference-parameters) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +## 1. Environment Configuration + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -LongCatVideo without deepspeed: - -```sh -export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/longcatvideo/train.py \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-05 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir_longcat_full_finetune" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --low_vram \ - --train_mode="normal" \ - --trainable_modules "." +**Method 2: Manual Dependency Installation** +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -LongCatVideo with Deepspeed Zero-2: +**Method 3: Using Docker** -```sh -export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +When using Docker, please ensure that the GPU driver and CUDA environment are correctly installed on your machine, then execute the following commands: + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official demo dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video dimensions (**recommended** to provide for bucket training. If not provided, they will be automatically read during training, which may slow down training when data is stored on slow systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height fields for JSON files without them, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, use absolute paths. + +--- + +## 3. Full Parameter Training + +### 3.1 Download Pretrained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download LongCat-Video official weights +modelscope download --model meituan-longcat/LongCat-Video --local_dir models/Diffusion_Transformer/LongCat-Video +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After downloading the data as per **2.1 Quick Test Dataset** and the weights as per **3.1 Download Pretrained Model**, you can directly copy and run the quick start command. + +We recommend using DeepSpeed-Zero-2 or FSDP for training. Here we use DeepSpeed-Zero-2 as an example. + +The difference between DeepSpeed-Zero-2 and FSDP lies in whether model weights are sharded. **If you run out of GPU memory with multiple GPUs using DeepSpeed-Zero-2**, you can switch to FSDP. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -119,7 +201,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir_longcat_full_finetune" \ + --output_dir="output_dir_longcatvideo" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -133,69 +215,100 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --low_vram \ --train_mode="normal" \ --trainable_modules "." - ``` -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. +### 3.3 Common Training Parameters -LongCatVideo with DeepSpeed Zero-3: +**Key Parameter Descriptions**: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +| Parameter | Description | Example Value | +|-----|------|-------| +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/LongCat-Video` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Batch size per GPU | 1 | +| `--image_sample_size` | Maximum image training resolution | 640 | +| `--video_sample_size` | Maximum video training resolution | 640 | +| `--token_sample_size` | Token sample size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Video sampling frames | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch) | 1 | +| `--dataloader_num_workers` | DataLoader worker processes | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate | 2e-05 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_longcatvideo` | +| `--gradient_checkpointing` | Enable gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon | 1e-10 | +| `--vae_mini_batch` | VAE encoding mini-batch size | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping, group by resolution | - | +| `--random_hw_adapt` | Auto-scale images/videos to random size in `[min_size, max_size]` range | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `normal` (standard) or `i2v` (image-to-video) | `normal` | +| `--resume_from_checkpoint` | Resume training path, use `"latest"` to auto-select latest checkpoint | None | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for video generation validation | `"A dog shaking head..."` | +| `--trainable_modules` | Trainable modules (`"."` means all modules) | `"."` | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Guide**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: a video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|------|------|--------| +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for video generation validation, space-separated for multiple prompts | Multiple space-separated prompts | + +**Example**: + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." ``` -Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +**Notes**: +- Validation videos are saved to the `output_dir` directory +- Multiple prompts format: `--validation_prompts "prompt1" "prompt2" "prompt3"` -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/longcatvideo/train.py \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-05 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir_longcat_full_finetune" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --low_vram \ - --train_mode="normal" \ - --trainable_modules "." +### 3.5 Training with FSDP -``` - -LongCatVideo with FSDP: +**If you run out of GPU memory with multiple GPUs using DeepSpeed-Zero-2**, you can switch to FSDP. ```sh export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -220,7 +333,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir_longcat_full_finetune" \ + --output_dir="output_dir_longcatvideo" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -234,5 +347,277 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --low_vram \ --train_mode="normal" \ --trainable_modules "." +``` -``` \ No newline at end of file +### 3.6 Training without DeepSpeed and FSDP + +**This approach is not recommended due to lack of memory-saving backends, which may easily cause out-of-memory errors**. Provided here only for reference. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/longcatvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.7 Multi-Machine Distributed Training + +**Suitable for**: Ultra-large datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/longcatvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Machine Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameters + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|------|------|-------| +| `GPU_memory_mode` | GPU memory mode, see table below for options | `model_group_offload` | +| `ulysses_degree` | Ulysses parallelism degree for multi-GPU inference | 1 | +| `ring_degree` | Ring parallelism degree for multi-GPU inference | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save memory | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective at fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/LongCat-Video` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow` | +| `transformer_path` | Path to trained Transformer weights | `None` | +| `vae_path` | Path to trained VAE weights | `None` | +| `lora_path` | LoRA weights path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of generated frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs without bf16 support | `torch.bfloat16` | +| `validation_image_start` | Reference image path for I2V mode | `"asset/1.png"` | +| `prompt` | Positive prompt describing generated content | `"The dog is shaking head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"Low quality, low resolution..."` | +| `guidance_scale` | Guidance strength | 4.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 25 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated videos | `samples/longcat-videos-i2v` or `samples/longcat-videos-t2v` | + +**GPU Memory Mode Descriptions**: + +| Mode | Description | Memory Usage | +|------|------|---------| +| `model_full_load` | Load entire model to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups offloaded between CPU/CUDA | Low | +| `sequential_cpu_offload` | Sequential offload layer by layer (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run single-GPU inference: + +```bash +python examples/longcatvideo/predict_t2v.py +``` + +Edit `examples/longcatvideo/predict_t2v.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "model_group_offload" +# Your actual model path +model_name = "models/Diffusion_Transformer/LongCat-Video" +# Path to trained weights, e.g., "output_dir_longcatvideo/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Write based on your generation content +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run single-GPU inference: + +```bash +python examples/longcatvideo/predict_i2v.py +``` + +Edit `examples/longcatvideo/predict_i2v.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "model_group_offload" +# Your actual model path +model_name = "models/Diffusion_Transformer/LongCat-Video" +# Path to trained weights, e.g., "output_dir_longcatvideo/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Reference image path +validation_image_start = "asset/1.png" +# Write based on your generation content +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/longcatvideo/predict_t2v.py` or `examples/longcatvideo/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must evenly divide the model's head count +- `ring_degree` splits on the sequence dimension, which affects communication overhead. Try to avoid using it when heads can be evenly divided. + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/longcatvideo/predict_t2v.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/longcatvideo/README_TRAIN_AVATAR.md b/scripts/longcatvideo/README_TRAIN_AVATAR.md index c98282a..8042567 100644 --- a/scripts/longcatvideo/README_TRAIN_AVATAR.md +++ b/scripts/longcatvideo/README_TRAIN_AVATAR.md @@ -164,7 +164,7 @@ If your data uses absolute paths, configure the training script as follows: ```bash export DATASET_NAME="" -export DATASET_META_NAME="/mnt/data/metadata_control.json" +export DATASET_META_NAME="/mnt/data/metadata.json" ``` > 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, use absolute paths. @@ -198,8 +198,8 @@ The difference between DeepSpeed-Zero-2 and FSDP lies in whether the model weigh ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -319,8 +319,8 @@ You can configure validation parameters to periodically generate test videos dur ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -367,8 +367,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -420,8 +420,8 @@ Assuming 2 machines with 8 GPUs each: ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Master machine IP export MASTER_PORT=10086 export WORLD_SIZE=2 # Total number of machines @@ -470,8 +470,8 @@ accelerate launch --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_POR ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Same as Master export MASTER_PORT=10086 export WORLD_SIZE=2 diff --git a/scripts/longcatvideo/README_TRAIN_AVATAR_LORA.md b/scripts/longcatvideo/README_TRAIN_AVATAR_LORA.md index 65bbdbc..924c328 100644 --- a/scripts/longcatvideo/README_TRAIN_AVATAR_LORA.md +++ b/scripts/longcatvideo/README_TRAIN_AVATAR_LORA.md @@ -197,8 +197,8 @@ The difference between DeepSpeed-Zero-2 and FSDP lies in whether the model weigh ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -327,8 +327,8 @@ You can configure validation parameters to periodically generate test videos dur ```sh export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -379,8 +379,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```sh export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -433,8 +433,8 @@ Assuming 2 machines with 8 GPUs each: ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Master machine IP export MASTER_PORT=10086 export WORLD_SIZE=2 # Total number of machines @@ -484,8 +484,8 @@ accelerate launch --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_POR ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Same as Master export MASTER_PORT=10086 export WORLD_SIZE=2 diff --git a/scripts/longcatvideo/README_TRAIN_AVATAR_LORA_zh-CN.md b/scripts/longcatvideo/README_TRAIN_AVATAR_LORA_zh-CN.md index bd2d710..1fc3f05 100644 --- a/scripts/longcatvideo/README_TRAIN_AVATAR_LORA_zh-CN.md +++ b/scripts/longcatvideo/README_TRAIN_AVATAR_LORA_zh-CN.md @@ -197,8 +197,8 @@ DeepSpeed-Zero-2 与 FSDP 的区别在于模型权重是否分片。**如果多 ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 和 NCCL_P2P_DISABLE=1 用于无 RDMA 的多机环境 # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -327,8 +327,8 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con ```sh export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 和 NCCL_P2P_DISABLE=1 用于无 RDMA 的多机环境 # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -379,8 +379,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```sh export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 和 NCCL_P2P_DISABLE=1 用于无 RDMA 的多机环境 # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -433,8 +433,8 @@ accelerate launch --mixed_precision="bf16" scripts/longcatvideo/train_avatar_lor ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Master 机器 IP export MASTER_PORT=10086 export WORLD_SIZE=2 # 总机器数 @@ -484,8 +484,8 @@ accelerate launch --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_POR ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # 与 Master 相同 export MASTER_PORT=10086 export WORLD_SIZE=2 diff --git a/scripts/longcatvideo/README_TRAIN_AVATAR_zh-CN.md b/scripts/longcatvideo/README_TRAIN_AVATAR_zh-CN.md index 7b38552..6ffa6cf 100644 --- a/scripts/longcatvideo/README_TRAIN_AVATAR_zh-CN.md +++ b/scripts/longcatvideo/README_TRAIN_AVATAR_zh-CN.md @@ -193,8 +193,8 @@ DeepSpeed-Zero-2 与 FSDP 的区别在于模型权重是否分片。**如果多 ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # 如果没有 RDMA 的多节点训练，取消注释以下两行 # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -301,8 +301,8 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap LongCatAvatarSingleStreamBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/longcatvideo/train_avatar.py \ @@ -346,8 +346,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" scripts/longcatvideo/train_avatar.py \ @@ -396,8 +396,8 @@ accelerate launch --mixed_precision="bf16" scripts/longcatvideo/train_avatar.py ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Master 机器 IP export MASTER_PORT=10086 export WORLD_SIZE=2 # 总机器数 @@ -446,8 +446,8 @@ accelerate launch --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_POR ```bash export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # 与 Master 相同 export MASTER_PORT=10086 export WORLD_SIZE=2 diff --git a/scripts/longcatvideo/README_TRAIN_LORA.md b/scripts/longcatvideo/README_TRAIN_LORA.md index 586baf5..965de39 100644 --- a/scripts/longcatvideo/README_TRAIN_LORA.md +++ b/scripts/longcatvideo/README_TRAIN_LORA.md @@ -1,102 +1,182 @@ -## Lora Training Code +# LongCat-Video LoRA Fine-tuning Training Guide -We can choose whether to use DeepSpeed and FSDP in LongCatVideo, which can save a lot of video memory. +This document provides a complete guide for LongCat-Video LoRA fine-tuning training, including environment configuration, data preparation, multiple distributed training strategies, and inference testing. -Some parameters in the sh file can be confusing, and they are explained in this document: +> **Note**: LongCat-Video is a video generation model that supports Text-to-Video (T2V) and Image-to-Video (I2V). This guide covers the LoRA fine-tuning training process, suitable for custom dataset fine-tuning scenarios. -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. -- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. -- `rank` means the dimension of the LoRA update matrices. -- `network_alpha` means the scale of the LoRA update matrices. +--- -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +## Table of Contents +- [1. Environment Configuration](#1-environment-configuration) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative Path vs Absolute Path Usage](#24-relative-path-vs-absolute-path-usage) +- [3. LoRA Training](#3-lora-training) + - [3.1 Download Pre-trained Model](#31-download-pre-trained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 LoRA-Specific Parameter Explanation](#33-lora-specific-parameter-explanation) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Training without DeepSpeed and FSDP](#36-training-without-deepspeed-and-fsdp) + - [3.7 Multi-Machine Distributed Training](#37-multi-machine-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameter Explanation](#41-inference-parameter-explanation) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +--- + +## 1. Environment Configuration + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -LongCatVideo without deepspeed: +**Method 2: Manual Installation** -```sh -export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/longcatvideo/train_lora.py \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-04 \ - --seed=42 \ - --output_dir="output_dir_longcat_lora" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --rank=64 \ - --network_alpha=32 \ - --target_name="qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3" \ - --use_peft_lora \ - --train_mode="normal" \ - --low_vram +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -LongCatVideo with Deepspeed Zero-2: +**Method 3: Using Docker** -```sh -export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +When using Docker, please ensure that the GPU driver and CUDA environment are correctly installed on your machine, then execute the following commands: + +```bash +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official example dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video width and height (**recommended to provide**, used for bucket training; if not provided, they will be automatically read during training, which may affect training speed when data is stored on slow systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height from JSON files without these fields, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative Path vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, relative paths are recommended. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, absolute paths are recommended. + +--- + +## 3. LoRA Training + +### 3.1 Download Pre-trained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download LongCat-Video official weights +modelscope download --model meituan-longcat/LongCat-Video --local_dir models/Diffusion_Transformer/LongCat-Video +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +If you have downloaded the data following **Section 2.1 Quick Test Dataset** and the weights following **Section 3.1 Download Pre-trained Model**, you can directly copy the quick start instructions to launch training. + +DeepSpeed-Zero-2 and FSDP are recommended for training. Here we use DeepSpeed-Zero-2 as an example to configure the shell file. + +The difference between DeepSpeed-Zero-2 and FSDP lies in whether model weights are sharded. **If you run out of VRAM when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -119,7 +199,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir_longcat_lora" \ + --output_dir="output_dir_longcatvideo_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -135,68 +215,104 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --target_name="qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3" \ --use_peft_lora \ --train_mode="normal" \ - --low_vram + --low_vram ``` -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. +### 3.3 LoRA-Specific Parameter Explanation -It is known that DeepSpeed Zero-3 is not compatible with PEFT. +**Key LoRA Parameter Descriptions**: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `--pretrained_model_name_or_path` | Pre-trained model path | `models/Diffusion_Transformer/LongCat-Video` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Batch size | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sampling size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Number of video frames to sample | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader worker processes | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (recommended for LoRA) | 1e-04 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed (for reproducibility) | 42 | +| `--output_dir` | Output directory | `output_dir_longcatvideo_lora` | +| `--gradient_checkpointing` | Enable gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini batch size for VAE encoding | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training; trains without cropping images/videos, groups by resolution | - | +| `--random_hw_adapt` | Automatically scale images/videos to random sizes within `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling (recommended) | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `normal` (regular) or `i2v` (image-to-video) | `normal` | +| `--resume_from_checkpoint` | Resume training from checkpoint path; use `"latest"` to auto-select the latest checkpoint | None | +| `--rank` | Dimension of LoRA update matrices (higher rank = stronger表达能力, but more VRAM) | 64 | +| `--network_alpha` | Scaling factor for LoRA update matrices (usually set to half of rank or same) | 32 | +| `--target_name` | Components/modules to apply LoRA, separated by commas | `qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3` | +| `--use_peft_lora` | Use PEFT module to add LoRA (more memory-efficient) | - | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for validation video generation | `"A young woman..."` | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: A video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to regularly generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|-----------|-------------|-------------------| +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for validation video generation, can separate multiple prompts with spaces | Multiple space-separated prompts | + +**Example**: + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A young woman stands on a sunlit coastline, her white dress gently fluttering in the sea breeze." ``` -Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +**Notes**: +- Validation videos will be saved in the `output_dir` directory. +- Multiple prompts validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/longcatvideo/train_lora.py \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-04 \ - --seed=42 \ - --output_dir="output_dir_longcat_lora" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --rank=64 \ - --network_alpha=32 \ - --target_name="qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3" \ - --train_mode="normal" \ - --low_vram -``` +### 3.5 Training with FSDP -LongCatVideo with FSDP: +**If you run out of VRAM when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. ```sh export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -219,7 +335,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir_longcat_lora" \ + --output_dir="output_dir_longcatvideo_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -235,5 +351,284 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --target_name="qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3" \ --use_peft_lora \ --train_mode="normal" \ - --low_vram -``` \ No newline at end of file + --low_vram +``` + +### 3.6 Training without DeepSpeed and FSDP + +**This approach is NOT recommended, as it lacks memory-saving backends and can easily cause VRAM issues**. It is only provided here as a reference. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/longcatvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +### 3.7 Multi-Machine Distributed Training + +**Suitable for**: Ultra-large datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/longcatvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Machine Training Considerations + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data path (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameter Explanation + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `GPU_memory_mode` | VRAM management mode, see table below for options | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism degree, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism degree, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save VRAM | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolutions) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/LongCat-Video` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow` | +| `transformer_path` | Path to load trained Transformer weights | `None` | +| `vae_path` | Path to load trained VAE weights | `None` | +| `lora_path` | LoRA weights path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of generated video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs not supporting bf16 | `torch.bfloat16` | +| `validation_image_start` | Reference image path for Image-to-Video (I2V mode) | `"asset/1.png"` | +| `prompt` | Positive prompt describing generated content | `"The dog is shaking head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"lowres, low quality..."` | +| `guidance_scale` | Guidance strength | 4.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 25 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated video | `samples/longcat-videos-i2v` or `samples/longcat-videos-t2v` | + +**VRAM Management Mode Descriptions**: + +| Mode | Description | VRAM Usage | +|------|-------------|------------| +| `model_full_load` | Load entire model to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/longcatvideo/predict_t2v.py +``` + +Modify `examples/longcatvideo/predict_t2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameter explanation above. + +```python +# Choose based on GPU VRAM +GPU_memory_mode = "model_group_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/LongCat-Video" +# Path to trained weights, e.g., "output_dir_longcatvideo_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA weight strength +lora_weight = 0.55 +# Write based on generated content +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/longcatvideo/predict_i2v.py +``` + +Modify `examples/longcatvideo/predict_i2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameter explanation above. + +```python +# Choose based on GPU VRAM +GPU_memory_mode = "model_group_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/LongCat-Video" +# LoRA weights path, e.g., "output_dir_longcatvideo_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA weight strength +lora_weight = 0.55 +# Starting image for Image-to-Video +validation_image_start = "asset/1.png" +# Write based on generated content +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/longcatvideo/predict_t2v.py` or `examples/longcatvideo/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be evenly divisible by the model's number of heads +- `ring_degree` splits along the sequence dimension and affects communication overhead; try to avoid it when heads are evenly divisible + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|-----------|---------------|-------------|-------------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/longcatvideo/predict_t2v.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/longcatvideo/README_TRAIN_LORA_zh-CN.md b/scripts/longcatvideo/README_TRAIN_LORA_zh-CN.md new file mode 100644 index 0000000..1b5d7a4 --- /dev/null +++ b/scripts/longcatvideo/README_TRAIN_LORA_zh-CN.md @@ -0,0 +1,634 @@ +# LongCat-Video LoRA 微调训练指南 + +本文档提供 LongCat-Video LoRA 微调训练的完整流程，包括环境配置、数据准备、多种分布式训练策略和推理测试。 + +> **说明**：LongCat-Video 是支持文生视频（T2V）和图生视频（I2V）的视频生成模型。本指南涵盖 LoRA 微调训练流程，适用于自定义数据集的微调场景。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、LoRA 训练](#三lora-训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 LoRA 专用参数解析](#33-lora-专用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 不使用 DeepSpeed 与 FSDP 训练](#36-不使用-deepspeed-与-fsdp-训练) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、LoRA 训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 LongCat-Video 官方权重 +modelscope download --model meituan-longcat/LongCat-Video --local_dir models/Diffusion_Transformer/LongCat-Video +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用 DeepSpeed-Zero-2 与 FSDP 方案进行训练。这里以 DeepSpeed-Zero-2 为例配置 shell 文件。 + +本文中 DeepSpeed-Zero-2 与 FSDP 的差别在于是否对模型权重进行分片，**如果使用多卡且使用 DeepSpeed-Zero-2 的情况下显存不足**，可以切换使用 FSDP 进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/longcatvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +### 3.3 LoRA 专用参数解析 + +**LoRA 关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/LongCat-Video` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率（LoRA 推荐值） | 1e-04 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子（可复现训练） | 42 | +| `--output_dir` | 输出目录 | `output_dir_longcatvideo_lora` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep（推荐启用） | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式：`normal`（普通）或 `i2v`（图生视频） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--rank` | LoRA 更新矩阵的维度（rank 越大表达能力越强，但显存占用越高） | 64 | +| `--network_alpha` | LoRA 更新矩阵的缩放系数（通常设置为 rank 的一半或相同） | 32 | +| `--target_name` | 应用 LoRA 的组件/模块，用逗号分隔 | `qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3` | +| `--use_peft_lora` | 使用 PEFT 模块添加 LoRA（更节省显存） | - | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一位年轻女子..."` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词,可用空格分隔多个提示词 | 多个空格分隔的提示词 | + +**示例**： + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="一位年轻女子站在阳光明媚的海岸线上，白裙在轻拂的海风中微微飘动。" +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用 DeepSpeed-Zero-2 的情况下显存不足**，可以切换使用 FSDP 进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=LongCatSingleStreamBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/longcatvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +### 3.6 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练 Shell 用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/longcatvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/longcatvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="qkv,q_linear,kv_linear,ffn.w1,ffn.w2,ffn.w3" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/LongCat-Video` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V 模式） | `"asset/1.png"` | +| `prompt` | 正向提示词，描述生成内容 | `"The dog is shaking head..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度 | 4.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 25 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/longcat-videos-i2v` 或 `samples/longcat-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/longcatvideo/predict_t2v.py +``` + +根据需求修改编辑 `examples/longcatvideo/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_group_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/LongCat-Video" +# 训练好的权重路径，如 "output_dir_longcatvideo_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA 权重强度 +lora_weight = 0.55 +# 根据生成内容编写 +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/longcatvideo/predict_i2v.py +``` + +根据需求修改编辑 `examples/longcatvideo/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_group_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/LongCat-Video" +# LoRA 权重路径，如 "output_dir_longcatvideo_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA 权重强度 +lora_weight = 0.55 +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/longcatvideo/predict_t2v.py` 或 `examples/longcatvideo/predict_i2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = GPU 数量 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/longcatvideo/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/longcatvideo/README_TRAIN_zh-CN.md b/scripts/longcatvideo/README_TRAIN_zh-CN.md new file mode 100644 index 0000000..0d96fb3 --- /dev/null +++ b/scripts/longcatvideo/README_TRAIN_zh-CN.md @@ -0,0 +1,623 @@ +# LongCat-Video 全参数训练指南 + +本文档提供 LongCat-Video Diffusion Transformer 全参数训练的完整流程，包括环境配置、数据准备、分布式训练和推理测试。 + +> **说明**：LongCat-Video 是一个支持文生视频（T2V）和图生视频（I2V）的长视频生成模型。本文档涵盖普通视频生成任务的训练流程。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、全量参数训练](#三全量参数训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 训练常用参数解析](#33-训练常用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 不使用 DeepSpeed 与 FSDP 训练](#36-不使用-deepspeed-与-fsdp-训练) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、全量参数训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 LongCat-Video 官方权重 +modelscope download --model meituan-longcat/LongCat-Video --local_dir models/Diffusion_Transformer/LongCat-Video +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用DeepSpeed-Zero-2与FSDP方案进行训练。这里使用DeepSpeed-Zero-2为例配置shell文件。 + +本文中DeepSpeed-Zero-2与FSDP的差别在于是否对模型权重进行分片，**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/longcatvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.3 训练常用参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/LongCat-Video` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率 | 2e-05 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_longcatvideo` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式：`normal`（普通）或 `i2v`（图生视频） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一位年轻女子..."` | +| `--trainable_modules` | 可训练模块（`"."` 表示所有模块） | `"."` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词,可用空格分隔多个提示词 | 多个空格分隔的提示词 | + +**示例**： + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="一只狗在摇头，视频质量很高，视野很清晰。高质量，杰作，最佳质量，高分辨率，超细节，极好的。" +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=LongCatSingleStreamBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/longcatvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.6 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/longcatvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/longcatvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_longcatvideo" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/LongCat-Video` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V 模式） | `"asset/1.png"` | +| `prompt` | 正向提示词，描述生成内容 | `"The dog is shaking head..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度 | 4.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 25 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/longcat-videos-i2v` 或 `samples/longcat-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/longcatvideo/predict_t2v.py +``` + +根据需求修改编辑 `examples/longcatvideo/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_group_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/LongCat-Video" +# 训练好的权重路径，如 "output_dir_longcatvideo/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 根据生成内容编写 +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/longcatvideo/predict_i2v.py +``` + +根据需求修改编辑 `examples/longcatvideo/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "model_group_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/LongCat-Video" +# 训练好的权重路径，如 "output_dir_longcatvideo/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/longcatvideo/predict_t2v.py` 或 `examples/longcatvideo/predict_i2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/longcatvideo/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/longcatvideo/train_avatar.sh b/scripts/longcatvideo/train_avatar.sh index 1085e11..10aed1f 100644 --- a/scripts/longcatvideo/train_avatar.sh +++ b/scripts/longcatvideo/train_avatar.sh @@ -1,7 +1,7 @@ export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 diff --git a/scripts/longcatvideo/train_avatar_lora.py b/scripts/longcatvideo/train_avatar_lora.py index c767758..ab148aa 100644 --- a/scripts/longcatvideo/train_avatar_lora.py +++ b/scripts/longcatvideo/train_avatar_lora.py @@ -602,12 +602,6 @@ def parse_args(): parser.add_argument( "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", ) - parser.add_argument( - "--noise_share_in_frames", action="store_true", help="Whether enable noise share in frames." - ) - parser.add_argument( - "--noise_share_in_frames_ratio", type=float, default=0.5, help="Noise share ratio.", - ) parser.add_argument( "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." ) diff --git a/scripts/longcatvideo/train_avatar_lora.sh b/scripts/longcatvideo/train_avatar_lora.sh index 38ec921..19bac3f 100644 --- a/scripts/longcatvideo/train_avatar_lora.sh +++ b/scripts/longcatvideo/train_avatar_lora.sh @@ -1,7 +1,7 @@ export MODEL_NAME="models/Diffusion_Transformer/LongCat-Video" export MODEL_NAME_AVATAR="models/Diffusion_Transformer/LongCat-Video-Avatar" export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 diff --git a/scripts/longcatvideo/train_lora.py b/scripts/longcatvideo/train_lora.py index 78bee78..b2991e1 100644 --- a/scripts/longcatvideo/train_lora.py +++ b/scripts/longcatvideo/train_lora.py @@ -547,12 +547,6 @@ def parse_args(): parser.add_argument( "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", ) - parser.add_argument( - "--noise_share_in_frames", action="store_true", help="Whether enable noise share in frames." - ) - parser.add_argument( - "--noise_share_in_frames_ratio", type=float, default=0.5, help="Noise share ratio.", - ) parser.add_argument( "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." ) diff --git a/scripts/turbodiffusion/README_TRAIN_DISTILL.md b/scripts/turbodiffusion/README_TRAIN_DISTILL.md index c2c579d..cc2261e 100755 --- a/scripts/turbodiffusion/README_TRAIN_DISTILL.md +++ b/scripts/turbodiffusion/README_TRAIN_DISTILL.md @@ -1,104 +1,182 @@ -## Training Code +# TurboDiffusion Distill Training Guide -The default training commands for the different versions are as follows: +This document provides a complete workflow for distilling Wan2.1 into TurboWan2.1, including environment configuration, data preparation, distributed training, and inference testing. -We can choose whether to use DeepSpeed and FSDP in Wan distill, which can save a lot of video memory. +> **Note**: TurboDiffusion is a knowledge distillation approach that reduces inference steps (e.g., from 25-50 steps to 4-8 steps) while maintaining video generation quality. -Some parameters in the sh file can be confusing, and they are explained in this document: +--- -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +## Table of Contents +- [1. Environment Configuration](#1-environment-configuration) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative vs Absolute Path Usage](#24-relative-vs-absolute-path-usage) +- [3. Distill Training](#3-distill-training) + - [3.1 Download Pretrained Model](#31-download-pretrained-model) + - [3.2 Quick Start (Without DeepSpeed/FSDP)](#32-quick-start-without-deepspeedfsdp) + - [3.3 Common Training Parameters](#33-common-training-parameters) + - [3.4 Training with DeepSpeed Zero-2](#34-training-with-deepspeed-zero-2) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Training with DeepSpeed Zero-3](#36-training-with-deepspeed-zero-3) + - [3.7 Multi-Machine Distributed Training](#37-multi-machine-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameters](#41-inference-parameters) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +## 1. Environment Configuration + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan distill without deepspeed: +**Method 2: Manual Dependency Installation** -Wan distill without DeepSpeed and FSDP is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/turbodiffusion/train_distill.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-06 \ - --learning_rate_critic=2e-07 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir_distill" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --train_mode="normal" \ - --trainable_modules "." \ - --low_vram +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -Wan distill with Deepspeed Zero-2: +**Method 3: Using Docker** -Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. +When using Docker, please ensure that the GPU driver and CUDA environment are correctly installed on your machine, then execute the following commands: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official demo dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video dimensions (**recommended** to provide for bucket training. If not provided, they will be automatically read during training, which may slow down training when data is stored on slow systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height fields for JSON files without them, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, use absolute paths. + +--- + +## 3. Distill Training + +### 3.1 Download Pretrained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download Wan2.1 official weights +modelscope download --model Wan-AI/Wan2.1-T2V-1.3B --local_dir models/Diffusion_Transformer/Wan2.1-T2V-1.3B +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After downloading the data as per **2.1 Quick Test Dataset** and the weights as per **3.1 Download Pretrained Model**, you can directly copy and run the quick start command. + +DeepSpeed-Zero-2 and FSDP are recommended for training. Here we use DeepSpeed-Zero-2 as an example. + +The difference between DeepSpeed-Zero-2 and FSDP is whether the model weights are sharded. **If you experience insufficient GPU memory when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -141,20 +219,199 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --low_vram ``` +### 3.3 Common Training Parameters + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--config_path` | Config file path | `config/wan2.1/wan_civitai.yaml` | +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B/` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Batch size per GPU | 1 | +| `--image_sample_size` | Maximum image training resolution | 640 | +| `--video_sample_size` | Maximum video training resolution | 640 | +| `--token_sample_size` | Token sample size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Video sampling frames | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch) | 1 | +| `--dataloader_num_workers` | DataLoader worker processes | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (generator) | 2e-06 | +| `--learning_rate_critic` | Initial learning rate (critic) | 2e-07 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_distill` | +| `--gradient_checkpointing` | Enable gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon | 1e-10 | +| `--vae_mini_batch` | VAE encoding mini-batch size | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping, group by resolution | - | +| `--random_hw_adapt` | Auto-scale images/videos to random size in `[min_size, max_size]` range | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `normal` (standard) or `i2v` (image-to-video) | `normal` | +| `--resume_from_checkpoint` | Resume training path, use `"latest"` to auto-select latest checkpoint | None | +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for video generation validation | `"A dog shaking head..."` | +| `--trainable_modules` | Trainable modules (`"."` means all modules) | `"."` | + +**Distill-Specific Parameters**: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--denoising_step_indices_list` | Denoising step list (core distill parameter) | `1000 750 500 250` | +| `--real_guidance_scale` | Real guidance scale for scoring | 6.0 | +| `--fake_guidance_scale` | Fake guidance scale for scoring | 0.0 | +| `--gen_update_interval` | Generator update interval | 5 | +| `--negative_prompt` | Negative prompt for distillation | Chinese negative prompt | +| `--validation_paths` | Validation image paths for I2V mode | Image path list | +| `--train_sampling_steps` | Training sampling steps | 1000 | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Guide**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: a video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +**Other Parameter Explanations**: +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|------|------|--------| +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for video generation validation | English prompts | +| `--validation_paths` | Validation image paths for I2V mode (i2v mode only) | `"asset/1.png"` | + +**Normal Mode Example** (T2V Validation): + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A dog shaking head. The video is of high quality, and the view is very clear." +``` + +**I2V Mode Example** (I2V Validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A dog shaking head. The video is of high quality, and the view is very clear." +``` + +**Notes**: +- Validation videos will be saved to the `output_dir` directory +- Multiple prompts validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` mode must provide `--validation_paths` parameter +- Distill model validation will use the steps defined in `denoising_step_indices_list` for inference + +### 3.5 Training with FSDP + +**If you experience insufficient GPU memory when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/turbodiffusion/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.6 Other Backends + +#### 3.6.1 Training with DeepSpeed-Zero-3 + DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. -Wan distill with DeepSpeed Zero-3: - -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: -```sh +DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```bash python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization ``` Training shell command is as follows: ```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -197,19 +454,20 @@ accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag --low_vram ``` -Wan distill with FSDP: +#### 3.6.2 Training Without DeepSpeed and FSDP + +**This approach is not recommended because without memory-saving backends, it easily causes insufficient GPU memory**. This is only provided as a reference shell for training. -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: ```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/turbodiffusion/train_distill.py \ +accelerate launch --mixed_precision="bf16" scripts/turbodiffusion/train_distill.py \ --config_path="config/wan2.1/wan_civitai.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ @@ -244,4 +502,237 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --train_mode="normal" \ --trainable_modules "." \ --low_vram -``` \ No newline at end of file +``` + +### 3.7 Multi-Machine Distributed Training + +**Suitable for**: Ultra-large datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/turbodiffusion/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Machine Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameters + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|------|------|-------| +| `GPU_memory_mode` | GPU memory mode, see table below for options | `sequential_cpu_offload` | +| `ulysses_degree` | Ulysses parallelism degree for multi-GPU inference | 1 | +| `ring_degree` | Ring parallelism degree for multi-GPU inference | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save memory | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective at fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow` | +| `transformer_path` | Path to trained Transformer weights | `models/Personalized_Model/TurboWan2.1-T2V-1.3B-480P.pth` | +| `vae_path` | Path to trained VAE weights | `None` | +| `lora_path` | LoRA weights path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` | +| `video_length` | Number of generated frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs without bf16 support | `torch.bfloat16` | +| `validation_image_start` | Reference image path for I2V mode | `"asset/1.png"` | +| `prompt` | Positive prompt describing generated content | `"A stylish woman walks..."` | +| `negative_prompt` | Negative prompt to avoid certain content | Chinese negative prompt | +| `guidance_scale` | Guidance strength (distill models typically use 1.0) | 1.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps (typically 4 for distill models) | 4 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated videos | `samples/turbowan-videos-t2v` | + +**GPU Memory Mode Descriptions**: + +| Mode | Description | Memory Usage | +|------|------|---------| +| `model_full_load` | Full model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run single-GPU inference: + +```bash +python examples/turbodiffusion/predict_t2v_wan2.1.py +``` + +Edit `examples/turbodiffusion/predict_t2v_wan2.1.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Your actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# Path to trained weights, e.g., "models/Personalized_Model/TurboWan2.1-T2V-1.3B-480P.pth" +transformer_path = "models/Personalized_Model/TurboWan2.1-T2V-1.3B-480P.pth" +# Write based on your generation content +prompt = "A stylish woman walks down a Tokyo street filled with warm glowing neon and animated city signage..." +# Distill models typically use 4 steps +num_inference_steps = 4 +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run single-GPU inference: + +```bash +python examples/turbodiffusion/predict_i2v_wan2.2.py +``` + +Edit `examples/turbodiffusion/predict_i2v_wan2.2.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Your actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# Path to trained weights +transformer_path = "models/Personalized_Model/TurboWan2.1-T2V-1.3B-480P.pth" +# Reference image path +validation_image_start = "asset/1.png" +# Write based on your generation content +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear..." +# Distill models typically use 4 steps +num_inference_steps = 4 +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/turbodiffusion/predict_t2v_wan2.1.py` or `examples/turbodiffusion/predict_i2v_wan2.2.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must evenly divide the model's head count +- `ring_degree` splits on the sequence dimension, which affects communication overhead. Try to avoid using it when heads can be evenly divided. + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | Single GPU | +| 2 | 2 | 1 | Head parallelism | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/turbodiffusion/predict_t2v_wan2.1.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun +- **TurboDiffusion Paper**: https://arxiv.org/abs/2411.19823 \ No newline at end of file diff --git a/scripts/turbodiffusion/README_TRAIN_DISTILL_zh-CN.md b/scripts/turbodiffusion/README_TRAIN_DISTILL_zh-CN.md new file mode 100755 index 0000000..ed7e1ec --- /dev/null +++ b/scripts/turbodiffusion/README_TRAIN_DISTILL_zh-CN.md @@ -0,0 +1,738 @@ +# TurboDiffusion 蒸馏训练指南 + +本文档提供了将 Wan2.1 蒸馏为 TurboWan2.1 的完整工作流，包括环境配置、数据准备、分布式训练和推理测试。 + +> **注意**: TurboDiffusion 是一种知识蒸馏方法，可以将推理步数从 25-50 步减少到 4-8 步，同时保持视频生成质量。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、蒸馏训练](#三蒸馏训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（不使用 DeepSpeed/FSDP）](#32-快速开始不使用-deepspeedfsdp) + - [3.3 常见训练参数](#33-常见训练参数) + - [3.4 使用 DeepSpeed Zero-2 训练](#34-使用-deepspeed-zero-2-训练) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文本生成视频（T2V）推理](#42-文本生成视频t2v推理) + - [4.3 图像生成视频（I2V）推理](#43-图像生成视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方法 1: 使用 requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方法 2: 手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方法 3: 使用 Docker** + +使用 Docker 时，请确保您的机器已正确安装 GPU 驱动和 CUDA 环境，然后执行以下命令： + +``` +# 拉取镜像 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# 进入容器 +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个包含多个训练样本的测试数据集。 + +```bash +# 下载官方演示数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`: 视频路径（相对路径或绝对路径） +- `text`: 视频描述（英文提示词） +- `type`: 数据类型，固定为 `"video"` +- `width` / `height`: 视频尺寸（**建议**提供以用于 bucket 训练。如果不提供，将在训练时自动读取，当数据存储在 OSS 等慢速系统时可能会降低训练速度）。 + - 您可以使用 `scripts/process_json_add_width_and_height.py` 为没有宽高字段的 JSON 文件提取宽高字段，支持图片和视频。 + - 用法：`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果您的数据使用相对路径，请在训练脚本中配置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果您的数据使用绝对路径，请在训练脚本中配置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，请使用相对路径。如果数据集存储在外部存储（如 NAS、OSS）或在多台机器间共享，请使用绝对路径。 + +--- + +## 三、蒸馏训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.1 官方权重 +modelscope download --model Wan-AI/Wan2.1-T2V-1.3B --local_dir models/Diffusion_Transformer/Wan2.1-T2V-1.3B +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用DeepSpeed-Zero-2与FSDP方案进行训练。这里使用DeepSpeed-Zero-2为例配置shell文件。 + +本文中DeepSpeed-Zero-2与FSDP的差别在于是否对模型权重进行分片，**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/turbodiffusion/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.3 常见训练参数 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--config_path` | 配置文件路径 | `config/wan2.1/wan_civitai.yaml` | +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B/` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元数据文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每个 GPU 的批次大小 | 1 | +| `--image_sample_size` | 最大图像训练分辨率 | 640 | +| `--video_sample_size` | 最大视频训练分辨率 | 640 | +| `--token_sample_size` | Token 样本大小 | 640 | +| `--video_sample_stride` | 视频采样步长 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（有效增加批次） | 1 | +| `--dataloader_num_workers` | DataLoader 工作进程数 | 8 | +| `--num_train_epochs` | 训练轮数 | 100 | +| `--checkpointing_steps` | 每 N 步保存检查点 | 50 | +| `--learning_rate` | 初始学习率（生成器） | 2e-06 | +| `--learning_rate_critic` | 初始学习率（判别器） | 2e-07 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_distill` | +| `--gradient_checkpointing` | 启用梯度检查点 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon | 1e-10 | +| `--vae_mini_batch` | VAE 编码小批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用 bucket 训练，不裁剪，按分辨率分组 | - | +| `--random_hw_adapt` | 自动将图像/视频缩放到 `[min_size, max_size]` 范围内的随机大小 | - | +| `--training_with_video_token_length` | 基于 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀时间步采样 | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式：`normal`（标准）或 `i2v`（图像生成视频） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新检查点 | None | +| `--validation_steps` | 每 N 步运行验证 | 2000 | +| `--validation_epochs` | 每 N 轮运行验证 | 5 | +| `--validation_prompts` | 用于视频生成验证的提示词 | `"A dog shaking head..."` | +| `--trainable_modules` | 可训练模块（`"."` 表示所有模块） | `"."` | + +**蒸馏特有参数**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--denoising_step_indices_list` | 去噪步骤列表（蒸馏核心参数） | `1000 750 500 250` | +| `--real_guidance_scale` | 用于评分的真实 guidance scale | 6.0 | +| `--fake_guidance_scale` | 用于评分的虚拟 guidance scale | 0.0 | +| `--gen_update_interval` | 生成器更新间隔 | 5 | +| `--negative_prompt` | 用于蒸馏的负向提示词 | 中文负向提示词 | +| `--validation_paths` | I2V 模式的验证图像路径 | 图像路径列表 | +| `--train_sampling_steps` | 训练采样步数 | 1000 | + +**样本大小配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率之间的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率之间的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果您不需要任意分辨率进行微调**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 表示 **240P**。 + - **全部设置为 480** 表示 **320P**。 + - **全部设置为 640** 表示 **480P**。 + - **全部设置为 960** 表示 **720P**。 + +**Token 长度训练指南**： +- 当启用 `training_with_video_token_length` 时，模型基于 token 长度进行训练。 +- 例如：分辨率为 512x512 且 49 帧的视频的 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与相应的帧数结合，使模型能够生成不同尺寸的视频。 + +**其他参数说明**： +- `enable_bucket` 用于启用 bucket 训练。启用后，模型不会在中心裁剪图像和视频，而是根据分辨率将它们分组到 bucket 中后对整个图像和视频进行训练。 +- `random_frame_crop` 用于对视频帧进行随机裁剪，以模拟不同帧数的视频。 +- `random_hw_adapt` 用于启用图像和视频的自动高度和宽度缩放。启用 `random_hw_adapt` 后，训练图像的高度和宽度将设置为 `image_sample_size` 作为最大值，`min(video_sample_size, 512)` 作为最小值。对于训练视频，高度和宽度将设置为 `image_sample_size` 作为最大值，`min(video_sample_size, 512)` 作为最小值。 + - 例如，启用 `random_hw_adapt` 后，设置 `video_sample_n_frames=49`、`video_sample_size=1024` 和 `image_sample_size=1024`，训练图像输入的分辨率为 `512x512` 到 `1024x1024`，训练视频输入的分辨率为 `512x512x49` 到 `1024x1024x49`。 + - 例如，启用 `random_hw_adapt` 后，设置 `video_sample_n_frames=49`、`video_sample_size=256` 和 `image_sample_size=1024`，训练图像输入的分辨率为 `256x256` 到 `1024x1024`，训练视频输入的分辨率为 `256x256x49`。 +- `training_with_video_token_length` 指定根据 token 长度训练模型。对于训练图像和视频，高度和宽度将设置为 `image_sample_size` 作为最大值，`video_sample_size` 作为最小值。 + - 例如，启用 `training_with_video_token_length` 后，设置 `video_sample_n_frames=49`、`token_sample_size=1024`、`video_sample_size=256` 和 `image_sample_size=1024`，训练图像输入的分辨率为 `256x256` 到 `1024x1024`，训练视频输入的分辨率为 `256x256x49` 到 `1024x1024x49`。 + - 例如，启用 `training_with_video_token_length` 后，设置 `video_sample_n_frames=49`、`token_sample_size=512`、`video_sample_size=256` 和 `image_sample_size=1024`，训练图像输入的分辨率为 `256x256` 到 `1024x1024`，训练视频输入的分辨率为 `256x256x49` 到 `1024x1024x9`。 + - 分辨率为 512x512 且 49 帧的视频的 token 长度为 13,312。我们需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与相应的长度结合，使模型能够生成不同尺寸的视频。 +- `train_mode` 用于指定训练模式，可以是 normal 或 i2v。由于 Wan 使用 inpaint 模型来实现图像生成视频，因此默认设置为 inpaint 模式。如果您只想实现文本生成视频，可以删除此行，它将默认为文本生成视频模式。 +- `resume_from_checkpoint` 用于设置是否应从先前的检查点恢复训练。使用路径或 `"latest"` 自动选择最后一个可用的检查点。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v 模式） | `"asset/1.png"` | + +**normal 模式示例**（T2V 验证）： + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A dog shaking head. The video is of high quality, and the view is very clear." +``` + +**i2v 模式示例**（I2V 验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A dog shaking head. The video is of high quality, and the view is very clear." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` 模式必须提供 `--validation_paths` 参数 +- 蒸馏模型的验证会使用 `denoising_step_indices_list` 中定义的步数进行推理 + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/turbodiffusion/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/turbodiffusion/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练 Shell 用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/turbodiffusion/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.7 多机分布式训练 + +**适用场景**：超大数据集、更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 个 GPU： + +**机器 0（主节点）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 主节点 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器排名（0 或 1） +# NCCL_IB_DISABLE=1 和 NCCL_P2P_DISABLE=1 用于没有 RDMA 的多节点 +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/turbodiffusion/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +**机器 1（工作节点）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与主节点相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这是 1 +# NCCL_IB_DISABLE=1 和 NCCL_P2P_DISABLE=1 用于没有 RDMA 的多节点 +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 建议使用 RDMA/InfiniBand（高性能） + - 没有 RDMA 时，添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | GPU 显存模式，见下表选项 | `sequential_cpu_offload` | +| `ulysses_degree` | 多 GPU 推理的 Ulysses 并行度 | 1 | +| `ring_degree` | 多 GPU 推理的 Ring 并行度 | 1 | +| `fsdp_dit` | 多 GPU 推理时对 Transformer 使用 FSDP 以节省显存 | `False` | +| `fsdp_text_encoder` | 多 GPU 推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 以加快推理（固定分辨率时有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow` | +| `transformer_path` | 训练好的 Transformer 权重路径 | `models/Personalized_Model/TurboWan2.1-T2V-1.3B-480P.pth` | +| `vae_path` | 训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[height, width]` | `[480, 832]` | +| `video_length` | 生成帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的 GPU 使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | I2V 模式的参考图像路径 | `"asset/1.png"` | +| `prompt` | 描述生成内容的正向提示词 | `"A stylish woman walks..."` | +| `negative_prompt` | 避免某些内容的负向提示词 | 中文负向提示词 | +| `guidance_scale` | 引导强度（蒸馏模型通常使用 1.0） | 1.0 | +| `seed` | 用于可重复性的随机种子 | 43 | +| `num_inference_steps` | 推理步数（蒸馏模型通常为 4） | 4 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 保存生成视频的路径 | `samples/turbowan-videos-t2v` | + +**GPU 显存模式说明**： + +| 模式 | 说明 | 显存使用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文本生成视频（T2V）推理 + +运行单 GPU 推理： + +```bash +python examples/turbodiffusion/predict_t2v_wan2.1.py +``` + +根据您的需要编辑 `examples/turbodiffusion/predict_t2v_wan2.1.py`。首次推理时，请关注以下关键参数。其他参数请参考上方的推理参数说明。 + +```python +# 根据 GPU 显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 您的实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# 训练好的权重路径，例如 "models/Personalized_Model/TurboWan2.1-T2V-1.3B-480P.pth" +transformer_path = "models/Personalized_Model/TurboWan2.1-T2V-1.3B-480P.pth" +# 根据您的生成内容编写 +prompt = "A stylish woman walks down a Tokyo street filled with warm glowing neon and animated city signage..." +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# ... +``` + +### 4.3 图像生成视频（I2V）推理 + +运行单 GPU 推理： + +```bash +python examples/turbodiffusion/predict_i2v_wan2.2.py +``` + +根据您的需要编辑 `examples/turbodiffusion/predict_i2v_wan2.2.py`。首次推理时，请关注以下关键参数。其他参数请参考上方的推理参数说明。 + +```python +# 根据 GPU 显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 您的实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# 训练好的权重路径 +transformer_path = "models/Personalized_Model/TurboWan2.1-T2V-1.3B-480P.pth" +# 参考图像路径 +validation_image_start = "asset/1.png" +# 根据您的生成内容编写 +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear..." +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# ... +``` + +### 4.4 多卡并行推理 + +**适用场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/turbodiffusion/predict_t2v_wan2.1.py` 或 `examples/turbodiffusion/predict_i2v_wan2.2.py`： + +```python +# 确保 ulysses_degree × ring_degree = GPU 数量 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的头数 +- `ring_degree` 在序列维度上拆分，会影响通信开销。当头可以均匀划分时，尽量避免使用它。 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 2 | 2 | 1 | 头并行 | +| 4 | 4 | 1 | 头并行 | +| 8 | 8 | 1 | 头并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多 GPU 推理 + +```bash +torchrun --nproc-per-node=2 examples/turbodiffusion/predict_t2v_wan2.1.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**: https://github.com/aigc-apps/VideoX-Fun +- **TurboDiffusion 论文**: https://arxiv.org/abs/2411.19823 diff --git a/scripts/turbodiffusion/train_distill.py b/scripts/turbodiffusion/train_distill.py index e743b0d..ec68899 100644 --- a/scripts/turbodiffusion/train_distill.py +++ b/scripts/turbodiffusion/train_distill.py @@ -53,7 +53,7 @@ from PIL import Image from torch.distributed.fsdp.fully_sharded_data_parallel import ( FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, ShardedStateDictConfig) -from torch.utils.data import Dataset, RandomSampler +from torch.utils.data import BatchSampler, Dataset, RandomSampler from torch.utils.tensorboard import SummaryWriter from torchvision import transforms from tqdm.auto import tqdm @@ -69,7 +69,7 @@ for project_root in project_roots: from videox_fun.data import (ASPECT_RATIO_512, ASPECT_RATIO_RANDOM_CROP_512, ASPECT_RATIO_RANDOM_CROP_PROB, - AspectRatioBatchImageVideoSampler, BatchSampler, + AspectRatioBatchImageVideoSampler, ImageVideoDataset, ImageVideoSampler, RandomSampler, TextDataset, get_closest_ratio, get_random_mask) diff --git a/scripts/wan2.1/README_TRAIN.md b/scripts/wan2.1/README_TRAIN.md index 63c82f8..3450a07 100755 --- a/scripts/wan2.1/README_TRAIN.md +++ b/scripts/wan2.1/README_TRAIN.md @@ -1,103 +1,185 @@ -## Training Code +# Wan2.1 Full Parameter Training Guide -The default training commands for the different versions are as follows: +This document provides a complete workflow for full parameter training of Wan2.1 Diffusion Transformer, including environment setup, data preparation, distributed training, and inference testing. -We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. +> **Note**: Wan2.1 is a video generation model that supports Text-to-Video (T2V) and Image-to-Video (I2V). This document covers the training workflow for general video generation tasks. -Some parameters in the sh file can be confusing, and they are explained in this document: +--- -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +## Table of Contents +- [1. Environment Setup](#1-environment-setup) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative vs Absolute Path Usage](#24-relative-vs-absolute-path-usage) +- [3. Full Parameter Training](#3-full-parameter-training) + - [3.1 Download Pretrained Model](#31-download-pretrained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 Training Parameter Reference](#33-training-parameter-reference) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-Node Distributed Training](#37-multi-node-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameter Reference](#41-inference-parameter-reference) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +## 1. Environment Setup + +**Option 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan T2V without deepspeed: +**Option 2: Manual Installation** -Wan without DeepSpeed is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/wan2.1/train.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-05 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --low_vram \ - --train_mode="normal" \ - --trainable_modules "." +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -Wan T2V with Deepspeed Zero-2: +**Option 3: Using Docker** -Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. +When using Docker, ensure that the GPU driver and CUDA environment are properly installed on your machine, then execute the following commands: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +```bash +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official demo dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, should be `"video"` +- `width` / `height`: Video width and height (**recommended to provide**, used for bucket training. If not provided, they will be automatically read during training, which may affect training speed when data is stored on slow storage systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height from JSON files without these fields. It supports processing both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, use absolute paths. + +--- + +## 3. Full Parameter Training + +### 3.1 Download Pretrained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download official Wan2.1 weights +# T2V model (Text-to-Video) +modelscope download --model Wan-AI/Wan2.1-T2V-1.3B --local_dir models/Diffusion_Transformer/Wan2.1-T2V-1.3B +# Or I2V model (Image-to-Video) +# modelscope download --model Wan-AI/Wan2.1-I2V-14B-480P --local_dir models/Diffusion_Transformer/Wan2.1-I2V-14B-480P +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After downloading the dataset as in **2.1 Quick Test Dataset** and the pretrained model as in **3.1 Download Pretrained Model**, you can directly copy and run the quick start command. + +We recommend using DeepSpeed-Zero-2 or FSDP for training. Here we use DeepSpeed-Zero-2 as an example. + +The difference between DeepSpeed-Zero-2 and FSDP in this repository is whether the model weights are sharded. **If GPU memory is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -123,7 +205,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.1" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -135,74 +217,114 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --enable_bucket \ --uniform_sampling \ --low_vram \ - --use_deepspeed \ --train_mode="normal" \ --trainable_modules "." ``` -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. +### 3.3 Training Parameter Reference -Wan T2V with DeepSpeed Zero-3: +**Key Parameter Descriptions**: -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Batch size | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sample size | 640 | +| `--video_sample_stride` | Video sample stride | 2 | +| `--video_sample_n_frames` | Number of video frames to sample | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader workers | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate | 2e-05 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.1` | +| `--gradient_checkpointing` | Gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 1 | +| `--max_grad_norm` | Maximum gradient norm for clipping | 0.05 | +| `--enable_bucket` | Enable bucket training without cropping, groups by resolution | - | +| `--random_hw_adapt` | Randomly scale images/videos to `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `normal` or `i2v` | `normal` | +| `--resume_from_checkpoint` | Resume training from checkpoint, use `"latest"` to auto-select | None | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 500 | +| `--validation_prompts` | Prompts for validation video generation | `"A brown dog shaking its head..."` | +| `--validation_paths` | Reference image paths for I2V validation (i2v mode only) | `"asset/1.png"` | +| `--trainable_modules` | Trainable modules (`"."` means all modules) | `"."` | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size for videos; when `random_hw_adapt` is True, it represents the minimum value for both video and image resolutions. +- `image_sample_size` represents the resolution size for images; when `random_hw_adapt` is True, it represents the maximum value for both video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: a video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|-----------|-------------|-------------------| +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 500 | +| `--validation_prompts` | Prompts for validation video generation | English prompts | +| `--validation_paths` | Reference image paths for I2V validation (i2v/inpaint mode only) | `"asset/1.png"` | + +**Normal Mode Example** (T2V validation): + +```bash + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." ``` -Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +**i2v/inpaint Mode Example** (I2V validation): -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-05 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --low_vram \ - --use_deepspeed \ - --train_mode="normal" \ - --trainable_modules "." +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." ``` -Wan T2V with FSDP: +**Notes**: +- Validation videos will be saved to the `output_dir` directory +- Multiple prompts format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` or `inpaint` mode requires the `--validation_paths` parameter + +### 3.5 Training with FSDP + +**If GPU memory is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: ```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -228,7 +350,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.1" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -240,7 +362,338 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --enable_bucket \ --uniform_sampling \ --low_vram \ - --use_deepspeed \ --train_mode="normal" \ --trainable_modules "." -``` \ No newline at end of file +``` + +### 3.6 Other Backends + +#### 3.6.1 Training with DeepSpeed-Zero-3 + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +#### 3.6.2 Training Without DeepSpeed and FSDP + +**This approach is not recommended as it lacks memory-saving backends and may easily cause out-of-memory errors**. Provided here for reference only. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.7 Multi-Node Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines with 8 GPUs each: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note: this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Node Training Notes + +- **Network Requirements**: + - Recommended: RDMA/InfiniBand (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must have access to the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameter Reference + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `GPU_memory_mode` | GPU memory management mode, see table below | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Path to trained Transformer weights | `None` | +| `vae_path` | Path to trained VAE weights | `None` | +| `lora_path` | LoRA weights path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs not supporting bf16 | `torch.bfloat16` | +| `validation_image_start` | Reference image path for I2V mode | `"asset/1.png"` | +| `prompt` | Positive prompt describing content | `"A brown dog shaking its head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"low resolution, low quality..."` | +| `guidance_scale` | Guidance strength | 6.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 50 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated video | `samples/wan-videos-i2v` or `samples/wan-videos-t2v` | + +**GPU Memory Management Modes**: + +| Mode | Description | Memory Usage | +|------|-------------|--------------| +| `model_full_load` | Full model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single GPU inference: + +```bash +python examples/wan2.1/predict_t2v.py +``` + +Edit `examples/wan2.1/predict_t2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, refer to the inference parameter reference above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# Path to trained weights, e.g., "output_dir_wan2.1/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Write based on the content you want to generate +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single GPU inference: + +```bash +python examples/wan2.1/predict_i2v.py +``` + +Edit `examples/wan2.1/predict_i2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, refer to the inference parameter reference above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" +# Path to trained weights, e.g., "output_dir_wan2.1/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Starting image for image-to-video +validation_image_start = "asset/1.png" +# Write based on the content you want to generate +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.1/predict_t2v.py` or `examples/wan2.1/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be divisible by the model's number of heads +- `ring_degree` splits along the sequence dimension and affects communication overhead; avoid using it when heads can be evenly divided + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|-----------|---------------|-------------|-------------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.1/predict_t2v.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun \ No newline at end of file diff --git a/scripts/wan2.1/README_TRAIN_DISTILL.md b/scripts/wan2.1/README_TRAIN_DISTILL.md index 4adb31d..c8e89f9 100755 --- a/scripts/wan2.1/README_TRAIN_DISTILL.md +++ b/scripts/wan2.1/README_TRAIN_DISTILL.md @@ -1,104 +1,185 @@ -## Training Code +# Wan2.1 Distillation Training Guide -The default training commands for the different versions are as follows: +This document provides a complete workflow for distilling Wan2.1 including environment setup, data preparation, distributed training, and inference testing. -We can choose whether to use DeepSpeed and FSDP in Wan distill, which can save a lot of video memory. +> **Note**: Wan2.1 is a video generation model that supports text-to-video (T2V) and image-to-video (I2V). This training code can reduce inference steps from 25-50 to 4-8 steps while maintaining video generation quality. -Some parameters in the sh file can be confusing, and they are explained in this document: +--- -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +## Table of Contents +- [1. Environment Setup](#1-environment-setup) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative vs Absolute Path Usage](#24-relative-vs-absolute-path-usage) +- [3. Distillation Training](#3-distillation-training) + - [3.1 Download Pretrained Models](#31-download-pretrained-models) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 Common Training Parameters](#33-common-training-parameters) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-Node Distributed Training](#37-multi-node-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameters](#41-inference-parameters) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +## 1. Environment Setup + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan distill without deepspeed: +**Method 2: Manual Installation** -Wan distill without DeepSpeed and FSDP is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_distill.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-06 \ - --learning_rate_critic=2e-07 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir_distill" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --train_mode="normal" \ - --trainable_modules "." \ - --low_vram +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -Wan distill with Deepspeed Zero-2: +**Method 3: Using Docker** -Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. +When using Docker, please ensure that your machine has correctly installed GPU drivers and CUDA environment, then execute the following commands: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset that contains several training data samples. + +```bash +# Download official example dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video dimensions (**recommended** to provide for bucket training. If not provided, it will be automatically read during training, which may affect training speed when data is stored on slower systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height fields for JSON files without these fields, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative vs Absolute Path Usage + +**Relative Paths**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Paths**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (such as NAS, OSS) or shared across multiple machines, use absolute paths. + +--- + +## 3. Distillation Training + +### 3.1 Download Pretrained Models + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download Wan2.1 official weights +# T2V model (text-to-video) +modelscope download --model Wan-AI/Wan2.1-T2V-1.3B --local_dir models/Diffusion_Transformer/Wan2.1-T2V-1.3B +# or I2V model (image-to-video) +# modelscope download --model Wan-AI/Wan2.1-I2V-14B-480P --local_dir models/Diffusion_Transformer/Wan2.1-I2V-14B-480P +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After downloading data according to **2.1 Quick Test Dataset** and downloading weights according to **3.1 Download Pretrained Models**, you can directly copy and run the quick start command. + +We recommend using DeepSpeed-Zero-2 and FSDP for training. Here we use DeepSpeed-Zero-2 as an example to configure the shell file. + +The difference between DeepSpeed-Zero-2 and FSDP lies in whether to shard model weights. **If you use multiple GPUs and encounter insufficient GPU memory with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -125,7 +206,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir_distill" \ + --output_dir="output_dir_wan2.1_distill" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -141,69 +222,122 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --low_vram ``` -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. +### 3.3 Common Training Parameters -Wan distill with DeepSpeed Zero-3: +**Key Parameter Descriptions**: -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +| Parameter | Description | Example Value | +|-----|------|-------| +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B/` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Batch size per GPU | 1 | +| `--image_sample_size` | Maximum image training resolution | 640 | +| `--video_sample_size` | Maximum video training resolution | 640 | +| `--token_sample_size` | Token sample size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Number of video frames | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch) | 1 | +| `--dataloader_num_workers` | DataLoader worker processes | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (generator) | 2e-06 | +| `--learning_rate_critic` | Initial learning rate (critic) | 2e-07 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.1_distill` | +| `--gradient_checkpointing` | Enable gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon | 1e-10 | +| `--vae_mini_batch` | VAE encoding mini-batch size | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping, group by resolution | - | +| `--random_hw_adapt` | Auto-scale images/videos to random sizes in `[min_size, max_size]` range | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `normal` (standard) or `i2v` (image-to-video) | `normal` | +| `--resume_from_checkpoint` | Resume training path, use `"latest"` to auto-select latest checkpoint | None | +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for video generation validation | `"A dog shaking head..."` | +| `--validation_paths` | Validation image paths for I2V mode (i2v mode only) | `"asset/1.png"` | +| `--trainable_modules` | Trainable modules (`"."` means all modules) | `"."` | + +**Distillation-Specific Parameters**: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--denoising_step_indices_list` | Denoising step indices list (core distillation parameter) | `1000 750 500 250` | +| `--real_guidance_scale` | Real guidance scale for scoring | 6.0 | +| `--fake_guidance_scale` | Fake guidance scale for scoring | 0.0 | +| `--gen_update_interval` | Generator update interval | 5 | +| `--negative_prompt` | Negative prompt for distillation | Chinese negative prompt | +| `--train_sampling_steps` | Training sampling steps | 1000 | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Guide**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: A video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|------|------|--------| +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for video generation validation | English prompts | +| `--validation_paths` | Validation image paths for I2V (i2v/inpaint mode only) | `"asset/1.png"` | + +**Normal Mode Example** (T2V Validation): + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on the shelf, surrounded by pink flowers. The soft and warm lighting in the room creates a comfortable atmosphere." ``` -Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +**I2V/Inpaint Mode Example** (I2V Validation): -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-06 \ - --learning_rate_critic=2e-07 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir_distill" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --train_mode="normal" \ - --trainable_modules "." \ - --low_vram +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on the shelf, surrounded by pink flowers. The soft and warm lighting in the room creates a comfortable atmosphere." ``` -Wan distill with FSDP: +**Notes**: +- Validation videos will be saved to the `output_dir` directory +- Multi-prompt validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` or `inpaint` mode must provide the `--validation_paths` parameter -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: -```sh +### 3.5 Training with FSDP + +**If you use multiple GPUs and encounter insufficient GPU memory with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -230,7 +364,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir_distill" \ + --output_dir="output_dir_wan2.1_distill" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -244,4 +378,347 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --train_mode="normal" \ --trainable_modules "." \ --low_vram -``` \ No newline at end of file +``` + +### 3.6 Other Backends + +#### 3.6.1 Training with DeepSpeed-Zero-3 + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +#### 3.6.2 Training without DeepSpeed and FSDP + +**This approach is not recommended because there is no memory-saving backend, which can easily cause out-of-memory errors**. We only provide the training shell for reference. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.7 Multi-Node Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Rank of this machine (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Node Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameters + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|------|------|-------| +| `GPU_memory_mode` | GPU memory mode, see options below | `model_group_offload` | +| `ulysses_degree` | Ulysses parallelism degree for multi-GPU inference | 1 | +| `ring_degree` | Ring parallelism degree for multi-GPU inference | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save memory | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective at fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Trained Transformer weight path | `None` or `output_dir_wan2.1_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors` | +| `vae_path` | Trained VAE weight path | `None` | +| `lora_path` | LoRA weight path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` | +| `video_length` | Number of frames to generate | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight dtype, use `torch.float16` for GPUs that don't support bf16 | `torch.bfloat16` | +| `validation_image_start` | Reference image path for I2V mode | `"asset/1.png"` | +| `prompt` | Positive prompt describing what to generate | `"A brown dog shaking its head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"Low resolution, low quality..."` | +| `guidance_scale` | Guidance strength (distillation models typically use 1.0) | 1.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps (typically 4 for distillation models) | 4 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated videos | `samples/wan-videos-t2v` or `samples/wan-videos-i2v` | + +**GPU Memory Mode Descriptions**: + +| Mode | Description | Memory Usage | +|------|------|---------| +| `model_full_load` | Entire model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run single GPU inference: + +```bash +python examples/wan2.1/predict_t2v.py +``` + +Edit `examples/wan2.1/predict_t2v.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, please refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Your actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# Trained weight path, e.g., "output_dir_wan2.1_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Distillation models typically use 4 steps +num_inference_steps = 4 +# Distillation models guidance_scale is typically 1.0 +guidance_scale = 1.0 +# Write according to your generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on the shelf, surrounded by pink flowers. The soft and warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run single GPU inference: + +```bash +python examples/wan2.1/predict_i2v.py +``` + +Edit `examples/wan2.1/predict_i2v.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, please refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Your actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" +# Trained weight path +transformer_path = None +# Distillation models typically use 4 steps +num_inference_steps = 4 +# Distillation models guidance_scale is typically 1.0 +guidance_scale = 1.0 +# Reference image path +validation_image_start = "asset/1.png" +# Write according to your generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on the shelf, surrounded by pink flowers. The soft and warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.1/predict_t2v.py` or `examples/wan2.1/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be divisible by the model's head count +- `ring_degree` splits on the sequence dimension, which affects communication overhead. Try to avoid using it when heads are evenly divisible. + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.1/predict_t2v.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun \ No newline at end of file diff --git a/scripts/wan2.1/README_TRAIN_DISTILL_LORA.md b/scripts/wan2.1/README_TRAIN_DISTILL_LORA.md index 3c3c6ea..ac608e5 100755 --- a/scripts/wan2.1/README_TRAIN_DISTILL_LORA.md +++ b/scripts/wan2.1/README_TRAIN_DISTILL_LORA.md @@ -1,109 +1,185 @@ -## Lora Training Code +# Wan2.1 Distillation LoRA Training Guide -The default training commands for the different versions are as follows: +This document provides a complete workflow for distilling and fine-tuning Wan2.1 with LoRA, including environment setup, data preparation, distributed training, and inference testing. -We can choose whether to use DeepSpeed and FSDP in Wan distill, which can save a lot of video memory. +> **Note**: This training method combines distillation (reducing inference steps) and LoRA (parameter-efficient fine-tuning) technologies. It can reduce inference steps from 25-50 to 4-8 steps with lower VRAM usage while maintaining or improving video generation quality. -Some parameters in the sh file can be confusing, and they are explained in this document: +--- -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. -- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. -- `rank` means the dimension of the LoRA update matrices. -- `network_alpha` means the scale of the LoRA update matrices. +## Table of Contents +- [1. Environment Setup](#1-environment-setup) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative vs Absolute Path Usage](#24-relative-vs-absolute-path-usage) +- [3. Distillation LoRA Training](#3-distillation-lora-training) + - [3.1 Download Pretrained Model](#31-download-pretrained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 Training Parameters Explanation](#33-training-parameters-explanation) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-Node Distributed Training](#37-multi-node-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameters Explanation](#41-inference-parameters-explanation) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. More Resources](#5-more-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +## 1. Environment Setup + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan distill without deepspeed: +**Method 2: Manual Installation** -Wan distill without DeepSpeed and FSDP is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_distill_lora.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-05 \ - --learning_rate_critic=1e-06 \ - --seed=42 \ - --output_dir="output_dir_distill" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --rank=64 \ - --network_alpha=32 \ - --target_name="q,k,v,ffn.0,ffn.2" \ - --use_peft_lora \ - --train_mode="normal" \ - --low_vram +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -Wan distill with Deepspeed Zero-2: +**Method 3: Using Docker** -Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. +When using Docker, please ensure that the graphics card driver and CUDA environment are correctly installed, then execute the following commands: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +``` +# Pull the image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# Enter the container +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download the official demo dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video width and height (**recommended to provide**, used for bucket training. If not provided, it will be automatically read during training, which may affect training speed when data is stored on slower systems like OSS). + - You can use the `scripts/process_json_add_width_and_height.py` file to extract width and height from JSON files without these fields. It supports processing both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, use absolute paths. + +--- + +## 3. Distillation LoRA Training + +### 3.1 Download Pretrained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download official Wan2.1 weights +# T2V model (Text-to-Video) +modelscope download --model Wan-AI/Wan2.1-T2V-1.3B --local_dir models/Diffusion_Transformer/Wan2.1-T2V-1.3B +# Or I2V model (Image-to-Video) +# modelscope download --model Wan-AI/Wan2.1-I2V-14B-480P --local_dir models/Diffusion_Transformer/Wan2.1-I2V-14B-480P +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After following **2.1 Quick Test Dataset** to download data and **3.1 Download Pretrained Model** to download weights, directly copy and run the quick start command. + +It is recommended to use DeepSpeed-Zero-2 or FSDP for training. Here we use DeepSpeed-Zero-2 as an example to configure the shell file. + +The difference between DeepSpeed-Zero-2 and FSDP in this repository is whether model weights are sharded. **If VRAM is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -128,7 +204,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --learning_rate=1e-05 \ --learning_rate_critic=1e-06 \ --seed=42 \ - --output_dir="output_dir_distill" \ + --output_dir="output_dir_wan2.1_distill_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -147,68 +223,137 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --low_vram ``` -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. +### 3.3 Training Parameters Explanation -It is known that DeepSpeed Zero-3 is not compatible with PEFT. +**LoRA-Specific Parameters**: -Wan distill with DeepSpeed Zero-3: +In addition to distillation training, LoRA training adds the following specific parameters: -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +| Parameter | Description | Example Value | +|-----|------|-------| +| `--use_peft_lora` | Whether to use PEFT module to add LoRA, this module saves more VRAM | - | +| `--rank` | Dimension (rank) of LoRA update matrix | 64 | +| `--network_alpha` | Scaling coefficient of LoRA update matrix | 32 | +| `--target_name` | Components/modules where LoRA is applied, comma-separated | `"q,k,v,ffn.0,ffn.2"` | +| `--lora_skip_name` | Modules skipped by LoRA (not trained) | None | + +**LoRA Configuration Recommendations**: +- **rank=64, network_alpha=32**: Suitable for most scenarios, balances quality and VRAM +- **rank=128, network_alpha=64**: Higher quality fine-tuning, but requires more VRAM +- **target_name="q,k,v,ffn.0,ffn.2"**: Fine-tunes attention layers and feed-forward networks, this is a common configuration +- **use_peft_lora**: Strongly recommended to enable, can significantly reduce VRAM usage + +**Key Parameters Explanation**: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B/` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Number of samples per batch | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sampling size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Number of video frames sampled | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch) | 1 | +| `--dataloader_num_workers` | Number of DataLoader subprocesses | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (generator) | 1e-05 | +| `--learning_rate_critic` | Initial learning rate (discriminator) | 1e-06 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.1_distill_lora` | +| `--gradient_checkpointing` | Activation recomputation | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping of images/videos, grouped by resolution | - | +| `--random_hw_adapt` | Automatically scale images/videos to random sizes within `[min_size, max_size]` range | - | +| `--training_with_video_token_length` | Train based on token length, supports any resolution | - | +| `--uniform_sampling` | Uniform timestep sampling | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `normal` (standard) or `i2v` (image-to-video) | `normal` | +| `--resume_from_checkpoint` | Resume training path, use `"latest"` to automatically select the latest checkpoint | None | +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for validation video generation | `"A brown dog shaking its head..."` | +| `--validation_paths` | Reference image paths for I2V validation (i2v mode only) | `"asset/1.png"` | + +**Distillation-Specific Parameters**: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--denoising_step_indices_list` | Denoising step list (core distillation parameter) | `1000 750 500 250` | +| `--real_guidance_scale` | Real guidance scale for scoring | 6.0 | +| `--fake_guidance_scale` | Fake guidance scale for scoring | 0.0 | +| `--gen_update_interval` | Generator update interval | 5 | +| `--negative_prompt` | Negative prompt for distillation | Chinese negative prompt | +| `--train_sampling_steps` | Training sampling steps | 1000 | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the video resolution size; when `random_hw_adapt` is True, it represents the minimum resolution for videos and images. +- `image_sample_size` represents the image resolution size; when `random_hw_adapt` is True, it represents the maximum resolution for videos and images. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Since configurations may cause confusion, **if you don't need arbitrary resolution fine-tuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: a video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, video frames are 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, video frames are 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, video frames are 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These combinations of resolutions and corresponding frame numbers enable the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to regularly generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameters Explanation**: + +| Parameter | Description | Recommended Value | +|------|------|--------| +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for validation video generation | English prompts | +| `--validation_paths` | Reference image paths for I2V validation (i2v/inpaint mode only) | `"asset/1.png"` | + +**normal Mode Example** (T2V Validation): + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="一只棕色的狗摇着头,坐在舒适房间里的浅色沙发上。在狗的后面,架子上有一幅镶框的画,周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" ``` -Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +**i2v/inpaint Mode Example** (I2V Validation): -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill_lora.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-05 \ - --learning_rate_critic=1e-06 \ - --seed=42 \ - --output_dir="output_dir_distill" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --train_mode="normal" \ - --low_vram +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on a shelf surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." ``` -Wan distill with FSDP: +**Notes**: +- Validation videos will be saved to the `output_dir` directory +- Multi-prompt validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` or `inpaint` mode must provide the `--validation_paths` parameter -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: -```sh +### 3.5 Training with FSDP + +If VRAM is insufficient when using multiple GPUs with DeepSpeed-Zero-2, you can switch to FSDP for training. + +```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -233,7 +378,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --learning_rate=1e-05 \ --learning_rate_critic=1e-06 \ --seed=42 \ - --output_dir="output_dir_distill" \ + --output_dir="output_dir_wan2.1_distill_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -250,4 +395,356 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --use_peft_lora \ --train_mode="normal" \ --low_vram -``` \ No newline at end of file +``` + +### 3.6 Other Backends + +#### 3.6.1 Training with DeepSpeed-Zero-3 + +DeepSpeed Zero-3 is not highly recommended at present. In this repository, using FSDP has fewer errors and is more stable. + +It is known that DeepSpeed Zero-3 is incompatible with PEFT. + +DeepSpeed Zero-3 is suitable for high-resolution 14B Wan models. After training, you can use the following command to obtain the final model: +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{your-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +The training shell command is as follows: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --low_vram +``` + +#### 3.6.2 Training without DeepSpeed and FSDP + +**This approach is not recommended because without VRAM-saving backends, it easily causes VRAM shortages**. This is only provided as a reference for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +### 3.7 Multi-Node Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Node Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameters Explanation + +**Key Parameters Explanation**: + +| Parameter | Description | Example Value | +|------|------|-------| +| `GPU_memory_mode` | VRAM management mode, see table below for options | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save VRAM | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer to accelerate inference (effective at fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Path to load trained Transformer weights | `None` or base model weights | +| `vae_path` | Path to load trained VAE weights | `None` | +| `lora_path` | LoRA weights path (distillation LoRA training output) | `output_dir_wan2.1_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of generated video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs that don't support bf16 | `torch.bfloat16` | +| `validation_image_start` | Reference image path for image-to-video (I2V mode) | `"asset/1.png"` | +| `prompt` | Positive prompt, describes the content to generate | `"A brown dog shaking its head..."` | +| `negative_prompt` | Negative prompt, content to avoid | `"Low resolution, low quality..."` | +| `guidance_scale` | Guidance strength (distillation models typically use 1.0) | 1.0 | +| `seed` | Random seed, for reproducing results | 43 | +| `num_inference_steps` | Number of inference steps (distillation models typically use 4) | 4 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated videos | `samples/wan-videos-i2v` or `samples/wan-videos-t2v` | + +**VRAM Management Mode Explanation**: + +| Mode | Description | VRAM Usage | +|------|------|---------| +| `model_full_load` | Entire model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.1/predict_t2v.py +``` + +Edit `examples/wan2.1/predict_t2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameters explanation above. + +```python +# Select based on GPU VRAM +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# Base model weight path (if you have trained full weights) +transformer_path = None +# LoRA weight path, e.g., "output_dir_wan2.1_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors" +lora_path = None +# Distillation models typically use 4 steps +num_inference_steps = 4 +# Distillation models guidance_scale is usually 1.0 +guidance_scale = 1.0 +# LoRA weight strength +lora_weight = 0.55 +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on a shelf surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.1/predict_i2v.py +``` + +Edit `examples/wan2.1/predict_i2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameters explanation above. + +```python +# Select based on GPU VRAM +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" +# Base model weight path (if you have trained full weights) +transformer_path = None +# LoRA weight path, e.g., "output_dir_wan2.1_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors" +lora_path = None +# Distillation models typically use 4 steps +num_inference_steps = 4 +# Distillation models guidance_scale is usually 1.0 +guidance_scale = 1.0 +# LoRA weight strength +lora_weight = 0.55 +# Starting image for image-to-video +validation_image_start = "asset/1.png" +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on a shelf surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.1/predict_t2v.py` or `examples/wan2.1/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be divisible by the model's head count +- `ring_degree` splits along the sequence dimension, which affects communication overhead. Try to avoid using it if heads are evenly divisible + +**Configuration Examples**: + +| Number of GPUs | ulysses_degree | ring_degree | Description | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.1/predict_t2v.py +``` + +--- + +## 5. More Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.1/README_TRAIN_DISTILL_LORA_zh-CN.md b/scripts/wan2.1/README_TRAIN_DISTILL_LORA_zh-CN.md new file mode 100755 index 0000000..4366b29 --- /dev/null +++ b/scripts/wan2.1/README_TRAIN_DISTILL_LORA_zh-CN.md @@ -0,0 +1,750 @@ +# Wan2.1 蒸馏 LoRA 训练指南 + +本文档提供了将 Wan2.1 进行蒸馏 + LoRA 微调的完整工作流,包括环境配置、数据准备、分布式训练和推理测试。 + +> **说明**:本训练方式结合了蒸馏(减少推理步数)和 LoRA(参数高效微调)两种技术,可以在显存占用较低的情况下,将推理步数从 25-50 步减少到 4-8 步,同时保持或提升视频生成质量。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、蒸馏 LoRA 训练](#三蒸馏-lora-训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 训练常用参数解析](#33-训练常用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1:使用 requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2:手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3:使用 docker** + +使用 docker 的情况下,请保证机器中已经正确安装显卡驱动与 CUDA 环境,然后以此执行以下命令: + +``` +# 拉取镜像 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# 进入容器 +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集,其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**(示例格式): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**: +- `file_path`:视频路径(相对或绝对路径) +- `text`:视频描述(英文提示词) +- `type`:数据类型,固定为 `"video"` +- `width` / `height`:视频宽高(**最好提供**,用于分桶训练,如果不提供则自动在训练时读取,当数据存储在如 oss 这样的速度较慢的系统上时,可能会影响训练速度)。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无 width 与 height 字段的 json 进行提取,支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**: + +如果数据的路径为相对路径,则在训练脚本中设置: + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**: + +如果数据的路径为绝对路径,则在训练脚本中设置: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**:如果数据集较小且存储在本地,推荐使用相对路径;如果数据集存储在外部存储(如 NAS、OSS)或多个机器共享存储,推荐使用绝对路径。 + +--- + +## 三、蒸馏 LoRA 训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.1 官方权重 +# T2V 模型(文生视频) +modelscope download --model Wan-AI/Wan2.1-T2V-1.3B --local_dir models/Diffusion_Transformer/Wan2.1-T2V-1.3B +# 或 I2V 模型(图生视频) +# modelscope download --model Wan-AI/Wan2.1-I2V-14B-480P --local_dir models/Diffusion_Transformer/Wan2.1-I2V-14B-480P +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用DeepSpeed-Zero-2与FSDP方案进行训练。这里使用DeepSpeed-Zero-2为例配置shell文件。 + +本文中DeepSpeed-Zero-2与FSDP的差别在于是否对模型权重进行分片，**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +### 3.3 训练常用参数解析 + +**LoRA 特有参数**: + +在蒸馏训练的基础上,LoRA 训练增加了以下特有参数: + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--use_peft_lora` | 是否使用 PEFT 模块添加 LoRA,使用该模块会更节省显存 | - | +| `--rank` | LoRA 更新矩阵的维度(秩) | 64 | +| `--network_alpha` | LoRA 更新矩阵的缩放系数 | 32 | +| `--target_name` | LoRA 应用的组件/模块,以逗号分隔 | `"q,k,v,ffn.0,ffn.2"` | +| `--lora_skip_name` | LoRA 跳过的模块(不训练) | None | + +**LoRA 配置建议**: +- **rank=64, network_alpha=32**:适用于大多数场景,在质量和显存之间取得平衡 +- **rank=128, network_alpha=64**:更高质量的微调,但需要更多显存 +- **target_name="q,k,v,ffn.0,ffn.2"**:对注意力层和前馈网络进行微调,这是常用的配置 +- **use_peft_lora**:强烈建议启用,可以显著降低显存占用 + +**关键参数说明**: + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B/` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数(等效增大 batch) | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率(生成器) | 1e-05 | +| `--learning_rate_critic` | 初始学习率(判别器) | 1e-06 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.1_distill_lora` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度:`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练,不裁剪图片/视频,按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练,支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式:`normal`(普通)或 `i2v`(图生视频) | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径,使用 `"latest"` 自动选择最新 checkpoint | None | +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个 epoch 执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证 I2V 的参考图像路径(仅 i2v 模式) | `"asset/1.png"` | + +**蒸馏特有参数**: + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--denoising_step_indices_list` | 去噪步骤列表(蒸馏核心参数) | `1000 750 500 250` | +| `--real_guidance_scale` | 用于评分的真实 guidance scale | 6.0 | +| `--fake_guidance_scale` | 用于评分的虚拟 guidance scale | 0.0 | +| `--gen_update_interval` | 生成器更新间隔 | 5 | +| `--negative_prompt` | 用于蒸馏的负向提示词 | 中文负向提示词 | +| `--train_sampling_steps` | 训练采样步数 | 1000 | + +**Sample Size 配置指南**: +- `video_sample_size` 表示视频的分辨率大小;当 `random_hw_adapt` 为 True 时,表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小;当 `random_hw_adapt` 为 True 时,表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时,最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆,**如果你不需要任意分辨率进行 finetuning**,建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值,例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**: +- 当启用 `training_with_video_token_length` 时,模型根据 token 长度进行训练。 +- 例如:512x512 分辨率、49 帧的视频,其 token 长度为 13,312,需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下,视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下,视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下,视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合,使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数,在训练过程中定期生成测试视频,以便监控训练进度和模型质量。 + +**验证参数说明**: + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个 epoch 执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证 I2V 的参考图像路径(仅 i2v/inpaint 模式) | `"asset/1.png"` | + +**normal 模式示例**(T2V 验证): + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**i2v/inpaint 模式示例**（I2V 验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**: +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式:`--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` 或 `inpaint` 模式必须提供 `--validation_paths` 参数 + +### 3.5 使用 FSDP 训练 + +如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +已知 DeepSpeed Zero-3 与 PEFT 不兼容。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{your-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --low_vram +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +### 3.7 多机分布式训练 + +**适合场景**:超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器,每台 8 张 GPU: + +**机器 0(Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank(0 或 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +**机器 1(Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**: + - 推荐 RDMA/InfiniBand(高性能) + - 无 RDMA 时添加环境变量: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**:所有机器必须能够访问相同的数据路径(NFS/共享存储) + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**: + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式,可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度,单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度,单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理(固定分辨率下有效) | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `sampler_name` | 采样器类型:`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` 或基础模型权重 | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径(蒸馏 LoRA 训练产出) | `output_dir_wan2.1_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度,不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径(I2V 模式) | `"asset/1.png"` | +| `prompt` | 正向提示词,描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词,避免生成的内容 | `"低分辨率,低画质..."` | +| `guidance_scale` | 引导强度(蒸馏模型通常使用 1.0) | 1.0 | +| `seed` | 随机种子,用于复现结果 | 43 | +| `num_inference_steps` | 推理步数(蒸馏模型通常为 4) | 4 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-i2v` 或 `samples/wan-videos-t2v` | + +**显存管理模式说明**: + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载(速度最慢) | 最低 | + +### 4.2 文生视频(T2V)推理 + +单卡推理运行如下命令: + +```bash +python examples/wan2.1/predict_t2v.py +``` + +根据需求修改编辑 `examples/wan2.1/predict_t2v.py`,初次推理重点关注如下参数,如果对其他参数感兴趣,请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# 基础模型权重路径(如果有训练过的全量权重) +transformer_path = None +# LoRA 权重路径,如 "output_dir_wan2.1_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors" +lora_path = None +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# 蒸馏模型 guidance_scale 通常为 1.0 +guidance_scale = 1.0 +# LoRA 权重强度 +lora_weight = 0.55 +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 图生视频(I2V)推理 + +单卡推理运行如下命令: + +```bash +python examples/wan2.1/predict_i2v.py +``` + +根据需求修改编辑 `examples/wan2.1/predict_i2v.py`,初次推理重点关注如下参数,如果对其他参数感兴趣,请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" +# 基础模型权重路径(如果有训练过的全量权重) +transformer_path = None +# LoRA 权重路径,如 "output_dir_wan2.1_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors" +lora_path = None +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# 蒸馏模型 guidance_scale 通常为 1.0 +guidance_scale = 1.0 +# LoRA 权重强度 +lora_weight = 0.55 +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**:高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.1/predict_t2v.py` 或 `examples/wan2.1/predict_i2v.py`: + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU: +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**: +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分,会影响通信开销,在 head 能整除的情况下尽量不要用 + +**配置示例**: + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.1/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**:https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.1/README_TRAIN_DISTILL_zh-CN.md b/scripts/wan2.1/README_TRAIN_DISTILL_zh-CN.md new file mode 100755 index 0000000..b167ead --- /dev/null +++ b/scripts/wan2.1/README_TRAIN_DISTILL_zh-CN.md @@ -0,0 +1,724 @@ +# Wan2.1 蒸馏训练指南 + +本文档提供了将 Wan2.1 蒸馏的完整工作流，包括环境配置、数据准备、分布式训练和推理测试。 + +> **说明**：Wan2.1 是一个支持文生视频（T2V）和图生视频（I2V）的视频生成模型。该训练代码可以将推理步数从 25-50 步减少到 4-8 步，同时保持视频生成质量。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、蒸馏训练](#三蒸馏训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 训练常用参数解析](#33-训练常用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# 拉取镜像 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# 进入容器 +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、蒸馏训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.1 官方权重 +# T2V 模型（文生视频） +modelscope download --model Wan-AI/Wan2.1-T2V-1.3B --local_dir models/Diffusion_Transformer/Wan2.1-T2V-1.3B +# 或 I2V 模型（图生视频） +# modelscope download --model Wan-AI/Wan2.1-I2V-14B-480P --local_dir models/Diffusion_Transformer/Wan2.1-I2V-14B-480P +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用DeepSpeed-Zero-2与FSDP方案进行训练。这里使用DeepSpeed-Zero-2为例配置shell文件。 + +本文中DeepSpeed-Zero-2与FSDP的差别在于是否对模型权重进行分片，**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.3 训练常用参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B/` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率（生成器） | 2e-06 | +| `--learning_rate_critic` | 初始学习率（判别器） | 2e-07 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.1_distill` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式：`normal`（普通）或 `i2v`（图生视频） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v 模式） | `"asset/1.png"` | +| `--trainable_modules` | 可训练模块（`"."` 表示所有模块） | `"."` | + +**蒸馏特有参数**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--denoising_step_indices_list` | 去噪步骤列表（蒸馏核心参数） | `1000 750 500 250` | +| `--real_guidance_scale` | 用于评分的真实 guidance scale | 6.0 | +| `--fake_guidance_scale` | 用于评分的虚拟 guidance scale | 0.0 | +| `--gen_update_interval` | 生成器更新间隔 | 5 | +| `--negative_prompt` | 用于蒸馏的负向提示词 | 中文负向提示词 | +| `--train_sampling_steps` | 训练采样步数 | 1000 | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v/inpaint 模式） | `"asset/1.png"` | + +**normal 模式示例**（T2V 验证）： + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**i2v/inpaint 模式示例**（I2V 验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` 或 `inpaint` 模式必须提供 `--validation_paths` 参数 + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` 或 `output_dir_wan2.1_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V 模式） | `"asset/1.png"` | +| `prompt` | 正向提示词，描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度（蒸馏模型通常使用 1.0） | 1.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数（蒸馏模型通常为 4） | 4 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-i2v` 或 `samples/wan-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.1/predict_t2v.py +``` + +根据需求修改编辑 `examples/wan2.1/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# 训练好的权重路径，如 "output_dir_wan2.1_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# 蒸馏模型 guidance_scale 通常为 1.0 +guidance_scale = 1.0 +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.1/predict_i2v.py +``` + +根据需求修改编辑 `examples/wan2.1/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" +# 训练好的权重路径，如 "output_dir_wan2.1_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# 蒸馏模型 guidance_scale 通常为 1.0 +guidance_scale = 1.0 +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.1/predict_t2v.py` 或 `examples/wan2.1/predict_i2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.1/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.1/README_TRAIN_LORA.md b/scripts/wan2.1/README_TRAIN_LORA.md index 345ffb9..148c94e 100755 --- a/scripts/wan2.1/README_TRAIN_LORA.md +++ b/scripts/wan2.1/README_TRAIN_LORA.md @@ -1,105 +1,185 @@ -## Lora Training Code +# Wan2.1 LoRA Fine-tuning Training Guide -We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. +This document provides a complete workflow for Wan2.1 LoRA fine-tuning training, including environment configuration, data preparation, various distributed training strategies, and inference testing. -Some parameters in the sh file can be confusing, and they are explained in this document: +> **Note**: Wan2.1 is a video generation model that supports text-to-video (T2V) and image-to-video (I2V). This guide covers the LoRA fine-tuning training workflow, suitable for fine-tuning scenarios with custom datasets. -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. -- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. -- `rank` means the dimension of the LoRA update matrices. -- `network_alpha` means the scale of the LoRA update matrices. +--- -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +## Table of Contents +- [1. Environment Configuration](#1-environment-configuration) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative Path vs Absolute Path Usage](#24-relative-path-vs-absolute-path-usage) +- [3. LoRA Training](#3-lora-training) + - [3.1 Download Pre-trained Model](#31-download-pre-trained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 LoRA-specific Parameter Explanation](#33-lora-specific-parameter-explanation) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-machine Distributed Training](#37-multi-machine-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameter Explanation](#41-inference-parameter-explanation) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. More Resources](#5-more-resources) -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +--- + +## 1. Environment Configuration + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan T2V without deepspeed: +**Method 2: Manual Dependency Installation** -Wan without DeepSpeed is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_lora.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-04 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --rank=64 \ - --network_alpha=32 \ - --target_name="q,k,v,ffn.0,ffn.2" \ - --use_peft_lora \ - --low_vram +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -Wan T2V with Deepspeed Zero-2: +**Method 3: Using Docker** -Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. +When using Docker, please ensure that the graphics card driver and CUDA environment are correctly installed on your machine, then execute the following commands: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official example dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video width and height (**highly recommended to provide**, used for bucket training. If not provided, they will be automatically read during training, which may affect training speed when data is stored on slow systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height fields from JSON files without them, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative Path vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, set in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, set in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Suggestion**: If the dataset is small and stored locally, relative paths are recommended. If the dataset is stored on external storage (such as NAS, OSS) or shared across multiple machines, absolute paths are recommended. + +--- + +## 3. LoRA Training + +### 3.1 Download Pre-trained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download Wan2.1 official weights +# T2V model (text-to-video) +modelscope download --model Wan-AI/Wan2.1-T2V-1.3B --local_dir models/Diffusion_Transformer/Wan2.1-T2V-1.3B +# or I2V model (image-to-video) +# modelscope download --model Wan-AI/Wan2.1-I2V-14B-480P --local_dir models/Diffusion_Transformer/Wan2.1-I2V-14B-480P +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After following **2.1 Quick Test Dataset** and **3.1 Download Pre-trained Model**, you can directly copy the quick start command to launch training. + +DeepSpeed-Zero-2 and FSDP are recommended for training. Here we use DeepSpeed-Zero-2 as an example. + +The difference between DeepSpeed-Zero-2 and FSDP in this repository lies in whether model weights are sharded. **If you encounter insufficient GPU memory when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -123,7 +203,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.1_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -138,69 +218,118 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --network_alpha=32 \ --target_name="q,k,v,ffn.0,ffn.2" \ --use_peft_lora \ - --low_vram + --low_vram \ + --train_mode="normal" ``` -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. +### 3.3 LoRA-specific Parameter Explanation -It is known that DeepSpeed Zero-3 is not compatible with PEFT. +**Key LoRA Parameters**: -Wan T2V with DeepSpeed Zero-3: +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `--config_path` | Configuration file path | `config/wan2.1/wan_civitai.yaml` | +| `--pretrained_model_name_or_path` | Pre-trained model path | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Number of samples per batch | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sampling size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Number of video frames to sample | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader subprocesses | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (recommended for LoRA) | 1e-04 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed (for reproducible training) | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.1_lora` | +| `--gradient_checkpointing` | Activation recomputation | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping, group by resolution | - | +| `--random_hw_adapt` | Auto-scale images/videos to random sizes within `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling (recommended) | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `normal` (text-to-video) or `i2v` (image-to-video) | `normal` | +| `--resume_from_checkpoint` | Resume training path, use `"latest"` to auto-select latest checkpoint | None | +| `--rank` | LoRA update matrix dimension (higher rank = stronger expression but more memory) | 64 | +| `--network_alpha` | LoRA update matrix scaling factor (usually set to half of rank or same) | 32 | +| `--target_name` | Components/modules to apply LoRA, comma-separated | `q,k,v,ffn.0,ffn.2` | +| `--use_peft_lora` | Use PEFT module to add LoRA (more memory-efficient) | - | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for validating video generation | `"A brown dog shaking head..."` | +| `--validation_paths` | Reference image paths for I2V validation (i2v mode only) | `"asset/1.png"` | -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. You must set save_state to True to save the model. After training, you can use the following command to get the final model: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: A video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameters**: + +| Parameter | Description | Recommended Value | +|-----------|-------------|-------------------| +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for video generation validation | English prompts | +| `--validation_paths` | Reference image paths for I2V validation (i2v/inpaint mode only) | `"asset/1.png"` | + +**Normal Mode Example** (T2V validation): + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." ``` -Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +**I2V/Inpaint Mode Example** (I2V validation): -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_lora.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-04 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --low_vram +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." ``` -Wan T2V with FSDP: +**Notes**: +- Validation videos are saved to the `output_dir` directory +- Multi-prompt validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` or `inpaint` mode must provide the `--validation_paths` parameter + +### 3.5 Training with FSDP + +**If you encounter insufficient GPU memory when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: ```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -224,7 +353,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.1_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -239,5 +368,345 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --network_alpha=32 \ --target_name="q,k,v,ffn.0,ffn.2" \ --use_peft_lora \ - --low_vram -``` \ No newline at end of file + --low_vram \ + --train_mode="normal" +``` + +### 3.6 Other Backends + +#### 3.6.1 Training with DeepSpeed-Zero-3 + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +DeepSpeed Zero-3 is suitable for high-resolution 14B Wan. After training, you can use the following command to obtain the final model: +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +#### 3.6.2 Training without DeepSpeed and FSDP + +**This approach is not recommended, as without memory-saving backends, it easily causes out-of-memory errors**. Only provided here for reference. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +### 3.7 Multi-machine Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note: this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-machine Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data path (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameter Explanation + +**Key Parameters**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `GPU_memory_mode` | Memory management mode, see table below for options | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism degree, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism degree, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save memory | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Path to load trained Transformer weights | `None` | +| `vae_path` | Path to load trained VAE weights | `None` | +| `lora_path` | LoRA weights path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs without bf16 support | `torch.bfloat16` | +| `validation_image_start` | Reference image path for image-to-video (I2V mode) | `"asset/1.png"` | +| `prompt` | Positive prompt describing generated content | `"A brown dog shaking head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"low resolution, low quality..."` | +| `guidance_scale` | Guidance strength | 6.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 50 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated videos | `samples/wan-videos-i2v` or `samples/wan-videos-t2v` | + +**Memory Management Modes**: + +| Mode | Description | Memory Usage | +|------|-------------|--------------| +| `model_full_load` | Entire model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.1/predict_t2v.py +``` + +Modify `examples/wan2.1/predict_t2v.py` according to your needs. For first-time inference, focus on the parameters below. If you're interested in other parameters, refer to the inference parameter explanation above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# Path to trained weights, e.g., "output_dir_wan2.1_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA weight strength +lora_weight = 0.55 +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.1/predict_i2v.py +``` + +Modify `examples/wan2.1/predict_i2v.py` according to your needs. For first-time inference, focus on the parameters below. If you're interested in other parameters, refer to the inference parameter explanation above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" +# LoRA weights path, e.g., "output_dir_wan2.1_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA weight strength +lora_weight = 0.55 +# Starting image for image-to-video +validation_image_start = "asset/1.png" +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.1/predict_t2v.py` or `examples/wan2.1/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must evenly divide the model's head count +- `ring_degree` splits along the sequence dimension, which affects communication overhead. Avoid using it when heads can be evenly divided. + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|-----------|----------------|-------------|-------------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.1/predict_t2v.py +``` + +--- + +## 5. More Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.1/README_TRAIN_LORA_zh-CN.md b/scripts/wan2.1/README_TRAIN_LORA_zh-CN.md new file mode 100644 index 0000000..5c98724 --- /dev/null +++ b/scripts/wan2.1/README_TRAIN_LORA_zh-CN.md @@ -0,0 +1,712 @@ +# Wan2.1 LoRA 微调训练指南 + +本文档提供 Wan2.1 LoRA 微调训练的完整流程,包括环境配置、数据准备、多种分布式训练策略和推理测试。 + +> **说明**：Wan2.1 是一个支持文生视频（T2V）和图生视频（I2V）的视频生成模型。本指南涵盖 LoRA 微调训练流程，适用于自定义数据集的微调场景。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、LoRA 训练](#三lora-训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 LoRA 专用参数解析](#33-lora-专用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、LoRA 训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.1 官方权重 +# T2V 模型（文生视频） +modelscope download --model Wan-AI/Wan2.1-T2V-1.3B --local_dir models/Diffusion_Transformer/Wan2.1-T2V-1.3B +# 或 I2V 模型（图生视频） +# modelscope download --model Wan-AI/Wan2.1-I2V-14B-480P --local_dir models/Diffusion_Transformer/Wan2.1-I2V-14B-480P +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用 DeepSpeed-Zero-2 与 FSDP 方案进行训练。这里以 DeepSpeed-Zero-2 为例配置 shell 文件。 + +本文中 DeepSpeed-Zero-2 与 FSDP 的差别在于是否对模型权重进行分片，**如果使用多卡且使用 DeepSpeed-Zero-2 的情况下显存不足**，可以切换使用 FSDP 进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +### 3.3 LoRA 专用参数解析 + +**LoRA 关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--config_path` | 配置文件路径 | `config/wan2.1/wan_civitai.yaml` | +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率（LoRA 推荐值） | 1e-04 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子（可复现训练） | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.1_lora` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep（推荐启用） | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式：`normal`（普通）或 `i2v`（图生视频） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--rank` | LoRA 更新矩阵的维度（rank 越大表达能力越强，但显存占用越高） | 64 | +| `--network_alpha` | LoRA 更新矩阵的缩放系数（通常设置为 rank 的一半或相同） | 32 | +| `--target_name` | 应用 LoRA 的组件/模块，用逗号分隔 | `q,k,v,ffn.0,ffn.2` | +| `--use_peft_lora` | 使用 PEFT 模块添加 LoRA（更节省显存） | - | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v 模式） | `"asset/1.png"` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v/inpaint 模式） | `"asset/1.png"` | + +**normal 模式示例**（T2V 验证）： + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**i2v/inpaint 模式示例**（I2V 验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` 或 `inpaint` 模式必须提供 `--validation_paths` 参数 + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V 模式） | `"asset/1.png"` | +| `prompt` | 正向提示词，描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度 | 6.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 50 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-i2v` 或 `samples/wan-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.1/predict_t2v.py +``` + +根据需求修改编辑 `examples/wan2.1/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# 训练好的权重路径，如 "output_dir_wan2.1_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA 权重强度 +lora_weight = 0.55 +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.1/predict_i2v.py +``` + +根据需求修改编辑 `examples/wan2.1/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" +# LoRA 权重路径，如 "output_dir_wan2.1_lora/checkpoint-xxx/lora_weights.safetensors" +lora_path = None +# LoRA 权重强度 +lora_weight = 0.55 +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.1/predict_t2v.py` 或 `examples/wan2.1/predict_i2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.1/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.1/README_TRAIN_zh-CN.md b/scripts/wan2.1/README_TRAIN_zh-CN.md new file mode 100644 index 0000000..ebff90e --- /dev/null +++ b/scripts/wan2.1/README_TRAIN_zh-CN.md @@ -0,0 +1,699 @@ +# Wan2.1 全参数训练指南 + +本文档提供 Wan2.1 Diffusion Transformer 全参数训练的完整流程，包括环境配置、数据准备、分布式训练和推理测试。 + +> **说明**：Wan2.1 是一个支持文生视频（T2V）和图生视频（I2V）的视频生成模型。本文档涵盖普通视频生成任务的训练流程。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、全量参数训练](#三全量参数训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 训练常用参数解析](#33-训练常用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、全量参数训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.1 官方权重 +# T2V 模型（文生视频） +modelscope download --model Wan-AI/Wan2.1-T2V-1.3B --local_dir models/Diffusion_Transformer/Wan2.1-T2V-1.3B +# 或 I2V 模型（图生视频） +# modelscope download --model Wan-AI/Wan2.1-I2V-14B-480P --local_dir models/Diffusion_Transformer/Wan2.1-I2V-14B-480P +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用DeepSpeed-Zero-2与FSDP方案进行训练。这里使用DeepSpeed-Zero-2为例配置shell文件。 + +本文中DeepSpeed-Zero-2与FSDP的差别在于是否对模型权重进行分片，**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.3 训练常用参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率 | 2e-05 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.1` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式：`normal`（普通）或 `i2v`（图生视频） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 500 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v 模式） | `"asset/1.png"` | +| `--trainable_modules` | 可训练模块（`"."` 表示所有模块） | `"."` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 500 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v/inpaint 模式） | `"asset/1.png"` | + +**normal 模式示例**（T2V 验证）： + +```bash + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**i2v/inpaint 模式示例**（I2V 验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` 或 `inpaint` 模式必须提供 `--validation_paths` 参数 + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.1-T2V-1.3B` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V 模式） | `"asset/1.png"` | +| `prompt` | 正向提示词，描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度 | 6.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 50 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-i2v` 或 `samples/wan-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.1/predict_t2v.py +``` + +根据需求修改编辑 `examples/wan2.1/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +# 训练好的权重路径，如 "output_dir_wan2.1/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.1/predict_i2v.py +``` + +根据需求修改编辑 `examples/wan2.1/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" +# 训练好的权重路径，如 "output_dir_wan2.1/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.1/predict_t2v.py` 或 `examples/wan2.1/predict_i2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.1/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.1/train_distill.py b/scripts/wan2.1/train_distill.py index 531a4a8..17ee068 100644 --- a/scripts/wan2.1/train_distill.py +++ b/scripts/wan2.1/train_distill.py @@ -53,7 +53,7 @@ from PIL import Image from torch.distributed.fsdp.fully_sharded_data_parallel import ( FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, ShardedStateDictConfig) -from torch.utils.data import Dataset, RandomSampler +from torch.utils.data import BatchSampler, Dataset, RandomSampler from torch.utils.tensorboard import SummaryWriter from torchvision import transforms from tqdm.auto import tqdm @@ -69,7 +69,7 @@ for project_root in project_roots: from videox_fun.data import (ASPECT_RATIO_512, ASPECT_RATIO_RANDOM_CROP_512, ASPECT_RATIO_RANDOM_CROP_PROB, - AspectRatioBatchImageVideoSampler, BatchSampler, + AspectRatioBatchImageVideoSampler, ImageVideoDataset, ImageVideoSampler, RandomSampler, TextDataset, get_closest_ratio, get_random_mask) diff --git a/scripts/wan2.1/train_distill_lora.py b/scripts/wan2.1/train_distill_lora.py index 6b1a0f8..e4ae846 100644 --- a/scripts/wan2.1/train_distill_lora.py +++ b/scripts/wan2.1/train_distill_lora.py @@ -53,7 +53,7 @@ from PIL import Image from torch.distributed.fsdp.fully_sharded_data_parallel import ( FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, ShardedStateDictConfig) -from torch.utils.data import Dataset, RandomSampler +from torch.utils.data import BatchSampler, Dataset, RandomSampler from torch.utils.tensorboard import SummaryWriter from torchvision import transforms from tqdm.auto import tqdm @@ -69,7 +69,7 @@ for project_root in project_roots: from videox_fun.data import (ASPECT_RATIO_512, ASPECT_RATIO_RANDOM_CROP_512, ASPECT_RATIO_RANDOM_CROP_PROB, - AspectRatioBatchImageVideoSampler, BatchSampler, + AspectRatioBatchImageVideoSampler, ImageVideoDataset, ImageVideoSampler, RandomSampler, TextDataset, get_closest_ratio, get_random_mask) diff --git a/scripts/wan2.1/train_lora.py b/scripts/wan2.1/train_lora.py index e61a7bb..85f68f3 100755 --- a/scripts/wan2.1/train_lora.py +++ b/scripts/wan2.1/train_lora.py @@ -594,12 +594,6 @@ def parse_args(): parser.add_argument( "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", ) - parser.add_argument( - "--noise_share_in_frames", action="store_true", help="Whether enable noise share in frames." - ) - parser.add_argument( - "--noise_share_in_frames_ratio", type=float, default=0.5, help="Noise share ratio.", - ) parser.add_argument( "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." ) diff --git a/scripts/wan2.1_fun/README_TRAIN.md b/scripts/wan2.1_fun/README_TRAIN.md index d3ddd7b..842ec1e 100755 --- a/scripts/wan2.1_fun/README_TRAIN.md +++ b/scripts/wan2.1_fun/README_TRAIN.md @@ -1,101 +1,185 @@ -## Training Code +# Wan2.1 Fun Full Parameter Training Guide -We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. +This document provides a complete workflow for full parameter training of Wan2.1 Fun Diffusion Transformer, including environment setup, data preparation, distributed training, and inference testing. -Some parameters in the sh file can be confusing, and they are explained in this document: +> **Note**: Wan2.1 Fun is a video generation model that supports Text-to-Video (T2V), Image-to-Video (I2V), and Video Inpainting. This document covers the training workflow for both normal video generation and video inpainting tasks. -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or inpaint. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +--- -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +## Table of Contents +- [1. Environment Setup](#1-environment-setup) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative vs Absolute Path Usage](#24-relative-vs-absolute-path-usage) +- [3. Full Parameter Training](#3-full-parameter-training) + - [3.1 Download Pretrained Model](#31-download-pretrained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 Training Parameter Reference](#33-training-parameter-reference) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-Node Distributed Training](#37-multi-node-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameter Reference](#41-inference-parameter-reference) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +--- + +## 1. Environment Setup + +**Option 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan T2V without deepspeed: +**Option 2: Manual Installation** -Wan without DeepSpeed is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-05 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --low_vram \ - --train_mode="normal" \ - --trainable_modules "." +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -Wan T2V with Deepspeed Zero-2: +**Option 3: Using Docker** -Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. +When using Docker, ensure that the GPU driver and CUDA environment are properly installed on your machine, then execute the following commands: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +```bash +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official demo dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, should be `"video"` +- `width` / `height`: Video width and height (**recommended to provide**, used for bucket training. If not provided, they will be automatically read during training, which may affect training speed when data is stored on slow storage systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height from JSON files without these fields. It supports processing both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, use absolute paths. + +--- + +## 3. Full Parameter Training + +### 3.1 Download Pretrained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download official Wan2.1 Fun weights +# Inpaint model (Video Inpainting) +modelscope download --model PAI/Wan2.1-Fun-V1.1-1.3B-InP --local_dir models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP + +# modelscope download --model PAI/Wan2.1-Fun-V1.1-14B-InP --local_dir models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After downloading the dataset as in **2.1 Quick Test Dataset** and the pretrained model as in **3.1 Download Pretrained Model**, you can directly copy and run the quick start command. + +We recommend using DeepSpeed-Zero-2 or FSDP for training. Here we use DeepSpeed-Zero-2 as an example. + +The difference between DeepSpeed-Zero-2 and FSDP in this repository is whether the model weights are sharded. **If GPU memory is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -121,7 +205,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.1_fun" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -133,74 +217,106 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --enable_bucket \ --uniform_sampling \ --low_vram \ - --use_deepspeed \ --train_mode="inpaint" \ --trainable_modules "." ``` -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. +### 3.3 Training Parameter Reference -Wan T2V with DeepSpeed Zero-3: +**Key Parameter Descriptions**: -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Batch size | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sample size | 640 | +| `--video_sample_stride` | Video sample stride | 2 | +| `--video_sample_n_frames` | Number of video frames to sample | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader workers | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate | 2e-05 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.1_fun` | +| `--gradient_checkpointing` | Gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 1 | +| `--max_grad_norm` | Maximum gradient norm for clipping | 0.05 | +| `--enable_bucket` | Enable bucket training without cropping, groups by resolution | - | +| `--random_hw_adapt` | Randomly scale images/videos to `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `inpaint` (video inpainting) | `inpaint` | +| `--resume_from_checkpoint` | Resume training from checkpoint, use `"latest"` to auto-select | None | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 500 | +| `--validation_prompts` | Prompts for validation video generation | `"A brown dog shaking its head..."` | +| `--validation_paths` | Reference image paths for inpaint validation (inpaint mode only) | `"asset/1.png"` | +| `--trainable_modules` | Trainable modules (`"."` means all modules) | `"."` | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size for videos; when `random_hw_adapt` is True, it represents the minimum value for both video and image resolutions. +- `image_sample_size` represents the resolution size for images; when `random_hw_adapt` is True, it represents the maximum value for both video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: a video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|-----------|-------------|-------------------| +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 500 | +| `--validation_prompts` | Prompts for validation video generation | English prompts | +| `--validation_paths` | Reference image paths for inpaint validation (inpaint mode only) | `"asset/1.png"` | + +**Inpaint Mode Example** (Video Inpainting Validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." ``` -Training shell command is as follows: +**Notes**: +- Validation videos will be saved to the `output_dir` directory +- Multiple prompts format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- `inpaint` mode requires the `--validation_paths` parameter + +### 3.5 Training with FSDP + +**If GPU memory is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + ```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-05 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --low_vram \ - --use_deepspeed \ - --train_mode="inpaint" \ - --trainable_modules "." -``` - -Wan T2V with FSDP: - -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -226,7 +342,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.1_fun" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -238,7 +354,341 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --enable_bucket \ --uniform_sampling \ --low_vram \ - --use_deepspeed \ --train_mode="inpaint" \ --trainable_modules "." -``` \ No newline at end of file +``` + +### 3.6 Other Backends + +#### 3.6.1 Training with DeepSpeed-Zero-3 + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +#### 3.6.2 Training Without DeepSpeed and FSDP + +**This approach is not recommended as it lacks memory-saving backends and may easily cause out-of-memory errors**. Provided here for reference only. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.7 Multi-Node Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines with 8 GPUs each: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note: this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Node Training Notes + +- **Network Requirements**: + - Recommended: RDMA/InfiniBand (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must have access to the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameter Reference + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `GPU_memory_mode` | GPU memory management mode, see table below | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Path to trained Transformer weights | `None` | +| `vae_path` | Path to trained VAE weights | `None` | +| `lora_path` | LoRA weights path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs not supporting bf16 | `torch.bfloat16` | +| `validation_image_start` | Reference image path for inpaint mode | `"asset/1.png"` | +| `validation_image_end` | Target image path for inpaint mode (set to `None` for pure I2V) | `None` | +| `prompt` | Positive prompt describing content | `"A brown dog shaking its head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"low resolution, low quality..."` | +| `guidance_scale` | Guidance strength | 6.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 50 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated video | `samples/wan-videos-fun-t2v` or `samples/wan-videos-fun-i2v` | + +**GPU Memory Management Modes**: + +| Mode | Description | Memory Usage | +|------|-------------|--------------| +| `model_full_load` | Full model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single GPU inference: + +```bash +python examples/wan2.1_fun/predict_t2v.py +``` + +Edit `examples/wan2.1_fun/predict_t2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, refer to the inference parameter reference above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +# Path to trained weights, e.g., "output_dir_wan2.1_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Write based on the content you want to generate +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single GPU inference: + +```bash +python examples/wan2.1_fun/predict_i2v.py +``` + +Edit `examples/wan2.1_fun/predict_i2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, refer to the inference parameter reference above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +# Path to trained weights, e.g., "output_dir_wan2.1_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Starting image for image-to-video / inpaint +validation_image_start = "asset/1.png" +# Target image for inpaint (set to None for pure image-to-video) +validation_image_end = None +# Write based on the content you want to generate +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.1_fun/predict_t2v.py` or `examples/wan2.1_fun/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be divisible by the model's number of heads +- `ring_degree` splits along the sequence dimension and affects communication overhead; avoid using it when heads can be evenly divided + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|-----------|---------------|-------------|-------------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.1_fun/predict_t2v.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun \ No newline at end of file diff --git a/scripts/wan2.1_fun/README_TRAIN_LORA.md b/scripts/wan2.1_fun/README_TRAIN_LORA.md index 19c1385..c899cf5 100755 --- a/scripts/wan2.1_fun/README_TRAIN_LORA.md +++ b/scripts/wan2.1_fun/README_TRAIN_LORA.md @@ -1,106 +1,185 @@ -## Lora Training Code +# Wan2.1 Fun LoRA Fine-tuning Training Guide -We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. +This document provides a complete workflow for Wan2.1 Fun LoRA fine-tuning training, including environment configuration, data preparation, various distributed training strategies, and inference testing. -Some parameters in the sh file can be confusing, and they are explained in this document: +> **Note**: Wan2.1 Fun is a video generation model that supports Text-to-Video (T2V), Image-to-Video (I2V), and Video Inpainting. This guide covers the LoRA fine-tuning training workflow for both normal video generation and video inpainting tasks. -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or inpaint. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. -- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. -- `rank` means the dimension of the LoRA update matrices. -- `network_alpha` means the scale of the LoRA update matrices. +--- -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +## Table of Contents +- [1. Environment Configuration](#1-environment-configuration) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative Path vs Absolute Path Usage](#24-relative-path-vs-absolute-path-usage) +- [3. LoRA Training](#3-lora-training) + - [3.1 Download Pre-trained Model](#31-download-pre-trained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 LoRA-specific Parameter Explanation](#33-lora-specific-parameter-explanation) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-machine Distributed Training](#37-multi-machine-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameter Explanation](#41-inference-parameter-explanation) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. More Resources](#5-more-resources) -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +--- + +## 1. Environment Configuration + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan T2V without deepspeed: +**Method 2: Manual Dependency Installation** -Wan without DeepSpeed is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_lora.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-04 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --train_mode="inpaint" \ - --rank=64 \ - --network_alpha=32 \ - --target_name="q,k,v,ffn.0,ffn.2" \ - --use_peft_lora \ - --low_vram +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -Wan T2V with Deepspeed Zero-2: +**Method 3: Using Docker** -Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. +When using Docker, please ensure that the graphics card driver and CUDA environment are correctly installed on your machine, then execute the following commands: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official example dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video width and height (**highly recommended to provide**, used for bucket training. If not provided, they will be automatically read during training, which may affect training speed when data is stored on slow systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height fields from JSON files without them, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative Path vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, set in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, set in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Suggestion**: If the dataset is small and stored locally, relative paths are recommended. If the dataset is stored on external storage (such as NAS, OSS) or shared across multiple machines, absolute paths are recommended. + +--- + +## 3. LoRA Training + +### 3.1 Download Pre-trained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download official Wan2.1 Fun weights +# Inpaint model (Video Inpainting) +modelscope download --model PAI/Wan2.1-Fun-V1.1-1.3B-InP --local_dir models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP + +# modelscope download --model PAI/Wan2.1-Fun-V1.1-14B-InP --local_dir models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After following **2.1 Quick Test Dataset** and **3.1 Download Pre-trained Model**, you can directly copy the quick start command to launch training. + +DeepSpeed-Zero-2 and FSDP are recommended for training. Here we use DeepSpeed-Zero-2 as an example. + +The difference between DeepSpeed-Zero-2 and FSDP in this repository lies in whether model weights are sharded. **If you encounter insufficient GPU memory when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -124,7 +203,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.1_fun_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -135,78 +214,114 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --training_with_video_token_length \ --enable_bucket \ --uniform_sampling \ - --use_deepspeed \ + --low_vram \ --train_mode="inpaint" \ --rank=64 \ --network_alpha=32 \ --target_name="q,k,v,ffn.0,ffn.2" \ - --use_peft_lora \ - --low_vram + --use_peft_lora ``` -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. +### 3.3 LoRA-specific Parameter Explanation -It is known that DeepSpeed Zero-3 is not compatible with PEFT. +**Key LoRA Parameters**: -Wan T2V with DeepSpeed Zero-3: +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `--config_path` | Configuration file path | `config/wan2.1/wan_civitai.yaml` | +| `--pretrained_model_name_or_path` | Pre-trained model path | `models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Number of samples per batch | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sampling size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Number of video frames to sample | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader subprocesses | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (recommended for LoRA) | 1e-04 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed (for reproducible training) | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.1_fun_lora` | +| `--gradient_checkpointing` | Activation recomputation | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping, group by resolution | - | +| `--random_hw_adapt` | Auto-scale images/videos to random sizes within `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling (recommended) | - | +| `--low_vram` | Low VRAM mode | - | +| `--train_mode` | Training mode: `normal` (text-to-video) or `inpaint` (video inpainting) | `inpaint` | +| `--resume_from_checkpoint` | Resume training path, use `"latest"` to auto-select latest checkpoint | None | +| `--rank` | LoRA update matrix dimension (higher rank = stronger expression but more memory) | 64 | +| `--network_alpha` | LoRA update matrix scaling factor (usually set to half of rank or same) | 32 | +| `--target_name` | Components/modules to apply LoRA, comma-separated | `q,k,v,ffn.0,ffn.2` | +| `--use_peft_lora` | Use PEFT module to add LoRA (more memory-efficient) | - | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for validating video generation | `"A brown dog shaking head..."` | +| `--validation_paths` | Reference image paths for inpaint validation (inpaint mode only) | `"asset/1.png"` | -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. You must set save_state to True to save the model. After training, you can use the following command to get the final model: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: A video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameters**: + +| Parameter | Description | Recommended Value | +|-----------|-------------|-------------------| +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for video generation validation | English prompts | +| `--validation_paths` | Reference image paths for inpaint validation (inpaint mode only) | `"asset/1.png"` | + +**Inpaint Mode Example** (Video Inpainting Validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." ``` -Training shell command is as follows: +**Notes**: +- Validation videos are saved to the `output_dir` directory +- Multi-prompt validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- `inpaint` mode must provide the `--validation_paths` parameter + +### 3.5 Training with FSDP + +**If you encounter insufficient GPU memory when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + ```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_lora.py \ - --config_path="config/wan2.1/wan_civitai.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-04 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --save_state \ - --use_deepspeed \ - --train_mode="inpaint" \ - --low_vram -``` - -Wan T2V with FSDP: - -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -230,7 +345,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.1_fun_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -241,12 +356,352 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --training_with_video_token_length \ --enable_bucket \ --uniform_sampling \ - --save_state \ - --use_deepspeed \ + --low_vram \ --train_mode="inpaint" \ --rank=64 \ --network_alpha=32 \ --target_name="q,k,v,ffn.0,ffn.2" \ - --use_peft_lora \ - --low_vram -``` \ No newline at end of file + --use_peft_lora +``` + +### 3.6 Other Backends + +#### 3.6.1 Training with DeepSpeed-Zero-3 + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +DeepSpeed Zero-3 is suitable for high-resolution 14B Wan. After training, you can use the following command to obtain the final model: +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora +``` + +#### 3.6.2 Training without DeepSpeed and FSDP + +**This approach is not recommended, as without memory-saving backends, it easily causes out-of-memory errors**. Only provided here for reference. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora +``` + +### 3.7 Multi-machine Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note: this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-machine Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data path (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameter Explanation + +**Key Parameters**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `GPU_memory_mode` | Memory management mode, see table below for options | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism degree, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism degree, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save memory | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Path to load trained Transformer weights | `None` | +| `vae_path` | Path to load trained VAE weights | `None` | +| `lora_path` | LoRA weights path | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs without bf16 support | `torch.bfloat16` | +| `validation_image_start` | Reference image path for inpaint mode | `"asset/1.png"` | +| `validation_image_end` | Target image path for inpaint mode (set to `None` for pure I2V) | `None` | +| `prompt` | Positive prompt describing generated content | `"A brown dog shaking head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"low resolution, low quality..."` | +| `guidance_scale` | Guidance strength | 6.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 50 | +| `lora_weight` | LoRA weight strength | 0.55 | +| `save_path` | Path to save generated videos | `samples/wan-videos-fun-t2v` or `samples/wan-videos-fun-i2v` | + +**Memory Management Modes**: + +| Mode | Description | Memory Usage | +|------|-------------|--------------| +| `model_full_load` | Entire model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.1_fun/predict_t2v.py +``` + +Modify `examples/wan2.1_fun/predict_t2v.py` according to your needs. For first-time inference, focus on the parameters below. If you're interested in other parameters, refer to the inference parameter explanation above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +# Path to trained weights, e.g., "output_dir_wan2.1_fun_lora/checkpoint-xxx/checkpoint-xxx.safetensors" +lora_path = None +# LoRA weight strength +lora_weight = 0.55 +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.1_fun/predict_i2v.py +``` + +Modify `examples/wan2.1_fun/predict_i2v.py` according to your needs. For first-time inference, focus on the parameters below. If you're interested in other parameters, refer to the inference parameter explanation above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +# LoRA weights path, e.g., "output_dir_wan2.1_fun_lora/checkpoint-xxx/checkpoint-xxx.safetensors" +lora_path = None +# LoRA weight strength +lora_weight = 0.55 +# Starting image for image-to-video / inpaint +validation_image_start = "asset/1.png" +# Target image for inpaint (set to None for pure image-to-video) +validation_image_end = None +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.1_fun/predict_t2v.py` or `examples/wan2.1_fun/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must evenly divide the model's head count +- `ring_degree` splits along the sequence dimension, which affects communication overhead. Avoid using it when heads can be evenly divided. + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|-----------|----------------|-------------|-------------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.1_fun/predict_t2v.py +``` + +--- + +## 5. More Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.1_fun/README_TRAIN_LORA_zh-CN.md b/scripts/wan2.1_fun/README_TRAIN_LORA_zh-CN.md new file mode 100644 index 0000000..d768410 --- /dev/null +++ b/scripts/wan2.1_fun/README_TRAIN_LORA_zh-CN.md @@ -0,0 +1,707 @@ +# Wan2.1 Fun LoRA 微调训练指南 + +本文档提供 Wan2.1 Fun LoRA 微调训练的完整流程,包括环境配置、数据准备、多种分布式训练策略和推理测试。 + +> **说明**：Wan2.1 Fun 是一个支持文生视频（T2V）、图生视频（I2V）和视频修复（Inpainting）的视频生成模型。本指南涵盖 LoRA 微调训练流程，适用于普通视频生成和视频修复任务的微调场景。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、LoRA 训练](#三lora-训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 LoRA 专用参数解析](#33-lora-专用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、LoRA 训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.1 Fun 官方权重 +# Inpaint 模型（视频修复） +modelscope download --model PAI/Wan2.1-Fun-V1.1-1.3B-InP --local_dir models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP + +# modelscope download --model PAI/Wan2.1-Fun-V1.1-14B-InP --local_dir models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用 DeepSpeed-Zero-2 与 FSDP 方案进行训练。这里以 DeepSpeed-Zero-2 为例配置 shell 文件。 + +本文中 DeepSpeed-Zero-2 与 FSDP 的差别在于是否对模型权重进行分片，**如果使用多卡且使用 DeepSpeed-Zero-2 的情况下显存不足**，可以切换使用 FSDP 进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora +``` + +### 3.3 LoRA 专用参数解析 + +**LoRA 关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--config_path` | 配置文件路径 | `config/wan2.1/wan_civitai.yaml` | +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率（LoRA 推荐值） | 1e-04 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子（可复现训练） | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.1_fun_lora` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep（推荐启用） | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式：`normal`（文生视频）或 `inpaint`（视频修复） | `inpaint` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--rank` | LoRA 更新矩阵的维度（rank 越大表达能力越强，但显存占用越高） | 64 | +| `--network_alpha` | LoRA 更新矩阵的缩放系数（通常设置为 rank 的一半或相同） | 32 | +| `--target_name` | 应用 LoRA 的组件/模块，用逗号分隔 | `q,k,v,ffn.0,ffn.2` | +| `--use_peft_lora` | 使用 PEFT 模块添加 LoRA（更节省显存） | - | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证 inpaint 的参考图像路径（仅 inpaint 模式） | `"asset/1.png"` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证 inpaint 的参考图像路径（仅 inpaint 模式） | `"asset/1.png"` | + +**Inpaint 模式示例**（视频修复验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` +- `inpaint` 模式必须提供 `--validation_paths` 参数 + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | inpaint 模式的参考图像路径 | `"asset/1.png"` | +| `validation_image_end` | inpaint 模式的目标图像路径（纯 I2V 时设为 `None`） | `None` | +| `prompt` | 正向提示词，描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度 | 6.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 50 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-fun-t2v` 或 `samples/wan-videos-fun-i2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.1_fun/predict_t2v.py +``` + +根据需求修改编辑 `examples/wan2.1_fun/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +# 训练好的权重路径，如 "output_dir_wan2.1_fun_lora/checkpoint-xxx/checkpoint-xxx.safetensors" +lora_path = None +# LoRA 权重强度 +lora_weight = 0.55 +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.1_fun/predict_i2v.py +``` + +根据需求修改编辑 `examples/wan2.1_fun/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +# LoRA 权重路径，如 "output_dir_wan2.1_fun_lora/checkpoint-xxx/checkpoint-xxx.safetensors" +lora_path = None +# LoRA 权重强度 +lora_weight = 0.55 +# 图生视频/修复的起始图像 +validation_image_start = "asset/1.png" +# inpaint 的目标图像（纯图生视频时设为 None） +validation_image_end = None +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.1_fun/predict_t2v.py` 或 `examples/wan2.1_fun/predict_i2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.1_fun/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.1_fun/README_TRAIN_zh-CN.md b/scripts/wan2.1_fun/README_TRAIN_zh-CN.md new file mode 100644 index 0000000..83f528a --- /dev/null +++ b/scripts/wan2.1_fun/README_TRAIN_zh-CN.md @@ -0,0 +1,695 @@ +# Wan2.1 Fun 全参数训练指南 + +本文档提供 Wan2.1 Fun Diffusion Transformer 全参数训练的完整流程，包括环境配置、数据准备、分布式训练和推理测试。 + +> **说明**：Wan2.1 Fun 是一个支持文生视频（T2V）、图生视频（I2V）和视频修复（Inpaint）的视频生成模型。本文档涵盖普通视频生成和视频修复任务的训练流程。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、全量参数训练](#三全量参数训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 训练常用参数解析](#33-训练常用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、全量参数训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.1 Fun 官方权重 +# T2V/I2V/Inpaint 模型（1.3B） +modelscope download --model PAI/Wan2.1-Fun-V1.1-1.3B-InP --local_dir models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP +# 或 14B 版本 +# modelscope download --model PAI/Wan2.1-Fun-V1.1-14B-InP --local_dir models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用DeepSpeed-Zero-2与FSDP方案进行训练。这里使用DeepSpeed-Zero-2为例配置shell文件。 + +本文中DeepSpeed-Zero-2与FSDP的差别在于是否对模型权重进行分片，**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.3 训练常用参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率 | 2e-05 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.1_fun` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep | - | +| `--low_vram` | 低显存模式 | - | +| `--train_mode` | 训练模式：`inpaint`（视频修复） | `inpaint` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 500 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 Inpaint 模式） | `"asset/1.png"` | +| `--trainable_modules` | 可训练模块（`"."` 表示所有模块） | `"."` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 500 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 inpaint 模式） | `"asset/1.png"` | + + +**inpaint 模式示例**（视频修复验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` +- `inpaint` 模式必须提供 `--validation_paths` 参数 + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.1_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的 Transformer 权重路径 | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | LoRA 权重路径 | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频/视频修复的参考图像路径（Inpaint 模式） | `"asset/1.png"` | +| `validation_image_end` | 视频修复的目标图像路径（仅 Inpaint 模式） | `None` | +| `prompt` | 正向提示词，描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度 | 6.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 50 | +| `lora_weight` | LoRA 权重强度 | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-fun-t2v` 或 `samples/wan-videos-fun-i2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.1_fun/predict_t2v.py +``` + +根据需求修改编辑 `examples/wan2.1_fun/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +# 训练好的权重路径，如 "output_dir_wan2.1_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.1_fun/predict_i2v.py +``` + +根据需求修改编辑 `examples/wan2.1_fun/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +# 训练好的权重路径，如 "output_dir_wan2.1_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 图生视频/视频修复的起始图像 +validation_image_start = "asset/1.png" +# 视频修复的目标图像（如果是纯图生视频，设置为 None） +validation_image_end = None +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.1_fun/predict_t2v.py` 或 `examples/wan2.1_fun/predict_i2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.1_fun/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.1_fun/train.py b/scripts/wan2.1_fun/train.py index 40f4ac7..6ac7bd4 100755 --- a/scripts/wan2.1_fun/train.py +++ b/scripts/wan2.1_fun/train.py @@ -656,7 +656,7 @@ def parse_args(): default="normal", help=( 'The format of training data. Support `"normal"`' - ' (default), `"i2v"`.' + ' (default), `"inpaint"`.' ), ) parser.add_argument( diff --git a/scripts/wan2.2/README_TRAIN.md b/scripts/wan2.2/README_TRAIN.md index aa76feb..257c7c9 100755 --- a/scripts/wan2.2/README_TRAIN.md +++ b/scripts/wan2.2/README_TRAIN.md @@ -1,63 +1,195 @@ -## Training Code +# Wan2.2 Full Parameter Training Guide -The default training commands for the different versions are as follows: +This document provides a complete workflow for full parameter training of Wan2.2 Diffusion Transformer, including environment setup, data preparation, distributed training, and inference testing. -We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. +> **Note**: Wan2.2 is a video generation model that supports Text-to-Video (T2V), Image-to-Video (I2V), and Text-Image-to-Video (TI2V). Wan2.2 adopts a dual-Transformer architecture (high-noise/low-noise models) for higher quality video generation. This document covers the training workflow for general video generation tasks. -Some parameters in the sh file can be confusing, and they are explained in this document: +--- -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal, i2v or ti2v. The t2v is used for 14B T2V model. The i2v is used for 14B I2V model. The ti2v is used in 5B TI2V model. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). +## Table of Contents +- [1. Environment Setup](#1-environment-setup) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative vs Absolute Path Usage](#24-relative-vs-absolute-path-usage) +- [3. Full Parameter Training](#3-full-parameter-training) + - [3.1 Download Pretrained Model](#31-download-pretrained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 Training Parameter Reference](#33-training-parameter-reference) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-Node Distributed Training](#37-multi-node-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameter Reference](#41-inference-parameter-reference) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -If you want to train 5B Wan2.2 TI2V model, please set config to `config/wan2.2/wan_civitai_5b.yaml`, set train_mode to `ti2v` and set boundary_type to `full`. +--- -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +## 1. Environment Setup -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +**Option 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan2.2 T2V without deepspeed: +**Option 2: Manual Installation** -Training 14B Wan2.2 without DeepSpeed may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**Option 3: Using Docker** + +When using Docker, ensure that the GPU driver and CUDA environment are properly installed on your machine, then execute the following commands: + +```bash +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official demo dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, should be `"video"` +- `width` / `height`: Video width and height (**recommended to provide**, used for bucket training. If not provided, they will be automatically read during training, which may affect training speed when data is stored on slow storage systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height from JSON files without these fields. It supports processing both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, use absolute paths. + +--- + +## 3. Full Parameter Training + +### 3.1 Download Pretrained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download official Wan2.2 weights +# T2V model (Text-to-Video) +modelscope download --model Wan-AI/Wan2.2-T2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-T2V-A14B +# Or I2V model (Image-to-Video) +# modelscope download --model Wan-AI/Wan2.2-I2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-I2V-A14B +# Or TI2V model (Text-Image-to-Video) +# modelscope download --model Wan-AI/Wan2.2-TI2V-5B --local_dir models/Diffusion_Transformer/Wan2.2-TI2V-5B +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After downloading the dataset as in **2.1 Quick Test Dataset** and the pretrained model as in **3.1 Download Pretrained Model**, you can directly copy and run the quick start command. + +We recommend using DeepSpeed-Zero-2 or FSDP for training. Here we use DeepSpeed-Zero-2 as an example. + +The difference between DeepSpeed-Zero-2 and FSDP in this repository is whether the model weights are sharded. **If GPU memory is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +**Wan2.2 T2V Training Example**: + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ @@ -77,7 +209,7 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -94,21 +226,19 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ --trainable_modules "." ``` -Wan T2V with Deepspeed Zero-2: +**Wan2.2 I2V Training Example**: -Wan with Deepspeed Zero-2 is suitable for training Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. - -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher scripts/wan2.2/train.py \ - --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -127,7 +257,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -139,33 +269,24 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --enable_bucket \ --uniform_sampling \ --low_vram \ - --use_deepspeed \ --boundary_type="low" \ - --train_mode="normal" \ + --train_mode="i2v" \ --trainable_modules "." ``` -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. +**Wan2.2 TI2V Training Example**: -Wan T2V with DeepSpeed Zero-3: - -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization -``` - -Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-TI2V-5B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ - --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -184,7 +305,7 @@ accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -196,19 +317,143 @@ accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag --enable_bucket \ --uniform_sampling \ --low_vram \ - --use_deepspeed \ - --boundary_type="low" \ - --train_mode="normal" \ + --boundary_type="full" \ + --train_mode="ti2v" \ --trainable_modules "." ``` -Wan T2V with FSDP: +### 3.3 Training Parameter Reference + +**Wan2.2 Dual-Transformer Architecture Explanation**: + +Wan2.2 adopts an innovative dual-Transformer architecture: +- **Low Noise Model**: Handles the low-noise stage (closer to final output) +- **High Noise Model**: Handles the high-noise stage (initial generation stage) +- **Boundary Type (boundary_type)**: + - `low`: Train low-noise model, high-noise model uses pretrained weights (recommended for T2V/I2V fine-tuning) + - `high`: Train high-noise model, low-noise model uses pretrained weights + - `full`: Single model training (for single-Transformer models like TI2V-5B) + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Batch size | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sample size | 640 | +| `--video_sample_stride` | Video sample stride | 2 | +| `--video_sample_n_frames` | Number of video frames to sample | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader workers | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate | 2e-05 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.2` | +| `--gradient_checkpointing` | Gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 1 | +| `--max_grad_norm` | Maximum gradient norm for clipping | 0.05 | +| `--enable_bucket` | Enable bucket training without cropping, groups by resolution | - | +| `--random_hw_adapt` | Randomly scale images/videos to `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling | - | +| `--low_vram` | Low VRAM mode | - | +| `--boundary_type` | Wan2.2 dual-Transformer boundary type: `low` (train low-noise), `high` (train high-noise), `full` (single model like TI2V-5B) | `low` | +| `--train_mode` | Training mode: `normal` (T2V), `i2v` (Image-to-Video), or `ti2v` (Text-Image-to-Video) | `normal` | +| `--resume_from_checkpoint` | Resume training from checkpoint, use `"latest"` to auto-select | None | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 500 | +| `--validation_prompts` | Prompts for validation video generation | `"A brown dog shaking its head..."` | +| `--validation_paths` | Reference image paths for I2V validation (i2v mode only) | `"asset/1.png"` | +| `--trainable_modules` | Trainable modules (`"."` means all modules) | `"."` | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size for videos; when `random_hw_adapt` is True, it represents the minimum value for both video and image resolutions. +- `image_sample_size` represents the resolution size for images; when `random_hw_adapt` is True, it represents the maximum value for both video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Wan2.2 Training Strategy Recommendations**: +- **T2V Model (Dual-Transformer)**: Use `boundary_type="low"` and `train_mode="normal"` to train the low-noise model. This maintains the generality of the high-noise part while fine-tuning the low-noise part for your data. +- **I2V Model (Dual-Transformer)**: Use `boundary_type="low"` and `train_mode="i2v"` to train the low-noise model. The dataset needs to include reference images. +- **TI2V Model (Single-Transformer)**: Use `boundary_type="full"` and `train_mode="ti2v"` for full training. The dataset needs to include reference images. +- **Memory Optimization**: Wan2.2 models are large (14B/5B parameters). It is strongly recommended to use `--low_vram` and `--gradient_checkpointing`. +- **Multi-GPU Training**: For 14B models, it is recommended to use FSDP or DeepSpeed-Zero-2/3 for multi-GPU training. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: a video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|-----------|-------------|-------------------| +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 500 | +| `--validation_prompts` | Prompts for validation video generation | English prompts | +| `--validation_paths` | Reference image paths for I2V validation (i2v mode only) | `"asset/1.png"` | + +**Normal Mode Example** (T2V validation): + +```bash + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**ti2v Mode Example** (TI2V validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**i2v Mode Example** (I2V validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**Notes**: +- Validation videos will be saved to the `output_dir` directory +- Multiple prompts format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` or `ti2v` mode requires the `--validation_paths` parameter +- Wan2.2 validation automatically selects single or dual-Transformer based on `boundary_type` + +### 3.5 Training with FSDP + +**If GPU memory is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: ```sh export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -234,7 +479,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -251,19 +496,29 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --trainable_modules "." ``` -If you want to train 5B Wan2.2 TI2V model, please set config to `config/wan2.2/wan_civitai_5b.yaml`, set train_mode to `ti2v` and set boundary_type to `full`. Training shell command is as follows: +### 3.6 Other Backends +#### 3.6.1 Training with DeepSpeed-Zero-3 + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: ```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-TI2V-5B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ - --config_path="config/wan2.2/wan_civitai_5b.yaml" \ +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -282,7 +537,7 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -294,7 +549,317 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ --enable_bucket \ --uniform_sampling \ --low_vram \ - --boundary_type="full" \ - --train_mode="ti2v" \ + --boundary_type="low" \ + --train_mode="normal" \ --trainable_modules "." -``` \ No newline at end of file +``` + +#### 3.6.2 Training Without DeepSpeed and FSDP + +**This approach is not recommended as it lacks memory-saving backends and may easily cause out-of-memory errors**. Provided here for reference only. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.7 Multi-Node Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines with 8 GPUs each: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note: this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Node Training Notes + +- **Network Requirements**: + - Recommended: RDMA/InfiniBand (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must have access to the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameter Reference + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `GPU_memory_mode` | GPU memory management mode, see table below | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Path to trained low-noise Transformer weights | `None` | +| `transformer_high_path` | Path to trained high-noise Transformer weights (dual-Transformer models only) | `None` | +| `vae_path` | Path to trained VAE weights | `None` | +| `lora_path` | Low-noise model LoRA weights path | `None` | +| `lora_high_path` | High-noise model LoRA weights path (dual-Transformer models only) | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs not supporting bf16 | `torch.bfloat16` | +| `validation_image_start` | Reference image path for I2V/TI2V mode | `"asset/1.png"` | +| `prompt` | Positive prompt describing content | `"A brown dog shaking its head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"low resolution, low quality..."` | +| `guidance_scale` | Guidance strength | 6.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 50 | +| `lora_weight` | Low-noise model LoRA weight strength | 0.55 | +| `lora_high_weight` | High-noise model LoRA weight strength (dual-Transformer models only) | 0.55 | +| `save_path` | Path to save generated video | `samples/wan-videos-i2v` or `samples/wan-videos-t2v` | + +**GPU Memory Management Modes**: + +| Mode | Description | Memory Usage | +|------|-------------|--------------| +| `model_full_load` | Full model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single GPU inference: + +```bash +python examples/wan2.2/predict_t2v.py +``` + +Edit `examples/wan2.2/predict_t2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, refer to the inference parameter reference above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.2-T2V-A14B" +# Path to trained low-noise weights, e.g., "output_dir_wan2.2/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Path to trained high-noise weights (if trained dual-Transformer) +transformer_high_path = None +# Write based on the content you want to generate +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single GPU inference: + +```bash +python examples/wan2.2/predict_i2v.py +``` + +Edit `examples/wan2.2/predict_i2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, refer to the inference parameter reference above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.2-I2V-A14B" +# Path to trained low-noise weights, e.g., "output_dir_wan2.2/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Path to trained high-noise weights +transformer_high_path = None +# Starting image for image-to-video +validation_image_start = "asset/1.png" +# Write based on the content you want to generate +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3.1 Text-Image-to-Video (TI2V) Inference + +Run the following command for single GPU inference: + +```bash +python examples/wan2.2/predict_ti2v.py +``` + +Edit `examples/wan2.2/predict_ti2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, refer to the inference parameter reference above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path (TI2V single model) +model_name = "models/Diffusion_Transformer/Wan2.2-TI2V-5B" +# Path to trained weights +transformer_path = None +# TI2V has only one model, transformer_high_path is not used +transformer_high_path = None +# Starting image for image-to-video +validation_image_start = "asset/1.png" +# Write based on the content you want to generate +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.2/predict_t2v.py`, `examples/wan2.2/predict_i2v.py`, or `examples/wan2.2/predict_ti2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be divisible by the model's number of heads +- `ring_degree` splits along the sequence dimension and affects communication overhead; avoid using it when heads can be evenly divided + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|-----------|---------------|-------------|-------------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.2/predict_t2v.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun \ No newline at end of file diff --git a/scripts/wan2.2/README_TRAIN_DISTILL.md b/scripts/wan2.2/README_TRAIN_DISTILL.md index bfffc59..6d81810 100755 --- a/scripts/wan2.2/README_TRAIN_DISTILL.md +++ b/scripts/wan2.2/README_TRAIN_DISTILL.md @@ -1,62 +1,197 @@ -## Training Code +# Wan2.2 Distillation Training Guide -The default training commands for the different versions are as follows: +This document provides a complete workflow for distilling Wan2.2 including environment setup, data preparation, distributed training, and inference testing. -We can choose whether to use DeepSpeed and FSDP in Wan distill, which can save a lot of video memory. +> **Note**: Wan2.2 is a video generation model that supports text-to-video (T2V), image-to-video (I2V), and text-image-to-video (TI2V). Wan2.2 adopts a dual-Transformer architecture (high-noise/low-noise models). This training code can reduce inference steps from 25-50 to 4-8 steps while maintaining video generation quality. -Some parameters in the sh file can be confusing, and they are explained in this document: +--- -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). +## Table of Contents +- [1. Environment Setup](#1-environment-setup) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative vs Absolute Path Usage](#24-relative-vs-absolute-path-usage) +- [3. Distillation Training](#3-distillation-training) + - [3.1 Download Pretrained Models](#31-download-pretrained-models) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 Common Training Parameters](#33-common-training-parameters) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-Node Distributed Training](#37-multi-node-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameters](#41-inference-parameters) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.3.1 Text-Image-to-Video (TI2V) Inference](#431-text-image-to-video-ti2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +## 1. Environment Setup + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan distill without deepspeed: +**Method 2: Manual Installation** -Wan distill without DeepSpeed and FSDP is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**Method 3: Using Docker** + +When using Docker, please ensure that your machine has correctly installed GPU drivers and CUDA environment, then execute the following commands: + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset that contains several training data samples. + +```bash +# Download official example dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video dimensions (**recommended** to provide for bucket training. If not provided, it will be automatically read during training, which may affect training speed when data is stored on slower systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height fields for JSON files without these fields, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative vs Absolute Path Usage + +**Relative Paths**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Paths**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (such as NAS, OSS) or shared across multiple machines, use absolute paths. + +--- + +## 3. Distillation Training + +### 3.1 Download Pretrained Models + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download Wan2.2 official weights +# T2V model (text-to-video) +modelscope download --model Wan-AI/Wan2.2-T2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-T2V-A14B +# or I2V model (image-to-video) +# modelscope download --model Wan-AI/Wan2.2-I2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-I2V-A14B +# or TI2V model (text-image-to-video) +# modelscope download --model Wan-AI/Wan2.2-TI2V-5B --local_dir models/Diffusion_Transformer/Wan2.2-TI2V-5B +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After downloading data according to **2.1 Quick Test Dataset** and downloading weights according to **3.1 Download Pretrained Models**, you can directly copy and run the quick start command. + +We recommend using DeepSpeed-Zero-2 and FSDP for training. Here we use DeepSpeed-Zero-2 as an example to configure the shell file. + +The difference between DeepSpeed-Zero-2 and FSDP lies in whether to shard model weights. **If you use multiple GPUs and encounter insufficient GPU memory with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +**Wan2.2 T2V Distillation Training Example**: + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill.py \ - --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -72,11 +207,11 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill.py \ --num_train_epochs=100 \ --checkpointing_steps=50 \ --learning_rate=2e-06 \ - --learning_rate_critic=2e-07\ + --learning_rate_critic=2e-07 \ --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir_distill" \ + --output_dir="output_dir_wan2.2_distill" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -88,19 +223,17 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill.py \ --enable_bucket \ --uniform_sampling \ --boundary_type="low" \ - --train_mode="i2v" \ + --train_mode="normal" \ --trainable_modules "." \ --low_vram ``` -Wan distill with Deepspeed Zero-2: +**Wan2.2 I2V Distillation Training Example**: -Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. - -```sh +```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -123,11 +256,11 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --num_train_epochs=100 \ --checkpointing_steps=50 \ --learning_rate=2e-06 \ - --learning_rate_critic=2e-07\ + --learning_rate_critic=2e-07 \ --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir_distill" \ + --output_dir="output_dir_wan2.2_distill" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -144,27 +277,19 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --low_vram ``` -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. +**Wan2.2 TI2V Distillation Training Example**: -Wan distill with DeepSpeed Zero-3: - -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization -``` - -Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-TI2V-5B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ - --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -180,11 +305,11 @@ accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag --num_train_epochs=100 \ --checkpointing_steps=50 \ --learning_rate=2e-06 \ - --learning_rate_critic=2e-07\ + --learning_rate_critic=2e-07 \ --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir_distill" \ + --output_dir="output_dir_wan2.2_distill" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -195,26 +320,163 @@ accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag --training_with_video_token_length \ --enable_bucket \ --uniform_sampling \ - --boundary_type="low" \ - --train_mode="i2v" \ + --boundary_type="full" \ + --train_mode="ti2v" \ --trainable_modules "." \ --low_vram ``` -Wan distill with FSDP: +### 3.3 Common Training Parameters -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +**Wan2.2 Dual-Transformer Architecture**: + +Wan2.2 adopts an innovative dual-Transformer architecture: +- **Low Noise Model**: Responsible for handling the low-noise stage (close to final output) +- **High Noise Model**: Responsible for handling the high-noise stage (initial generation stage) +- **Boundary Type (boundary_type)**: + - `low`: Train low noise model, high noise model uses pretrained weights (recommended for T2V/I2V distillation) + - `high`: Train high noise model, low noise model uses pretrained weights + - `full`: Single model training (for TI2V-5B and other single-Transformer models) + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Batch size per GPU | 1 | +| `--image_sample_size` | Maximum image training resolution | 640 | +| `--video_sample_size` | Maximum video training resolution | 640 | +| `--token_sample_size` | Token sample size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Number of video frames | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch) | 1 | +| `--dataloader_num_workers` | DataLoader worker processes | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (generator) | 2e-06 | +| `--learning_rate_critic` | Initial learning rate (critic) | 2e-07 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.2_distill` | +| `--gradient_checkpointing` | Enable gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon | 1e-10 | +| `--vae_mini_batch` | VAE encoding mini-batch size | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping, group by resolution | - | +| `--random_hw_adapt` | Auto-scale images/videos to random sizes in `[min_size, max_size]` range | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling | - | +| `--low_vram` | Low VRAM mode | - | +| `--boundary_type` | Wan2.2 dual-Transformer boundary type: `low` (train low noise model), `high` (train high noise model), `full` (train single model like TI2V-5B) | `low` | +| `--train_mode` | Training mode: `normal` (T2V), `i2v` (image-to-video), or `ti2v` (text-image-to-video) | `normal` | +| `--resume_from_checkpoint` | Resume training path, use `"latest"` to auto-select latest checkpoint | None | +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for video generation validation | `"A dog shaking head..."` | +| `--validation_paths` | Validation image paths for I2V mode (i2v mode only) | `"asset/1.png"` | +| `--trainable_modules` | Trainable modules (`"."` means all modules) | `"."` | + +**Distillation-Specific Parameters**: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--denoising_step_indices_list` | Denoising step indices list (core distillation parameter) | `1000 750 500 250` | +| `--real_guidance_scale` | Real guidance scale for scoring | 6.0 | +| `--fake_guidance_scale` | Fake guidance scale for scoring | 0.0 | +| `--gen_update_interval` | Generator update interval | 5 | +| `--train_sampling_steps` | Training sampling steps | 1000 | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Wan2.2 Distillation Training Strategy**: +- **T2V Model (Dual-Transformer)**: Use `boundary_type="low"` and `train_mode="normal"` to distill the low noise model. This maintains the generality of the high noise part while fine-tuning the low noise part for fast inference. +- **I2V Model (Dual-Transformer)**: Use `boundary_type="low"` and `train_mode="i2v"` to distill the low noise model. The dataset needs to include reference images. +- **TI2V Model (Single-Transformer)**: Use `boundary_type="full"` and `train_mode="ti2v"` to distill the single model. The dataset needs to include reference images. TI2V model supports dynamic switching between T2V and I2V modes during inference. +- **Memory Optimization**: Wan2.2 models are large (14B/5B parameters). It is highly recommended to use `--low_vram` and `--gradient_checkpointing`. +- **Multi-GPU Training**: For 14B models, use FSDP or DeepSpeed-Zero-2/3 for multi-GPU training. For 5B models, single GPU or fewer GPUs can be used. +- **Distillation Steps**: The default `--denoising_step_indices_list=1000 750 500 250` corresponds to 4-step distillation. You can adjust to 8 steps or other configurations as needed. + +**Token Length Training Guide**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: A video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|------|------|--------| +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for video generation validation | English prompts | +| `--validation_paths` | Validation image paths for I2V (i2v mode only) | `"asset/1.png"` | + +**Normal Mode Example** (T2V Validation): + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on the shelf, surrounded by pink flowers. The soft and warm lighting in the room creates a comfortable atmosphere." +``` + +**I2V Mode Example** (I2V Validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on the shelf, surrounded by pink flowers. The soft and warm lighting in the room creates a comfortable atmosphere." +``` + +**TI2V Mode Example** (TI2V Validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on the shelf, surrounded by pink flowers. The soft and warm lighting in the room creates a comfortable atmosphere." +``` + +**Notes**: +- Validation videos will be saved to the `output_dir` directory +- Multi-prompt validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` or `ti2v` mode must provide the `--validation_paths` parameter +- Wan2.2 validation will automatically select single or dual Transformer based on `boundary_type` + +### 3.5 Training with FSDP + +**If you use multiple GPUs and encounter insufficient GPU memory with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train_distill.py \ - --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -230,11 +492,11 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --num_train_epochs=100 \ --checkpointing_steps=50 \ --learning_rate=2e-06 \ - --learning_rate_critic=2e-07\ + --learning_rate_critic=2e-07 \ --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir_distill" \ + --output_dir="output_dir_wan2.2_distill" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -246,7 +508,390 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --enable_bucket \ --uniform_sampling \ --boundary_type="low" \ - --train_mode="i2v" \ + --train_mode="normal" \ --trainable_modules "." \ --low_vram -``` \ No newline at end of file +``` + +### 3.6 Other Backends + +#### 3.6.1 Training with DeepSpeed-Zero-3 + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +#### 3.6.2 Training without DeepSpeed and FSDP + +**This approach is not recommended because there is no memory-saving backend, which can easily cause out-of-memory errors**. We only provide the training shell for reference. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.7 Multi-Node Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Rank of this machine (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Node Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameters + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|------|------|-------| +| `GPU_memory_mode` | GPU memory mode, see options below | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism degree, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism degree, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save memory | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective at fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Trained low-noise Transformer weight path | `None` or `output_dir_wan2.2_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors` | +| `transformer_high_path` | Trained high-noise Transformer weight path (dual-Transformer models only) | `None` | +| `vae_path` | Trained VAE weight path | `None` | +| `lora_path` | Low-noise model LoRA weight path | `None` | +| `lora_high_path` | High-noise model LoRA weight path (dual-Transformer models only) | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of frames to generate | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight dtype, use `torch.float16` for GPUs that don't support bf16 | `torch.bfloat16` | +| `validation_image_start` | Reference image path for I2V mode | `"asset/1.png"` | +| `prompt` | Positive prompt describing what to generate | `"A brown dog shaking its head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"Low resolution, low quality..."` | +| `guidance_scale` | Guidance strength (distillation models typically use 1.0) | 1.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps (typically 4 for distillation models) | 4 | +| `lora_weight` | Low-noise model LoRA weight strength | 0.55 | +| `lora_high_weight` | High-noise model LoRA weight strength (dual-Transformer models only) | 0.55 | +| `save_path` | Path to save generated videos | `samples/wan-videos-i2v` or `samples/wan-videos-t2v` | + +**GPU Memory Mode Descriptions**: + +| Mode | Description | Memory Usage | +|------|------|---------| +| `model_full_load` | Entire model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run single GPU inference: + +```bash +python examples/wan2.2/predict_t2v.py +``` + +Edit `examples/wan2.2/predict_t2v.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, please refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Your actual model path +model_name = "models/Diffusion_Transformer/Wan2.2-T2V-A14B" +# Trained low-noise weight path, e.g., "output_dir_wan2.2_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Trained high-noise weight path (if dual-Transformer was trained) +transformer_high_path = None +# Distillation models typically use 4 steps +num_inference_steps = 4 +# Distillation models guidance_scale is typically 1.0 +guidance_scale = 1.0 +# Write according to your generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on the shelf, surrounded by pink flowers. The soft and warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run single GPU inference: + +```bash +python examples/wan2.2/predict_i2v.py +``` + +Edit `examples/wan2.2/predict_i2v.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, please refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Your actual model path +model_name = "models/Diffusion_Transformer/Wan2.2-I2V-A14B" +# Trained low-noise weight path, e.g., "output_dir_wan2.2_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Trained high-noise weight path +transformer_high_path = None +# Distillation models typically use 4 steps +num_inference_steps = 4 +# Distillation models guidance_scale is typically 1.0 +guidance_scale = 1.0 +# Reference image path +validation_image_start = "asset/1.png" +# Write according to your generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on the shelf, surrounded by pink flowers. The soft and warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3.1 Text-Image-to-Video (TI2V) Inference + +Run single GPU inference: + +```bash +python examples/wan2.2/predict_ti2v.py +``` + +Edit `examples/wan2.2/predict_ti2v.py` according to your needs. For first-time inference, focus on the following key parameters. For other parameters, please refer to the inference parameter descriptions above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Your actual model path (TI2V single model) +model_name = "models/Diffusion_Transformer/Wan2.2-TI2V-5B" +# Trained weight path, e.g., "output_dir_wan2.2_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# TI2V has only one model, transformer_high_path is not used +transformer_high_path = None +# Distillation models typically use 4 steps +num_inference_steps = 4 +# Distillation models guidance_scale is typically 1.0 +guidance_scale = 1.0 +# Reference image path +validation_image_start = "asset/1.png" +# Write according to your generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on the shelf, surrounded by pink flowers. The soft and warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.2/predict_t2v.py`, `examples/wan2.2/predict_i2v.py`, or `examples/wan2.2/predict_ti2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be divisible by the model's head count +- `ring_degree` splits on the sequence dimension, which affects communication overhead. Try to avoid using it when heads are evenly divisible. + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.2/predict_t2v.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.2/README_TRAIN_DISTILL_LORA.md b/scripts/wan2.2/README_TRAIN_DISTILL_LORA.md index 8dc6a19..34c5b01 100755 --- a/scripts/wan2.2/README_TRAIN_DISTILL_LORA.md +++ b/scripts/wan2.2/README_TRAIN_DISTILL_LORA.md @@ -1,66 +1,207 @@ -## Lora Training Code +# Wan2.2 Distillation LoRA Training Guide -The default training commands for the different versions are as follows: +This document provides a complete workflow for distilling and fine-tuning Wan2.2 with LoRA, including environment setup, data preparation, distributed training, and inference testing. -We can choose whether to use DeepSpeed and FSDP in Wan distill, which can save a lot of video memory. +> **Note**: Wan2.2 is a video generation model that supports Text-to-Video (T2V), Image-to-Video (I2V), and Text-Image-to-Video (TI2V). Wan2.2 adopts a dual-Transformer architecture (high-noise/low-noise models). This training method combines distillation (reducing inference steps) and LoRA (parameter-efficient fine-tuning) technologies. It can reduce inference steps from 25-50 to 4-8 steps with lower VRAM usage while maintaining or improving video generation quality. -Some parameters in the sh file can be confusing, and they are explained in this document: +--- -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). -- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. -- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. -- `rank` means the dimension of the LoRA update matrices. -- `network_alpha` means the scale of the LoRA update matrices. +## Table of Contents +- [1. Environment Setup](#1-environment-setup) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative vs Absolute Path Usage](#24-relative-vs-absolute-path-usage) +- [3. Distillation LoRA Training](#3-distillation-lora-training) + - [3.1 Download Pretrained Model](#31-download-pretrained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 Training Parameters Explanation](#33-training-parameters-explanation) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-Node Distributed Training](#37-multi-node-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameters Explanation](#41-inference-parameters-explanation) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.3.1 Text-Image-to-Video (TI2V) Inference](#431-text-image-to-video-ti2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. More Resources](#5-more-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +## 1. Environment Setup + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan distill without deepspeed: +**Method 2: Manual Installation** -Wan distill without DeepSpeed and FSDP is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**Method 3: Using Docker** + +When using Docker, please ensure that the graphics card driver and CUDA environment are correctly installed, then execute the following commands: + +``` +# Pull the image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# Enter the container +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download the official demo dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video width and height (**recommended to provide**, used for bucket training. If not provided, it will be automatically read during training, which may affect training speed when data is stored on slower systems like OSS). + - You can use the `scripts/process_json_add_width_and_height.py` file to extract width and height from JSON files without these fields. It supports processing both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, use absolute paths. + +--- + +## 3. Distillation LoRA Training + +### 3.1 Download Pretrained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download official Wan2.2 weights +# T2V model (Text-to-Video) +modelscope download --model Wan-AI/Wan2.2-T2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-T2V-A14B +# Or I2V model (Image-to-Video) +# modelscope download --model Wan-AI/Wan2.2-I2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-I2V-A14B +# Or TI2V model (Text-Image-to-Video) +# modelscope download --model Wan-AI/Wan2.2-TI2V-5B --local_dir models/Diffusion_Transformer/Wan2.2-TI2V-5B +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After following **2.1 Quick Test Dataset** to download data and **3.1 Download Pretrained Model** to download weights, directly copy and run the quick start command. + +It is recommended to use DeepSpeed-Zero-2 or FSDP for training. Here we use DeepSpeed-Zero-2 as an example to configure the shell file. + +The difference between DeepSpeed-Zero-2 and FSDP in this repository is whether model weights are sharded. **If VRAM is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +**Wan2.2 Dual-Transformer Architecture Explanation**: + +Wan2.2 adopts an innovative dual-Transformer architecture: +- **Low Noise Model**: Responsible for processing the low-noise stage (close to final output) +- **High Noise Model**: Responsible for processing the high-noise stage (initial generation stage) +- **Boundary Type (boundary_type)**: + - `low`: Train the low-noise model, high-noise model uses pretrained weights (recommended for T2V/I2V distillation) + - `high`: Train the high-noise model, low-noise model uses pretrained weights + - `full`: Single model training (for single-Transformer models like TI2V-5B) + +**Wan2.2 T2V Distillation LoRA Training Example**: + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill_lora.py \ - --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -78,7 +219,7 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill_lora.py --learning_rate=1e-05 \ --learning_rate_critic=1e-06 \ --seed=42 \ - --output_dir="output_dir_distill" \ + --output_dir="output_dir_wan2.2_distill_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -90,22 +231,20 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill_lora.py --enable_bucket \ --uniform_sampling \ --boundary_type="low" \ - --train_mode="i2v" \ --rank=64 \ --network_alpha=32 \ --target_name="q,k,v,ffn.0,ffn.2" \ --use_peft_lora \ + --train_mode="normal" \ --low_vram ``` -Wan distill with Deepspeed Zero-2: +**Wan2.2 I2V Distillation LoRA Training Example**: -Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. - -```sh +```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -130,7 +269,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --learning_rate=1e-05 \ --learning_rate_critic=1e-06 \ --seed=42 \ - --output_dir="output_dir_distill" \ + --output_dir="output_dir_wan2.2_distill_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -142,84 +281,152 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --enable_bucket \ --uniform_sampling \ --boundary_type="low" \ - --train_mode="i2v" \ --rank=64 \ --network_alpha=32 \ --target_name="q,k,v,ffn.0,ffn.2" \ --use_peft_lora \ - --low_vram -``` - -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. - -It is known that DeepSpeed Zero-3 is not compatible with PEFT. - -Wan distill with DeepSpeed Zero-3: - -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization -``` - -Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill_lora.py \ - --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-05 \ - --learning_rate_critic=1e-06 \ - --seed=42 \ - --output_dir="output_dir_distill" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --boundary_type="low" \ --train_mode="i2v" \ --low_vram ``` -Wan distill with FSDP: +### 3.3 Training Parameters Explanation -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +**LoRA-Specific Parameters**: + +In addition to distillation training, LoRA training adds the following specific parameters: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--use_peft_lora` | Whether to use PEFT module to add LoRA, this module saves more VRAM | - | +| `--rank` | Dimension (rank) of LoRA update matrix | 64 | +| `--network_alpha` | Scaling coefficient of LoRA update matrix | 32 | +| `--target_name` | Components/modules where LoRA is applied, comma-separated | `"q,k,v,ffn.0,ffn.2"` | +| `--lora_skip_name` | Modules skipped by LoRA (not trained) | None | + +**LoRA Configuration Recommendations**: +- **rank=64, network_alpha=32**: Suitable for most scenarios, balances quality and VRAM +- **rank=128, network_alpha=64**: Higher quality fine-tuning, but requires more VRAM +- **target_name="q,k,v,ffn.0,ffn.2"**: Fine-tunes attention layers and feed-forward networks, this is a common configuration +- **use_peft_lora**: Strongly recommended to enable, can significantly reduce VRAM usage + +**Key Parameters Explanation**: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Number of samples per batch | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sampling size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Number of video frames sampled | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch) | 1 | +| `--dataloader_num_workers` | Number of DataLoader subprocesses | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (generator) | 1e-05 | +| `--learning_rate_critic` | Initial learning rate (discriminator) | 1e-06 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.2_distill_lora` | +| `--gradient_checkpointing` | Activation recomputation | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping of images/videos, grouped by resolution | - | +| `--random_hw_adapt` | Automatically scale images/videos to random sizes within `[min_size, max_size]` range | - | +| `--training_with_video_token_length` | Train based on token length, supports any resolution | - | +| `--uniform_sampling` | Uniform timestep sampling | - | +| `--low_vram` | Low VRAM mode | - | +| `--boundary_type` | Wan2.2 dual-Transformer boundary type: `low` (train low-noise model), `high` (train high-noise model), `full` (train single model like TI2V-5B) | `low` | +| `--train_mode` | Training mode: `normal` (standard T2V) or `i2v` (image-to-video) | `normal` | +| `--resume_from_checkpoint` | Resume training path, use `"latest"` to automatically select the latest checkpoint | None | +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for validation video generation | `"A brown dog shaking its head..."` | +| `--validation_paths` | Reference image paths for I2V validation (i2v mode only) | `"asset/1.png"` | + +**Distillation-Specific Parameters**: + +| Parameter | Description | Example Value | +|-----|------|-------| +| `--denoising_step_indices_list` | Denoising step list (core distillation parameter) | `1000 750 500 250` | +| `--real_guidance_scale` | Real guidance scale for scoring | 6.0 | +| `--fake_guidance_scale` | Fake guidance scale for scoring | 0.0 | +| `--gen_update_interval` | Generator update interval | 5 | +| `--train_sampling_steps` | Training sampling steps | 1000 | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the video resolution size; when `random_hw_adapt` is True, it represents the minimum resolution for videos and images. +- `image_sample_size` represents the image resolution size; when `random_hw_adapt` is True, it represents the maximum resolution for videos and images. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Since configurations may cause confusion, **if you don't need arbitrary resolution fine-tuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: a video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, video frames are 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, video frames are 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, video frames are 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These combinations of resolutions and corresponding frame numbers enable the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to regularly generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameters Explanation**: + +| Parameter | Description | Recommended Value | +|------|------|--------| +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for validation video generation | English prompts | +| `--validation_paths` | Reference image paths for I2V validation (i2v/inpaint mode only) | `"asset/1.png"` | + +**normal Mode Example** (T2V Validation): + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**i2v/inpaint Mode Example** (I2V Validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on a shelf surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**Notes**: +- Validation videos will be saved to the `output_dir` directory +- Multi-prompt validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` or `inpaint` mode must provide the `--validation_paths` parameter + +### 3.5 Training with FSDP + +If VRAM is insufficient when using multiple GPUs with DeepSpeed-Zero-2, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train_distill_lora.py \ - --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -237,7 +444,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --learning_rate=1e-05 \ --learning_rate_critic=1e-06 \ --seed=42 \ - --output_dir="output_dir_distill" \ + --output_dir="output_dir_wan2.2_distill_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -249,10 +456,416 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --enable_bucket \ --uniform_sampling \ --boundary_type="low" \ - --train_mode="i2v" \ --rank=64 \ --network_alpha=32 \ --target_name="q,k,v,ffn.0,ffn.2" \ --use_peft_lora \ + --train_mode="normal" \ --low_vram -``` \ No newline at end of file +``` + +### 3.6 Other Backends + +#### 3.6.1 Training with DeepSpeed-Zero-3 + +DeepSpeed Zero-3 is not highly recommended at present. In this repository, using FSDP has fewer errors and is more stable. + +It is known that DeepSpeed Zero-3 is incompatible with PEFT. + +DeepSpeed Zero-3 is suitable for high-resolution 14B Wan models. After training, you can use the following command to obtain the final model: +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{your-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +The training shell command is as follows: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="normal" \ + --low_vram +``` + +#### 3.6.2 Training without DeepSpeed and FSDP + +**This approach is not recommended because without VRAM-saving backends, it easily causes VRAM shortages**. This is only provided as a reference for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +### 3.7 Multi-Node Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Node Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameters Explanation + +**Key Parameters Explanation**: + +| Parameter | Description | Example Value | +|------|------|-------| +| `GPU_memory_mode` | VRAM management mode, see table below for options | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save VRAM | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer to accelerate inference (effective at fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Path to load trained low-noise Transformer weights | `None` or base model weights | +| `transformer_high_path` | Path to load trained high-noise Transformer weights (dual-Transformer models only) | `None` | +| `vae_path` | Path to load trained VAE weights | `None` | +| `lora_path` | LoRA weights path for low-noise model (distillation LoRA training output) | `output_dir_wan2.2_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors` | +| `lora_high_path` | LoRA weights path for high-noise model (dual-Transformer models only) | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of generated video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs that don't support bf16 | `torch.bfloat16` | +| `validation_image_start` | Reference image path for image-to-video (I2V mode) | `"asset/1.png"` | +| `prompt` | Positive prompt, describes the content to generate | `"A brown dog shaking its head..."` | +| `negative_prompt` | Negative prompt, content to avoid | `"Low resolution, low quality..."` | +| `guidance_scale` | Guidance strength (distillation models typically use 1.0) | 1.0 | +| `seed` | Random seed, for reproducing results | 43 | +| `num_inference_steps` | Number of inference steps (distillation models typically use 4) | 4 | +| `lora_weight` | LoRA weight strength for low-noise model | 0.55 | +| `lora_high_weight` | LoRA weight strength for high-noise model (dual-Transformer models only) | 0.55 | +| `save_path` | Path to save generated videos | `samples/wan-videos-i2v` or `samples/wan-videos-t2v` | + +**VRAM Management Mode Explanation**: + +| Mode | Description | VRAM Usage | +|------|------|---------| +| `model_full_load` | Entire model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.2/predict_t2v.py +``` + +Edit `examples/wan2.2/predict_t2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameters explanation above. + +```python +# Select based on GPU VRAM +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.2-T2V-A14B" +# Base model weight path (if you have trained full weights) +transformer_path = None +# Trained high-noise weight path (if trained dual-Transformer) +transformer_high_path = None +# LoRA weight path, e.g., "output_dir_wan2.2_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors" +lora_path = None +# LoRA weight path for high-noise model (if trained) +lora_high_path = None +# Distillation models typically use 4 steps +num_inference_steps = 4 +# Distillation models guidance_scale is usually 1.0 +guidance_scale = 1.0 +# LoRA weight strength +lora_weight = 0.55 +# LoRA weight strength for high-noise model +lora_high_weight = 0.55 +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on a shelf surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.2/predict_i2v.py +``` + +Edit `examples/wan2.2/predict_i2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameters explanation above. + +```python +# Select based on GPU VRAM +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.2-I2V-A14B" +# Base model weight path (if you have trained full weights) +transformer_path = None +# Trained high-noise weight path +transformer_high_path = None +# LoRA weight path, e.g., "output_dir_wan2.2_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors" +lora_path = None +# LoRA weight path for high-noise model +lora_high_path = None +# Distillation models typically use 4 steps +num_inference_steps = 4 +# Distillation models guidance_scale is usually 1.0 +guidance_scale = 1.0 +# LoRA weight strength +lora_weight = 0.55 +# LoRA weight strength for high-noise model +lora_high_weight = 0.55 +# Starting image for image-to-video +validation_image_start = "asset/1.png" +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on a shelf surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3.1 Text-Image-to-Video (TI2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.2/predict_ti2v.py +``` + +Edit `examples/wan2.2/predict_ti2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, please refer to the inference parameters explanation above. + +```python +# Select based on GPU VRAM +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path (TI2V single model) +model_name = "models/Diffusion_Transformer/Wan2.2-TI2V-5B" +# Trained weight path, e.g., "output_dir_wan2.2_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors" +transformer_path = None +# TI2V has only one model, transformer_high_path is not used +transformer_high_path = None +# LoRA weight path +lora_path = None +# LoRA weight path for high-noise model (not used for TI2V) +lora_high_path = None +# Distillation models typically use 4 steps +num_inference_steps = 4 +# Distillation models guidance_scale is usually 1.0 +guidance_scale = 1.0 +# LoRA weight strength +lora_weight = 0.55 +# Starting image for image-to-video +validation_image_start = "asset/1.png" +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed picture on a shelf surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.2/predict_t2v.py`, `examples/wan2.2/predict_i2v.py` or `examples/wan2.2/predict_ti2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be divisible by the model's head count +- `ring_degree` splits along the sequence dimension, which affects communication overhead. Try to avoid using it if heads are evenly divisible + +**Configuration Examples**: + +| Number of GPUs | ulysses_degree | ring_degree | Description | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.2/predict_t2v.py +``` + +--- + +## 5. More Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.2/README_TRAIN_DISTILL_LORA_zh-CN.md b/scripts/wan2.2/README_TRAIN_DISTILL_LORA_zh-CN.md new file mode 100755 index 0000000..c9bdfe9 --- /dev/null +++ b/scripts/wan2.2/README_TRAIN_DISTILL_LORA_zh-CN.md @@ -0,0 +1,873 @@ +# Wan2.2 蒸馏 LoRA 训练指南 + +本文档提供了将 Wan2.2 进行蒸馏 + LoRA 微调的完整工作流,包括环境配置、数据准备、分布式训练和推理测试。 + +> **说明**:Wan2.2 是一个支持文生视频(T2V)、图生视频(I2V)和文本图生视频(TI2V)的视频生成模型。Wan2.2采用双Transformer架构(高噪声/低噪声模型)。本训练方式结合了蒸馏(减少推理步数)和 LoRA(参数高效微调)两种技术,可以在显存占用较低的情况下,将推理步数从 25-50 步减少到 4-8 步,同时保持或提升视频生成质量。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、蒸馏 LoRA 训练](#三蒸馏-lora-训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 训练常用参数解析](#33-训练常用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.3.1 文本图生视频（TI2V）推理](#431-文本图生视频ti2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1:使用 requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2:手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3:使用 docker** + +使用 docker 的情况下,请保证机器中已经正确安装显卡驱动与 CUDA 环境,然后以此执行以下命令: + +``` +# 拉取镜像 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# 进入容器 +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集,其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**(示例格式): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**: +- `file_path`:视频路径(相对或绝对路径) +- `text`:视频描述(英文提示词) +- `type`:数据类型,固定为 `"video"` +- `width` / `height`:视频宽高(**最好提供**,用于分桶训练,如果不提供则自动在训练时读取,当数据存储在如 oss 这样的速度较慢的系统上时,可能会影响训练速度)。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无 width 与 height 字段的 json 进行提取,支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**: + +如果数据的路径为相对路径,则在训练脚本中设置: + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**: + +如果数据的路径为绝对路径,则在训练脚本中设置: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**:如果数据集较小且存储在本地,推荐使用相对路径;如果数据集存储在外部存储(如 NAS、OSS)或多个机器共享存储,推荐使用绝对路径。 + +--- + +## 三、蒸馏 LoRA 训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.2 官方权重 +# T2V 模型(文生视频) +modelscope download --model Wan-AI/Wan2.2-T2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-T2V-A14B +# 或 I2V 模型(图生视频) +# modelscope download --model Wan-AI/Wan2.2-I2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-I2V-A14B +# 或 TI2V 模型(文本图生视频) +# modelscope download --model Wan-AI/Wan2.2-TI2V-5B --local_dir models/Diffusion_Transformer/Wan2.2-TI2V-5B +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用DeepSpeed-Zero-2与FSDP方案进行训练。这里使用DeepSpeed-Zero-2为例配置shell文件。 + +本文中DeepSpeed-Zero-2与FSDP的差别在于是否对模型权重进行分片，**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +**Wan2.2 双Transformer架构说明**： + +Wan2.2采用了创新的双Transformer架构： +- **低噪声模型（Low Noise Model）**：负责处理低噪声阶段（接近最终输出） +- **高噪声模型（High Noise Model）**：负责处理高噪声阶段（初始生成阶段） +- **边界类型（boundary_type）**： + - `low`：训练低噪声模型，高噪声模型使用预训练权重（推荐用于T2V/I2V蒸馏） + - `high`：训练高噪声模型，低噪声模型使用预训练权重 + - `full`：单模型训练（用于TI2V-5B等单Transformer模型） + +**Wan2.2 T2V 蒸馏 LoRA 训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +**Wan2.2 I2V 蒸馏 LoRA 训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="i2v" \ + --low_vram +``` + +### 3.3 训练常用参数解析 + +**LoRA 特有参数**: + +在蒸馏训练的基础上,LoRA 训练增加了以下特有参数: + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--use_peft_lora` | 是否使用 PEFT 模块添加 LoRA,使用该模块会更节省显存 | - | +| `--rank` | LoRA 更新矩阵的维度(秩) | 64 | +| `--network_alpha` | LoRA 更新矩阵的缩放系数 | 32 | +| `--target_name` | LoRA 应用的组件/模块,以逗号分隔 | `"q,k,v,ffn.0,ffn.2"` | +| `--lora_skip_name` | LoRA 跳过的模块(不训练) | None | + +**LoRA 配置建议**: +- **rank=64, network_alpha=32**:适用于大多数场景,在质量和显存之间取得平衡 +- **rank=128, network_alpha=64**:更高质量的微调,但需要更多显存 +- **target_name="q,k,v,ffn.0,ffn.2"**:对注意力层和前馈网络进行微调,这是常用的配置 +- **use_peft_lora**:强烈建议启用,可以显著降低显存占用 + +**关键参数说明**: + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数(等效增大 batch) | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率(生成器) | 1e-05 | +| `--learning_rate_critic` | 初始学习率(判别器) | 1e-06 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.2_distill_lora` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度:`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练,不裁剪图片/视频,按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练,支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep | - | +| `--low_vram` | 低显存模式 | - | +| `--boundary_type` | Wan2.2双Transformer边界类型：`low`（训练低噪声模型）、`high`（训练高噪声模型）、`full`（训练单模型如TI2V-5B） | `low` | +| `--train_mode` | 训练模式：`normal`（普通T2V）或 `i2v`（图生视频） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径,使用 `"latest"` 自动选择最新 checkpoint | None | +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个 epoch 执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证 I2V 的参考图像路径(仅 i2v 模式) | `"asset/1.png"` | + +**蒸馏特有参数**: + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--denoising_step_indices_list` | 去噪步骤列表(蒸馏核心参数) | `1000 750 500 250` | +| `--real_guidance_scale` | 用于评分的真实 guidance scale | 6.0 | +| `--fake_guidance_scale` | 用于评分的虚拟 guidance scale | 0.0 | +| `--gen_update_interval` | 生成器更新间隔 | 5 | +| `--train_sampling_steps` | 训练采样步数 | 1000 | + +**Sample Size 配置指南**: +- `video_sample_size` 表示视频的分辨率大小;当 `random_hw_adapt` 为 True 时,表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小;当 `random_hw_adapt` 为 True 时,表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时,最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆,**如果你不需要任意分辨率进行 finetuning**,建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值,例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**: +- 当启用 `training_with_video_token_length` 时,模型根据 token 长度进行训练。 +- 例如:512x512 分辨率、49 帧的视频,其 token 长度为 13,312,需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下,视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下,视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下,视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合,使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数,在训练过程中定期生成测试视频,以便监控训练进度和模型质量。 + +**验证参数说明**: + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个 epoch 执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证 I2V 的参考图像路径(仅 i2v/inpaint 模式) | `"asset/1.png"` | + +**normal 模式示例**(T2V 验证): + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**i2v/inpaint 模式示例**(I2V 验证): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**: +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式:`--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` 或 `inpaint` 模式必须提供 `--validation_paths` 参数 + +### 3.5 使用 FSDP 训练 + +如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +已知 DeepSpeed Zero-3 与 PEFT 不兼容。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{your-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +### 3.7 多机分布式训练 + +**适合场景**:超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器,每台 8 张 GPU: + +**机器 0(Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank(0 或 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +**机器 1(Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**: + - 推荐 RDMA/InfiniBand(高性能) + - 无 RDMA 时添加环境变量: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**:所有机器必须能够访问相同的数据路径(NFS/共享存储) + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**: + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式,可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度,单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度,单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理(固定分辨率下有效) | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的低噪声 Transformer 权重路径 | `None` 或基础模型权重 | +| `transformer_high_path` | 加载训练好的高噪声 Transformer 权重路径（仅双Transformer模型） | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | 低噪声模型 LoRA 权重路径（蒸馏 LoRA 训练产出） | `output_dir_wan2.2_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors` | +| `lora_high_path` | 高噪声模型 LoRA 权重路径（仅双Transformer模型） | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度,不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径(I2V 模式) | `"asset/1.png"` | +| `prompt` | 正向提示词,描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词,避免生成的内容 | `"低分辨率,低画质..."` | +| `guidance_scale` | 引导强度(蒸馏模型通常使用 1.0) | 1.0 | +| `seed` | 随机种子,用于复现结果 | 43 | +| `num_inference_steps` | 推理步数（蒸馏模型通常为 4） | 4 | +| `lora_weight` | 低噪声模型 LoRA 权重强度 | 0.55 | +| `lora_high_weight` | 高噪声模型 LoRA 权重强度（仅双Transformer模型） | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-i2v` 或 `samples/wan-videos-t2v` | + +**显存管理模式说明**: + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载(速度最慢) | 最低 | + +### 4.2 文生视频(T2V)推理 + +单卡推理运行如下命令: + +```bash +python examples/wan2.2/predict_t2v.py +``` + +根据需求修改编辑 `examples/wan2.2/predict_t2v.py`,初次推理重点关注如下参数,如果对其他参数感兴趣,请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-T2V-A14B" +# 基础模型权重路径(如果有训练过的全量权重) +transformer_path = None +# 训练好的高噪声权重路径（如果训练了双Transformer） +transformer_high_path = None +# LoRA 权重路径,如 "output_dir_wan2.2_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors" +lora_path = None +# 高噪声模型的 LoRA 权重路径（如果训练了） +lora_high_path = None +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# 蒸馏模型 guidance_scale 通常为 1.0 +guidance_scale = 1.0 +# LoRA 权重强度 +lora_weight = 0.55 +# 高噪声模型的 LoRA 权重强度 +lora_high_weight = 0.55 +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 图生视频(I2V)推理 + +单卡推理运行如下命令: + +```bash +python examples/wan2.2/predict_i2v.py +``` + +根据需求修改编辑 `examples/wan2.2/predict_i2v.py`,初次推理重点关注如下参数,如果对其他参数感兴趣,请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-I2V-A14B" +# 基础模型权重路径(如果有训练过的全量权重) +transformer_path = None +# 训练好的高噪声权重路径 +transformer_high_path = None +# LoRA 权重路径,如 "output_dir_wan2.2_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors" +lora_path = None +# 高噪声模型的 LoRA 权重路径 +lora_high_path = None +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# 蒸馏模型 guidance_scale 通常为 1.0 +guidance_scale = 1.0 +# LoRA 权重强度 +lora_weight = 0.55 +# 高噪声模型的 LoRA 权重强度 +lora_high_weight = 0.55 +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3.1 文本图生视频（TI2V）推理 + +单卡推理运行如下命令: + +```bash +python examples/wan2.2/predict_ti2v.py +``` + +根据需求修改编辑 `examples/wan2.2/predict_ti2v.py`,初次推理重点关注如下参数,如果对其他参数感兴趣,请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径（TI2V单模型） +model_name = "models/Diffusion_Transformer/Wan2.2-TI2V-5B" +# 训练好的权重路径,如 "output_dir_wan2.2_distill_lora/checkpoint-xxx/pytorch_lora_weights.safetensors" +transformer_path = None +# TI2V只有一个模型,transformer_high_path不使用 +transformer_high_path = None +# LoRA 权重路径 +lora_path = None +# 高噪声模型的 LoRA 权重路径（TI2V不使用） +lora_high_path = None +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# 蒸馏模型 guidance_scale 通常为 1.0 +guidance_scale = 1.0 +# LoRA 权重强度 +lora_weight = 0.55 +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**:高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.2/predict_t2v.py`、`examples/wan2.2/predict_i2v.py` 或 `examples/wan2.2/predict_ti2v.py`: + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU: +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**: +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分,会影响通信开销,在 head 能整除的情况下尽量不要用 + +**配置示例**: + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.2/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**:https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.2/README_TRAIN_DISTILL_zh-CN.md b/scripts/wan2.2/README_TRAIN_DISTILL_zh-CN.md new file mode 100755 index 0000000..0494174 --- /dev/null +++ b/scripts/wan2.2/README_TRAIN_DISTILL_zh-CN.md @@ -0,0 +1,897 @@ +# Wan2.2 蒸馏训练指南 + +本文档提供了将 Wan2.2 蒸馏的完整工作流,包括环境配置、数据准备、分布式训练和推理测试。 + +> **说明**：Wan2.2 是一个支持文生视频（T2V）、图生视频（I2V）和文本图生视频（TI2V）的视频生成模型。Wan2.2采用双Transformer架构（高噪声/低噪声模型），该训练代码可以将推理步数从 25-50 步减少到 4-8 步，同时保持视频生成质量。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、蒸馏训练](#三蒸馏训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 训练常用参数解析](#33-训练常用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.3.1 文本图生视频（TI2V）推理](#431-文本图生视频ti2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# 拉取镜像 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# 进入容器 +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、蒸馏训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.2 官方权重 +# T2V 模型（文生视频） +modelscope download --model Wan-AI/Wan2.2-T2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-T2V-A14B +# 或 I2V 模型（图生视频） +# modelscope download --model Wan-AI/Wan2.2-I2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-I2V-A14B +# 或 TI2V 模型（文本图生视频） +# modelscope download --model Wan-AI/Wan2.2-TI2V-5B --local_dir models/Diffusion_Transformer/Wan2.2-TI2V-5B +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用DeepSpeed-Zero-2与FSDP方案进行训练。这里使用DeepSpeed-Zero-2为例配置shell文件。 + +本文中DeepSpeed-Zero-2与FSDP的差别在于是否对模型权重进行分片，**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +**Wan2.2 T2V 蒸馏训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +**Wan2.2 I2V 蒸馏训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="i2v" \ + --trainable_modules "." \ + --low_vram +``` + +**Wan2.2 TI2V 蒸馏训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-TI2V-5B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --train_mode="ti2v" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.3 训练常用参数解析 + +**Wan2.2 双Transformer架构说明**： + +Wan2.2采用了创新的双Transformer架构： +- **低噪声模型（Low Noise Model）**：负责处理低噪声阶段（接近最终输出） +- **高噪声模型（High Noise Model）**：负责处理高噪声阶段（初始生成阶段） +- **边界类型（boundary_type）**： + - `low`：训练低噪声模型，高噪声模型使用预训练权重（推荐用于T2V/I2V蒸馏） + - `high`：训练高噪声模型，低噪声模型使用预训练权重 + - `full`：单模型训练（用于TI2V-5B等单Transformer模型） + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率（生成器） | 2e-06 | +| `--learning_rate_critic` | 初始学习率（判别器） | 2e-07 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.2_distill` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep | - | +| `--low_vram` | 低显存模式 | - | +| `--boundary_type` | Wan2.2双Transformer边界类型：`low`（训练低噪声模型）、`high`（训练高噪声模型）、`full`（训练单模型如TI2V-5B） | `low` | +| `--train_mode` | 训练模式：`normal`（普通T2V）、`i2v`（图生视频）或 `ti2v`（文本图生视频） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v 模式） | `"asset/1.png"` | +| `--trainable_modules` | 可训练模块（`"."` 表示所有模块） | `"."` | + +**蒸馏特有参数**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--denoising_step_indices_list` | 去噪步骤列表（蒸馏核心参数） | `1000 750 500 250` | +| `--real_guidance_scale` | 用于评分的真实 guidance scale | 6.0 | +| `--fake_guidance_scale` | 用于评分的虚拟 guidance scale | 0.0 | +| `--gen_update_interval` | 生成器更新间隔 | 5 | +| `--train_sampling_steps` | 训练采样步数 | 1000 | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Wan2.2 蒸馏训练策略建议**： +- **T2V模型（双Transformer）**：使用 `boundary_type="low"` 和 `train_mode="normal"` 蒸馏低噪声模型，这样可以保持高噪声部分的通用性，同时微调低噪声部分以实现快速推理。 +- **I2V模型（双Transformer）**：使用 `boundary_type="low"` 和 `train_mode="i2v"` 蒸馏低噪声模型，数据集需要包含参考图像。 +- **TI2V模型（单Transformer）**：使用 `boundary_type="full"` 和 `train_mode="ti2v"` 蒸馏单模型，数据集需要包含参考图像。TI2V模型支持在推理时动态切换T2V和I2V模式。 +- **显存优化**：Wan2.2模型较大（14B/5B参数），强烈建议使用 `--low_vram` 和 `--gradient_checkpointing`。 +- **多卡训练**：对于14B模型，推荐使用FSDP或DeepSpeed-Zero-2/3进行多卡训练；5B模型可以使用单卡或少数卡训练。 +- **蒸馏步数**：默认的 `--denoising_step_indices_list=1000 750 500 250` 对应4步蒸馏，可以根据需要调整为8步或其他配置。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v 模式） | `"asset/1.png"` | + +**normal 模式示例**（T2V 验证）： + +```bash + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**i2v 模式示例**（I2V 验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**ti2v 模式示例**（TI2V 验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=2000 \ + --validation_epochs=5 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` 或 `ti2v` 模式必须提供 `--validation_paths` 参数 +- Wan2.2的验证会根据`boundary_type`自动选择使用单Transformer或双Transformer + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的低噪声 Transformer 权重路径 | `None` 或 `output_dir_wan2.2_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors` | +| `transformer_high_path` | 加载训练好的高噪声 Transformer 权重路径（仅双Transformer模型） | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | 低噪声模型 LoRA 权重路径 | `None` | +| `lora_high_path` | 高噪声模型 LoRA 权重路径（仅双Transformer模型） | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V 模式） | `"asset/1.png"` | +| `prompt` | 正向提示词，描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度（蒸馏模型通常使用 1.0） | 1.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数（蒸馏模型通常为 4） | 4 | +| `lora_weight` | 低噪声模型 LoRA 权重强度 | 0.55 | +| `lora_high_weight` | 高噪声模型 LoRA 权重强度（仅双Transformer模型） | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-i2v` 或 `samples/wan-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2/predict_t2v.py +``` + +根据需求修改编辑 `examples/wan2.2/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-T2V-A14B" +# 训练好的低噪声权重路径，如 "output_dir_wan2.2_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 训练好的高噪声权重路径（如果训练了双Transformer） +transformer_high_path = None +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# 蒸馏模型 guidance_scale 通常为 1.0 +guidance_scale = 1.0 +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2/predict_i2v.py +``` + +根据需求修改编辑 `examples/wan2.2/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-I2V-A14B" +# 训练好的低噪声权重路径，如 "output_dir_wan2.2_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 训练好的高噪声权重路径 +transformer_high_path = None +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# 蒸馏模型 guidance_scale 通常为 1.0 +guidance_scale = 1.0 +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3.1 文本图生视频（TI2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2/predict_ti2v.py +``` + +根据需求修改编辑 `examples/wan2.2/predict_ti2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径（TI2V单模型） +model_name = "models/Diffusion_Transformer/Wan2.2-TI2V-5B" +# 训练好的权重路径，如 "output_dir_wan2.2_distill/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# TI2V只有一个模型，transformer_high_path不使用 +transformer_high_path = None +# 蒸馏模型通常使用 4 步 +num_inference_steps = 4 +# 蒸馏模型 guidance_scale 通常为 1.0 +guidance_scale = 1.0 +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.2/predict_t2v.py`、`examples/wan2.2/predict_i2v.py` 或 `examples/wan2.2/predict_ti2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.2/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.2/README_TRAIN_LORA.md b/scripts/wan2.2/README_TRAIN_LORA.md index 69aa36e..3a89981 100755 --- a/scripts/wan2.2/README_TRAIN_LORA.md +++ b/scripts/wan2.2/README_TRAIN_LORA.md @@ -1,110 +1,189 @@ -## Lora Training Code +# Wan2.2 LoRA Fine-tuning Training Guide -We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. +This document provides a complete workflow for Wan2.2 LoRA fine-tuning training, including environment configuration, data preparation, various distributed training strategies, and inference testing. -Some parameters in the sh file can be confusing, and they are explained in this document: +> **Note**: Wan2.2 is a video generation model that supports text-to-video (T2V), image-to-video (I2V), and text-image-to-video (TI2V). Wan2.2 adopts a dual-Transformer architecture (high-noise/low-noise models) for higher quality video generation. This guide covers the LoRA fine-tuning training workflow, suitable for fine-tuning scenarios with custom datasets. -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal, i2v or ti2v. The t2v is used for 14B T2V model. The i2v is used for 14B I2V model. The ti2v is used in 5B TI2V model. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). -- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. -- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. -- `rank` means the dimension of the LoRA update matrices. -- `network_alpha` means the scale of the LoRA update matrices. +--- -If you want to train 5B Wan2.2 TI2V model, please set config to `config/wan2.2/wan_civitai_5b.yaml`, set train_mode to `ti2v` and set boundary_type to `full`. +## Table of Contents +- [1. Environment Configuration](#1-environment-configuration) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative Path vs Absolute Path Usage](#24-relative-path-vs-absolute-path-usage) +- [3. LoRA Training](#3-lora-training) + - [3.1 Download Pre-trained Model](#31-download-pre-trained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 LoRA-specific Parameter Explanation](#33-lora-specific-parameter-explanation) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-machine Distributed Training](#37-multi-machine-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameter Explanation](#41-inference-parameter-explanation) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. More Resources](#5-more-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +## 1. Environment Configuration + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan2.2 T2V without deepspeed: +**Method 2: Manual Dependency Installation** -Training 14B Wan2.2 without DeepSpeed may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_lora.py \ - --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-04 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --boundary_type="low" \ - --rank=64 \ - --network_alpha=32 \ - --target_name="q,k,v,ffn.0,ffn.2" \ - --use_peft_lora \ - --train_mode="normal" \ - --low_vram +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -Wan T2V with Deepspeed Zero-2: +**Method 3: Using Docker** -Wan with Deepspeed Zero-2 is suitable for training Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. +When using Docker, please ensure that the graphics card driver and CUDA environment are correctly installed on your machine, then execute the following commands: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official example dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video width and height (**highly recommended to provide**, used for bucket training. If not provided, they will be automatically read during training, which may affect training speed when data is stored on slow systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height fields from JSON files without them, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative Path vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, set in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, set in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Suggestion**: If the dataset is small and stored locally, relative paths are recommended. If the dataset is stored on external storage (such as NAS, OSS) or shared across multiple machines, absolute paths are recommended. + +--- + +## 3. LoRA Training + +### 3.1 Download Pre-trained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download Wan2.2 official weights +# T2V model (text-to-video) +modelscope download --model Wan-AI/Wan2.2-T2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-T2V-A14B +# or I2V model (image-to-video) +# modelscope download --model Wan-AI/Wan2.2-I2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-I2V-A14B +# or TI2V model (text-image-to-video) +# modelscope download --model Wan-AI/Wan2.2-TI2V-5B --local_dir models/Diffusion_Transformer/Wan2.2-TI2V-5B +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After following **2.1 Quick Test Dataset** and **3.1 Download Pre-trained Model**, you can directly copy the quick start command to launch training. + +DeepSpeed-Zero-2 and FSDP are recommended for training. Here we use DeepSpeed-Zero-2 as an example. + +The difference between DeepSpeed-Zero-2 and FSDP in this repository lies in whether model weights are sharded. **If you encounter insufficient GPU memory when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +**Wan2.2 T2V LoRA Training Example**: + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -128,7 +207,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -145,32 +224,22 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --target_name="q,k,v,ffn.0,ffn.2" \ --use_peft_lora \ --train_mode="normal" \ - --low_vram + --low_vram ``` -DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. +**Wan2.2 I2V LoRA Training Example**: -It is known that DeepSpeed Zero-3 is not compatible with PEFT. - -Wan T2V with DeepSpeed Zero-3: - -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. You must set save_state to True to save the model. After training, you can use the following command to get the final model: -```sh -python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization -``` - -Training shell command is as follows: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ - --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -187,7 +256,7 @@ accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -199,17 +268,192 @@ accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag --enable_bucket \ --uniform_sampling \ --boundary_type="low" \ - --train_mode="normal" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="i2v" \ --low_vram ``` -Wan T2V with FSDP: +**Wan2.2 TI2V LoRA Training Example**: + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-TI2V-5B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="ti2v" \ + --low_vram +``` + +### 3.3 LoRA-specific Parameter Explanation + +**Wan2.2 Dual-Transformer Architecture Explanation**: + +Wan2.2 adopts an innovative dual-Transformer architecture: +- **Low Noise Model**: Responsible for handling the low-noise stage (closer to final output) +- **High Noise Model**: Responsible for handling the high-noise stage (initial generation stage) +- **Boundary Type (boundary_type)**: + - `low`: Train low noise model, high noise model uses pre-trained weights (recommended for T2V/I2V LoRA fine-tuning) + - `high`: Train high noise model, low noise model uses pre-trained weights + - `full`: Single model training (for single-Transformer models like TI2V-5B) + +**Key LoRA Parameters**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `--config_path` | Configuration file path | `config/wan2.2/wan_civitai_t2v.yaml` | +| `--pretrained_model_name_or_path` | Pre-trained model path | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Number of samples per batch | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sampling size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Number of video frames to sample | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader subprocesses | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (recommended for LoRA) | 1e-04 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed (for reproducible training) | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.2_lora` | +| `--gradient_checkpointing` | Activation recomputation | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping, group by resolution | - | +| `--random_hw_adapt` | Auto-scale images/videos to random sizes within `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling (recommended) | - | +| `--low_vram` | Low VRAM mode | - | +| `--boundary_type` | Wan2.2 dual-Transformer boundary type: `low` (train low noise model), `high` (train high noise model), `full` (train single model like TI2V-5B) | `low` | +| `--train_mode` | Training mode: `normal` (T2V), `i2v` (image-to-video), or `ti2v` (text-image-to-video) | `normal` | +| `--resume_from_checkpoint` | Resume training path, use `"latest"` to auto-select latest checkpoint | None | +| `--rank` | LoRA update matrix dimension (higher rank = stronger expression but more memory) | 64 | +| `--network_alpha` | LoRA update matrix scaling factor (usually set to half of rank or same) | 32 | +| `--target_name` | Components/modules to apply LoRA, comma-separated | `q,k,v,ffn.0,ffn.2` | +| `--use_peft_lora` | Use PEFT module to add LoRA (more memory-efficient) | - | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for validating video generation | `"A brown dog shaking head..."` | +| `--validation_paths` | Reference image paths for I2V/TI2V validation (i2v/ti2v mode only) | `"asset/1.png"` | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: A video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameters**: + +| Parameter | Description | Recommended Value | +|-----------|-------------|-------------------| +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 100 | +| `--validation_prompts` | Prompts for video generation validation | English prompts | +| `--validation_paths` | Reference image paths for I2V validation (i2v/inpaint mode only) | `"asset/1.png"` | + +**Normal Mode Example** (T2V validation): + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**TI2V Mode Example** (TI2V validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**I2V Mode Example** (I2V validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**Notes**: +- Validation videos are saved to the `output_dir` directory +- Multi-prompt validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` or `ti2v` mode must provide the `--validation_paths` parameter +- Wan2.2 validation automatically selects single-Transformer or dual-Transformer based on `boundary_type` + +### 3.5 Training with FSDP + +**If you encounter insufficient GPU memory when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: ```sh export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -233,7 +477,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -249,23 +493,33 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --network_alpha=32 \ --target_name="q,k,v,ffn.0,ffn.2" \ --use_peft_lora \ - --train_mode="normal" \ - --low_vram + --low_vram \ + --train_mode="normal" ``` -If you want to train 5B Wan2.2 TI2V model, please set config to `config/wan2.2/wan_civitai_5b.yaml`, set train_mode to `ti2v` and set boundary_type to `full`. Training shell command is as follows: +### 3.6 Other Backends +#### 3.6.1 Training with DeepSpeed-Zero-3 + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +DeepSpeed Zero-3 is suitable for high-resolution 14B Wan. After training, you can use the following command to obtain the final model: +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: ```sh export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_lora.py \ - --config_path="config/wan2.2/wan_civitai_5b.yaml" \ +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -282,7 +536,7 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_lora.py \ --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -293,11 +547,297 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_lora.py \ --training_with_video_token_length \ --enable_bucket \ --uniform_sampling \ - --boundary_type="full" \ + --boundary_type="low" \ --rank=64 \ --network_alpha=32 \ --target_name="q,k,v,ffn.0,ffn.2" \ --use_peft_lora \ - --train_mode="ti2v" \ - --low_vram -``` \ No newline at end of file + --low_vram \ + --train_mode="normal" +``` + +#### 3.6.2 Training without DeepSpeed and FSDP + +**This approach is not recommended, as without memory-saving backends, it easily causes out-of-memory errors**. Only provided here for reference. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +### 3.7 Multi-machine Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note: this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-machine Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data path (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameter Explanation + +**Key Parameters**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `GPU_memory_mode` | Memory management mode, see table below for options | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism degree, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism degree, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save memory | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Path to load trained low-noise Transformer weights | `None` | +| `transformer_high_path` | Path to load trained high-noise Transformer weights (dual-Transformer models only) | `None` | +| `vae_path` | Path to load trained VAE weights | `None` | +| `lora_path` | Low-noise model LoRA weights path | `None` | +| `lora_high_path` | High-noise model LoRA weights path (dual-Transformer models only) | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs without bf16 support | `torch.bfloat16` | +| `validation_image_start` | Reference image path for image-to-video (I2V/TI2V mode) | `"asset/1.png"` | +| `prompt` | Positive prompt describing generated content | `"A brown dog shaking head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"low resolution, low quality..."` | +| `guidance_scale` | Guidance strength | 6.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 50 | +| `lora_weight` | Low-noise model LoRA weight strength | 0.55 | +| `lora_high_weight` | High-noise model LoRA weight strength (dual-Transformer models only) | 0.55 | +| `save_path` | Path to save generated videos | `samples/wan-videos-i2v` or `samples/wan-videos-t2v` | + +**Memory Management Modes**: + +| Mode | Description | Memory Usage | +|------|-------------|--------------| +| `model_full_load` | Entire model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.2/predict_t2v.py +``` + +Modify `examples/wan2.2/predict_t2v.py` according to your needs. For first-time inference, focus on the parameters below. If you're interested in other parameters, refer to the inference parameter explanation above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.2-T2V-A14B" +# Path to trained low-noise weights, e.g., "output_dir_wan2.2_lora/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Path to trained high-noise weights (if trained dual-Transformer) +transformer_high_path = None +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 Image-to-Video (I2V) Inference + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.2/predict_i2v.py +``` + +Modify `examples/wan2.2/predict_i2v.py` according to your needs. For first-time inference, focus on the parameters below. If you're interested in other parameters, refer to the inference parameter explanation above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.2-I2V-A14B" +# Path to trained low-noise weights, e.g., "output_dir_wan2.2_lora/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Path to trained high-noise weights +transformer_high_path = None +# Starting image for image-to-video +validation_image_start = "asset/1.png" +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.2/predict_t2v.py`, `examples/wan2.2/predict_i2v.py`, or `examples/wan2.2/predict_ti2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must evenly divide the model's head count +- `ring_degree` splits along the sequence dimension, which affects communication overhead. Avoid using it when heads can be evenly divided. + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|-----------|----------------|-------------|-------------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.2/predict_t2v.py +``` + +--- + +## 5. More Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.2/README_TRAIN_LORA_zh-CN.md b/scripts/wan2.2/README_TRAIN_LORA_zh-CN.md new file mode 100644 index 0000000..02487aa --- /dev/null +++ b/scripts/wan2.2/README_TRAIN_LORA_zh-CN.md @@ -0,0 +1,843 @@ +# Wan2.2 LoRA 微调训练指南 + +本文档提供 Wan2.2 LoRA 微调训练的完整流程,包括环境配置、数据准备、多种分布式训练策略和推理测试。 + +> **说明**：Wan2.2 是一个支持文生视频（T2V）、图生视频（I2V）和文本图生视频（TI2V）的视频生成模型。Wan2.2采用双Transformer架构（高噪声/低噪声模型），支持更高质量的视频生成。本指南涵盖 LoRA 微调训练流程，适用于自定义数据集的微调场景。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、LoRA 训练](#三lora-训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 LoRA 专用参数解析](#33-lora-专用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、LoRA 训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.2 官方权重 +# T2V 模型（文生视频） +modelscope download --model Wan-AI/Wan2.2-T2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-T2V-A14B +# 或 I2V 模型（图生视频） +# modelscope download --model Wan-AI/Wan2.2-I2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-I2V-A14B +# 或 TI2V 模型（文本图生视频） +# modelscope download --model Wan-AI/Wan2.2-TI2V-5B --local_dir models/Diffusion_Transformer/Wan2.2-TI2V-5B +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用 DeepSpeed-Zero-2 与 FSDP 方案进行训练。这里以 DeepSpeed-Zero-2 为例配置 shell 文件。 + +本文中 DeepSpeed-Zero-2 与 FSDP 的差别在于是否对模型权重进行分片，**如果使用多卡且使用 DeepSpeed-Zero-2 的情况下显存不足**，可以切换使用 FSDP 进行训练。 + +**Wan2.2 T2V LoRA 训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +**Wan2.2 I2V LoRA 训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="i2v" \ + --low_vram +``` + +**Wan2.2 TI2V LoRA 训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-TI2V-5B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="ti2v" \ + --low_vram +``` + +### 3.3 LoRA 专用参数解析 + +**Wan2.2 双Transformer架构说明**： + +Wan2.2采用了创新的双Transformer架构： +- **低噪声模型（Low Noise Model）**：负责处理低噪声阶段（接近最终输出） +- **高噪声模型（High Noise Model）**：负责处理高噪声阶段（初始生成阶段） +- **边界类型（boundary_type）**： + - `low`：训练低噪声模型，高噪声模型使用预训练权重（推荐用于T2V/I2V LoRA微调） + - `high`：训练高噪声模型，低噪声模型使用预训练权重 + - `full`：单模型训练（用于TI2V-5B等单Transformer模型） + +**LoRA 关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--config_path` | 配置文件路径 | `config/wan2.2/wan_civitai_t2v.yaml` | +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率（LoRA 推荐值） | 1e-04 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子（可复现训练） | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.2_lora` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep（推荐启用） | - | +| `--low_vram` | 低显存模式 | - | +| `--boundary_type` | Wan2.2双Transformer边界类型：`low`（训练低噪声模型）、`high`（训练高噪声模型）、`full`（训练单模型如TI2V-5B） | `low` | +| `--train_mode` | 训练模式：`normal`（普通T2V）、`i2v`（图生视频）或 `ti2v`（文本图生视频） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--rank` | LoRA 更新矩阵的维度（rank 越大表达能力越强，但显存占用越高） | 64 | +| `--network_alpha` | LoRA 更新矩阵的缩放系数（通常设置为 rank 的一半或相同） | 32 | +| `--target_name` | 应用 LoRA 的组件/模块，用逗号分隔 | `q,k,v,ffn.0,ffn.2` | +| `--use_peft_lora` | 使用 PEFT 模块添加 LoRA（更节省显存） | - | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证 I2V/TI2V 的参考图像路径（仅 i2v/ti2v 模式） | `"asset/1.png"` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 100 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v/inpaint 模式） | `"asset/1.png"` | + +**normal 模式示例**（T2V 验证）： + +```bash + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**ti2v 模式示例**（TI2V 验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**i2v 模式示例**（I2V 验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` 或 `ti2v` 模式必须提供 `--validation_paths` 参数 +- Wan2.2的验证会根据`boundary_type`自动选择使用单Transformer或双Transformer + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的低噪声 Transformer 权重路径 | `None` | +| `transformer_high_path` | 加载训练好的高噪声 Transformer 权重路径（仅双Transformer模型） | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | 低噪声模型 LoRA 权重路径 | `None` | +| `lora_high_path` | 高噪声模型 LoRA 权重路径（仅双Transformer模型） | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V/TI2V 模式） | `"asset/1.png"` | +| `prompt` | 正向提示词，描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度 | 6.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 50 | +| `lora_weight` | 低噪声模型 LoRA 权重强度 | 0.55 | +| `lora_high_weight` | 高噪声模型 LoRA 权重强度（仅双Transformer模型） | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-i2v` 或 `samples/wan-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2/predict_t2v.py +``` + +根据需求修改编辑 `examples/wan2.2/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-T2V-A14B" +# 训练好的低噪声权重路径，如 "output_dir_wan2.2_lora/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 训练好的高噪声权重路径（如果训练了双Transformer） +transformer_high_path = None +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2/predict_i2v.py +``` + +根据需求修改编辑 `examples/wan2.2/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-I2V-A14B" +# 训练好的低噪声权重路径，如 "output_dir_wan2.2_lora/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 训练好的高噪声权重路径 +transformer_high_path = None +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.2/predict_t2v.py`、`examples/wan2.2/predict_i2v.py` 或 `examples/wan2.2/predict_ti2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.2/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.2/README_TRAIN_S2V.md b/scripts/wan2.2/README_TRAIN_S2V.md index 31788fb..047a555 100755 --- a/scripts/wan2.2/README_TRAIN_S2V.md +++ b/scripts/wan2.2/README_TRAIN_S2V.md @@ -194,8 +194,8 @@ The difference between DeepSpeed-Zero-2 and FSDP lies in whether the model weigh ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -314,8 +314,8 @@ You can configure validation parameters to periodically generate test videos dur ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -362,8 +362,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -415,8 +415,8 @@ Assuming 2 machines with 8 GPUs each: **Machine 0 (Master)**: ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Master machine IP export MASTER_PORT=10086 export WORLD_SIZE=2 # Total number of machines @@ -465,8 +465,8 @@ accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main **Machine 1 (Worker)**: ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Same as Master export MASTER_PORT=10086 export WORLD_SIZE=2 diff --git a/scripts/wan2.2/README_TRAIN_S2V_zh-CN.md b/scripts/wan2.2/README_TRAIN_S2V_zh-CN.md index f48ecf9..d68ac9f 100755 --- a/scripts/wan2.2/README_TRAIN_S2V_zh-CN.md +++ b/scripts/wan2.2/README_TRAIN_S2V_zh-CN.md @@ -189,8 +189,8 @@ DeepSpeed-Zero-2 与 FSDP 的区别在于模型权重是否分片。**如果多 ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" # 如果没有 RDMA 的多节点训练，取消注释以下两行 # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -298,8 +298,8 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_s2v.py \ ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap WanS2VAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train_s2v.py \ @@ -343,8 +343,8 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" NCCL_DEBUG=INFO accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_s2v.py \ @@ -393,8 +393,8 @@ accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_s2v.py \ **机器 0（Master）**： ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # Master 机器 IP export MASTER_PORT=10086 export WORLD_SIZE=2 # 总机器数 @@ -443,8 +443,8 @@ accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main **机器 1（Worker）**： ```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +export DATASET_NAME="datasets/X-Fun-Videos-Audios-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Audios-Demo/metadata_add_width_height.json" export MASTER_ADDR="192.168.1.100" # 与 Master 相同 export MASTER_PORT=10086 export WORLD_SIZE=2 diff --git a/scripts/wan2.2/README_TRAIN_zh-CN.md b/scripts/wan2.2/README_TRAIN_zh-CN.md new file mode 100644 index 0000000..aaec812 --- /dev/null +++ b/scripts/wan2.2/README_TRAIN_zh-CN.md @@ -0,0 +1,865 @@ +# Wan2.2 全参数训练指南 + +本文档提供 Wan2.2 Diffusion Transformer 全参数训练的完整流程，包括环境配置、数据准备、分布式训练和推理测试。 + +> **说明**：Wan2.2 是一个支持文生视频（T2V）、图生视频（I2V）和文本图生视频（TI2V）的视频生成模型。Wan2.2采用双Transformer架构（高噪声/低噪声模型），支持更高质量的视频生成。本文档涵盖普通视频生成任务的训练流程。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、全量参数训练](#三全量参数训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 训练常用参数解析](#33-训练常用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、全量参数训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.2 官方权重 +# T2V 模型（文生视频） +modelscope download --model Wan-AI/Wan2.2-T2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-T2V-A14B +# 或 I2V 模型（图生视频） +# modelscope download --model Wan-AI/Wan2.2-I2V-A14B --local_dir models/Diffusion_Transformer/Wan2.2-I2V-A14B +# 或 TI2V 模型（文本图生视频） +# modelscope download --model Wan-AI/Wan2.2-TI2V-5B --local_dir models/Diffusion_Transformer/Wan2.2-TI2V-5B +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用DeepSpeed-Zero-2与FSDP方案进行训练。这里使用DeepSpeed-Zero-2为例配置shell文件。 + +本文中DeepSpeed-Zero-2与FSDP的差别在于是否对模型权重进行分片，**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +**Wan2.2 T2V 训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." +``` + +**Wan2.2 I2V 训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="i2v" \ + --trainable_modules "." +``` + +**Wan2.2 TI2V 训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-TI2V-5B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="full" \ + --train_mode="ti2v" \ + --trainable_modules "." +``` + +### 3.3 训练常用参数解析 + +**Wan2.2 双Transformer架构说明**： + +Wan2.2采用了创新的双Transformer架构： +- **低噪声模型（Low Noise Model）**：负责处理低噪声阶段（接近最终输出） +- **高噪声模型（High Noise Model）**：负责处理高噪声阶段（初始生成阶段） +- **边界类型（boundary_type）**： + - `low`：训练低噪声模型，高噪声模型使用预训练权重（推荐用于T2V/I2V微调） + - `high`：训练高噪声模型，低噪声模型使用预训练权重 + - `full`：单模型训练（用于TI2V-5B等单Transformer模型） + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率 | 2e-05 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.2` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep | - | +| `--low_vram` | 低显存模式 | - | +| `--boundary_type` | Wan2.2双Transformer边界类型：`low`（训练低噪声模型）、`high`（训练高噪声模型）、`full`（训练单模型如TI2V-5B） | `low` | +| `--train_mode` | 训练模式：`normal`（普通T2V）、`i2v`（图生视频）或 `ti2v`（文本图生视频） | `normal` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 500 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v 模式） | `"asset/1.png"` | +| `--trainable_modules` | 可训练模块（`"."` 表示所有模块） | `"."` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Wan2.2 训练策略建议**： +- **T2V模型（双Transformer）**：使用 `boundary_type="low"` 和 `train_mode="normal"` 训练低噪声模型，这样可以保持高噪声部分的通用性，同时微调低噪声部分以适应您的数据。 +- **I2V模型（双Transformer）**：使用 `boundary_type="low"` 和 `train_mode="i2v"` 训练低噪声模型，数据集需要包含参考图像。 +- **TI2V模型（单Transformer）**：使用 `boundary_type="full"` 和 `train_mode="ti2v"` 进行完整训练，数据集需要包含参考图像。 +- **显存优化**：Wan2.2模型较大（14B/5B参数），强烈建议使用 `--low_vram` 和 `--gradient_checkpointing`。 +- **多卡训练**：对于14B模型，推荐使用FSDP或DeepSpeed-Zero-2/3进行多卡训练。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 500 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 i2v 模式） | `"asset/1.png"` | + +**normal 模式示例**（T2V 验证）： + +```bash + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**ti2v 模式示例**（TI2V 验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**i2v 模式示例**（I2V 验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` +- `i2v` 或 `ti2v` 模式必须提供 `--validation_paths` 参数 +- Wan2.2的验证会根据`boundary_type`自动选择使用单Transformer或双Transformer + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的低噪声 Transformer 权重路径 | `None` | +| `transformer_high_path` | 加载训练好的高噪声 Transformer 权重路径（仅双Transformer模型） | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | 低噪声模型 LoRA 权重路径 | `None` | +| `lora_high_path` | 高噪声模型 LoRA 权重路径（仅双Transformer模型） | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V/TI2V 模式） | `"asset/1.png"` | +| `prompt` | 正向提示词，描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度 | 6.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 50 | +| `lora_weight` | 低噪声模型 LoRA 权重强度 | 0.55 | +| `lora_high_weight` | 高噪声模型 LoRA 权重强度（仅双Transformer模型） | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-i2v` 或 `samples/wan-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2/predict_t2v.py +``` + +根据需求修改编辑 `examples/wan2.2/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-T2V-A14B" +# 训练好的低噪声权重路径，如 "output_dir_wan2.2/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 训练好的高噪声权重路径（如果训练了双Transformer） +transformer_high_path = None +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3 图生视频（I2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2/predict_i2v.py +``` + +根据需求修改编辑 `examples/wan2.2/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-I2V-A14B" +# 训练好的低噪声权重路径，如 "output_dir_wan2.2/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 训练好的高噪声权重路径 +transformer_high_path = None +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.3.1 文本图生视频（TI2V）推理 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2/predict_ti2v.py +``` + +根据需求修改编辑 `examples/wan2.2/predict_ti2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径（TI2V单模型） +model_name = "models/Diffusion_Transformer/Wan2.2-TI2V-5B" +# 训练好的权重路径 +transformer_path = None +# TI2V只有一个模型，transformer_high_path不使用 +transformer_high_path = None +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.2/predict_t2v.py`、`examples/wan2.2/predict_i2v.py` 或 `examples/wan2.2/predict_ti2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.2/predict_t2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.2/train.py b/scripts/wan2.2/train.py index efd0dec..ebf56ad 100644 --- a/scripts/wan2.2/train.py +++ b/scripts/wan2.2/train.py @@ -709,7 +709,9 @@ def parse_args(): type=str, default="low", help=( - 'The format of training data. Support `"low"` and `"high"`' + 'The training boundary type for dual-Transformer architecture. ' + 'Support `"low"` (train low-noise model), `"high"` (train high-noise model), ' + 'and `"full"` (single model training like TI2V-5B).' ), ) parser.add_argument( @@ -717,8 +719,8 @@ def parse_args(): type=str, default="normal", help=( - 'The format of training data. Support `"normal"`' - ' (default), `"i2v"`.' + 'The training mode. Support `"normal"` (T2V, default), ' + '`"i2v"` (Image-to-Video), and `"ti2v"` (Text-Image-to-Video).' ), ) parser.add_argument( diff --git a/scripts/wan2.2/train_distill.py b/scripts/wan2.2/train_distill.py index 5786eed..367f994 100644 --- a/scripts/wan2.2/train_distill.py +++ b/scripts/wan2.2/train_distill.py @@ -53,7 +53,7 @@ from PIL import Image from torch.distributed.fsdp.fully_sharded_data_parallel import ( FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, ShardedStateDictConfig) -from torch.utils.data import Dataset, RandomSampler +from torch.utils.data import Dataset, RandomSampler, BatchSampler from torch.utils.tensorboard import SummaryWriter from torchvision import transforms from tqdm.auto import tqdm @@ -69,7 +69,7 @@ for project_root in project_roots: from videox_fun.data import (ASPECT_RATIO_512, ASPECT_RATIO_RANDOM_CROP_512, ASPECT_RATIO_RANDOM_CROP_PROB, - AspectRatioBatchImageVideoSampler, BatchSampler, + AspectRatioBatchImageVideoSampler, ImageVideoDataset, ImageVideoSampler, RandomSampler, TextDataset, get_closest_ratio, get_random_mask) @@ -726,7 +726,9 @@ def parse_args(): type=str, default="low", help=( - 'The format of training data. Support `"low"` and `"high"`' + 'The training boundary type for dual-Transformer architecture. ' + 'Support `"low"` (train low-noise model), `"high"` (train high-noise model), ' + 'and `"full"` (single model training like TI2V-5B).' ), ) parser.add_argument( @@ -734,8 +736,8 @@ def parse_args(): type=str, default="normal", help=( - 'The format of training data. Support `"normal"`' - ' (default), `"i2v"`.' + 'The training mode. Support `"normal"` (T2V, default), ' + '`"i2v"` (Image-to-Video), and `"ti2v"` (Text-Image-to-Video).' ), ) parser.add_argument( diff --git a/scripts/wan2.2/train_distill_lora.py b/scripts/wan2.2/train_distill_lora.py index b22cee9..a7d5d93 100644 --- a/scripts/wan2.2/train_distill_lora.py +++ b/scripts/wan2.2/train_distill_lora.py @@ -53,7 +53,7 @@ from PIL import Image from torch.distributed.fsdp.fully_sharded_data_parallel import ( FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, ShardedStateDictConfig) -from torch.utils.data import Dataset, RandomSampler +from torch.utils.data import BatchSampler, Dataset, RandomSampler from torch.utils.tensorboard import SummaryWriter from torchvision import transforms from tqdm.auto import tqdm @@ -69,7 +69,7 @@ for project_root in project_roots: from videox_fun.data import (ASPECT_RATIO_512, ASPECT_RATIO_RANDOM_CROP_512, ASPECT_RATIO_RANDOM_CROP_PROB, - AspectRatioBatchImageVideoSampler, BatchSampler, + AspectRatioBatchImageVideoSampler, ImageVideoDataset, ImageVideoSampler, RandomSampler, TextDataset, get_closest_ratio, get_random_mask) @@ -740,7 +740,9 @@ def parse_args(): type=str, default="low", help=( - 'The format of training data. Support `"low"` and `"high"`' + 'The training boundary type for dual-Transformer architecture. ' + 'Support `"low"` (train low-noise model), `"high"` (train high-noise model), ' + 'and `"full"` (single model training like TI2V-5B).' ), ) parser.add_argument( @@ -748,8 +750,8 @@ def parse_args(): type=str, default="normal", help=( - 'The format of training data. Support `"normal"`' - ' (default), `"i2v"`.' + 'The training mode. Support `"normal"` (T2V, default), ' + '`"i2v"` (Image-to-Video), and `"ti2v"` (Text-Image-to-Video).' ), ) parser.add_argument( diff --git a/scripts/wan2.2/train_lora.py b/scripts/wan2.2/train_lora.py index 1404be5..2fb1015 100755 --- a/scripts/wan2.2/train_lora.py +++ b/scripts/wan2.2/train_lora.py @@ -617,12 +617,6 @@ def parse_args(): parser.add_argument( "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", ) - parser.add_argument( - "--noise_share_in_frames", action="store_true", help="Whether enable noise share in frames." - ) - parser.add_argument( - "--noise_share_in_frames_ratio", type=float, default=0.5, help="Noise share ratio.", - ) parser.add_argument( "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." ) @@ -723,7 +717,9 @@ def parse_args(): type=str, default="low", help=( - 'The format of training data. Support `"low"` and `"high"`' + 'The training boundary type for dual-Transformer architecture. ' + 'Support `"low"` (train low-noise model), `"high"` (train high-noise model), ' + 'and `"full"` (single model training like TI2V-5B).' ), ) parser.add_argument( @@ -731,8 +727,8 @@ def parse_args(): type=str, default="normal", help=( - 'The format of training data. Support `"normal"`' - ' (default), `"i2v"`.' + 'The training mode. Support `"normal"` (T2V, default), ' + '`"i2v"` (Image-to-Video), and `"ti2v"` (Text-Image-to-Video).' ), ) parser.add_argument( diff --git a/scripts/wan2.2_fun/README_TRAIN.md b/scripts/wan2.2_fun/README_TRAIN.md index a938759..db13c53 100755 --- a/scripts/wan2.2_fun/README_TRAIN.md +++ b/scripts/wan2.2_fun/README_TRAIN.md @@ -1,105 +1,187 @@ -## Training Code +# Wan2.2 Fun Full Parameter Training Guide -We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. +This document provides a complete workflow for full parameter training of Wan2.2 Fun (Inpainting), including environment setup, data preparation, distributed training, and inference testing. -Some parameters in the sh file can be confusing, and they are explained in this document: +> **Note**: Wan2.2 Fun is a video inpainting model based on the Wan2.2 architecture, supporting video inpainting tasks. Wan2.2 adopts a dual-Transformer architecture (high-noise/low-noise models), and the 5B version uses a single-Transformer architecture. This document covers the training workflow for Wan2.2 Fun, supporting both A14B and 5B model variants. -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or inpaint. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). +--- -If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. +## Table of Contents +- [1. Environment Setup](#1-environment-setup) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative vs Absolute Path Usage](#24-relative-vs-absolute-path-usage) +- [3. Full Parameter Training](#3-full-parameter-training) + - [3.1 Download Pretrained Model](#31-download-pretrained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 Training Parameter Reference](#33-training-parameter-reference) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-Node Distributed Training](#37-multi-node-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameter Reference](#41-inference-parameter-reference) + - [4.2 Text-to-Video (T2V) Inference](#42-text-to-video-t2v-inference) + - [4.3 Image-to-Video (I2V) Inference](#43-image-to-video-i2v-inference) + - [4.4 Multi-GPU Parallel Inference](#44-multi-gpu-parallel-inference) +- [5. Additional Resources](#5-additional-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/wan2.2_fun/xxx.py +## 1. Environment Setup + +**Option 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan T2V without deepspeed: +**Option 2: Manual Installation** -Training 14B Wan2.2 without DeepSpeed may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train.py \ - --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=2e-05 \ - --lr_scheduler="constant_with_warmup" \ - --lr_warmup_steps=100 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --boundary_type="low" \ - --low_vram \ - --train_mode="normal" \ - --trainable_modules "." +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless ``` -Wan T2V with Deepspeed Zero-2: +**Option 3: Using Docker** -Wan with Deepspeed Zero-2 is suitable for training 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. +When using Docker, ensure that the GPU driver and CUDA environment are properly installed on your machine, then execute the following commands: -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +```bash +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official demo dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, should be `"video"` +- `width` / `height`: Video width and height (**recommended to provide**, used for bucket training. If not provided, they will be automatically read during training, which may affect training speed when data is stored on slow storage systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height from JSON files without these fields. It supports processing both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, configure in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, configure in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Recommendation**: If the dataset is small and stored locally, use relative paths. If the dataset is stored on external storage (e.g., NAS, OSS) or shared across multiple machines, use absolute paths. + +--- + +## 3. Full Parameter Training + +### 3.1 Download Pretrained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download official Wan2.2 weights +# A14B model (dual-Transformer architecture) +modelscope download --model PAI/Wan2.2-Fun-A14B-InP --local_dir models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP +# or 5B model (single-Transformer architecture) +# modelscope download --model PAI/Wan2.2-Fun-5B-InP --local_dir models/Diffusion_Transformer/Wan2.2-Fun-5B-InP +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After downloading the dataset as in **2.1 Quick Test Dataset** and the pretrained model as in **3.1 Download Pretrained Model**, you can directly copy and run the quick start command. + +We recommend using DeepSpeed-Zero-2 or FSDP for training. Here we use DeepSpeed-Zero-2 as an example. + +The difference between DeepSpeed-Zero-2 and FSDP in this repository is whether the model weights are sharded. **If GPU memory is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +**Wan2.2 T2V Training Example**: + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -125,7 +207,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_fun" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -136,27 +218,290 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --training_with_video_token_length \ --enable_bucket \ --uniform_sampling \ - --boundary_type="low" \ --low_vram \ - --use_deepspeed \ + --boundary_type="low" \ --train_mode="inpaint" \ --trainable_modules "." ``` +**Wan2.2 I2V Training Example**: + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +**Wan2.2 Fun 5B Training Example**: + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-5B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="full" \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.3 Training Parameter Reference + +**Wan2.2 Dual-Transformer Architecture Explanation**: + +Wan2.2 adopts an innovative dual-Transformer architecture: +- **Low Noise Model**: Handles the low-noise stage (closer to final output) +- **High Noise Model**: Handles the high-noise stage (initial generation stage) +- **Boundary Type (boundary_type)**: + - `low`: Train low-noise model, high-noise model uses pretrained weights (recommended for T2V/I2V fine-tuning) + - `high`: Train high-noise model, low-noise model uses pretrained weights + - `full`: Single model training (for single-Transformer models like TI2V-5B) + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `--pretrained_model_name_or_path` | Pretrained model path | `models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP` | +| `--train_data_dir` | Training data directory | `datasets/internal_datasets/` | +| `--train_data_meta` | Training data metadata file | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | Batch size | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sample size | 640 | +| `--video_sample_stride` | Video sample stride | 2 | +| `--video_sample_n_frames` | Number of video frames to sample | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader workers | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate | 2e-05 | +| `--lr_scheduler` | Learning rate scheduler | `constant_with_warmup` | +| `--lr_warmup_steps` | Learning rate warmup steps | 100 | +| `--seed` | Random seed | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.2` | +| `--gradient_checkpointing` | Gradient checkpointing | - | +| `--mixed_precision` | Mixed precision: `fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 1 | +| `--max_grad_norm` | Maximum gradient norm for clipping | 0.05 | +| `--enable_bucket` | Enable bucket training without cropping, groups by resolution | - | +| `--random_hw_adapt` | Randomly scale images/videos to `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling | - | +| `--low_vram` | Low VRAM mode | - | +| `--boundary_type` | Wan2.2 dual-Transformer boundary type: `low` (train low-noise), `high` (train high-noise), `full` (single model like TI2V-5B) | `low` | +| `--train_mode` | Training mode: fixed to `inpaint` (video inpainting) | `inpaint` | +| `--resume_from_checkpoint` | Resume training from checkpoint, use `"latest"` to auto-select | None | +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 500 | +| `--validation_prompts` | Prompts for validation video generation | `"A brown dog shaking its head..."` | +| `--validation_paths` | Reference image paths for I2V validation (i2v mode only) | `"asset/1.png"` | +| `--trainable_modules` | Trainable modules (`"."` means all modules) | `"."` | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size for videos; when `random_hw_adapt` is True, it represents the minimum value for both video and image resolutions. +- `image_sample_size` represents the resolution size for images; when `random_hw_adapt` is True, it represents the maximum value for both video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Wan2.2 Training Strategy Recommendations**: +- **T2V Model (Dual-Transformer)**: Use `boundary_type="low"` and `train_mode="inpaint"` to train the low-noise model. This maintains the generality of the high-noise part while fine-tuning the low-noise part for your data. +- **I2V Model (Dual-Transformer)**: Use `boundary_type="low"` and `train_mode="inpaint"` to train the low-noise model. The dataset needs to include reference images. +- **TI2V Model (Single-Transformer)**: Use `boundary_type="full"` and `train_mode="inpaint"` for full training. The dataset needs to include reference images. +- **Memory Optimization**: Wan2.2 models are large (14B/5B parameters). It is strongly recommended to use `--low_vram` and `--gradient_checkpointing`. +- **Multi-GPU Training**: For 14B models, it is recommended to use FSDP or DeepSpeed-Zero-2/3 for multi-GPU training. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: a video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameter Descriptions**: + +| Parameter | Description | Recommended Value | +|-----------|-------------|-------------------| +| `--validation_steps` | Run validation every N steps | 100 | +| `--validation_epochs` | Run validation every N epochs | 500 | +| `--validation_prompts` | Prompts for validation video generation | English prompts | +| `--validation_paths` | Reference image paths for I2V validation (i2v mode only) | `"asset/1.png"` | + +**A14B Mode Example** (dual-Transformer validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**5B Mode Example** (single-Transformer validation): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**Notes**: +- Validation videos will be saved to the `output_dir` directory +- Multiple prompts format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- Wan2.2 Fun validation automatically selects single or dual-Transformer based on `boundary_type` +- Validation uses inpainting mode with reference image as the starting frame + +### 3.5 Training with FSDP + +**If GPU memory is insufficient when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.6 Other Backends + +#### 3.6.1 Training with DeepSpeed-Zero-3 + DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. -Wan T2V with DeepSpeed Zero-3: - -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: -```sh +DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```bash python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization ``` Training shell command is as follows: ```sh export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 @@ -182,7 +527,7 @@ accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_fun" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -193,26 +538,25 @@ accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag --training_with_video_token_length \ --enable_bucket \ --uniform_sampling \ - --boundary_type="low" \ --low_vram \ - --use_deepspeed \ --train_mode="inpaint" \ --trainable_modules "." ``` -Wan T2V with FSDP: +#### 3.6.2 Training Without DeepSpeed and FSDP + +**This approach is not recommended as it lacks memory-saving backends and may easily cause out-of-memory errors**. Provided here for reference only. -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: ```sh export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train.py \ +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train.py \ --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ @@ -232,7 +576,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_fun" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -243,25 +587,36 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --training_with_video_token_length \ --enable_bucket \ --uniform_sampling \ - --boundary_type="low" \ --low_vram \ --train_mode="inpaint" \ --trainable_modules "." ``` -If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. Training shell command is as follows: +### 3.7 Multi-Node Distributed Training -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-5B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines with 8 GPUs each: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train.py \ - --config_path="config/wan2.2/wan_civitai_5b.yaml" \ +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -280,7 +635,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --lr_scheduler="constant_with_warmup" \ --lr_warmup_steps=100 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_fun" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -291,8 +646,253 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --training_with_video_token_length \ --enable_bucket \ --uniform_sampling \ - --boundary_type="full" \ --low_vram \ --train_mode="inpaint" \ --trainable_modules "." -``` \ No newline at end of file +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note: this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-Node Training Notes + +- **Network Requirements**: + - Recommended: RDMA/InfiniBand (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must have access to the same data paths (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameter Reference + +**Key Parameter Descriptions**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `GPU_memory_mode` | GPU memory management mode, see table below | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Path to trained low-noise Transformer weights | `None` | +| `transformer_high_path` | Path to trained high-noise Transformer weights (dual-Transformer models only) | `None` | +| `vae_path` | Path to trained VAE weights | `None` | +| `lora_path` | Low-noise model LoRA weights path | `None` | +| `lora_high_path` | High-noise model LoRA weights path (dual-Transformer models only) | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs not supporting bf16 | `torch.bfloat16` | +| `validation_image_start` | Reference image path for I2V/TI2V mode | `"asset/1.png"` | +| `prompt` | Positive prompt describing content | `"A brown dog shaking its head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"low resolution, low quality..."` | +| `guidance_scale` | Guidance strength | 6.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 50 | +| `lora_weight` | Low-noise model LoRA weight strength | 0.55 | +| `lora_high_weight` | High-noise model LoRA weight strength (dual-Transformer models only) | 0.55 | +| `save_path` | Path to save generated video | `samples/wan-videos-i2v` or `samples/wan-videos-t2v` | + +**GPU Memory Management Modes**: + +| Mode | Description | Memory Usage | +|------|-------------|--------------| +| `model_full_load` | Full model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Text-to-Video (T2V) Inference + +#### 4.2.1 A14B Dual-Transformer Model + +Run the following command for single GPU inference: + +```bash +python examples/wan2.2_fun/predict_t2v.py +``` + +Edit `examples/wan2.2_fun/predict_t2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, refer to the inference parameter reference above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +# Path to trained low-noise weights, e.g., "output_dir_wan2.2/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Path to trained high-noise weights (if trained dual-Transformer) +transformer_high_path = None +# Write based on the content you want to generate +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +#### 4.2.2 5B Single-Transformer Model + +Run the following command for single GPU inference: + +```bash +python examples/wan2.2_fun/predict_t2v_5b.py +``` + +Edit `examples/wan2.2_fun/predict_t2v_5b.py` according to your needs. For initial inference, focus on the following parameters. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path (5B model) +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-5B-InP" +# Config path for 5B model +config_path = "config/wan2.2/wan_civitai_5b.yaml" +# Since 5B is a single-Transformer model, only transformer_path is used +transformer_path = None +transformer_high_path = None # Not used for 5B model +# Write based on the content you want to generate +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# 5B model typically uses higher resolution +sample_size = [704, 1280] +video_length = 121 +fps = 21 +# ... +``` + +**Key Differences Between A14B and 5B**: +- **Architecture**: A14B uses dual-Transformer (high/low noise), 5B uses single-Transformer +- **Config**: A14B uses `wan_civitai_i2v.yaml`, 5B uses `wan_civitai_5b.yaml` +- **Parameters**: For 5B, `transformer_high_path` and `lora_high_path` are not used +- **Resolution**: 5B model supports higher resolutions (e.g., 704x1280) +- **Frame Count**: 5B model can generate longer videos (e.g., 121 frames) + +### 4.3 Image-to-Video (I2V) Inference + +#### 4.3.1 A14B Dual-Transformer Model + +Run the following command for single GPU inference: + +```bash +python examples/wan2.2_fun/predict_i2v.py +``` + +Edit `examples/wan2.2_fun/predict_i2v.py` according to your needs. For initial inference, focus on the following parameters. If you're interested in other parameters, refer to the inference parameter reference above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +# Path to trained low-noise weights, e.g., "output_dir_wan2.2/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# Path to trained high-noise weights +transformer_high_path = None +# Starting image for image-to-video +validation_image_start = "asset/1.png" +# Write based on the content you want to generate +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +#### 4.3.2 5B Single-Transformer Model + +Run the following command for single GPU inference: + +```bash +python examples/wan2.2_fun/predict_i2v_5b.py +``` + +Edit `examples/wan2.2_fun/predict_i2v_5b.py` according to your needs. For initial inference, focus on the following parameters. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path (5B model) +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-5B-InP" +# Config path for 5B model +config_path = "config/wan2.2/wan_civitai_5b.yaml" +# Since 5B is a single-Transformer model, only transformer_path is used +transformer_path = None +transformer_high_path = None # Not used for 5B model +# Starting image for image-to-video +validation_image_start = "asset/1.png" +validation_image_end = None # Optional ending image +# Write based on the content you want to generate +prompt = "A brown dog licking its tongue, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# 5B model typically uses higher resolution +sample_size = [704, 1280] +video_length = 121 +fps = 24 +# ... +``` + +### 4.4 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.2_fun/predict_t2v.py` or `examples/wan2.2_fun/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must be divisible by the model's number of heads +- `ring_degree` splits along the sequence dimension and affects communication overhead; avoid using it when heads can be evenly divided + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|-----------|---------------|-------------|-------------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.2_fun/predict_i2v.py +``` + +--- + +## 5. Additional Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun \ No newline at end of file diff --git a/scripts/wan2.2_fun/README_TRAIN_LORA.md b/scripts/wan2.2_fun/README_TRAIN_LORA.md index 018b768..e8fba28 100755 --- a/scripts/wan2.2_fun/README_TRAIN_LORA.md +++ b/scripts/wan2.2_fun/README_TRAIN_LORA.md @@ -1,114 +1,201 @@ -## Lora Training Code +# Wan2.2 Fun LoRA Fine-tuning Training Guide -We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. +This document provides a complete workflow for Wan2.2 Fun (Inpainting) LoRA fine-tuning training, including environment configuration, data preparation, various distributed training strategies, and inference testing. -Some parameters in the sh file can be confusing, and they are explained in this document: +> **Note**: Wan2.2 Fun is a video inpainting model based on the Wan2.2 architecture, supporting video inpainting tasks. Wan2.2 adopts a dual-Transformer architecture (high-noise/low-noise models), and the 5B version uses a single-Transformer architecture. This guide covers the LoRA fine-tuning training workflow for Wan2.2 Fun, supporting both A14B and 5B model variants. -- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. -- Sample size Configuration Guide - - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. - - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. - - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. - - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. - - **All set to 320** represents **240P**. - - **All set to 480** represents **320P**. - - **All set to 640** represents **480P**. - - **All set to 960** represents **720P**. -- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. -- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. - - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. -- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. - - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. - - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. - - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). - - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). - - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). - - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. -- `train_mode` is used to specify the training mode, which can be either normal or inpaint. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. -- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. -- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). -- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. -- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. -- `rank` means the dimension of the LoRA update matrices. -- `network_alpha` means the scale of the LoRA update matrices. +--- -If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. +## Table of Contents +- [1. Environment Configuration](#1-environment-configuration) +- [2. Data Preparation](#2-data-preparation) + - [2.1 Quick Test Dataset](#21-quick-test-dataset) + - [2.2 Dataset Structure](#22-dataset-structure) + - [2.3 metadata.json Format](#23-metadatajson-format) + - [2.4 Relative Path vs Absolute Path Usage](#24-relative-path-vs-absolute-path-usage) +- [3. LoRA Training](#3-lora-training) + - [3.1 Download Pre-trained Model](#31-download-pre-trained-model) + - [3.2 Quick Start (DeepSpeed-Zero-2)](#32-quick-start-deepspeed-zero-2) + - [3.3 LoRA-specific Parameter Explanation](#33-lora-specific-parameter-explanation) + - [3.4 Training Validation](#34-training-validation) + - [3.5 Training with FSDP](#35-training-with-fsdp) + - [3.6 Other Backends](#36-other-backends) + - [3.7 Multi-machine Distributed Training](#37-multi-machine-distributed-training) +- [4. Inference Testing](#4-inference-testing) + - [4.1 Inference Parameter Explanation](#41-inference-parameter-explanation) + - [4.2 Inpainting Inference](#42-inpainting-inference) + - [4.3 Multi-GPU Parallel Inference](#43-multi-gpu-parallel-inference) +- [5. More Resources](#5-more-resources) -When train model with multi machines, please set the params as follows: -```sh -export MASTER_ADDR="your master address" -export MASTER_PORT=10086 -export WORLD_SIZE=1 # The number of machines -export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 -export RANK=0 # The rank of this machine +--- -accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/wan2.2_fun/xxx.py +## 1. Environment Configuration + +**Method 1: Using requirements.txt** + +```bash +pip install -r requirements.txt ``` -Wan T2V without deepspeed: +**Method 2: Manual Dependency Installation** -Training 14B Wan2.2 without DeepSpeed may result in insufficient GPU memory. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**Method 3: Using Docker** + +When using Docker, please ensure that the graphics card driver and CUDA environment are correctly installed on your machine, then execute the following commands: + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 2. Data Preparation + +### 2.1 Quick Test Dataset + +We provide a test dataset containing several training samples. + +```bash +# Download official example dataset +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 Dataset Structure + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json Format + +**Relative Path Format** (example format): +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**Absolute Path Format**: +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**Key Field Descriptions**: +- `file_path`: Video path (relative or absolute path) +- `text`: Video description (English prompt) +- `type`: Data type, fixed as `"video"` +- `width` / `height`: Video width and height (**highly recommended to provide**, used for bucket training. If not provided, they will be automatically read during training, which may affect training speed when data is stored on slow systems like OSS). + - You can use `scripts/process_json_add_width_and_height.py` to extract width and height fields from JSON files without them, supporting both images and videos. + - Usage: `python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`. + +### 2.4 Relative Path vs Absolute Path Usage + +**Relative Path**: + +If your data uses relative paths, set in the training script: + +```bash export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**Absolute Path**: + +If your data uses absolute paths, set in the training script: + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **Suggestion**: If the dataset is small and stored locally, relative paths are recommended. If the dataset is stored on external storage (such as NAS, OSS) or shared across multiple machines, absolute paths are recommended. + +--- + +## 3. LoRA Training + +### 3.1 Download Pre-trained Model + +```bash +# Create model directory +mkdir -p models/Diffusion_Transformer + +# Download Wan2.2 Fun official weights +# A14B model (dual-Transformer architecture) +modelscope download --model PAI/Wan2.2-Fun-A14B-InP --local_dir models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP +# or 5B model (single-Transformer architecture) +# modelscope download --model PAI/Wan2.2-Fun-5B-InP --local_dir models/Diffusion_Transformer/Wan2.2-Fun-5B-InP +``` + +### 3.2 Quick Start (DeepSpeed-Zero-2) + +After following **2.1 Quick Test Dataset** and **3.1 Download Pre-trained Model**, you can directly copy the quick start command to launch training. + +DeepSpeed-Zero-2 and FSDP are recommended for training. Here we use DeepSpeed-Zero-2 as an example. + +**DeepSpeed-Zero-2 vs FSDP**: +- Both are distributed training strategies that help reduce GPU memory usage +- DeepSpeed-Zero-2: Optimizer states and gradients are sharded across GPUs +- FSDP (Fully Sharded Data Parallel): Model weights, optimizer states, and gradients are all sharded +- **If you encounter insufficient GPU memory with DeepSpeed-Zero-2**, switch to FSDP for better memory efficiency + +> **About train_lora.sh**: The `train_lora.sh` script in this directory provides a basic training template **without DeepSpeed or FSDP**. It's suitable for: +> - Single-GPU training +> - Quick testing and debugging +> - Custom modifications for your specific needs +> +> For production training with multiple GPUs, **use the DeepSpeed-Zero-2 or FSDP commands below** for better performance and memory efficiency. + +**Wan2.2 Fun LoRA Training Example (DeepSpeed-Zero-2)**: + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO - -accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train_lora.py \ - --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --train_data_dir=$DATASET_NAME \ - --train_data_meta=$DATASET_META_NAME \ - --image_sample_size=640 \ - --video_sample_size=640 \ - --token_sample_size=640 \ - --video_sample_stride=2 \ - --video_sample_n_frames=81 \ - --train_batch_size=1 \ - --video_repeat=1 \ - --gradient_accumulation_steps=1 \ - --dataloader_num_workers=8 \ - --num_train_epochs=100 \ - --checkpointing_steps=50 \ - --learning_rate=1e-04 \ - --seed=42 \ - --output_dir="output_dir" \ - --gradient_checkpointing \ - --mixed_precision="bf16" \ - --adam_weight_decay=3e-2 \ - --adam_epsilon=1e-10 \ - --vae_mini_batch=1 \ - --max_grad_norm=0.05 \ - --random_hw_adapt \ - --training_with_video_token_length \ - --enable_bucket \ - --uniform_sampling \ - --boundary_type="low" \ - --rank=64 \ - --network_alpha=32 \ - --target_name="q,k,v,ffn.0,ffn.2" \ - --use_peft_lora \ - --train_mode="inpaint" \ - --low_vram -``` - -Wan T2V with Deepspeed Zero-2: - -Wan with Deepspeed Zero-2 is suitable for training 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. - -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" -# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. -# export NCCL_IB_DISABLE=1 -# export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +export NCCL_DEBUG=INFO accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_lora.py \ --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ @@ -128,7 +215,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_fun_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -148,26 +235,196 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --low_vram ``` +> **Note**: The `train_lora.sh` script in this directory provides a basic training template without DeepSpeed. For better multi-GPU training performance and memory efficiency, use the DeepSpeed-Zero-2 command above. + +### 3.3 LoRA-specific Parameter Explanation + +**Wan2.2 Dual-Transformer Architecture Explanation**: + +Wan2.2 adopts an innovative dual-Transformer architecture: +- **Low Noise Model**: Responsible for handling the low-noise stage (closer to final output) +- **High Noise Model**: Responsible for handling the high-noise stage (initial generation stage) +- **Boundary Type (boundary_type)**: + - `low`: Train low noise model, high noise model uses pre-trained weights (recommended for T2V/I2V LoRA fine-tuning) + - `high`: Train high noise model, low noise model uses pre-trained weights + - `full`: Single model training (for single-Transformer models like TI2V-5B) + +**Key LoRA Parameters**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `--config_path` | Configuration file path | `config/wan2.2/wan_civitai_i2v.yaml` | +| `--pretrained_model_name_or_path` | Pre-trained model path | `models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP` | +| `--train_data_dir` | Training data directory | `datasets/X-Fun-Videos-Demo/` | +| `--train_data_meta` | Training data metadata file | `datasets/X-Fun-Videos-Demo/metadata_add_width_height.json` | +| `--train_batch_size` | Number of samples per batch | 1 | +| `--image_sample_size` | Maximum training resolution for images | 640 | +| `--video_sample_size` | Maximum training resolution for videos | 640 | +| `--token_sample_size` | Token sampling size | 640 | +| `--video_sample_stride` | Video sampling stride | 2 | +| `--video_sample_n_frames` | Number of video frames to sample | 81 | +| `--gradient_accumulation_steps` | Gradient accumulation steps (effectively increases batch size) | 1 | +| `--dataloader_num_workers` | Number of DataLoader subprocesses | 8 | +| `--num_train_epochs` | Number of training epochs | 100 | +| `--checkpointing_steps` | Save checkpoint every N steps | 50 | +| `--learning_rate` | Initial learning rate (recommended for LoRA) | 1e-04 | +| `--lr_scheduler` | Learning rate scheduler: `linear`, `cosine`, `cosine_with_restarts`, `polynomial`, `constant`, `constant_with_warmup` | `constant` | +| `--lr_warmup_steps` | Learning rate warmup steps | 500 | +| `--seed` | Random seed (for reproducible training) | 42 | +| `--output_dir` | Output directory | `output_dir_wan2.2_fun_lora` | +| `--gradient_checkpointing` | Activation recomputation to save memory | - | +| `--mixed_precision` | Mixed precision: `no`, `fp16`, `bf16` | `bf16` | +| `--adam_weight_decay` | AdamW weight decay | 3e-2 | +| `--adam_epsilon` | AdamW epsilon value | 1e-10 | +| `--vae_mini_batch` | Mini-batch size for VAE encoding | 1 | +| `--max_grad_norm` | Gradient clipping threshold | 0.05 | +| `--enable_bucket` | Enable bucket training, no cropping, group by resolution | - | +| `--random_hw_adapt` | Auto-scale images/videos to random sizes within `[min_size, max_size]` | - | +| `--training_with_video_token_length` | Train based on token length, supports arbitrary resolutions | - | +| `--uniform_sampling` | Uniform timestep sampling (recommended) | - | +| `--low_vram` | Low VRAM mode for memory efficiency | - | +| `--boundary_type` | Wan2.2 dual-Transformer boundary type: `low` (train low noise model), `high` (train high noise model), `full` (train single model like TI2V-5B) | `low` | +| `--train_mode` | Training mode: `normal` (T2V), `inpaint` (video inpainting) | `inpaint` | +| `--resume_from_checkpoint` | Resume training path, use `"latest"` to auto-select latest checkpoint | None | +| `--rank` | LoRA update matrix dimension (higher rank = stronger expression but more memory) | 64 | +| `--network_alpha` | LoRA update matrix scaling factor (usually set to half of rank or same) | 32 | +| `--target_name` | Components/modules to apply LoRA, comma-separated (e.g., `q,k,v,ffn.0,ffn.2`) | `q,k,v,ffn.0,ffn.2` | +| `--lora_skip_name` | Components to skip in LoRA training, comma-separated | None | +| `--use_peft_lora` | Use PEFT module to add LoRA (more memory-efficient) | - | +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for validating video generation | `"A brown dog shaking head..."` | +| `--validation_paths` | Reference image paths for I2V validation (inpaint mode only) | `"asset/1.png"` | +| `--use_deepspeed` | Enable DeepSpeed for distributed training | - | +| `--use_fsdp` | Enable FSDP for distributed training | - | +| `--use_8bit_adam` | Use 8-bit Adam optimizer to save memory | - | +| `--use_came` | Use CAME optimizer | - | +| `--multi_stream` | Use CUDA multi-stream for performance | - | +| `--snr_loss` | Use SNR loss function | - | +| `--weighting_scheme` | Timestep weighting scheme: `sigma_sqrt`, `logit_normal`, `mode`, `cosmap`, `none` | `none` | +| `--motion_sub_loss` | Enable motion sub-loss for better temporal consistency | - | +| `--motion_sub_loss_ratio` | Motion sub-loss ratio | 0.25 | + +**Sample Size Configuration Guide**: +- `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. +- `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. +- `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. +- Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. + +**Token Length Training Explanation**: +- When `training_with_video_token_length` is enabled, the model trains based on token length. +- For example: A video with 512x512 resolution and 49 frames has a token length of 13,312, requiring `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding frame counts allow the model to generate videos of different sizes. + +### 3.4 Training Validation + +You can configure validation parameters to periodically generate test videos during training, allowing you to monitor training progress and model quality. + +**Validation Parameters**: + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `--validation_steps` | Run validation every N steps | 2000 | +| `--validation_epochs` | Run validation every N epochs | 5 | +| `--validation_prompts` | Prompts for video generation validation | None | +| `--validation_paths` | Reference image paths for I2V validation (inpaint mode only) | None | + +**Validation Example** (with inpaint mode): + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**Notes**: +- Validation videos are saved to the `output_dir` directory +- Multi-prompt validation format: `--validation_prompts "prompt1" "prompt2" "prompt3"` +- Wan2.2 Fun validation automatically selects single or dual-Transformer based on `boundary_type` +- Validation uses inpainting mode with reference image as the starting frame when `train_mode="inpaint"` +- For T2V validation (without reference image), set `train_mode="normal"` and omit `--validation_paths` + +### 3.5 Training with FSDP + +**If you encounter insufficient GPU memory when using multiple GPUs with DeepSpeed-Zero-2**, you can switch to FSDP for training. + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +export NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +> **Note**: FSDP is more stable in this repository and has fewer errors compared to DeepSpeed-Zero-3. Use FSDP when DeepSpeed-Zero-2 encounters memory issues with multiple GPUs. + +### 3.6 Other Backends + +#### 3.6.1 Training with DeepSpeed-Zero-3 + DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. -It is known that DeepSpeed Zero-3 is not compatible with PEFT. - -Wan T2V with DeepSpeed Zero-3: - -Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. You must set save_state to True to save the model. After training, you can use the following command to get the final model: -```sh +DeepSpeed Zero-3 is suitable for high-resolution 14B Wan. After training, you can use the following command to obtain the final model: +```bash python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization ``` Training shell command is as follows: -```sh +```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +export NCCL_DEBUG=INFO accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_lora.py \ --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ @@ -187,7 +444,7 @@ accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_fun_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -199,23 +456,28 @@ accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag --enable_bucket \ --uniform_sampling \ --boundary_type="low" \ - --train_mode="inpaint" \ - --low_vram + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" ``` -Wan T2V with FSDP: +#### 3.6.2 Training without DeepSpeed and FSDP -Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: -```sh +**This approach is not recommended, as without memory-saving backends, it easily causes out-of-memory errors**. Only provided here for reference. + +```bash export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +export NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train_lora.py \ +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train_lora.py \ --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ @@ -233,7 +495,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_fun_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -249,23 +511,37 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --network_alpha=32 \ --target_name="q,k,v,ffn.0,ffn.2" \ --use_peft_lora \ - --train_mode="inpaint" \ - --low_vram + --low_vram \ + --train_mode="inpaint" ``` -If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. Training shell command is as follows: +> **Note**: This is similar to the `train_lora.sh` script but with the correct dataset paths. The `train_lora.sh` script can be used as a starting point for single-GPU training. -```sh -export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-5B-InP" -export DATASET_NAME="datasets/internal_datasets/" -export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +### 3.7 Multi-machine Distributed Training + +**Suitable for**: Ultra-large-scale datasets, faster training speed + +#### 3.7.1 Environment Configuration + +Assuming 2 machines, each with 8 GPUs: + +**Machine 0 (Master)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master machine IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # Total number of machines +export NUM_PROCESS=16 # Total processes = machines × 8 +export RANK=0 # Current machine rank (0 or 1) # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. # export NCCL_IB_DISABLE=1 # export NCCL_P2P_DISABLE=1 -NCCL_DEBUG=INFO +export NCCL_DEBUG=INFO -accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train_lora.py \ - --config_path="config/wan2.2/wan_civitai_5b.yaml" \ +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATASET_NAME \ --train_data_meta=$DATASET_META_NAME \ @@ -282,7 +558,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --checkpointing_steps=50 \ --learning_rate=1e-04 \ --seed=42 \ - --output_dir="output_dir" \ + --output_dir="output_dir_wan2.2_fun_lora" \ --gradient_checkpointing \ --mixed_precision="bf16" \ --adam_weight_decay=3e-2 \ @@ -293,11 +569,223 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --training_with_video_token_length \ --enable_bucket \ --uniform_sampling \ - --boundary_type="full" \ + --boundary_type="low" \ --rank=64 \ --network_alpha=32 \ --target_name="q,k,v,ffn.0,ffn.2" \ --use_peft_lora \ - --train_mode="inpaint" \ - --low_vram -``` \ No newline at end of file + --low_vram \ + --train_mode="inpaint" +``` + +**Machine 1 (Worker)**: +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Same as Master +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # Note: this is 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +export NCCL_DEBUG=INFO + +# Use the same accelerate launch command as Machine 0 +``` + +#### 3.7.2 Multi-machine Training Notes + +- **Network Requirements**: + - RDMA/InfiniBand recommended (high performance) + - Without RDMA, add environment variables: + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **Data Synchronization**: All machines must be able to access the same data path (NFS/shared storage) + +--- + +## 4. Inference Testing + +### 4.1 Inference Parameter Explanation + +**Key Parameters**: + +| Parameter | Description | Example Value | +|-----------|-------------|---------------| +| `GPU_memory_mode` | Memory management mode, see table below for options | `model_group_offload` | +| `ulysses_degree` | Head dimension parallelism degree, 1 for single GPU | 1 | +| `ring_degree` | Sequence dimension parallelism degree, 1 for single GPU | 1 | +| `fsdp_dit` | Use FSDP for Transformer during multi-GPU inference to save memory | `False` | +| `fsdp_text_encoder` | Use FSDP for text encoder during multi-GPU inference | `True` | +| `compile_dit` | Compile Transformer for faster inference (effective for fixed resolution) | `False` | +| `model_name` | Model path | `models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP` | +| `sampler_name` | Sampler type: `Flow`, `Flow_Unipc`, `Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | Path to load trained low-noise Transformer weights | `None` | +| `transformer_high_path` | Path to load trained high-noise Transformer weights (dual-Transformer models only) | `None` | +| `vae_path` | Path to load trained VAE weights | `None` | +| `lora_path` | Low-noise model LoRA weights path | `None` | +| `lora_high_path` | High-noise model LoRA weights path (dual-Transformer models only) | `None` | +| `sample_size` | Generated video resolution `[height, width]` | `[480, 832]` or `[832, 480]` | +| `video_length` | Number of video frames | `81` | +| `fps` | Frames per second | `16` | +| `weight_dtype` | Model weight precision, use `torch.float16` for GPUs without bf16 support | `torch.bfloat16` | +| `validation_image_start` | Reference image path for image-to-video (I2V mode) | `"asset/1.png"` | +| `validation_image_end` | Reference image path for end frame (optional) | `None` | +| `validation_mask` | Mask path for inpainting (inpaint mode) | `None` | +| `prompt` | Positive prompt describing generated content | `"A brown dog shaking head..."` | +| `negative_prompt` | Negative prompt to avoid certain content | `"low resolution, low quality..."` | +| `guidance_scale` | Guidance strength | 6.0 | +| `seed` | Random seed for reproducibility | 43 | +| `num_inference_steps` | Number of inference steps | 50 | +| `lora_weight` | Low-noise model LoRA weight strength | 0.55 | +| `lora_high_weight` | High-noise model LoRA weight strength (dual-Transformer models only) | 0.55 | +| `save_path` | Path to save generated videos | `samples/wan-videos-i2v` or `samples/wan-videos-t2v` | + +**Memory Management Modes**: + +| Mode | Description | Memory Usage | +|------|-------------|--------------| +| `model_full_load` | Entire model loaded to GPU | Highest | +| `model_full_load_and_qfloat8` | Full load + FP8 quantization | High | +| `model_cpu_offload` | Offload model to CPU after use | Medium | +| `model_cpu_offload_and_qfloat8` | CPU offload + FP8 quantization | Medium-Low | +| `model_group_offload` | Layer groups switch between CPU/CUDA | Low | +| `sequential_cpu_offload` | Layer-by-layer offload (slowest) | Lowest | + +### 4.2 Inpainting Inference + +#### 4.2.1 Inference Script Selection + +Wan2.2 Fun provides multiple inference scripts. Choose based on your model version and task type: + +| Script | Model Version | Architecture | Primary Use | +|--------|--------------|--------------|-------------| +| `predict_i2v.py` | A14B | Dual-Transformer | Image-to-Video/Inpainting (I2V/Inpaint) | +| `predict_t2v.py` | A14B | Dual-Transformer | Text-to-Video (T2V) | +| `predict_i2v_5b.py` | 5B | Single-Transformer | Image-to-Video/Inpainting (I2V/Inpaint) | +| `predict_t2v_5b.py` | 5B | Single-Transformer | Text-to-Video (T2V) | + +> **Note**: +> - A14B model uses dual-Transformer architecture (low-noise + high-noise models), requiring both `transformer_path` and `transformer_high_path` +> - 5B model uses single-Transformer architecture, only `transformer_path` is needed, keep `transformer_high_path` as `None` + +#### 4.2.2 A14B Model Inference (Dual-Transformer) + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.2_fun/predict_i2v.py +``` + +Modify `examples/wan2.2_fun/predict_i2v.py` according to your needs. For first-time inference, focus on the parameters below. If you're interested in other parameters, refer to the inference parameter explanation above. + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# Based on actual model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +# Path to trained low-noise weights +transformer_path = None +# Path to trained high-noise weights +transformer_high_path = None +# Path to trained LoRA weights, e.g., "output_dir_wan2.2_fun_lora/checkpoint-xxx/diffusion_pytorch_model.safetensors" +lora_path = None +lora_high_path = None +# Starting image for inpainting +validation_image_start = "asset/1.png" +# Mask for inpainting (optional, will be auto-generated if not provided) +validation_mask = None +# Write based on generated content +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +> **Note**: Wan2.2 Fun is primarily designed for video inpainting tasks. Use `predict_i2v.py` for both I2V and inpainting workflows. The model will automatically handle inpainting when a mask is provided or generated. + +#### 4.2.3 5B Model Inference (Single-Transformer) + +Run the following command for single-GPU inference: + +```bash +python examples/wan2.2_fun/predict_i2v_5b.py +``` + +Modify `examples/wan2.2_fun/predict_i2v_5b.py` according to your needs, focusing on the parameters below: + +```python +# Choose based on GPU memory +GPU_memory_mode = "sequential_cpu_offload" +# 5B model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-5B-InP/" +# Path to trained weights (5B is single-Transformer, only set transformer_path) +transformer_path = None +# 5B model doesn't use high-noise Transformer, keep as None +transformer_high_path = None +# Path to trained LoRA weights, e.g., "output_dir_wan2.2_fun_lora/checkpoint-xxx/diffusion_pytorch_model.safetensors" +lora_path = None +# 5B model doesn't use high-noise LoRA, keep as None +lora_high_path = None +# Starting image for inpainting +validation_image_start = "asset/1.png" +validation_image_end = None +# Write based on generated content +prompt = "A brown dog licking its tongue, sitting on a light-colored sofa in a cozy room. Behind the dog, there is a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +> **Note**: +> - 5B model uses single-Transformer architecture with simpler configuration and lower memory usage +> - If you trained with `boundary_type="full"`, only load `transformer_path` during inference, no need to set `transformer_high_path` +> - For LoRA training, only set `lora_path`, keep `lora_high_path` as `None` + +### 4.3 Multi-GPU Parallel Inference + +**Suitable for**: High-resolution generation, accelerated inference + +#### Install Parallel Inference Dependencies + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### Configure Parallel Strategy + +Edit `examples/wan2.2_fun/predict_i2v.py`: + +```python +# Ensure ulysses_degree × ring_degree = number of GPUs used +# For example, using 2 GPUs: +ulysses_degree = 2 # Head dimension parallelism +ring_degree = 1 # Sequence dimension parallelism +``` + +**Configuration Principles**: +- `ulysses_degree` must evenly divide the model's head count +- `ring_degree` splits along the sequence dimension, which affects communication overhead. Avoid using it when heads can be evenly divided. + +**Configuration Examples**: + +| GPU Count | ulysses_degree | ring_degree | Description | +|-----------|----------------|-------------|-------------| +| 1 | 1 | 1 | Single GPU | +| 4 | 4 | 1 | Head parallelism | +| 8 | 8 | 1 | Head parallelism | +| 8 | 4 | 2 | Hybrid parallelism | + +#### Run Multi-GPU Inference + +```bash +torchrun --nproc-per-node=2 examples/wan2.2_fun/predict_i2v.py +``` + +--- + +## 5. More Resources + +- **Official GitHub**: https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.2_fun/README_TRAIN_LORA_zh-CN.md b/scripts/wan2.2_fun/README_TRAIN_LORA_zh-CN.md new file mode 100644 index 0000000..ad5f07e --- /dev/null +++ b/scripts/wan2.2_fun/README_TRAIN_LORA_zh-CN.md @@ -0,0 +1,791 @@ +# Wan2.2 Fun LoRA 微调训练指南 + +本文档提供 Wan2.2 Fun（视频修复）LoRA 微调训练的完整流程，包括环境配置、数据准备、多种分布式训练策略和推理测试。 + +> **说明**：Wan2.2 Fun 是一个基于 Wan2.2 架构的视频修复模型，支持视频修复任务。Wan2.2采用双Transformer架构（高噪声/低噪声模型），5B版本使用单Transformer架构。本指南涵盖 Wan2.2 Fun 的 LoRA 微调训练流程，支持 A14B 和 5B 两种模型变体。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、LoRA 训练](#三lora-训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 LoRA 专用参数解析](#33-lora-专用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 视频修复推理](#42-视频修复推理) + - [4.3 多卡并行推理](#43-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、LoRA 训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.2 Fun 官方权重 +# A14B 模型（双Transformer架构） +modelscope download --model PAI/Wan2.2-Fun-A14B-InP --local_dir models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP +# 或 5B 模型（单Transformer架构） +# modelscope download --model PAI/Wan2.2-Fun-5B-InP --local_dir models/Diffusion_Transformer/Wan2.2-Fun-5B-InP +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用 DeepSpeed-Zero-2 与 FSDP 方案进行训练。这里以 DeepSpeed-Zero-2 为例配置 shell 文件。 + +**DeepSpeed-Zero-2 与 FSDP 对比**： +- 两者都是分布式训练策略，帮助降低GPU显存占用 +- DeepSpeed-Zero-2：优化器状态和梯度在GPU间分片 +- FSDP（完全分片数据并行）：模型权重、优化器状态和梯度全部分片 +- **如果使用多卡且 DeepSpeed-Zero-2 显存不足**，切换使用 FSDP 以获得更好的显存效率 + +> **关于 train_lora.sh**：本目录中的 `train_lora.sh` 脚本提供了一个**不带 DeepSpeed 或 FSDP** 的基础训练模板。适用于： +> - 单GPU训练 +> - 快速测试和调试 +> - 根据特定需求进行自定义修改 +> +> 对于多GPU的生产环境训练，**请使用下方的 DeepSpeed-Zero-2 或 FSDP 命令**以获得更好的性能和显存效率。 + +**Wan2.2 Fun LoRA 训练示例（DeepSpeed-Zero-2）**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +export NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="inpaint" \ + --low_vram +``` + +> **说明**：本目录中的 `train_lora.sh` 脚本提供了不带 DeepSpeed 的基础训练模板。对于更好的多GPU训练性能和显存效率，请使用上方的 DeepSpeed-Zero-2 命令。 + +### 3.3 LoRA 专用参数解析 + +**Wan2.2 双Transformer架构说明**： + +Wan2.2采用了创新的双Transformer架构： +- **低噪声模型（Low Noise Model）**：负责处理低噪声阶段（接近最终输出） +- **高噪声模型（High Noise Model）**：负责处理高噪声阶段（初始生成阶段） +- **边界类型（boundary_type）**： + - `low`：训练低噪声模型，高噪声模型使用预训练权重（推荐用于T2V/I2V LoRA微调） + - `high`：训练高噪声模型，低噪声模型使用预训练权重 + - `full`：单模型训练（用于TI2V-5B等单Transformer模型） + +**LoRA 关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--config_path` | 配置文件路径 | `config/wan2.2/wan_civitai_i2v.yaml` | +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP` | +| `--train_data_dir` | 训练数据目录 | `datasets/X-Fun-Videos-Demo/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/X-Fun-Videos-Demo/metadata_add_width_height.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率（LoRA 推荐值） | 1e-04 | +| `--lr_scheduler` | 学习率调度器：`linear`、`cosine`、`cosine_with_restarts`、`polynomial`、`constant`、`constant_with_warmup` | `constant` | +| `--lr_warmup_steps` | 学习率预热步数 | 500 | +| `--seed` | 随机种子（可复现训练） | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.2_fun_lora` | +| `--gradient_checkpointing` | 激活重计算以节省显存 | - | +| `--mixed_precision` | 混合精度：`no`、`fp16`、`bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep（推荐启用） | - | +| `--low_vram` | 低显存模式，提高显存效率 | - | +| `--boundary_type` | Wan2.2双Transformer边界类型：`low`（训练低噪声模型）、`high`（训练高噪声模型）、`full`（训练单模型如TI2V-5B） | `low` | +| `--train_mode` | 训练模式：`normal`（普通T2V）、`inpaint`（视频修复） | `inpaint` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--rank` | LoRA 更新矩阵的维度（rank 越大表达能力越强，但显存占用越高） | 64 | +| `--network_alpha` | LoRA 更新矩阵的缩放系数（通常设置为 rank 的一半或相同） | 32 | +| `--target_name` | 应用 LoRA 的组件/模块，用逗号分隔（如 `q,k,v,ffn.0,ffn.2`） | `q,k,v,ffn.0,ffn.2` | +| `--lora_skip_name` | LoRA训练中跳过的组件，用逗号分隔 | None | +| `--use_peft_lora` | 使用 PEFT 模块添加 LoRA（更节省显存） | - | +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 inpaint 模式） | `"asset/1.png"` | +| `--use_deepspeed` | 启用 DeepSpeed 分布式训练 | - | +| `--use_fsdp` | 启用 FSDP 分布式训练 | - | +| `--use_8bit_adam` | 使用 8-bit Adam 优化器节省显存 | - | +| `--use_came` | 使用 CAME 优化器 | - | +| `--multi_stream` | 使用 CUDA 多流提升性能 | - | +| `--snr_loss` | 使用 SNR 损失函数 | - | +| `--weighting_scheme` | Timestep 加权方案：`sigma_sqrt`、`logit_normal`、`mode`、`cosmap`、`none` | `none` | +| `--motion_sub_loss` | 启用运动子损失以提升时序一致性 | - | +| `--motion_sub_loss_ratio` | 运动子损失比例 | 0.25 | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 默认值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 2000 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 5 | +| `--validation_prompts` | 验证视频生成的提示词 | None | +| `--validation_paths` | 验证 I2V 的参考图像路径（仅 inpaint 模式） | None | + +**验证示例**（inpaint 模式）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=100 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` +- Wan2.2 Fun 的验证会根据 `boundary_type` 自动选择使用单Transformer或双Transformer +- 当 `train_mode="inpaint"` 时，验证使用修复模式，参考图像作为起始帧 +- 对于 T2V 验证（不带参考图像），设置 `train_mode="normal"` 并省略 `--validation_paths` + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +export NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +> **说明**：在本仓库中，FSDP 比 DeepSpeed-Zero-3 更稳定且出错更少。当 DeepSpeed-Zero-2 在多GPU情况下遇到显存问题时，请使用 FSDP。 + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +export NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +export NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +> **说明**：这与 `train_lora.sh` 脚本类似，但使用了正确的数据集路径。`train_lora.sh` 脚本可以用作单GPU训练的起点。 + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +export NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +export NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.2-T2V-A14B` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的低噪声 Transformer 权重路径 | `None` | +| `transformer_high_path` | 加载训练好的高噪声 Transformer 权重路径（仅双Transformer模型） | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | 低噪声模型 LoRA 权重路径 | `None` | +| `lora_high_path` | 高噪声模型 LoRA 权重路径（仅双Transformer模型） | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V 模式） | `"asset/1.png"` | +| `validation_image_end` | 结束帧的参考图像路径（可选） | `None` | +| `validation_mask` | 修复的掩码路径（inpaint 模式） | `None` | +| `prompt` | 正向提示词，描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度 | 6.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 50 | +| `lora_weight` | 低噪声模型 LoRA 权重强度 | 0.55 | +| `lora_high_weight` | 高噪声模型 LoRA 权重强度（仅双Transformer模型） | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-i2v` 或 `samples/wan-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 视频修复推理 + +#### 4.2.1 推理脚本选择 + +Wan2.2 Fun 提供多种推理脚本，请根据您的模型版本和任务类型选择： + +| 脚本 | 模型版本 | 架构 | 主要用途 | +|------|---------|------|---------| +| `predict_i2v.py` | A14B | 双Transformer | 图生视频/视频修复（I2V/Inpaint） | +| `predict_t2v.py` | A14B | 双Transformer | 文生视频（T2V） | +| `predict_i2v_5b.py` | 5B | 单Transformer | 图生视频/视频修复（I2V/Inpaint） | +| `predict_t2v_5b.py` | 5B | 单Transformer | 文生视频（T2V） | + +> **说明**： +> - A14B 模型使用双Transformer架构（低噪声+高噪声模型），需要配置 `transformer_path` 和 `transformer_high_path` +> - 5B 模型使用单Transformer架构，只需配置 `transformer_path`，`transformer_high_path` 保持为 `None` + +#### 4.2.2 A14B模型推理（双Transformer） + +单卡推理运行如下命令： + +```bash +python examples/wan2.2_fun/predict_i2v.py +``` + +根据需求修改 `examples/wan2.2_fun/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +# 训练好的低噪声权重路径 +transformer_path = None +# 训练好的高噪声权重路径 +transformer_high_path = None +# 训练好的权重路径，如 "output_dir_wan2.2_fun_lora/checkpoint-xxx/diffusion_pytorch_model.safetensors" +lora_path = None +lora_high_path = None +# 修复的起始图像 +validation_image_start = "asset/1.png" +# 修复的掩码（可选，如果不提供将自动生成） +validation_mask = None +# 根据生成内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +> **说明**：Wan2.2 Fun 主要用于视频修复任务。使用 `predict_i2v.py` 进行 I2V 和修复工作流。当提供或生成掩码时，模型将自动处理修复任务。 + +#### 4.2.3 5B模型推理（单Transformer） + +单卡推理运行如下命令： + +```bash +python examples/wan2.2_fun/predict_i2v_5b.py +``` + +根据需求修改 `examples/wan2.2_fun/predict_i2v_5b.py`，重点关注如下参数： + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 5B模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-5B-InP/" +# 训练好的权重路径（5B为单Transformer，只需设置transformer_path） +transformer_path = None +# 5B模型不使用高噪声Transformer，保持为None +transformer_high_path = None +# 训练好的权重路径，如 "output_dir_wan2.2_fun_lora/checkpoint-xxx/diffusion_pytorch_model.safetensors" +lora_path = None +# 5B模型不使用高噪声LoRA，保持为None +lora_high_path = None +# 修复的起始图像 +validation_image_start = "asset/1.png" +validation_image_end = None +# 根据生成内容编写 +prompt = "A brown dog licking its tongue, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +> **说明**： +> - 5B 模型采用单Transformer架构，配置更简单，显存占用更低 +> - 训练时如果使用 `boundary_type="full"`，推理时只需加载 `transformer_path`，无需设置 `transformer_high_path` +> - LoRA 训练时也只需设置 `lora_path`，`lora_high_path` 保持为 `None` + +### 4.3 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.2_fun/predict_i2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.2_fun/predict_i2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.2_fun/README_TRAIN_zh-CN.md b/scripts/wan2.2_fun/README_TRAIN_zh-CN.md new file mode 100644 index 0000000..66120be --- /dev/null +++ b/scripts/wan2.2_fun/README_TRAIN_zh-CN.md @@ -0,0 +1,906 @@ +# Wan2.2 Fun 全参数训练指南 + +本文档提供 Wan2.2 Fun (Inpainting) 全参数训练的完整流程，包括环境配置、数据准备、分布式训练和推理测试。 + +> **说明**：Wan2.2 Fun 是一个基于 Wan2.2 架构的视频修复模型，支持视频 inpainting任务。Wan2.2采用双Transformer架构（高噪声/低噪声模型），5B版本采用单Transformer架构。本文档涵盖 Wan2.2 Fun 的训练流程，支持 A14B 和 5B 两种模型规格。 + +--- + +## 目录 +- [一、环境配置](#一环境配置) +- [二、数据准备](#二数据准备) + - [2.1 快速测试数据集](#21-快速测试数据集) + - [2.2 数据集结构](#22-数据集结构) + - [2.3 metadata.json 格式](#23-metadatajson-格式) + - [2.4 相对路径与绝对路径使用方案](#24-相对路径与绝对路径使用方案) +- [三、全量参数训练](#三全量参数训练) + - [3.1 下载预训练模型](#31-下载预训练模型) + - [3.2 快速开始（DeepSpeed-Zero-2）](#32-快速开始deepspeed-zero-2) + - [3.3 训练常用参数解析](#33-训练常用参数解析) + - [3.4 训练验证](#34-训练验证) + - [3.5 使用 FSDP 训练](#35-使用-fsdp-训练) + - [3.6 其他后端](#36-其他后端) + - [3.7 多机分布式训练](#37-多机分布式训练) +- [四、推理测试](#四推理测试) + - [4.1 推理参数解析](#41-推理参数解析) + - [4.2 文生视频（T2V）推理](#42-文生视频t2v推理) + - [4.3 图生视频（I2V）推理](#43-图生视频i2v推理) + - [4.4 多卡并行推理](#44-多卡并行推理) +- [五、更多资源](#五更多资源) + +--- + +## 一、环境配置 + +**方式 1：使用requirements.txt** + +```bash +pip install -r requirements.txt +``` + +**方式 2：手动安装依赖** + +```bash +pip install Pillow einops safetensors timm tomesd librosa "torch>=2.1.2" torchdiffeq torchsde decord datasets numpy scikit-image +pip install omegaconf SentencePiece imageio[ffmpeg] imageio[pyav] tensorboard beautifulsoup4 ftfy func_timeout onnxruntime +pip install "peft>=0.17.0" "accelerate>=0.25.0" "gradio>=3.41.2" "diffusers>=0.30.1" "transformers>=4.46.2" +pip install yunchang xfuser modelscope openpyxl deepspeed==0.17.0 numpy==1.26.4 +pip uninstall opencv-python opencv-contrib-python opencv-python-headless -y +pip install opencv-python-headless +``` + +**方式 3：使用docker** + +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun +``` + +--- + +## 二、数据准备 + +### 2.1 快速测试数据集 + +我们提供了一个测试的数据集，其中包含若干训练数据。 + +```bash +# 下载官方示例数据集 +modelscope download --dataset PAI/X-Fun-Videos-Demo --local_dir ./datasets/X-Fun-Videos-Demo +``` + +### 2.2 数据集结构 + +``` +📦 datasets/ +├── 📂 my_dataset/ +│ ├── 📂 train/ +│ │ ├── 📄 video001.mp4 +│ │ ├── 📄 video002.mp4 +│ │ └── 📄 ... +│ └── 📄 metadata.json +``` + +### 2.3 metadata.json 格式 + +**相对路径格式**（示例格式）： +```json +[ + { + "file_path": "train/video001.mp4", + "text": "A beautiful sunset over the ocean, golden hour lighting", + "type": "video", + "width": 1024, + "height": 1024 + }, + { + "file_path": "train/video002.mp4", + "text": "A person walking through a forest, cinematic view", + "type": "video", + "width": 1328, + "height": 1328 + } +] +``` + +**绝对路径格式**： +```json +[ + { + "file_path": "/mnt/data/videos/sunset.mp4", + "text": "A beautiful sunset over the ocean", + "type": "video", + "width": 1024, + "height": 1024 + } +] +``` + +**关键字段说明**： +- `file_path`：视频路径（相对或绝对路径） +- `text`：视频描述（英文提示词） +- `type`：数据类型，固定为 `"video"` +- `width` / `height`：视频宽高（**最好提供**，用于分桶训练，如果不提供则自动在训练时读取，当数据存储在如oss这样的速度较慢的系统上时，可能会影响训练速度）。 + - 可以使用`scripts/process_json_add_width_and_height.py`文件对无width与height字段的json进行提取，支持处理图片与视频。 + - 使用方案为`python scripts/process_json_add_width_and_height.py --input_file datasets/X-Fun-Videos-Demo/metadata.json --output_file datasets/X-Fun-Videos-Demo/metadata_add_width_height.json`。 + +### 2.4 相对路径与绝对路径使用方案 + +**相对路径**： + +如果数据的路径为相对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +``` + +**绝对路径**： + +如果数据的路径为绝对路径，则在训练脚本中设置： + +```bash +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/metadata.json" +``` + +> 💡 **建议**：如果数据集较小且存储在本地，推荐使用相对路径；如果数据集存储在外部存储（如 NAS、OSS）或多个机器共享存储，推荐使用绝对路径。 + +--- + +## 三、全量参数训练 + +### 3.1 下载预训练模型 + +```bash +# 创建模型目录 +mkdir -p models/Diffusion_Transformer + +# 下载 Wan2.2 Fun 官方权重 +# A14B 模型（双Transformer架构） +modelscope download --model PAI/Wan2.2-Fun-A14B-InP --local_dir models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP +# 或 5B 模型（单Transformer架构） +# modelscope download --model PAI/Wan2.2-Fun-5B-InP --local_dir models/Diffusion_Transformer/Wan2.2-Fun-5B-InP +``` + +### 3.2 快速开始（DeepSpeed-Zero-2） + +如果按照 **2.1 快速测试数据集下载数据** 与 **3.1 下载预训练模型下载权重**后，直接复制快速开始的启动指令进行启动。 + +推荐使用 DeepSpeed-Zero-2 与 FSDP 方案进行训练。这里以 DeepSpeed-Zero-2 为例配置 shell 文件。 + +DeepSpeed-Zero-2 与 FSDP 在本仓库中的区别在于是否对模型权重进行分片。**如果使用多卡且 DeepSpeed-Zero-2 显存不足**，可以切换使用 FSDP 进行训练。 + +**Wan2.2 T2V 训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +**Wan2.2 I2V 训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +**Wan2.2 Fun 5B 训练示例**： + +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-5B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="full" \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.3 训练常用参数解析 + +**Wan2.2 Fun 双Transformer架构说明**： + +Wan2.2 Fun采用了创新的双Transformer架构（A14B版本）： +- **低噪声模型（Low Noise Model）**：负责处理低噪声阶段（接近最终输出） +- **高噪声模型（High Noise Model）**：负责处理高噪声阶段（初始生成阶段） +- **5B版本**：采用单Transformer架构，更轻量级 +- **边界类型（boundary_type）**： + - `low`：训练低噪声模型，高噪声模型使用预训练权重（推荐用于A14B微调） + - `high`：训练高噪声模型，低噪声模型使用预训练权重 + - `full`：单模型训练（用于5B等单Transformer模型） + +**训练模式说明**： +- `train_mode="inpaint"`：视频修复模式，会自动启用 inpainting 数据增强和掩码生成 +- 训练时会随机生成掩码，将部分视频区域遮盖，模型学习修复这些区域 +- 支持 T2V 风格的修复（掩码全覆盖）和局部修复（部分掩码）混合训练 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|-----|------|-------| +| `--pretrained_model_name_or_path` | 预训练模型路径 | `models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP` | +| `--train_data_dir` | 训练数据目录 | `datasets/internal_datasets/` | +| `--train_data_meta` | 训练数据元文件 | `datasets/internal_datasets/metadata.json` | +| `--train_batch_size` | 每批次样本数 | 1 | +| `--image_sample_size` | 图像最大训练分辨率 | 640 | +| `--video_sample_size` | 视频最大训练分辨率 | 640 | +| `--token_sample_size` | Token 采样尺寸 | 640 | +| `--video_sample_stride` | 视频采样步幅 | 2 | +| `--video_sample_n_frames` | 视频采样帧数 | 81 | +| `--gradient_accumulation_steps` | 梯度累积步数（等效增大 batch） | 1 | +| `--dataloader_num_workers` | DataLoader 子进程数 | 8 | +| `--num_train_epochs` | 训练 epoch 数 | 100 | +| `--checkpointing_steps` | 每 N 步保存 checkpoint | 50 | +| `--learning_rate` | 初始学习率 | 2e-05 | +| `--lr_scheduler` | 学习率调度器 | `constant_with_warmup` | +| `--lr_warmup_steps` | 学习率预热步数 | 100 | +| `--seed` | 随机种子 | 42 | +| `--output_dir` | 输出目录 | `output_dir_wan2.2_fun` | +| `--gradient_checkpointing` | 激活重计算 | - | +| `--mixed_precision` | 混合精度：`fp16/bf16` | `bf16` | +| `--adam_weight_decay` | AdamW 权重衰减 | 3e-2 | +| `--adam_epsilon` | AdamW epsilon 值 | 1e-10 | +| `--vae_mini_batch` | VAE 编码时的迷你批次大小 | 1 | +| `--max_grad_norm` | 梯度裁剪阈值 | 0.05 | +| `--enable_bucket` | 启用分桶训练，不裁剪图片/视频，按分辨率分组训练 | - | +| `--random_hw_adapt` | 自动缩放图片/视频到 `[min_size, max_size]` 范围内的随机尺寸 | - | +| `--training_with_video_token_length` | 根据 token 长度训练，支持任意分辨率 | - | +| `--uniform_sampling` | 均匀采样 timestep | - | +| `--low_vram` | 低显存模式 | - | +| `--boundary_type` | Wan2.2双Transformer边界类型：`low`（训练低噪声模型）、`high`（训练高噪声模型）、`full`（训练单模型如5B） | `low` | +| `--train_mode` | 训练模式：固定为 `inpaint`（视频修复） | `inpaint` | +| `--resume_from_checkpoint` | 恢复训练路径，使用 `"latest"` 自动选择最新 checkpoint | None | +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 500 | +| `--validation_prompts` | 验证视频生成的提示词 | `"一只棕色的狗摇着头..."` | +| `--validation_paths` | 验证的参考图像路径 | `"asset/1.png"` | +| `--trainable_modules` | 可训练模块（`.` 表示所有模块） | `"."` | + +**Sample Size 配置指南**： +- `video_sample_size` 表示视频的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最小值。 +- `image_sample_size` 表示图像的分辨率大小；当 `random_hw_adapt` 为 True 时，表示视频和图像分辨率的最大值。 +- `token_sample_size` 表示当 `training_with_video_token_length` 为 True 时，最大 token 长度对应的分辨率。 +- 由于配置可能产生混淆，**如果你不需要任意分辨率进行 finetuning**，建议将 `video_sample_size`、`image_sample_size` 和 `token_sample_size` 设置为相同的固定值，例如 **(320, 480, 512, 640, 960)**。 + - **全部设置为 320** 代表 **240P**。 + - **全部设置为 480** 代表 **320P**。 + - **全部设置为 640** 代表 **480P**。 + - **全部设置为 960** 代表 **720P**。 + +**Wan2.2 Fun 训练策略建议**： +- **A14B模型（双Transformer）**：使用 `boundary_type="low"` 和 `train_mode="inpaint"` 训练低噪声模型，这样可以保持高噪声部分的通用性，同时微调低噪声部分以适应您的修复任务。 +- **5B模型（单Transformer）**：使用 `boundary_type="full"` 和 `train_mode="inpaint"` 进行完整训练。 +- **显存优化**：Wan2.2模型较大（14B/5B参数），强烈建议使用 `--low_vram` 和 `--gradient_checkpointing`。 +- **多卡训练**：对于14B模型，如果单卡显存不足，可以使用 FSDP 或 DeepSpeed 进行多卡训练。 + +**Token Length 训练说明**： +- 当启用 `training_with_video_token_length` 时，模型根据 token 长度进行训练。 +- 例如：512x512 分辨率、49 帧的视频，其 token 长度为 13,312，需要设置 `token_sample_size = 512`。 + - 在 512x512 分辨率下，视频帧数为 49 (~= 512 * 512 * 49 / 512 / 512)。 + - 在 768x768 分辨率下，视频帧数为 21 (~= 512 * 512 * 49 / 768 / 768)。 + - 在 1024x1024 分辨率下，视频帧数为 9 (~= 512 * 512 * 49 / 1024 / 1024)。 + - 这些分辨率与对应帧数的组合，使模型能够生成不同尺寸的视频。 + +### 3.4 训练验证 + +你可以配置验证参数，在训练过程中定期生成测试视频，以便监控训练进度和模型质量。 + +**验证参数说明**： + +| 参数 | 说明 | 推荐值 | +|------|------|--------| +| `--validation_steps` | 每 N 步执行一次验证 | 100 | +| `--validation_epochs` | 每 N 个epoch执行一次验证 | 500 | +| `--validation_prompts` | 验证视频生成的提示词 | 英文提示词 | +| `--validation_paths` | 验证的参考图像路径 | `"asset/1.png"` | + +**A14B 模式示例**（双Transformer验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**5B 模式示例**（单Transformer验证）： + +```bash + --validation_paths "asset/1.png" \ + --validation_steps=100 \ + --validation_epochs=500 \ + --validation_prompts="A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +``` + +**注意事项**： +- 验证视频会保存到 `output_dir` 目录中 +- 多提示词验证格式：`--validation_prompts "prompt1" "prompt2" "prompt3"` +- Wan2.2 Fun的验证会根据`boundary_type`自动选择使用单Transformer或双Transformer +- 验证时会使用 inpainting 模式，提供参考图像作为起始帧 + +### 3.5 使用 FSDP 训练 + +**如果使用多卡且使用DeepSpeed-Zero-2的情况下显存不足**，可以切换使用FSDP进行训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.6 其他后端 + +#### 3.6.1 使用DeepSpeed-Zero-3进行训练 + +目前不太推荐使用 DeepSpeed Zero-3。在本仓库中，使用 FSDP 出错更少且更稳定。 + +DeepSpeed Zero-3 适合高分辨率的 14B Wan。训练后，您可以使用以下命令获取最终模型： +```bash +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +训练 shell 命令如下： +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +#### 3.6.2 不使用 DeepSpeed 与 FSDP 训练 + +**该方案并不被推荐，因为没有显存节约后端，容易造成显存不足**。这里仅提供训练Shell用于参考训练。 + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +### 3.7 多机分布式训练 + +**适合场景**：超大规模数据集、需要更快的训练速度 + +#### 3.7.1 环境配置 + +假设有 2 台机器，每台 8 张 GPU： + +**机器 0（Master）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # Master 机器 IP +export MASTER_PORT=10086 +export WORLD_SIZE=2 # 机器总数 +export NUM_PROCESS=16 # 总进程数 = 机器数 × 8 +export RANK=0 # 当前机器 rank（0 或 1） +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_wan2.2_fun" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +**机器 1（Worker）**： +```bash +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/X-Fun-Videos-Demo/" +export DATASET_META_NAME="datasets/X-Fun-Videos-Demo/metadata_add_width_height.json" +export MASTER_ADDR="192.168.1.100" # 与 Master 相同 +export MASTER_PORT=10086 +export WORLD_SIZE=2 +export NUM_PROCESS=16 +export RANK=1 # 注意这里是 1 +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +# 使用与机器 0 相同的 accelerate launch 命令 +``` + +#### 3.7.2 多机训练注意事项 + +- **网络要求**： + - 推荐 RDMA/InfiniBand（高性能） + - 无 RDMA 时添加环境变量： + ```bash + export NCCL_IB_DISABLE=1 + export NCCL_P2P_DISABLE=1 + ``` + +- **数据同步**：所有机器必须能够访问相同的数据路径（NFS/共享存储） + +--- + +## 四、推理测试 + +### 4.1 推理参数解析 + +**关键参数说明**： + +| 参数 | 说明 | 示例值 | +|------|------|-------| +| `GPU_memory_mode` | 显存管理模式，可选值见下表 | `model_group_offload` | +| `ulysses_degree` | Head 维度并行度，单卡时为 1 | 1 | +| `ring_degree` | Sequence 维度并行度，单卡时为 1 | 1 | +| `fsdp_dit` | 多卡推理时对 Transformer 使用 FSDP 节省显存 | `False` | +| `fsdp_text_encoder` | 多卡推理时对文本编码器使用 FSDP | `True` | +| `compile_dit` | 编译 Transformer 加速推理（固定分辨率下有效） | `False` | +| `model_name` | 模型路径 | `models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP` | +| `sampler_name` | 采样器类型：`Flow`、`Flow_Unipc`、`Flow_DPM++` | `Flow_Unipc` | +| `transformer_path` | 加载训练好的低噪声 Transformer 权重路径 | `None` | +| `transformer_high_path` | 加载训练好的高噪声 Transformer 权重路径（仅双Transformer模型） | `None` | +| `vae_path` | 加载训练好的 VAE 权重路径 | `None` | +| `lora_path` | 低噪声模型 LoRA 权重路径 | `None` | +| `lora_high_path` | 高噪声模型 LoRA 权重路径（仅双Transformer模型） | `None` | +| `sample_size` | 生成视频分辨率 `[高度, 宽度]` | `[480, 832]` 或 `[832, 480]` | +| `video_length` | 生成视频帧数 | `81` | +| `fps` | 每秒帧数 | `16` | +| `weight_dtype` | 模型权重精度，不支持 bf16 的显卡使用 `torch.float16` | `torch.bfloat16` | +| `validation_image_start` | 图生视频的参考图像路径（I2V/TI2V 模式） | `"asset/1.png"` | +| `prompt` | 正向提示词，描述生成内容 | `"一只棕色的狗摇着头..."` | +| `negative_prompt` | 负向提示词，避免生成的内容 | `"低分辨率，低画质..."` | +| `guidance_scale` | 引导强度 | 6.0 | +| `seed` | 随机种子，用于复现结果 | 43 | +| `num_inference_steps` | 推理步数 | 50 | +| `lora_weight` | 低噪声模型 LoRA 权重强度 | 0.55 | +| `lora_high_weight` | 高噪声模型 LoRA 权重强度（仅双Transformer模型） | 0.55 | +| `save_path` | 生成视频保存路径 | `samples/wan-videos-i2v` 或 `samples/wan-videos-t2v` | + +**显存管理模式说明**： + +| 模式 | 说明 | 显存占用 | +|------|------|---------| +| `model_full_load` | 整个模型加载到 GPU | 最高 | +| `model_full_load_and_qfloat8` | 全量加载 + FP8 量化 | 高 | +| `model_cpu_offload` | 使用后将模型卸载到 CPU | 中等 | +| `model_cpu_offload_and_qfloat8` | CPU 卸载 + FP8 量化 | 中低 | +| `model_group_offload` | 层组在 CPU/CUDA 间切换 | 低 | +| `sequential_cpu_offload` | 逐层卸载（速度最慢） | 最低 | + +### 4.2 文生视频（T2V）推理 + +#### 4.2.1 A14B 双Transformer模型 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2_fun/predict_t2v.py +``` + +根据需求修改 `examples/wan2.2_fun/predict_t2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +# 训练好的低噪声权重路径，如 "output_dir_wan2.2_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 训练好的高噪声权重路径（如果训练了双Transformer） +transformer_high_path = None +# 根据想生成的内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +#### 4.2.2 5B 单Transformer模型 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2_fun/predict_t2v_5b.py +``` + +根据需求修改 `examples/wan2.2_fun/predict_t2v_5b.py`，初次推理重点关注如下参数。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径（5B模型） +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-5B-InP" +# 5B模型的配置文件路径 +config_path = "config/wan2.2/wan_civitai_5b.yaml" +# 由于5B是单Transformer模型，只使用transformer_path +transformer_path = None +transformer_high_path = None # 5B模型不使用 +# 根据想生成的内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# 5B模型通常使用更高的分辨率 +sample_size = [704, 1280] +video_length = 121 +fps = 21 +# ... +``` + +**A14B与5B模型的主要区别**： +- **架构**：A14B使用双Transformer（高/低噪声），5B使用单Transformer +- **配置文件**：A14B使用 `wan_civitai_i2v.yaml`，5B使用 `wan_civitai_5b.yaml` +- **参数**：5B模型不使用 `transformer_high_path` 和 `lora_high_path` +- **分辨率**：5B模型支持更高分辨率（如704x1280） +- **帧数**：5B模型可生成更长视频（如121帧） + +### 4.3 图生视频（I2V）推理 + +#### 4.3.1 A14B 双Transformer模型 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2_fun/predict_i2v.py +``` + +根据需求修改 `examples/wan2.2_fun/predict_i2v.py`，初次推理重点关注如下参数，如果对其他参数感兴趣，请查看上方的推理参数解析。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径 +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +# 训练好的低噪声权重路径，如 "output_dir_wan2.2_fun/checkpoint-xxx/diffusion_pytorch_model.safetensors" +transformer_path = None +# 训练好的高噪声权重路径 +transformer_high_path = None +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +# 根据想生成的内容编写 +prompt = "A brown dog shaking its head, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# ... +``` + +#### 4.3.2 5B 单Transformer模型 + +单卡推理运行如下命令： + +```bash +python examples/wan2.2_fun/predict_i2v_5b.py +``` + +根据需求修改 `examples/wan2.2_fun/predict_i2v_5b.py`，初次推理重点关注如下参数。 + +```python +# 根据显卡显存选择 +GPU_memory_mode = "sequential_cpu_offload" +# 根据实际模型路径（5B模型） +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-5B-InP" +# 5B模型的配置文件路径 +config_path = "config/wan2.2/wan_civitai_5b.yaml" +# 由于5B是单Transformer模型，只使用transformer_path +transformer_path = None +transformer_high_path = None # 5B模型不使用 +# 图生视频的起始图像 +validation_image_start = "asset/1.png" +validation_image_end = None # 可选的结束图像 +# 根据想生成的内容编写 +prompt = "A brown dog licking its tongue, sitting on a light-colored sofa in a cozy room. Behind the dog, there's a framed painting on a shelf, surrounded by pink flowers. The soft, warm lighting in the room creates a comfortable atmosphere." +# 5B模型通常使用更高的分辨率 +sample_size = [704, 1280] +video_length = 121 +fps = 24 +# ... +``` + +### 4.4 多卡并行推理 + +**适合场景**：高分辨率生成、加速推理 + +#### 安装并行推理依赖 + +```bash +pip install xfuser==0.4.2 yunchang==0.6.2 +``` + +#### 配置并行策略 + +编辑 `examples/wan2.2_fun/predict_t2v.py` 或 `examples/wan2.2_fun/predict_i2v.py`： + +```python +# 确保 ulysses_degree × ring_degree = 使用的 GPU 数 +# 例如使用 2 张 GPU： +ulysses_degree = 2 # Head 维度并行 +ring_degree = 1 # Sequence 维度并行 +``` + +**配置原则**： +- `ulysses_degree` 必须能整除模型的 head 数 +- `ring_degree` 是在 sequence 维度切分，会影响通信开销，在 head 能整除的情况下尽量不要用 + +**配置示例**： + +| GPU 数量 | ulysses_degree | ring_degree | 说明 | +|---------|---------------|-------------|------| +| 1 | 1 | 1 | 单 GPU | +| 4 | 4 | 1 | Head 并行 | +| 8 | 8 | 1 | Head 并行 | +| 8 | 4 | 2 | 混合并行 | + +#### 运行多卡推理 + +```bash +torchrun --nproc-per-node=2 examples/wan2.2_fun/predict_i2v.py +``` + +--- + +## 五、更多资源 + +- **官方 GitHub**：https://github.com/aigc-apps/VideoX-Fun diff --git a/scripts/wan2.2_fun/train.py b/scripts/wan2.2_fun/train.py index 2ad14f6..50985b1 100644 --- a/scripts/wan2.2_fun/train.py +++ b/scripts/wan2.2_fun/train.py @@ -688,7 +688,7 @@ def parse_args(): default="normal", help=( 'The format of training data. Support `"normal"`' - ' (default), `"i2v"`.' + ' (default), `"inpaint"`.' ), ) parser.add_argument( diff --git a/videox_fun/data/__init__.py b/videox_fun/data/__init__.py index def102a..06eecca 100644 --- a/videox_fun/data/__init__.py +++ b/videox_fun/data/__init__.py @@ -2,8 +2,9 @@ from .bucket_sampler import (ASPECT_RATIO_512, ASPECT_RATIO_RANDOM_CROP_512, ASPECT_RATIO_RANDOM_CROP_PROB, AspectRatioBatchImageSampler, AspectRatioBatchImageVideoSampler, - AspectRatioBatchSampler, RandomSampler, - get_closest_ratio, get_image_size_without_loading) + AspectRatioBatchSampler, BatchSampler, + RandomSampler, get_closest_ratio, + get_image_size_without_loading) from .dataset_image import CC15M, ImageEditDataset from .dataset_image_video import (ImageVideoControlDataset, ImageVideoDataset, ImageVideoSafetensorsDataset, diff --git a/videox_fun/data/bucket_sampler.py b/videox_fun/data/bucket_sampler.py index 4cfc323..a0889b8 100755 --- a/videox_fun/data/bucket_sampler.py +++ b/videox_fun/data/bucket_sampler.py @@ -260,9 +260,9 @@ class AspectRatioBatchSampler(BatchSampler): video_dir = os.path.join(self.video_folder, f"{videoid}.mp4") cap = cv2.VideoCapture(video_dir) - # 获取视频尺寸 - width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) # 浮点数转换为整数 - height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # 浮点数转换为整数 + # Get video dimensions + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) # Convert float to integer + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # Convert float to integer ratio = height / width # self.dataset[idx] else: @@ -332,11 +332,19 @@ class AspectRatioBatchImageVideoSampler(BatchSampler): width, height = image_dict.get("width", None), image_dict.get("height", None) if width is None or height is None: - image_id, name = image_dict['file_path'], image_dict['text'] - if self.train_folder is None: - image_dir = image_id + image_id = image_dict['file_path'] + # Handle multiview: file_path can be list or str + if isinstance(image_id, list): + image_dir = image_id[0] else: - image_dir = os.path.join(self.train_folder, image_id) + image_dir = image_id + + if self.train_folder is None: + pass # image_dir is already absolute path + elif isinstance(self.train_folder, list): + pass # train_folder is list, use image_dir directly + else: + image_dir = os.path.join(self.train_folder, image_dir) width, height = get_image_size_without_loading(image_dir) @@ -364,16 +372,24 @@ class AspectRatioBatchImageVideoSampler(BatchSampler): width, height = video_dict.get("width", None), video_dict.get("height", None) if width is None or height is None: - video_id, name = video_dict['file_path'], video_dict['text'] - if self.train_folder is None: - video_dir = video_id + video_id = video_dict['file_path'] + # Handle multiview: file_path can be list or str + if isinstance(video_id, list): + video_dir = video_id[0] # Use first view for aspect ratio else: - video_dir = os.path.join(self.train_folder, video_id) + video_dir = video_id + + if self.train_folder is None: + pass # video_dir is already absolute path + elif isinstance(self.train_folder, list): + pass # train_folder is list, use video_dir directly + else: + video_dir = os.path.join(self.train_folder, video_dir) cap = cv2.VideoCapture(video_dir) - # 获取视频尺寸 - width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) # 浮点数转换为整数 - height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # 浮点数转换为整数 + # Get video dimensions + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) # Convert float to integer + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # Convert float to integer ratio = height / width # self.dataset[idx] else: diff --git a/videox_fun/reward/aesthetic_v2_5_predictor.py b/videox_fun/reward/aesthetic_v2_5_predictor.py index 867f429..78169cf 100644 --- a/videox_fun/reward/aesthetic_v2_5_predictor.py +++ b/videox_fun/reward/aesthetic_v2_5_predictor.py @@ -76,7 +76,7 @@ class AestheticPredictorV2_5Model(SiglipVisionModel): loss = None if labels is not None: loss_fct = nn.MSELoss() - loss = loss_fct() + loss = loss_fct(prediction.squeeze(-1), labels) if not return_dict: return (loss, prediction, image_embeds) @@ -125,7 +125,22 @@ def convert_v2_5_from_siglip( else: state_dict = torch.load(predictor_name_or_path, map_location="cpu") - assert isinstance(state_dict, OrderedDict) + # Handle checkpoint with multiple fields (e.g., from training) + if isinstance(state_dict, dict) and "model_state_dict" in state_dict: + state_dict = state_dict["model_state_dict"] + + if not isinstance(state_dict, OrderedDict): + state_dict = OrderedDict(state_dict) + + # Remove _orig_mod. prefix if present (from torch.compile) + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k.startswith("_orig_mod."): + new_key = k[len("_orig_mod."):] + else: + new_key = k + new_state_dict[new_key] = v + state_dict = new_state_dict model.layers.load_state_dict(state_dict) model.eval() diff --git a/videox_fun/reward/hpsv3_predictor.py b/videox_fun/reward/hpsv3_predictor.py index 543d528..29d50b0 100644 --- a/videox_fun/reward/hpsv3_predictor.py +++ b/videox_fun/reward/hpsv3_predictor.py @@ -417,7 +417,14 @@ class Qwen2VLRewardModelBT(Qwen2VLForConditionalGeneration): ) if inputs_embeds is None: - inputs_embeds = self.model.embed_tokens(input_ids) + # Support both old and new transformer versions + if hasattr(self.model, 'embed_tokens'): + inputs_embeds = self.model.embed_tokens(input_ids) + elif hasattr(self.model, 'language_model'): + inputs_embeds = self.model.language_model.embed_tokens(input_ids) + else: + raise AttributeError("Cannot find embed_tokens in model structure") + if pixel_values is not None: pixel_values = pixel_values.type(self.visual.get_dtype()) image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) @@ -548,7 +555,7 @@ def create_model_and_processor(model_config, peft_lora_config, training_args, ca ) processor = AutoProcessor.from_pretrained( - model_config.model_name_or_path, padding_side="right", cache_dir=cache_dir + model_config.model_name_or_path, padding_side="left", cache_dir=cache_dir ) special_token_ids = None @@ -703,7 +710,87 @@ class HPSv3RewardInferencer: if "model" in state_dict: state_dict = state_dict["model"] - model.load_state_dict(state_dict, strict=True) + + # --- Start of Key Mapping Fix for Transformers Version Compatibility --- + # Detect model architecture to determine key mapping strategy + # transformers >= 4.52: model structure uses model.language_model.* + # transformers < 4.52: model structure uses model.layers.*, visual.* + + # Check if model uses new architecture by inspecting actual model structure + # Method 1: Check if model.model has 'language_model' as a direct child module + has_language_model_attr = hasattr(model.model, 'language_model') + # Method 2: Check if 'language_model' is in named modules + has_language_model_module = any(name == 'language_model' for name, _ in model.model.named_children()) + uses_new_arch = has_language_model_attr or has_language_model_module + + # Detect checkpoint format by scanning all keys + has_legacy_visual = any(k.startswith("visual.") for k in state_dict.keys()) + has_new_visual = any(k.startswith("model.visual.") for k in state_dict.keys()) + has_new_language = any(k.startswith("model.language_model.") for k in state_dict.keys()) + has_old_language = any( + k.startswith("model.layers.") and "language_model" not in k + for k in state_dict.keys() + ) + + # Log detection results for debugging + logger.debug(f"Model architecture: {'new' if uses_new_arch else 'old'} (has_language_model={has_language_model_attr}, has_language_module={has_language_model_module})") + logger.debug(f"Checkpoint format: legacy_visual={has_legacy_visual}, new_visual={has_new_visual}, new_language={has_new_language}, old_language={has_old_language}") + + needs_fix = False + mapping_direction = None # 'old_to_new' or 'new_to_old' + + # Case 1: New transformers but old checkpoint format + if uses_new_arch and (has_legacy_visual or has_old_language): + needs_fix = True + mapping_direction = 'old_to_new' + logger.info("Detected old checkpoint format with new Transformers (>=4.52). Remapping old->new...") + + # Case 2: Old transformers but new checkpoint format (your case with 4.51.3) + elif not uses_new_arch and (has_new_language or has_new_visual): + needs_fix = True + mapping_direction = 'new_to_old' + logger.info("Detected new checkpoint format with old Transformers (<4.52). Remapping new->old...") + + if needs_fix: + new_state_dict = {} + for k, v in state_dict.items(): + new_k = k + + if mapping_direction == 'old_to_new': + # Old format -> New format + # 1. visual.xxx -> model.visual.xxx + if k.startswith("visual."): + new_k = "model." + k + # 2. model.layers.xxx -> model.language_model.layers.xxx + elif k.startswith("model.layers.") or k.startswith("model.norm.") or k.startswith("model.embed_tokens."): + new_k = k.replace("model.", "model.language_model.", 1) + + elif mapping_direction == 'new_to_old': + # New format -> Old format + # 1. model.language_model.xxx -> model.xxx + if k.startswith("model.language_model."): + new_k = k.replace("model.language_model.", "model.", 1) + # 2. model.visual.xxx -> visual.xxx + elif k.startswith("model.visual."): + new_k = k[len("model."):] + else: + new_k = k # Keep unchanged + + new_state_dict[new_k] = v + + state_dict = new_state_dict + logger.info(f"Key remapping completed successfully ({mapping_direction}).") + # --- End of Key Mapping Fix --- + + # Load state dict - use strict=False for compatibility + missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False) + + # Log warnings if there are mismatches + if missing_keys: + logger.warning(f"Missing keys in state_dict ({len(missing_keys)}): {missing_keys[:5]}...") + if unexpected_keys: + logger.warning(f"Unexpected keys in state_dict ({len(unexpected_keys)}): {unexpected_keys[:5]}...") + model.eval() self.model = model diff --git a/videox_fun/reward/video_align_predictor.py b/videox_fun/reward/video_align_predictor.py index a640fd7..979f2e4 100644 --- a/videox_fun/reward/video_align_predictor.py +++ b/videox_fun/reward/video_align_predictor.py @@ -557,7 +557,14 @@ class Qwen2VLRewardModelBT(Qwen2VLForConditionalGeneration): return_dict = return_dict if return_dict is not None else self.config.use_return_dict if inputs_embeds is None: - inputs_embeds = self.model.embed_tokens(input_ids) + # Support both old and new transformer versions + if hasattr(self.model, 'embed_tokens'): + inputs_embeds = self.model.embed_tokens(input_ids) + elif hasattr(self.model, 'language_model'): + inputs_embeds = self.model.language_model.embed_tokens(input_ids) + else: + raise AttributeError("Cannot find embed_tokens in model structure") + if pixel_values is not None: pixel_values = pixel_values.type(self.visual.get_dtype()) image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) @@ -627,31 +634,144 @@ class Qwen2VLRewardModelBT(Qwen2VLForConditionalGeneration): return {"logits": pooled_logits} -# ========================= Model Utils ========================= - -def _insert_adapter_name_into_state_dict( - state_dict: dict[str, torch.Tensor], adapter_name: str, parameter_prefix: str -) -> dict[str, torch.Tensor]: - """Utility function to remap the state_dict keys to fit the PEFT model by inserting the adapter name.""" - peft_model_state_dict = {} - for key, val in state_dict.items(): - if parameter_prefix in key: - suffix = key.split(parameter_prefix)[1] - if "." in suffix: - suffix_to_replace = ".".join(suffix.split(".")[1:]) - key = key.replace(suffix_to_replace, f"{adapter_name}.{suffix_to_replace}") - else: - key = f"{key}.{adapter_name}" - peft_model_state_dict[key] = val +def _insert_adapter_name_into_state_dict(state_dict, adapter_name="default", parameter_prefix="lora_"): + """Insert adapter name into LoRA state dict keys.""" + new_state_dict = {} + for k, v in state_dict.items(): + if ("lora_A" in k or "lora_B" in k) and f".{adapter_name}." not in k: + new_k = k.replace(".lora_A.", f".lora_A.{adapter_name}.").replace(".lora_B.", f".lora_B.{adapter_name}.") + new_state_dict[new_k] = v else: - peft_model_state_dict[key] = val - return peft_model_state_dict + new_state_dict[k] = v + return new_state_dict +def _remap_checkpoint_keys(state_dict, model=None): + """Remap checkpoint keys to match current Transformers architecture. + + Supports bidirectional mapping: + - Old format (transformers < 4.52): visual.*, model.layers.* + - New format (transformers >= 4.52): model.visual.*, model.language_model.* + + Args: + state_dict: The checkpoint state dict + model: Optional model instance to detect architecture. If None, auto-detect from keys. + """ + # Detect model architecture if model is provided + if model is not None: + # Check if model uses new architecture (has language_model attribute) + has_language_model_attr = hasattr(model.model, 'language_model') + has_language_model_module = any(name == 'language_model' for name, _ in model.model.named_children()) + uses_new_arch = has_language_model_attr or has_language_model_module + else: + # Auto-detect from state_dict keys (assume new format if contains language_model) + uses_new_arch = any('language_model' in k for k in state_dict.keys()) + + # Detect checkpoint format + has_legacy_visual = any(k.startswith("base_model.model.visual.") for k in state_dict.keys()) or \ + any(k.startswith("visual.") for k in state_dict.keys()) + has_new_visual = any(k.startswith("base_model.model.model.visual.") for k in state_dict.keys()) + has_new_language = any(k.startswith("base_model.model.model.language_model.") for k in state_dict.keys()) or \ + any(k.startswith("model.language_model.") for k in state_dict.keys()) + has_old_language = any( + (k.startswith("base_model.model.model.layers.") and "language_model" not in k) or + (k.startswith("model.layers.") and "language_model" not in k) + for k in state_dict.keys() + ) + + # Log detection results + logger.debug(f"Model architecture: {'new' if uses_new_arch else 'old'}") + logger.debug(f"Checkpoint format: legacy_visual={has_legacy_visual}, new_visual={has_new_visual}, " + f"new_language={has_new_language}, old_language={has_old_language}") + + # Determine mapping direction + needs_fix = False + mapping_direction = None + + if uses_new_arch and (has_legacy_visual or has_old_language): + needs_fix = True + mapping_direction = 'old_to_new' + logger.info("Detected old checkpoint format with new Transformers (>=4.52). Remapping old->new...") + elif not uses_new_arch and (has_new_language or has_new_visual): + needs_fix = True + mapping_direction = 'new_to_old' + logger.info("Detected new checkpoint format with old Transformers (<4.52). Remapping new->old...") + + if not needs_fix: + logger.info("Checkpoint format matches model architecture. No remapping needed.") + return state_dict + + # Perform key remapping + mapped_state_dict = {} + remapped_count = 0 + + for k, v in state_dict.items(): + new_k = k + + if mapping_direction == 'old_to_new': + # Old format -> New format + # Handle both "visual." and "base_model.model.visual." prefixes + if k.startswith("base_model.model.visual."): + new_k = k.replace("base_model.model.visual.", "base_model.model.model.visual.", 1) + remapped_count += 1 + elif k.startswith("visual."): + new_k = k.replace("visual.", "model.visual.", 1) + remapped_count += 1 + elif k.startswith("base_model.model.model.layers."): + new_k = k.replace("base_model.model.model.layers.", "base_model.model.model.language_model.layers.", 1) + remapped_count += 1 + elif k.startswith("base_model.model.model.embed_tokens"): + new_k = k.replace("base_model.model.model.embed_tokens", "base_model.model.model.language_model.embed_tokens", 1) + remapped_count += 1 + elif k.startswith("base_model.model.model.norm"): + new_k = k.replace("base_model.model.model.norm", "base_model.model.model.language_model.norm", 1) + remapped_count += 1 + elif k.startswith("model.layers."): + new_k = k.replace("model.layers.", "model.language_model.layers.", 1) + remapped_count += 1 + elif k.startswith("model.embed_tokens"): + new_k = k.replace("model.embed_tokens", "model.language_model.embed_tokens", 1) + remapped_count += 1 + elif k.startswith("model.norm"): + new_k = k.replace("model.norm", "model.language_model.norm", 1) + remapped_count += 1 + + elif mapping_direction == 'new_to_old': + # New format -> Old format + if k.startswith("base_model.model.model.language_model."): + new_k = k.replace("base_model.model.model.language_model.", "base_model.model.model.", 1) + remapped_count += 1 + elif k.startswith("base_model.model.model.visual."): + new_k = k.replace("base_model.model.model.visual.", "base_model.model.visual.", 1) + remapped_count += 1 + elif k.startswith("model.language_model."): + new_k = k.replace("model.language_model.", "model.", 1) + remapped_count += 1 + elif k.startswith("model.visual."): + new_k = k.replace("model.visual.", "visual.", 1) + remapped_count += 1 + + mapped_state_dict[new_k] = v + + logger.info(f"Key remapping completed ({mapping_direction}). {remapped_count} keys were modified.") + return mapped_state_dict -def load_model_from_checkpoint( - model, checkpoint_dir, checkpoint_step -): + +def _merge_state_dicts(model_state_dict, non_lora_dict, lora_dict): + """Merge non-LoRA and LoRA state dicts into model state dict.""" + for k, v in non_lora_dict.items(): + if k in model_state_dict: + model_state_dict[k] = v + + for k, v in lora_dict.items(): + if k in model_state_dict: + model_state_dict[k] = v + + return model_state_dict + + +def load_model_from_checkpoint(model, checkpoint_dir, checkpoint_step): + """Load model from checkpoint directory.""" checkpoint_paths = glob.glob(os.path.join(checkpoint_dir, "checkpoint-*")) checkpoint_paths.sort(key=lambda x: int(x.split("-")[-1]), reverse=True) @@ -672,19 +792,36 @@ def load_model_from_checkpoint( full_ckpt = os.path.join(checkpoint_path, "model.pth") lora_ckpt = os.path.join(checkpoint_path, "adapter_model.safetensors") non_lora_ckpt = os.path.join(checkpoint_path, "non_lora_state_dict.pth") - if os.path.exists(full_ckpt): + use_lora_path = os.path.exists(lora_ckpt) and os.path.exists(non_lora_ckpt) + + if not use_lora_path and os.path.exists(full_ckpt): + print(f"===> No LoRA files found. Loading full model from: {full_ckpt}") model_state_dict = torch.load(full_ckpt, map_location="cpu") - model.load_state_dict(model_state_dict) - else: + + # Use model-aware remapping for better compatibility + mapped_state_dict = _remap_checkpoint_keys(model_state_dict, model=model) + + model.load_state_dict(mapped_state_dict, strict=True) + + elif use_lora_path: + print(f"===> Loading LoRA components from: {checkpoint_path}") + lora_state_dict = safetensors.torch.load_file(lora_ckpt) non_lora_state_dict = torch.load(non_lora_ckpt, map_location="cpu") - lora_state_dict = _insert_adapter_name_into_state_dict(lora_state_dict, adapter_name="default", parameter_prefix="lora_") + # Use model-aware remapping for better compatibility + non_lora_state_dict = _remap_checkpoint_keys(non_lora_state_dict, model=model) + + lora_state_dict = _insert_adapter_name_into_state_dict(lora_state_dict, adapter_name="default") model_state_dict = model.state_dict() - model_state_dict.update(non_lora_state_dict) - model_state_dict.update(lora_state_dict) - model.load_state_dict(model_state_dict) + model_state_dict = _merge_state_dicts(model_state_dict, non_lora_state_dict, lora_state_dict) + + model.load_state_dict(model_state_dict, strict=True) + print("===> Model loaded successfully with strict=True.") + + else: + raise FileNotFoundError(f"No valid checkpoint files found in {checkpoint_path}") return model, checkpoint_step @@ -719,7 +856,7 @@ def create_model_and_processor(model_config, peft_lora_config, training_args, de ) processor = AutoProcessor.from_pretrained( - model_config.model_name_or_path, padding_side="right", cache_dir=cache_dir + model_config.model_name_or_path, padding_side="left", cache_dir=cache_dir ) special_token_ids = None