diff --git a/README.md b/README.md index 372b47713..a54dfa24f 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,6 @@ bash scripts/text_condition/gpu/train_t2v.sh ## TODO +- [ ] Create dummy debug data. - [ ] Add Mochi - [ ] Add Mochi VAE -- [ ] Create dummy debug data. \ No newline at end of file diff --git a/scripts/text_condition/npu/sample_inpaint.sh b/scripts/text_condition/npu/sample_inpaint.sh deleted file mode 100644 index 005be4739..000000000 --- a/scripts/text_condition/npu/sample_inpaint.sh +++ /dev/null @@ -1,34 +0,0 @@ -WEIGHT_PATH="/home/opensora/pre_weights/" - -export PYTHONPATH="${PYTHONPATH:+$PYTHONPATH:}$(pwd)" -export MASTER_PORT=12359 - -if [ -z "$SAMPLE_SAVE_PATH" ]; then - export SAMPLE_SAVE_PATH="/home/image_data/sample_videos" -fi - -if [ -z "$SAMPLE_HEIGHT" ]; then - echo "You should set both envs of SAMPLE_HEIGHT and SAMPLE_WIDTH" - return -fi - -torchrun --nproc_per_node=8 opensora/sample/sample_inpaint_on_npu.py \ - --model_path /home/image_data/checkpoints/${PROJECT_NAME} \ - --num_frames ${NUM_FRAME} \ - --height $SAMPLE_HEIGHT \ - --width $SAMPLE_WIDTH \ - --cache_dir "../cache_dir" \ - --text_encoder_name ${WEIGHT_PATH}/google/mt5-xxl \ - --text_prompt "/home/image_data/checkpoints/i2v_validation_dir/prompt.txt" \ - --ae CausalVAEModel_4x8x8 \ - --ae_path "${WEIGHT_PATH}/test140k/" \ - --save_img_path "${SAMPLE_SAVE_PATH}/${PROJECT_NAME}" \ - --fps 24 \ - --max_sequence_length 512 \ - --guidance_scale 5.0 \ - --num_sampling_steps 50 \ - --sample_method PNDM \ - --enable_tiling \ - --tile_overlap_factor 0.125 \ - --model_3d \ - --validation_dir "/home/image_data/checkpoints/i2v_validation_dir" \ No newline at end of file diff --git a/scripts/text_condition/npu/train_image3d_480p.sh b/scripts/text_condition/npu/train_image3d_480p.sh deleted file mode 100644 index c320b0bad..000000000 --- a/scripts/text_condition/npu/train_image3d_480p.sh +++ /dev/null @@ -1,54 +0,0 @@ -export PROJECT=$PROJECT_NAME -WEIGHT_PATH="/home/opensora/pre_weights/" -env -#export WANDB_MODE='offline' -export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE -export HCCL_ALGO="level0:NA;level1:H-D_R" -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -accelerate launch \ - --config_file scripts/accelerate_configs/multi_node_example_by_deepspeed.yaml \ - --machine_rank=${MACHINE_RANK} \ - --main_process_ip=${MAIN_PROCESS_IP_VALUE} \ - opensora/train/train_t2v_diffusers.py \ - --model OpenSoraT2V-ROPE-L/122 \ - --text_encoder_name ${WEIGHT_PATH}/google/mt5-xxl \ - --cache_dir "../cache_dir" \ - --dataset t2v \ - --data "scripts/train_data/merge_data_on_npu.txt" \ - --ae CausalVAEModel_D4_4x8x8 \ - --ae_path "${WEIGHT_PATH}/test140k/" \ - --sample_rate 1 \ - --num_frames 1 \ - --max_height 480 \ - --max_width 640 \ - --interpolation_scale_t 1.0 \ - --interpolation_scale_h 1.0 \ - --interpolation_scale_w 1.0 \ - --attention_mode xformers \ - --gradient_checkpointing \ - --train_batch_size=8 \ - --dataloader_num_workers 20 \ - --gradient_accumulation_steps=1 \ - --max_train_steps=1000000 \ - --learning_rate=1e-4 \ - --lr_scheduler="constant" \ - --seed=10 \ - --lr_warmup_steps=500 \ - --mixed_precision="bf16" \ - --report_to="wandb" \ - --checkpointing_steps=2000 \ - --output_dir="/home/image_data/checkpoints/${PROJECT}/" \ - --allow_tf32 \ - --model_max_length 512 \ - --use_image_num 0 \ - --snr_gamma 5.0 \ - --use_ema \ - --ema_start_step 0 \ - --cfg 0.1 \ - --enable_tiling \ - --tile_overlap_factor 0.0 \ - --use_rope \ - --noise_offset 0.02 \ - --resume_from_checkpoint="latest" - diff --git a/scripts/text_condition/npu/train_image3d_720p.sh b/scripts/text_condition/npu/train_image3d_720p.sh deleted file mode 100644 index 2562dac26..000000000 --- a/scripts/text_condition/npu/train_image3d_720p.sh +++ /dev/null @@ -1,54 +0,0 @@ -export PROJECT=$PROJECT_NAME -WEIGHT_PATH="/home/opensora/pre_weights/" -env -#export WANDB_MODE='offline' -export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE -export HCCL_ALGO="level0:NA;level1:H-D_R" -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -accelerate launch \ - --config_file scripts/accelerate_configs/multi_node_example_by_deepspeed.yaml \ - --machine_rank=${MACHINE_RANK} \ - --main_process_ip=${MAIN_PROCESS_IP_VALUE} \ - opensora/train/train_t2v_diffusers.py \ - --model OpenSoraT2V-ROPE-L/122 \ - --text_encoder_name ${WEIGHT_PATH}/google/mt5-xxl \ - --cache_dir "../cache_dir" \ - --dataset t2v \ - --data "scripts/train_data/merge_data_on_npu.txt" \ - --ae CausalVAEModel_D4_4x8x8 \ - --ae_path "${WEIGHT_PATH}/test140k/" \ - --sample_rate 1 \ - --num_frames 1 \ - --max_height 720 \ - --max_width 1280 \ - --interpolation_scale_t 1.0 \ - --interpolation_scale_h 1.5 \ - --interpolation_scale_w 2.0 \ - --attention_mode xformers \ - --gradient_checkpointing \ - --train_batch_size=1 \ - --dataloader_num_workers 8 \ - --gradient_accumulation_steps=1 \ - --max_train_steps=1000000 \ - --learning_rate=1e-4 \ - --lr_scheduler="constant" \ - --seed=10 \ - --lr_warmup_steps=500 \ - --mixed_precision="bf16" \ - --report_to="wandb" \ - --checkpointing_steps=1000 \ - --output_dir="/home/image_data/checkpoints/${PROJECT}/" \ - --allow_tf32 \ - --model_max_length 512 \ - --use_image_num 0 \ - --snr_gamma 5.0 \ - --use_ema \ - --ema_start_step 0 \ - --cfg 0.1 \ - --enable_tiling \ - --tile_overlap_factor 0.0 \ - --use_rope \ - --noise_offset 0.02 \ - --resume_from_checkpoint="latest" - diff --git a/scripts/text_condition/npu/train_inpaint_video3d_nx480p.sh b/scripts/text_condition/npu/train_inpaint_video3d_nx480p.sh deleted file mode 100644 index b7f751312..000000000 --- a/scripts/text_condition/npu/train_inpaint_video3d_nx480p.sh +++ /dev/null @@ -1,61 +0,0 @@ -export WANDB_KEY="720d886d8c437c2142c88056a1eab8ef78d64a1f" -# # export WANDB_MODE="offline" -export ENTITY="yunyangge" -WEIGHT_PATH="/home/opensora/pre_weights/" -env -# export WANDB_MODE='offline' -export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE -export HCCL_ALGO="level0:NA;level1:H-D_R" - -accelerate launch \ - --config_file scripts/accelerate_configs/multi_node_example_by_deepspeed.yaml \ - --machine_rank=${MACHINE_RANK} \ - --main_process_ip=${MAIN_PROCESS_IP_VALUE} \ - opensora/train/train_inpaint.py \ - --model OpenSoraInpaint-ROPE-L/122 \ - --text_encoder_name ${WEIGHT_PATH}/google/mt5-xxl \ - --cache_dir "../cache_dir" \ - --dataset i2v \ - --ae CausalVAEModel_4x8x8 \ - --ae_path "${WEIGHT_PATH}/test140k/" \ - --video_data "./scripts/train_data/video_data_on_npu.txt" \ - --image_data "./scripts/train_data/image_data_on_npu.txt" \ - --sample_rate 1 \ - --num_frames ${NUM_FRAME} \ - --max_height 480 \ - --max_width 640 \ - --interpolation_scale_t 1.0 \ - --interpolation_scale_h 1.0 \ - --interpolation_scale_w 1.0 \ - --attention_mode xformers \ - --gradient_checkpointing \ - --train_batch_size=1 \ - --dataloader_num_workers 8 \ - --gradient_accumulation_steps=1 \ - --max_train_steps=1000000 \ - --learning_rate=1e-5 \ - --lr_scheduler="constant" \ - --seed=42 \ - --lr_warmup_steps=0 \ - --mixed_precision="bf16" \ - --report_to="wandb" \ - --checkpointing_steps=200 \ - --output_dir="/home/image_data/checkpoints/${PROJECT}/" \ - --allow_tf32 \ - --model_max_length 512 \ - --use_image_num 0 \ - --snr_gamma 5.0 \ - --use_ema \ - --ema_start_step 0 \ - --cfg 0.1 \ - --enable_tiling \ - --tile_overlap_factor 0.125 \ - --use_rope \ - --noise_offset 0.02 \ - --pretrained "/home/image_data/checkpoints/gyy_pretrained_f125/model_ema/diffusion_pytorch_model.safetensors" \ - --resume_from_checkpoint="latest" \ - --i2v_ratio 0.5 \ - --transition_ratio 0.4 \ - --default_text_ratio 0.5 \ - --group_frame \ - --speed_factor 1.1 \ diff --git a/scripts/text_condition/npu/train_video3d_nx480p.sh b/scripts/text_condition/npu/train_video3d_nx480p.sh deleted file mode 100644 index d406a1ccb..000000000 --- a/scripts/text_condition/npu/train_video3d_nx480p.sh +++ /dev/null @@ -1,57 +0,0 @@ -export PROJECT=$PROJECT_NAME -WEIGHT_PATH="/home/opensora/pre_weights/" -env -#export WANDB_MODE='offline' -export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE -export HCCL_ALGO="level0:NA;level1:H-D_R" -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -accelerate launch \ - --config_file scripts/accelerate_configs/multi_node_example_by_deepspeed.yaml \ - --machine_rank=${MACHINE_RANK} \ - --main_process_ip=${MAIN_PROCESS_IP_VALUE} \ - opensora/train/train_t2v_diffusers.py \ - --model OpenSoraT2V-ROPE-L/122 \ - --text_encoder_name ${WEIGHT_PATH}/google/mt5-xxl \ - --cache_dir "../cache_dir" \ - --dataset t2v \ - --data "scripts/train_data/merge_data_on_npu.txt" \ - --ae CausalVAEModel_D4_4x8x8 \ - --ae_path "${WEIGHT_PATH}/test140k/" \ - --sample_rate 1 \ - --num_frames ${NUM_FRAME} \ - --max_height 480 \ - --max_width 640 \ - --interpolation_scale_t 1.0 \ - --interpolation_scale_h 1.0 \ - --interpolation_scale_w 1.0 \ - --attention_mode xformers \ - --gradient_checkpointing \ - --train_batch_size=1 \ - --dataloader_num_workers 8 \ - --gradient_accumulation_steps=1 \ - --max_train_steps=1000000 \ - --learning_rate=1e-4 \ - --lr_scheduler="constant" \ - --seed=10 \ - --lr_warmup_steps=500 \ - --mixed_precision="bf16" \ - --report_to="wandb" \ - --checkpointing_steps=1000 \ - --output_dir="/home/image_data/checkpoints/${PROJECT}/" \ - --allow_tf32 \ - --model_max_length 512 \ - --use_image_num 0 \ - --snr_gamma 5.0 \ - --use_ema \ - --ema_start_step 0 \ - --cfg 0.1 \ - --enable_tiling \ - --tile_overlap_factor 0.125 \ - --use_rope \ - --noise_offset 0.02 \ - --resume_from_checkpoint="latest" \ - --enable_stable_fp32 \ - --ema_decay 0.999 \ - --speed_factor 1.0 \ - --drop_short_ratio 1.0 diff --git a/scripts/text_condition/npu/train_video3d_nx720p.sh b/scripts/text_condition/npu/train_video3d_nx720p.sh deleted file mode 100644 index 415fb5728..000000000 --- a/scripts/text_condition/npu/train_video3d_nx720p.sh +++ /dev/null @@ -1,57 +0,0 @@ -export PROJECT=$PROJECT_NAME -WEIGHT_PATH="/home/opensora/pre_weights/" -env -#export WANDB_MODE='offline' -export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE -export HCCL_ALGO="level0:NA;level1:H-D_R" -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -accelerate launch \ - --config_file scripts/accelerate_configs/multi_node_example_by_deepspeed.yaml \ - --machine_rank=${MACHINE_RANK} \ - --main_process_ip=${MAIN_PROCESS_IP_VALUE} \ - opensora/train/train_t2v_diffusers.py \ - --model OpenSoraT2V-ROPE-L/122 \ - --text_encoder_name ${WEIGHT_PATH}/google/mt5-xxl \ - --cache_dir "../cache_dir" \ - --dataset t2v \ - --data "scripts/train_data/merge_data_on_npu.txt" \ - --ae CausalVAEModel_D4_4x8x8 \ - --ae_path "${WEIGHT_PATH}/test140k/" \ - --sample_rate 1 \ - --num_frames ${NUM_FRAME} \ - --max_height 720 \ - --max_width 1280 \ - --interpolation_scale_t 1.0 \ - --interpolation_scale_h 1.5 \ - --interpolation_scale_w 2.0 \ - --attention_mode xformers \ - --gradient_checkpointing \ - --train_batch_size=1 \ - --dataloader_num_workers 8 \ - --gradient_accumulation_steps=1 \ - --max_train_steps=1000000 \ - --learning_rate=1e-4 \ - --lr_scheduler="constant" \ - --seed=10 \ - --lr_warmup_steps=500 \ - --mixed_precision="bf16" \ - --report_to="wandb" \ - --checkpointing_steps=1000 \ - --output_dir="/home/image_data/checkpoints/${PROJECT}/" \ - --allow_tf32 \ - --model_max_length 512 \ - --use_image_num 0 \ - --snr_gamma 5.0 \ - --use_ema \ - --ema_start_step 0 \ - --cfg 0.1 \ - --enable_tiling \ - --tile_overlap_factor 0.125 \ - --use_rope \ - --noise_offset 0.02 \ - --resume_from_checkpoint="latest" \ - --enable_stable_fp32 \ - --ema_decay 0.999 \ - --speed_factor 1.0 \ - --drop_short_ratio 1.0 diff --git a/scripts/text_condition/npu/train_video3d_sp_nx480p.sh b/scripts/text_condition/npu/train_video3d_sp_nx480p.sh deleted file mode 100644 index 39268b88a..000000000 --- a/scripts/text_condition/npu/train_video3d_sp_nx480p.sh +++ /dev/null @@ -1,59 +0,0 @@ -export PROJECT=$PROJECT_NAME -WEIGHT_PATH="/home/opensora/pre_weights/" -env -#export WANDB_MODE='offline' -export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE -export HCCL_ALGO="level0:NA;level1:H-D_R" -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -accelerate launch \ - --config_file scripts/accelerate_configs/multi_node_example_by_deepspeed.yaml \ - --machine_rank=${MACHINE_RANK} \ - --main_process_ip=${MAIN_PROCESS_IP_VALUE} \ - opensora/train/train_t2v_diffusers.py \ - --model OpenSoraT2V-ROPE-L/122 \ - --text_encoder_name ${WEIGHT_PATH}/google/mt5-xxl \ - --cache_dir "../cache_dir" \ - --dataset t2v \ - --data "scripts/train_data/merge_data_on_npu.txt" \ - --ae CausalVAEModel_D4_4x8x8 \ - --ae_path "${WEIGHT_PATH}/test140k/" \ - --sample_rate 1 \ - --num_frames ${NUM_FRAME} \ - --max_height 480 \ - --max_width 640 \ - --interpolation_scale_t 1.0 \ - --interpolation_scale_h 1.0 \ - --interpolation_scale_w 1.0 \ - --attention_mode xformers \ - --gradient_checkpointing \ - --train_batch_size=1 \ - --dataloader_num_workers 8 \ - --gradient_accumulation_steps=1 \ - --max_train_steps=1000000 \ - --learning_rate=1e-4 \ - --lr_scheduler="constant" \ - --seed=10 \ - --lr_warmup_steps=500 \ - --mixed_precision="bf16" \ - --report_to="wandb" \ - --checkpointing_steps=1000 \ - --output_dir="/home/image_data/checkpoints/${PROJECT}/" \ - --allow_tf32 \ - --model_max_length 512 \ - --use_image_num 0 \ - --snr_gamma 5.0 \ - --use_ema \ - --ema_start_step 0 \ - --cfg 0.1 \ - --enable_tiling \ - --tile_overlap_factor 0.125 \ - --use_rope \ - --noise_offset 0.02 \ - --resume_from_checkpoint="latest" \ - --enable_stable_fp32 \ - --ema_decay 0.999 \ - --speed_factor 1.0 \ - --drop_short_ratio 1.0 \ - --sp_size 8 \ - --train_sp_batch_size 1 diff --git a/scripts/text_condition/npu/train_video3d_sp_nx720p.sh b/scripts/text_condition/npu/train_video3d_sp_nx720p.sh deleted file mode 100644 index d5b08e540..000000000 --- a/scripts/text_condition/npu/train_video3d_sp_nx720p.sh +++ /dev/null @@ -1,59 +0,0 @@ -export PROJECT=$PROJECT_NAME -WEIGHT_PATH="/home/opensora/pre_weights/" -env -#export WANDB_MODE='offline' -export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE -export HCCL_ALGO="level0:NA;level1:H-D_R" -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -accelerate launch \ - --config_file scripts/accelerate_configs/multi_node_example_by_deepspeed.yaml \ - --machine_rank=${MACHINE_RANK} \ - --main_process_ip=${MAIN_PROCESS_IP_VALUE} \ - opensora/train/train_t2v_diffusers.py \ - --model OpenSoraT2V-ROPE-L/122 \ - --text_encoder_name ${WEIGHT_PATH}/google/mt5-xxl \ - --cache_dir "../cache_dir" \ - --dataset t2v \ - --data "scripts/train_data/merge_data_on_npu.txt" \ - --ae CausalVAEModel_D4_4x8x8 \ - --ae_path "${WEIGHT_PATH}/test140k/" \ - --sample_rate 1 \ - --num_frames ${NUM_FRAME} \ - --max_height 720 \ - --max_width 1280 \ - --interpolation_scale_t 1.0 \ - --interpolation_scale_h 1.5 \ - --interpolation_scale_w 2.0 \ - --attention_mode xformers \ - --gradient_checkpointing \ - --train_batch_size=1 \ - --dataloader_num_workers 8 \ - --gradient_accumulation_steps=1 \ - --max_train_steps=1000000 \ - --learning_rate=1e-4 \ - --lr_scheduler="constant" \ - --seed=10 \ - --lr_warmup_steps=500 \ - --mixed_precision="bf16" \ - --report_to="wandb" \ - --checkpointing_steps=1000 \ - --output_dir="/home/image_data/checkpoints/${PROJECT}/" \ - --allow_tf32 \ - --model_max_length 512 \ - --use_image_num 0 \ - --snr_gamma 5.0 \ - --use_ema \ - --ema_start_step 0 \ - --cfg 0.1 \ - --enable_tiling \ - --tile_overlap_factor 0.125 \ - --use_rope \ - --noise_offset 0.02 \ - --resume_from_checkpoint="latest" \ - --enable_stable_fp32 \ - --ema_decay 0.999 \ - --speed_factor 1.0 \ - --drop_short_ratio 1.0 \ - --sp_size 8 \ - --train_sp_batch_size 1 diff --git a/scripts/text_condition/gpu/sample_inpaint.sh b/scripts/text_condition/sample_inpaint.sh similarity index 100% rename from scripts/text_condition/gpu/sample_inpaint.sh rename to scripts/text_condition/sample_inpaint.sh diff --git a/scripts/text_condition/gpu/sample_inpaint_ddp.sh b/scripts/text_condition/sample_inpaint_ddp.sh similarity index 100% rename from scripts/text_condition/gpu/sample_inpaint_ddp.sh rename to scripts/text_condition/sample_inpaint_ddp.sh diff --git a/scripts/text_condition/gpu/sample_inpaint_sp.sh b/scripts/text_condition/sample_inpaint_sp.sh similarity index 100% rename from scripts/text_condition/gpu/sample_inpaint_sp.sh rename to scripts/text_condition/sample_inpaint_sp.sh diff --git a/scripts/text_condition/gpu/sample_t2v.sh b/scripts/text_condition/sample_t2v.sh similarity index 100% rename from scripts/text_condition/gpu/sample_t2v.sh rename to scripts/text_condition/sample_t2v.sh diff --git a/scripts/text_condition/gpu/sample_t2v_ddp.sh b/scripts/text_condition/sample_t2v_ddp.sh similarity index 100% rename from scripts/text_condition/gpu/sample_t2v_ddp.sh rename to scripts/text_condition/sample_t2v_ddp.sh diff --git a/scripts/text_condition/gpu/sample_t2v_sp.sh b/scripts/text_condition/sample_t2v_sp.sh similarity index 100% rename from scripts/text_condition/gpu/sample_t2v_sp.sh rename to scripts/text_condition/sample_t2v_sp.sh diff --git a/scripts/text_condition/gpu/train_inpaint.sh b/scripts/text_condition/train_inpaint.sh similarity index 100% rename from scripts/text_condition/gpu/train_inpaint.sh rename to scripts/text_condition/train_inpaint.sh diff --git a/scripts/text_condition/gpu/train_t2v.sh b/scripts/text_condition/train_t2v.sh similarity index 100% rename from scripts/text_condition/gpu/train_t2v.sh rename to scripts/text_condition/train_t2v.sh