Peiyuan Zhang
8ec82cbc37
Delete Open Sora Plan Modeling amend log validation Change name to fast video Remove OSP modeling clean up name changing remove files Deleted unnecessary files commit first commit first training ok Update generate_synthetic.sh and deepspeed_zero2_config.yaml 14 debug overfitting .... debugging .. Debug successful! Add zero3 OK update optimizer load small bug random seed args sp enable & still has bug rename update inference sp code / can run / still has bug / don't output normal mp4 switch to deepspeed dummyoptim fix some bugs; still output green SP inference done! fix typos in readme and latent dataset debug file
66 lines
1.9 KiB
Bash
66 lines
1.9 KiB
Bash
# export WANDB_MODE="offline"
|
|
export ENTITY="linbin"
|
|
export HF_DATASETS_OFFLINE=1
|
|
export TRANSFORMERS_OFFLINE=1
|
|
export PDSH_RCMD_TYPE=ssh
|
|
# NCCL setting
|
|
export GLOO_SOCKET_IFNAME=bond0
|
|
export NCCL_SOCKET_IFNAME=bond0
|
|
export NCCL_IB_HCA=mlx5_10:1,mlx5_11:1,mlx5_12:1,mlx5_13:1
|
|
export NCCL_IB_GID_INDEX=3
|
|
export NCCL_IB_TC=162
|
|
export NCCL_IB_TIMEOUT=22
|
|
export NCCL_PXN_DISABLE=0
|
|
export NCCL_IB_QPS_PER_CONNECTION=4
|
|
export NCCL_ALGO=Ring
|
|
export OMP_NUM_THREADS=1
|
|
export MKL_NUM_THREADS=1
|
|
# export NCCL_ALGO=Tree
|
|
|
|
accelerate launch \
|
|
--config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
|
|
opensora/train/train_t2v_diffusers.py \
|
|
--model OpenSoraT2V-ROPE-L/122 \
|
|
--text_encoder_name google/mt5-xxl \
|
|
--cache_dir "./cache_dir" \
|
|
--dataset t2v \
|
|
--data "scripts/train_data/merge_data.txt" \
|
|
--ae CausalVAEModel_D4_4x8x8 \
|
|
--ae_path "/storage/dataset/test140k" \
|
|
--sample_rate 1 \
|
|
--num_frames 29 \
|
|
--max_height 480 \
|
|
--max_width 640 \
|
|
--interpolation_scale_t 1.0 \
|
|
--interpolation_scale_h 1.0 \
|
|
--interpolation_scale_w 1.0 \
|
|
--attention_mode xformers \
|
|
--gradient_checkpointing \
|
|
--train_batch_size=1 \
|
|
--dataloader_num_workers 10 \
|
|
--gradient_accumulation_steps=1 \
|
|
--max_train_steps=1000000 \
|
|
--learning_rate=1e-4 \
|
|
--lr_scheduler="constant" \
|
|
--lr_warmup_steps=0 \
|
|
--mixed_precision="bf16" \
|
|
--report_to="wandb" \
|
|
--checkpointing_steps=250 \
|
|
--allow_tf32 \
|
|
--text_max_length 512 \
|
|
--use_image_num 0 \
|
|
--tile_sample_stride_height 0.125 \
|
|
--snr_gamma 5.0 \
|
|
--use_ema \
|
|
--ema_start_step 0 \
|
|
--cfg 0.1 \
|
|
--noise_offset 0.02 \
|
|
--use_rope \
|
|
--resume_from_checkpoint="latest" \
|
|
--ema_decay 0.999 \
|
|
--enable_tiling \
|
|
--speed_factor 1.0 \
|
|
--group_frame \
|
|
--sp_size 8 \
|
|
--train_sp_batch_size 2 \
|
|
--output_dir="bs16x8x1_29x480p_lr1e-4_snr5_ema999_opensora122_rope_mt5xxl_speed1.0" |