merge main branch.

Former-commit-id: 5f78da5eae2b18305bb4a0a6dc7ab771e9b96f60

scripts/accelerate_configs/hostfile

@@ -0,0 +1,2 @@
+server1 slots=8  # your server name  and  GPU in total
+server2 slots=8

scripts/accelerate_configs/multi_node_example.yaml

@@ -0,0 +1,18 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: DEEPSPEED
+deepspeed_config:
+ deepspeed_config_file: scripts/accelerate_configs/zero3_offload.json
+ deepspeed_hostfile: # FullPath here
+fsdp_config: {}
+machine_rank: 0
+main_process_ip: # main process ip
+main_process_port: 29501
+main_training_function: main
+num_machines: 2
+num_processes: 16
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

scripts/sample.sh

@@ -1,5 +1,6 @@
 accelerate launch --num_processes 1 --main_process_port 29501 --mixed_precision bf16 opensora/sample/sample.py \
-  --model Latte-XL/122 --ae stabilityai/sd-vae-ft-mse \
+  --model Latte-XL/122 \
+  --ae stabilityai/sd-vae-ft-mse \
   --ckpt results/011-Latte-XL-122-F128S3-landscope_feature-Gc-Amp/checkpoints/0005000.pt --extras 1 \
-  --fps 10 --num-frames 128 --image-size 256 --num-sampling-steps 250 --attention_mode flash
+  --fps 10 --num-frames 128 --image-size 256 --num-sampling-steps 250 --attention-mode flash
 

scripts/sky/train_256.sh

@@ -8,7 +8,7 @@ accelerate launch \
     --data-path /remote-home/yeyang/sky_timelapse/sky_train/ \
     --extras 1 \
     --sample-rate 3 \
-    --num-frames 16 \
+    --num-frames 128 \
     --max-image-size 256 \
     --max-train-steps 1000000 \
     --local-batch-size 5 \

scripts/ucf101/train_256.sh

@@ -0,0 +1,23 @@
+
+accelerate launch \
+    --config_file scripts/accelerate_configs/ddp_config.yaml \
+    opensora/train/train.py \
+    --model Latte-XL/122 \
+    --dataset ucf101 \
+    --num-classes 101 \
+    --ae stabilityai/sd-vae-ft-mse \
+    --data-path /remote-home/yeyang/UCF-101 \
+    --extras 2 \
+    --sample-rate 3 \
+    --num-frames 16 \
+    --max-image-size 256 \
+    --max-train-steps 1000000 \
+    --local-batch-size 5 \
+    --lr 1e-4 \
+    --ckpt-every 500 \
+    --log-every 50 \
+    --gradient-checkpointing \
+    --attention-mode flash \
+    --mixed-precision bf16
+
+

scripts/videogpt/train_videogpt.sh

@@ -1,26 +1,30 @@
+export ACCELERATE_GRADIENT_ACCUMULATION_STEPS=1
+
 accelerate launch \
-  --config_file scripts/accelerate_configs/default_config.yaml \
+  --config_file scripts/accelerate_configs/ddp_config.yaml \
   opensora/train/train_videogpt.py \
   --do_train \
   --seed 1234 \
-  --data_path UCF-101 \
-  --per_device_train_batch_size 8 \
-  --gradient_accumulation_steps 8 \
-  --learning_rate 1e-5 \
+  --data_path "datasets/UCF-101/" \
+  --per_device_train_batch_size 32  \
+  --gradient_accumulation_steps $ACCELERATE_GRADIENT_ACCUMULATION_STEPS \
+  --learning_rate 7e-4 \
   --weight_decay 0. \
   --num_train_epochs 2 \
-  --embedding_dim 512 \
   --lr_scheduler_type cosine \
   --max_grad_norm 1.0 \
   --save_strategy steps \
   --save_total_limit 5 \
   --logging_steps 5 \
-  --save_steps 5 \
-  --n_codes 2048 \
+  --save_steps 1000 \
+  --n_codes 1024 \
   --n_hiddens 240 \
+  --embedding_dim 4 \
   --n_res_layers 4 \
   --downsample "4,4,4" \
   --resolution 128 \
   --sequence_length 16 \
-  --output_dir results/videogpt \
-  --report_to tensorboard
+  --output_dir results/videogpt_444_128 \
+  --bf16 True \
+  --fp16 False \
+  --report_to tensorboard

scripts/videogpt/train_videogpt_dsz2.sh

@@ -0,0 +1,30 @@
+export ACCELERATE_GRADIENT_ACCUMULATION_STEPS=1
+
+accelerate launch \
+  --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
+  opensora/train/train_videogpt.py \
+  --do_train \
+  --seed 1234 \
+  --data_path "datasets/UCF-101/" \
+  --per_device_train_batch_size 32  \
+  --gradient_accumulation_steps $ACCELERATE_GRADIENT_ACCUMULATION_STEPS \
+  --learning_rate 7e-4 \
+  --weight_decay 0. \
+  --num_train_epochs 2 \
+  --lr_scheduler_type cosine \
+  --max_grad_norm 1.0 \
+  --save_strategy steps \
+  --save_total_limit 5 \
+  --logging_steps 5 \
+  --save_steps 10000 \
+  --n_codes 1024 \
+  --n_hiddens 240 \
+  --embedding_dim 4 \
+  --n_res_layers 4 \
+  --downsample "4,4,4" \
+  --resolution 128 \
+  --sequence_length 16 \
+  --output_dir results/videogpt_444_128 \
+  --bf16 True \
+  --fp16 False \
+  --report_to tensorboard

scripts/videogpt/train_videogpt_dsz3.sh

@@ -0,0 +1,30 @@
+export ACCELERATE_GRADIENT_ACCUMULATION_STEPS=1
+
+accelerate launch \
+  --config_file scripts/accelerate_configs/deepspeed_zero3_config.yaml \
+  opensora/train/train_videogpt.py \
+  --do_train \
+  --seed 1234 \
+  --data_path "datasets/UCF-101/" \
+  --per_device_train_batch_size 32  \
+  --gradient_accumulation_steps $ACCELERATE_GRADIENT_ACCUMULATION_STEPS \
+  --learning_rate 7e-4 \
+  --weight_decay 0. \
+  --num_train_epochs 2 \
+  --lr_scheduler_type cosine \
+  --max_grad_norm 1.0 \
+  --save_strategy steps \
+  --save_total_limit 5 \
+  --logging_steps 5 \
+  --save_steps 10000 \
+  --n_codes 1024 \
+  --n_hiddens 240 \
+  --embedding_dim 4 \
+  --n_res_layers 4 \
+  --downsample "4,4,4" \
+  --resolution 128 \
+  --sequence_length 16 \
+  --output_dir results/videogpt_444_128 \
+  --bf16 True \
+  --fp16 False \
+  --report_to tensorboard