Merge main branch.

Former-commit-id: 800ca74fad0d9021f1e96ddf32cd93cdb37cbbe0

scripts/accelerate_configs/ddp_config.yaml

@@ -0,0 +1,11 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: MULTI_GPU
+fsdp_config: {}
+machine_rank: 0
+main_process_ip: null
+main_process_port: 29501
+main_training_function: main
+num_machines: 1
+num_processes: 8
+gpu_ids: 0,1,2,3,4,5,6,7
+use_cpu: false

scripts/accelerate_configs/deepspeed_zero2_offload_config.yaml

@@ -0,0 +1,13 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: DEEPSPEED
+deepspeed_config:
+ deepspeed_config_file: scripts/accelerate_configs/zero2_offload.json
+fsdp_config: {}
+machine_rank: 0
+main_process_ip: null
+main_process_port: 29501
+main_training_function: main
+num_machines: 1
+num_processes: 8
+gpu_ids: 0,1,2,3,4,5,6,7
+use_cpu: false

scripts/accelerate_configs/deepspeed_zero3_config.yaml

@@ -2,7 +2,6 @@ compute_environment: LOCAL_MACHINE
 distributed_type: DEEPSPEED
 deepspeed_config:
  deepspeed_config_file: scripts/accelerate_configs/zero3.json
- zero3_init_flag: true
 fsdp_config: {}
 machine_rank: 0
 main_process_ip: null

scripts/accelerate_configs/deepspeed_zero3_offload_config.yaml

@@ -0,0 +1,13 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: DEEPSPEED
+deepspeed_config:
+ deepspeed_config_file: scripts/accelerate_configs/zero3_offload.json
+fsdp_config: {}
+machine_rank: 0
+main_process_ip: null
+main_process_port: 29501
+main_training_function: main
+num_machines: 1
+num_processes: 8
+gpu_ids: 0,1,2,3,4,5,6,7
+use_cpu: false

scripts/accelerate_configs/zero2_offload.json

@@ -0,0 +1,26 @@
+{
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": "auto"
+    },
+    "train_micro_batch_size_per_gpu": "auto",
+    "train_batch_size": "auto",
+    "gradient_accumulation_steps": "auto",
+    "zero_optimization": {
+        "stage": 2,
+        "offload_optimizer": {
+            "device": "cpu"
+        },
+        "overlap_comm": true,
+        "contiguous_gradients": true,
+        "sub_group_size": 1e9,
+        "reduce_bucket_size": "auto"
+    }
+}

scripts/accelerate_configs/zero3_offload.json

@@ -0,0 +1,39 @@
+{
+  "fp16": {
+    "enabled": "auto",
+    "loss_scale": 0,
+    "loss_scale_window": 1000,
+    "initial_scale_power": 16,
+    "hysteresis": 2,
+    "min_loss_scale": 1
+  },
+  "bf16": {
+    "enabled": "auto"
+  },
+  "zero_optimization": {
+    "stage": 3,
+    "offload_optimizer": {
+      "device": "cpu",
+      "pin_memory": true
+    },
+    "offload_param": {
+      "device": "cpu",
+      "pin_memory": true
+    },
+    "overlap_comm": true,
+    "contiguous_gradients": true,
+    "sub_group_size": 1e9,
+    "reduce_bucket_size": "auto",
+    "stage3_prefetch_bucket_size": "auto",
+    "stage3_param_persistence_threshold": "auto",
+    "stage3_max_live_parameters": 1e9,
+    "stage3_max_reuse_distance": 1e9,
+    "gather_16bit_weights_on_model_save": true
+  },
+  "gradient_accumulation_steps": "auto",
+  "gradient_clipping": "auto",
+  "train_batch_size": "auto",
+  "train_micro_batch_size_per_gpu": "auto",
+  "steps_per_print": 1e5,
+  "wall_clock_breakdown": false
+}

scripts/land_scope/train_feature.sh

@@ -1,9 +0,0 @@
-
-
-accelerate launch --multi_gpu --num_processes 7  --mixed_precision bf16 opensora/train/train_with_feature.py \
-  --model Latte-XL/122 --dataset landscope_feature \
-  --ae stabilityai/sd-vae-ft-mse \
-  --data-path "/remote-home/yeyang/train_movie/*/*256.npy" --extras 1 \
-  --sample-rate 3 --num-frames 128 --max-image-size 256 \
-  --max-train-steps 1000000 --local-batch-size 5 --lr 1e-4 \
-  --ckpt-every 500 --log-every 50  --gradient-checkpointing

scripts/land_scope/train_videofeature.sh

@@ -1,9 +0,0 @@
-
-
-accelerate launch --multi_gpu --num_processes 7 opensora/train/train_with_feature.py \
-  --model Latte-XL/122 --dataset landscope_video_feature \
-  --ae ucf101_stride4x4x4 \
-  --data-path "/remote-home/yeyang/train_movie/*/*s256-videogpt-f16*.npy" --extras 1 \
-  --sample-rate 3 --num-frames 128 --max-image-size 256 \
-  --max-train-steps 1000000 --local-batch-size 16 --lr 1e-4 \
-  --ckpt-every 500 --log-every 50

scripts/sky/train.sh

@@ -1,9 +0,0 @@
-
-
-accelerate launch --multi_gpu --num_processes 8 --mixed_precision bf16 opensora/train/train.py \
-  --model Latte-XL/122 --dataset sky \
-  --ae stabilityai/sd-vae-ft-mse \
-  --data-path "/remote-home/yeyang/sky_timelapse/sky_train/" --extras 1 \
-  --sample-rate 1 --num-frames 128 --max-image-size 256 \
-  --max-train-steps 1000000 --local-batch-size 2 --lr 1e-4 --num-workers 10 \
-  --ckpt-every 500 --log-every 50 --gradient-checkpointing --attention_mode flash

scripts/sky/train_256.sh

@@ -0,0 +1,21 @@
+
+accelerate launch \
+    --config_file scripts/accelerate_configs/ddp_config.yaml \
+    opensora/train/train.py \
+    --model Latte-XL/122 \
+    --dataset sky \
+    --ae stabilityai/sd-vae-ft-mse \
+    --data-path /remote-home/yeyang/sky_timelapse/sky_train/ \
+    --extras 1 \
+    --sample-rate 3 \
+    --num-frames 16 \
+    --max-image-size 256 \
+    --max-train-steps 1000000 \
+    --local-batch-size 5 \
+    --lr 1e-4 \
+    --ckpt-every 500 \
+    --log-every 50 \
+    --gradient-checkpointing \
+    --attention_mode flash \
+    --mixed-precision bf16 \
+

scripts/sky/train_256_dsz2.sh

@@ -0,0 +1,20 @@
+
+accelerate launch \
+    --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
+    opensora/train/train.py \
+    --model Latte-XL/122 \
+    --dataset sky \
+    --ae stabilityai/sd-vae-ft-mse \
+    --data-path /remote-home/yeyang/sky_timelapse/sky_train/ \
+    --extras 1 \
+    --sample-rate 3 \
+    --num-frames 16 \
+    --max-image-size 256 \
+    --max-train-steps 1000000 \
+    --local-batch-size 5 \
+    --lr 1e-4 \
+    --ckpt-every 500 \
+    --log-every 50 \
+    --gradient-checkpointing \
+    --attention_mode flash \
+    --mixed-precision bf16

scripts/sky/train_256_dsz2_off.sh

@@ -0,0 +1,20 @@
+
+accelerate launch \
+    --config_file scripts/accelerate_configs/deepspeed_zero2_offload_config.yaml \
+    opensora/train/train.py \
+    --model Latte-XL/122 \
+    --dataset sky \
+    --ae stabilityai/sd-vae-ft-mse \
+    --data-path /remote-home/yeyang/sky_timelapse/sky_train/ \
+    --extras 1 \
+    --sample-rate 3 \
+    --num-frames 16 \
+    --max-image-size 256 \
+    --max-train-steps 1000000 \
+    --local-batch-size 5 \
+    --lr 1e-4 \
+    --ckpt-every 500 \
+    --log-every 50 \
+    --gradient-checkpointing \
+    --attention_mode flash \
+    --mixed-precision bf16

scripts/sky/train_256_dsz3.sh

@@ -0,0 +1,20 @@
+
+accelerate launch \
+    --config_file scripts/accelerate_configs/deepspeed_zero3_config.yaml \
+    opensora/train/train.py \
+    --model Latte-XL/122 \
+    --dataset sky \
+    --ae stabilityai/sd-vae-ft-mse \
+    --data-path /remote-home/yeyang/sky_timelapse/sky_train/ \
+    --extras 1 \
+    --sample-rate 3 \
+    --num-frames 16 \
+    --max-image-size 256 \
+    --max-train-steps 1000000 \
+    --local-batch-size 5 \
+    --lr 1e-4 \
+    --ckpt-every 500 \
+    --log-every 50 \
+    --gradient-checkpointing \
+    --attention_mode flash \
+    --mixed-precision bf16

scripts/sky/train_256_dsz3_off.sh

@@ -0,0 +1,20 @@
+
+accelerate launch \
+    --config_file scripts/accelerate_configs/deepspeed_zero3_offload_config.yaml \
+    opensora/train/train.py \
+    --model Latte-XL/122 \
+    --dataset sky \
+    --ae stabilityai/sd-vae-ft-mse \
+    --data-path /remote-home/yeyang/sky_timelapse/sky_train/ \
+    --extras 1 \
+    --sample-rate 3 \
+    --num-frames 16 \
+    --max-image-size 256 \
+    --max-train-steps 1000000 \
+    --local-batch-size 5 \
+    --lr 1e-4 \
+    --ckpt-every 500 \
+    --log-every 50 \
+    --gradient-checkpointing \
+    --attention_mode flash \
+    --mixed-precision bf16

scripts/sky/train_512.sh

@@ -1,8 +0,0 @@
-
-accelerate launch --multi_gpu --num_processes 8 --mixed_precision bf16 opensora/train/train.py \
-  --model Latte-XL/122 --dataset sky \
-  --ae stabilityai/sd-vae-ft-mse \
-  --data-path "/remote-home/yeyang/sky_timelapse/sky_train/" --extras 1 \
-  --sample-rate 3 --num-frames 16 --max-image-size 512 \
-  --max-train-steps 1000000 --local-batch-size 2 --lr 1e-4 \
-  --ckpt-every 500 --log-every 50 --gradient-checkpointing --attention_mode flash

scripts/sky/train_feature.sh

@@ -1,9 +0,0 @@
-
-
-accelerate launch  --multi_gpu --num_processes 8 --mixed_precision bf16 opensora/train/train_with_feature.py \
-  --model Latte-XL/122 --dataset sky_feature \
-  --ae stabilityai/sd-vae-ft-mse \
-  --data-path "/remote-home/yeyang/sky_timelapse/sky_train/" --extras 1 \
-  --sample-rate 3 --num-frames 128 --max-image-size 256 \
-  --max-train-steps 1000000 --local-batch-size 5 --lr 1e-4 \
-  --ckpt-every 500 --log-every 50 --gradient-checkpointing --attention_mode flash

scripts/sky/train_feature_512.sh

@@ -1,9 +0,0 @@
-
-
-accelerate launch --multi_gpu --num_processes 8 --mixed_precision bf16 opensora/train/train_with_feature.py \
-  --model Latte-XL/122 --dataset landscope_feature \
-  --ae stabilityai/sd-vae-ft-mse \
-  --data-path "/remote-home/yeyang/sky_timelapse/sky_train/" --extras 1 \
-  --sample-rate 3 --num-frames 64 --max-image-size 512 \
-  --max-train-steps 1000000 --local-batch-size 5 --lr 1e-4 \
-  --ckpt-every 500 --log-every 50 --gradient-checkpointing --attention_mode flash