Support deepspeed for videogpt training.

Former-commit-id: 1f3dfe038845d91b81b0ecc9b6f3cabb06eed5d5
2024-03-09 17:33:30 +08:00
parent a60217616e
commit 3b4e881b11
5 changed files with 86 additions and 4 deletions
@@ -0,0 +1,13 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: DEEPSPEED
+deepspeed_config:
+ deepspeed_config_file: scripts/accelerate_configs/zero2.json
+fsdp_config: {}
+machine_rank: 0
+main_process_ip: null
+main_process_port: 29501
+main_training_function: main
+num_machines: 1
+num_processes: 1
+# gpu_ids: 0,1,2,3,4,5,6,7
+use_cpu: false
@@ -0,0 +1,14 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: DEEPSPEED
+deepspeed_config:
+ deepspeed_config_file: scripts/accelerate_configs/zero3.json
+ zero3_init_flag: true
+fsdp_config: {}
+machine_rank: 0
+main_process_ip: null
+main_process_port: 29501
+main_training_function: main
+num_machines: 1
+num_processes: 8
+gpu_ids: 0,1,2,3,4,5,6,7
+use_cpu: false
@@ -0,0 +1,23 @@
+{
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": "auto"
+    },
+    "train_micro_batch_size_per_gpu": "auto",
+    "train_batch_size": "auto",
+    "gradient_accumulation_steps": "auto",
+    "zero_optimization": {
+        "stage": 2,
+        "overlap_comm": true,
+        "contiguous_gradients": true,
+        "sub_group_size": 1e9,
+        "reduce_bucket_size": "auto"
+    }
+}
@@ -0,0 +1,28 @@
+{
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": "auto"
+    },
+    "train_micro_batch_size_per_gpu": "auto",
+    "train_batch_size": "auto",
+    "gradient_accumulation_steps": "auto",
+    "zero_optimization": {
+        "stage": 3,
+        "overlap_comm": true,
+        "contiguous_gradients": true,
+        "sub_group_size": 1e9,
+        "reduce_bucket_size": "auto",
+        "stage3_prefetch_bucket_size": "auto",
+        "stage3_param_persistence_threshold": "auto",
+        "stage3_max_live_parameters": 1e9,
+        "stage3_max_reuse_distance": 1e9,
+        "stage3_gather_16bit_weights_on_model_save": true
+    }
+}
@@ -1,17 +1,21 @@
+export ACCELERATE_GRADIENT_ACCUMULATION_STEPS=8
+
 accelerate launch \
-  --config_file scripts/accelerate_configs/default_config.yaml \
+  --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
  opensora/train/train_videogpt.py \
  --do_train \
  --seed 1234 \
-  --data_path UCF-101 \
+  --data_path datasets/UCF-101 \
  --per_device_train_batch_size 8 \
-  --gradient_accumulation_steps 8 \
+  --gradient_accumulation_steps $ACCELERATE_GRADIENT_ACCUMULATION_STEPS \
  --learning_rate 1e-5 \
  --weight_decay 0. \
  --num_train_epochs 2 \
  --embedding_dim 512 \
  --lr_scheduler_type cosine \
  --max_grad_norm 1.0 \
+  --bf16 True \
+  --fp16 False \
  --save_strategy steps \
  --save_total_limit 5 \
  --logging_steps 5 \
@@ -23,4 +27,4 @@ accelerate launch \
  --resolution 128 \
  --sequence_length 16 \
  --output_dir results/videogpt \
-  --report_to tensorboard
+  --report_to tensorboard