Merge pull request #121 from sysuyy/add_multi_node_script

[feat]: use accelerate on multi-node Former-commit-id: 67b01dc1d934d3d86d7ecca00c5dd1a11fd4d22d
2024-03-12 14:05:08 +08:00
parent aa3ff1e9c3 330e46e9e8
commit 429391840f
2 changed files with 20 additions and 0 deletions
@@ -0,0 +1,2 @@
+server1 slots=8  # your server name  and  GPU in total
+server2 slots=8
@@ -0,0 +1,18 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: DEEPSPEED
+deepspeed_config:
+ deepspeed_config_file: scripts/accelerate_configs/zero3_offload.json
+ deepspeed_hostfile: # FullPath here
+fsdp_config: {}
+machine_rank: 0
+main_process_ip: # main process ip
+main_process_port: 29501
+main_training_function: main
+num_machines: 2
+num_processes: 16
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false