From 330e46e9e808ba7106e9618da9dd9790dd404cbd Mon Sep 17 00:00:00 2001
From: sysuyy <2607384125@qq.com>
Date: Mon, 11 Mar 2024 14:45:23 +0000
Subject: [PATCH] [feat]: use accelerate on multi-node

Former-commit-id: 0b12fc31f1d2a63f9da38821174ab63d9b3a2d72
---
 scripts/accelerate_configs/hostfile            |  2 ++
 .../accelerate_configs/multi_node_example.yaml | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 scripts/accelerate_configs/hostfile
 create mode 100644 scripts/accelerate_configs/multi_node_example.yaml

diff --git a/scripts/accelerate_configs/hostfile b/scripts/accelerate_configs/hostfile
new file mode 100644
index 000000000..5e384ed0f
--- /dev/null
+++ b/scripts/accelerate_configs/hostfile
@@ -0,0 +1,2 @@
+server1 slots=8  # your server name  and  GPU in total
+server2 slots=8
diff --git a/scripts/accelerate_configs/multi_node_example.yaml b/scripts/accelerate_configs/multi_node_example.yaml
new file mode 100644
index 000000000..c462d62cc
--- /dev/null
+++ b/scripts/accelerate_configs/multi_node_example.yaml
@@ -0,0 +1,18 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: DEEPSPEED
+deepspeed_config:
+ deepspeed_config_file: scripts/accelerate_configs/zero3_offload.json
+ deepspeed_hostfile: # FullPath here
+fsdp_config: {}
+machine_rank: 0
+main_process_ip: # main process ip
+main_process_port: 29501
+main_training_function: main
+num_machines: 2
+num_processes: 16
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false