init

2024-10-10 13:16:35 +03:00
parent 9ac59a4cc3
commit 5e352445d6
31 changed files with 7668 additions and 0 deletions
@@ -0,0 +1,25 @@
+from .utils import (
+    create_optimizer,
+    get_rank,
+    get_world_size,
+    is_main_process,
+    is_dist_avail_and_initialized,
+    init_distributed_mode, 
+    setup_for_distributed, 
+    cosine_scheduler,
+    constant_scheduler,
+)
+
+from .sp_utils import (
+    is_sequence_parallel_initialized,
+    init_sequence_parallel_group,
+    get_sequence_parallel_group,
+    get_sequence_parallel_world_size,
+    get_sequence_parallel_rank,
+    get_sequence_parallel_group_rank,
+    get_sequence_parallel_proc_num,
+    init_sync_input_group,
+    get_sync_input_group,
+)
+
+from .communicate import all_to_all