Fix paralel parallel (#61034)

co63oc · web-flow · commit 4110bb1dbf79 · 2024-01-23T14:25:45.000+08:00
diff --git a/paddle/phi/api/yaml/generator/dist_api_gen.py b/paddle/phi/api/yaml/generator/dist_api_gen.py
@@ -1730,8 +1730,8 @@ def generate_output_dist_attr_setting(self) -> str:
     def generate_return_code(self) -> str:
         return self.gene_return_code()
 
-    def generate_auto_paralel_branch(self) -> str:
-        # if no tensor input, do not genetate auto parallel branch
+    def generate_auto_parallel_branch(self) -> str:
+        # if no tensor input, do not generate auto parallel branch
         if len(self.inputs['names']) == 0:
             return ""
 
@@ -1815,7 +1815,7 @@ def gene_base_api_code(self, inplace_flag=False):
                     and not self.api.endswith("_double_grad")
                     and not self.api.endswith("_triple_grad")
                 ):
-                    dist_branch_code += self.generate_auto_paralel_branch()
+                    dist_branch_code += self.generate_auto_parallel_branch()
             kernel_dispatch_code += dist_branch_code
             for kernel_name in self.kernel['func']:
                 kernel_dispatch_code += self.gene_dispatch_code(
@@ -1837,7 +1837,7 @@ def gene_base_api_code(self, inplace_flag=False):
                 and not self.api.endswith("_double_grad")
                 and not self.api.endswith("_triple_grad")
             ):
-                dist_branch_code = self.generate_auto_paralel_branch()
+                dist_branch_code = self.generate_auto_parallel_branch()
             return API_IMPL_TEMPLATE.format(
                 self.get_return_type(inplace_flag),
                 api_func_name,
diff --git a/paddle/phi/api/yaml/generator/dist_bw_api_gen.py b/paddle/phi/api/yaml/generator/dist_bw_api_gen.py
@@ -355,8 +355,8 @@ def generate_reshard_output_code(self):
 
         return reshard_output_code
 
-    def generate_auto_paralel_branch(self) -> str:
-        # if no tensor input, do not genetate auto parallel branch
+    def generate_auto_parallel_branch(self) -> str:
+        # if no tensor input, do not generate auto parallel branch
         if len(self.inputs['names']) == 0:
             return ""
         infer_spmd_code = self.generate_infer_spmd_code()
diff --git a/python/paddle/distributed/auto_parallel/static/operators/common.py b/python/paddle/distributed/auto_parallel/static/operators/common.py
@@ -66,13 +66,13 @@ class ParallelMode:
 
     DataParallel = "auto_parallel/data_parallel"
     TensorParallel = "auto_parallel/tensor_parallel"
-    PipelineParalel = "auto_parallel/pipeline_paralel"
+    PipelineParallel = "auto_parallel/pipeline_parallel"
     MoEParallel = "auto_parallel/moe_parallel"
 
 
 class SyncMode:
     """
-    the synchorization mode for communication or auxiliary operator
+    the synchronization mode for communication or auxiliary operator
     """
 
     AmpFlagSync = "auto_parallel/amp_flag_synchorization"
diff --git a/python/paddle/distributed/auto_tuner/prune.py b/python/paddle/distributed/auto_tuner/prune.py
@@ -127,7 +127,7 @@ def prune_by_mp(tuner_cfg, cur_cfg, history_cfgs=[]):
         "num_attention_heads", None
     )
     seq_length = tuner_cfg["model_cfg"].get("seq_length", None)
-    use_sequence_paralel = tuner_cfg.get("use_sequence_paralel", False)
+    use_sequence_parallel = tuner_cfg.get("use_sequence_parallel", False)
 
     if mp_degree is None:
         return False
@@ -141,7 +141,7 @@ def prune_by_mp(tuner_cfg, cur_cfg, history_cfgs=[]):
     if num_attention_heads and num_attention_heads % mp_degree != 0:
         return True
 
-    if seq_length and seq_length % mp_degree != 0 and use_sequence_paralel:
+    if seq_length and seq_length % mp_degree != 0 and use_sequence_parallel:
         return True
 
     mp_degree_candidates = tuner_cfg.get("mp_degree", None)
diff --git a/python/paddle/distributed/auto_tuner/utils.py b/python/paddle/distributed/auto_tuner/utils.py
@@ -107,7 +107,9 @@ def dist_degree(mode, num_gpus, num_nodes, tuner_cfg=None):
                 "num_attention_heads", None
             )
             seq_length = tuner_cfg["model_cfg"].get("seq_length", None)
-            use_sequence_paralel = tuner_cfg.get("use_sequence_paralel", False)
+            use_sequence_parallel = tuner_cfg.get(
+                "use_sequence_parallel", False
+            )
 
             if hidden_size and hidden_size % mp_degree != 0:
                 prune_flag = True
@@ -121,7 +123,7 @@ def dist_degree(mode, num_gpus, num_nodes, tuner_cfg=None):
             if (
                 seq_length
                 and seq_length % mp_degree != 0
-                and use_sequence_paralel
+                and use_sequence_parallel
             ):
                 prune_flag = True
 
diff --git a/python/paddle/distributed/fleet/base/topology.py b/python/paddle/distributed/fleet/base/topology.py
@@ -291,7 +291,7 @@ def __init__(self, topology):
         _HYBRID_PARALLEL_GROUP = self
 
     def get_parallel_mode(self):
-        # there are five modes : DataParallel / TensorParallel / PipelineParallel / ShardingParallel / SepParalel
+        # there are five modes : DataParallel / TensorParallel / PipelineParallel / ShardingParallel / SepParallel
         # NOTE when sharding conjugates with other parallel, sharding should act like a optimizer and
         # adding its parallel logic within that parallelism
         # when use sharding alone, it should have its own parallelism for its parallel logic
diff --git a/test/legacy_test/test_directory_migration.py b/test/legacy_test/test_directory_migration.py
@@ -137,7 +137,7 @@ def test_old_directory(self):
             'paddle.imperative.load',
             'paddle.imperative.ParallelEnv',
             'paddle.imperative.prepare_context',
-            'paddle.imperative.DataParalell',
+            'paddle.imperative.DataParallel',
             'paddle.imperative.jit',
             'paddle.imperative.TracedLayer',
             'paddle.imperative.declarative',
