Fix some typos (adapot, swith, etc.) (#61652)

co63oc · web-flow · commit 121d55cd6d74 · 2024-02-19T11:38:51.000+08:00
* Fix

* ci

* ci
diff --git a/python/paddle/distributed/passes/auto_parallel_fp16.py b/python/paddle/distributed/passes/auto_parallel_fp16.py
@@ -90,7 +90,7 @@ def set_auto_cast_attr(cast_op, block):
         out_var.desc.set_dtype(out_var_fw.dtype)
 
 
-# adapot for backward op
+# adapt for backward op
 # TODO check if bf16 and fp16 still share the same logic
 def _keep_fp32_input(op, in_name):
     if not op.amp_options.enable:
@@ -831,7 +831,7 @@ def _apply_single_impl(self, main_program, startup_program, context):
             self.use_optimizer_fp16 = self.get_attr("level", None) == "o3"
 
         AMPList = amp_utils.AutoMixedPrecisionLists
-        # swith enviroment for fp16 / bf16.
+        # switch environment for fp16 / bf16.
         if self.target_dtype == "float16":
             __target_dtype = core.VarDesc.VarType.FP16
         elif self.target_dtype == "bfloat16":
@@ -851,7 +851,7 @@ def _apply_single_impl(self, main_program, startup_program, context):
             dtype=self.target_dtype,
         )
 
-        # NOTE don't not change input data dtype, since it is controled by dataloader
+        # NOTE don't not change input data dtype, since it is controlled by dataloader
         # and which is out of control of FP16 Pass
         input_data_var_names = [var.name for var in self.get_attr("input_data")]
         with paddle.static.program_guard(main_program, startup_program):
diff --git a/python/paddle/distributed/passes/auto_parallel_fused_linear_promotion.py b/python/paddle/distributed/passes/auto_parallel_fused_linear_promotion.py
@@ -196,7 +196,7 @@ def _apply_single_impl(self, main_program, startup_program, context):
             self._dist_context, self._global_rank
         )
         logger.debug(f"before main_program: {main_program}")
-        # 2. get the forward and backward op list idexs in source patterns
+        # 2. get the forward and backward op list indexes in source patterns
         (
             forward_segments,
             backward_segments,
@@ -313,7 +313,7 @@ def can_match_pattern(
 
         global_block = main_program.global_block()
         forward_segments = []
-        backward_segmnets = []
+        backward_segments = []
         ops_len = len(global_block.ops)
 
         self._forward_patterns_len = len(self._source_patterns["forward"])
@@ -343,17 +343,17 @@ def can_match_pattern(
                     forward_matmul_inputs,
                     is_backward=True,
                 ):
-                    backward_segmnets.append(
+                    backward_segments.append(
                         [id, id + self._backward_patterns_len]
                     )
             else:
                 pass
         assert len(forward_segments) >= len(
-            backward_segmnets
+            backward_segments
         ), "The number of forward segments should be not shorter than the number of backward segments."
         logger.info(f"forward_segments: {forward_segments}")
-        logger.info(f"backward_segmnets: {backward_segmnets}")
-        return forward_segments, backward_segmnets
+        logger.info(f"backward_segments: {backward_segments}")
+        return forward_segments, backward_segments
 
     def _collective_data_parallel_groups(self, main_block):
         for op in main_block.ops:
@@ -636,15 +636,15 @@ def _transform_backward(
                     add_grad_op = global_block.ops[segment[0]]
                     matmul_grad_op = global_block.ops[segment[-1] - 1]
                     origin_comm_op_id = segment[-1] - 2
-                    orgin_comm_op = global_block.ops[origin_comm_op_id]
+                    origin_comm_op = global_block.ops[origin_comm_op_id]
                     new_comm_op = global_block._insert_op(
                         segment[0],
                         type="nop",
                     )
-                    new_comm_op.desc.copy_from(orgin_comm_op.desc)
+                    new_comm_op.desc.copy_from(origin_comm_op.desc)
                     # rename input and output
                     new_comm_op._rename_input(
-                        orgin_comm_op.input_arg_names[0],
+                        origin_comm_op.input_arg_names[0],
                         add_grad_op.input_arg_names[0],
                     )
                     add_grad_op._rename_input(
@@ -669,7 +669,7 @@ def _transform_backward(
                         )  # c_allreduce_sum
                 global_block._sync_with_cpp()
         else:  # not is_first_rank_in tp or sp
-            # need to delete the grad op assosiated with the deleted bias var
+            # need to delete the grad op associated with the deleted bias var
             if not is_sp:
                 for segment in reversed(backward_segments):
                     add_grad_op = global_block.ops[segment[0]]
@@ -690,12 +690,12 @@ def _transform_backward(
             else:
                 for segment in reversed(backward_segments):
                     add_grad_op = global_block.ops[segment[0]]
-                    orgin_comm_op = global_block.ops[segment[-1] - 2]
+                    origin_comm_op = global_block.ops[segment[-1] - 2]
                     rename_var_names_map[
                         add_grad_op.output_arg_names[0]
                     ] = add_grad_op.input_arg_names[0]
-                    orgin_comm_op._rename_input(
-                        orgin_comm_op.input_arg_names[0],
+                    origin_comm_op._rename_input(
+                        origin_comm_op.input_arg_names[0],
                         add_grad_op.input_arg_names[0],
                     )
                     global_block._remove_var(add_grad_op.output_arg_names[0])
@@ -718,7 +718,7 @@ def _transform_backward(
                     )  # elementwise_add_grad op
                 global_block._sync_with_cpp()
 
-        # rename input vars in gloabl_block
+        # rename input vars in global_block
         for op in global_block.ops:
             if is_optimize_op(op):
                 continue
@@ -811,12 +811,12 @@ def _transform_startup_program(
         self, startup_program, deleted_bias_names, dp_group, is_first_rank
     ):
         """
-        Delete the vars and ops assosiated with deleted_bias_names in startup program.
+        Delete the vars and ops associated with deleted_bias_names in startup program.
         """
         logger.debug(f"Before transform startup_program: {startup_program}")
         cur_glock = startup_program.global_block()
         to_delete_op_ids = []
-        # for variables assosiated with deleted_bias_names in amp-o2, such as 'opt_linear_1.b_0_fp32_master_0'
+        # for variables associated with deleted_bias_names in amp-o2, such as 'opt_linear_1.b_0_fp32_master_0'
         to_delete_extra_vars = []
         for id, op in enumerate(cur_glock.ops):
             if not is_first_rank:
diff --git a/python/paddle/distributed/passes/auto_parallel_grad_clip.py b/python/paddle/distributed/passes/auto_parallel_grad_clip.py
@@ -173,9 +173,9 @@ def __init__(
         self.pure_data_parallel = self._is_pure_data_parallel()
         self.rank_to_params = self._partition_parameters(params)
 
-    def is_calcuate_norm(self, name):
+    def is_calculate_norm(self, name):
         """
-        whether the param_name@GRAD paticipate in the calculation of global_norm
+        whether the param_name@GRAD participate in the calculation of global_norm
         """
         if not self.is_local_param(name):
             return False
@@ -354,7 +354,9 @@ def _remove_no_need_ops_vars(self, block):
                     # if the param@GRAD in cur_rank does not participate in the calculation of global_norm
                     param_name = input_name[: input_name.find("@GRAD")]
                     is_local = self.clip_helper.is_local_param(param_name)
-                    is_calculate = self.clip_helper.is_calcuate_norm(param_name)
+                    is_calculate = self.clip_helper.is_calculate_norm(
+                        param_name
+                    )
                     if not is_local or not is_calculate:
                         removed_op_idx.add(idx)
                         removed_tmp_var.update(set(op.output_arg_names))
diff --git a/python/paddle/distributed/passes/auto_parallel_pipeline.py b/python/paddle/distributed/passes/auto_parallel_pipeline.py
@@ -320,7 +320,7 @@ def _task_stream(self):
         recv_vars_name = {}
         for ib, src_block in enumerate(self._program.blocks):
             if ib == 0:
-                strat_block = start_prog.block(0)
+                start_block = start_prog.block(0)
                 end_block = end_prog.block(0)
 
                 is_after_while_op = False
@@ -333,7 +333,7 @@ def _task_stream(self):
 
                     if not is_after_while_op:
                         _create_program(
-                            src_block, strat_block, op, force_create=True
+                            src_block, start_block, op, force_create=True
                         )
                     else:
                         _create_program(
@@ -344,11 +344,11 @@ def _task_stream(self):
                 # The while block will be split to two separate blocks:
                 #     while{transformer_layer(send_block), generation_and_broadcast(recv_block)}
                 # The send_block:
-                #     include all ops about tansformer layers computation
-                #     execlude the nccl op about the while cond var(the last pp stage).
+                #     include all ops about transformer layers computation
+                #     exclude the nccl op about the while cond var(the last pp stage).
                 # The recv_block:
                 #     include all computation ops about generation and while cond var
-                #     execlude the nccl op about the while cond var(the pp stages exclude the last one)
+                #     exclude the nccl op about the while cond var(the pp stages exclude the last one)
                 # the nccl op about the while cond var:
                 #     put these varnames in the recv task node and do communication with brpc instead of nccl.
                 send_block = send_prog.block(0)
diff --git a/python/paddle/distributed/passes/auto_parallel_quantization.py b/python/paddle/distributed/passes/auto_parallel_quantization.py
@@ -169,7 +169,7 @@ def _apply_single_impl(self, main_program, startup_program, context):
 
         # 7. Convert Graph back to Program
         quant_program = main_graph.to_program()
-        quant_program = self.move_presist_var_to_global_block(quant_program)
+        quant_program = self.move_persist_var_to_global_block(quant_program)
 
         # 8.1 get new prams_grads from quant_program
         new_params_grads = []
@@ -203,7 +203,7 @@ def _apply_single_impl(self, main_program, startup_program, context):
         context.set_attr("params_grads", new_params_grads)
         context.set_attr("loss", new_loss)
 
-    def move_presist_var_to_global_block(self, program):
+    def move_persist_var_to_global_block(self, program):
         global_block = program.global_block()
         for _op in global_block.ops:
             if _op.type == "while":
diff --git a/python/paddle/distributed/passes/auto_parallel_sequence_parallel_optimization.py b/python/paddle/distributed/passes/auto_parallel_sequence_parallel_optimization.py
@@ -65,7 +65,7 @@ def _apply_single_impl(self, main_program, startup_program, context):
         with paddle.static.program_guard(main_program, startup_program):
             # TODO remove this pass when we use local reshard for all communication
             self._fuse_allreduce_split()
-            self._memory_opimization()
+            self._memory_optimization()
             self._overlap()
 
     def _fuse_allreduce_split(self):
@@ -157,7 +157,7 @@ def is_valid_split_op(idx, block):
 
         block._sync_with_cpp()
 
-    def _memory_opimization(self):
+    def _memory_optimization(self):
         pass
 
     def _overlap(self):
diff --git a/python/paddle/distributed/passes/auto_parallel_sharding.py b/python/paddle/distributed/passes/auto_parallel_sharding.py
@@ -194,7 +194,7 @@ def _apply_single_impl(self, main_program, startup_program, context):
         # NOTE Multi / Sub-Block Support
         # we assume that only parameter are present and partitioned in main_block,
         # there is NO new param in sub_block, and all params in sub_block follows the same
-        # partition as main_block. the above contraint fullfill the 3 most common use-cases in Paddle sub_block:
+        # partition as main_block. the above constraint fullfill the 3 most common use-cases in Paddle sub_block:
         # 1. subblock for lr scheduler
         # 2. sub-block uses the same or partial network of main-block, e.g. GPT3 generation model
         # 3. sub-block used for double backward
@@ -227,7 +227,7 @@ def _collective_data_parallel_groups(self, main_block):
                 self.dp_groups.add(group)
 
         # TODO(JZ-LIANG) allow more than one dp groups in network, support more general distribution
-        # genetated by auto search
+        # generated by auto search
         if len(self.dp_groups) != 1:
             raise NotImplementedError(
                 "So far Only and Exactly one data parallel group in network are supported, but got [{}] different data parallel groups".format(
@@ -271,7 +271,7 @@ def _build_sharding_infos(self, main_block, params_grads):
                 assert self.grad_comm_stream_num < 2
                 assert (
                     len(self.dp_groups) == 1
-                ), "hybrid sharding and data parallelism are supported only when there is excatly one data parallel group in the network"
+                ), "hybrid sharding and data parallelism are supported only when there is exactly one data parallel group in the network"
                 outer_dp_group, sharding_group = _get_dp_and_sharding_groups(
                     dp_group.ranks, self.sharding_world_size, self.global_rank
                 )
@@ -1095,13 +1095,13 @@ def op_depend_on_group(op, group):
         coalesce_op_set = set(coalesce_op_map.keys())
         modify_op_set = set(modify_reduce_op_map.keys())
         remove_op_set = set(remove_reduce_op_indices)
-        confilct = coalesce_op_set.intersection(modify_op_set)
+        conflict = coalesce_op_set.intersection(modify_op_set)
 
-        assert len(confilct) == 0
-        confilct = coalesce_op_set.intersection(remove_op_set)
-        assert len(confilct) == 0
-        confilct = modify_op_set.intersection(remove_op_set)
-        assert len(confilct) == 0
+        assert len(conflict) == 0
+        conflict = coalesce_op_set.intersection(remove_op_set)
+        assert len(conflict) == 0
+        conflict = modify_op_set.intersection(remove_op_set)
+        assert len(conflict) == 0
 
         # update block
         for idx, op in reversed(list(enumerate(block.ops))):
@@ -1625,7 +1625,7 @@ def _inference_data_parallel_group_for_operator(rank_id, op, dist_context):
     dp_group = None
     for input_name in op.input_arg_names:
         # TODO(zhaoyingli): maintain a dict in dist_context to record all variables which are renamed,
-        # to solve the param@RESHARD cannot be identifed.
+        # to solve the param@RESHARD cannot be identified.
         if not is_parameter_related(input_name, op.block, dist_context):
             dist_attr = dist_context.get_op_dist_attr_for_program(op)
             process_mesh = dist_attr.process_mesh
@@ -1651,7 +1651,7 @@ def _inference_data_parallel_group_for_operator(rank_id, op, dist_context):
 
 def partition_by_use_order(params, group_size):
     """
-    shard the continouse param into same rank and divide the forward&backward computation into segement,
+    shard the continuous param into same rank and divide the forward&backward computation into segment,
     which will favor the fuse pass in later.
 
     we assume that the params is already sorted by utilization order.
@@ -1677,7 +1677,7 @@ def partition_by_use_order(params, group_size):
 
 def partition_by_greedy_even(params, group_size):
     """
-    use greedy alogrithm to partition parameter as even as possible.
+    use greedy algorithm to partition parameter as even as possible.
     """
     mapping = {}
     for rank_ in range(group_size):
@@ -1836,8 +1836,8 @@ def get_var_rank(self, varname):
     def is_in_local_shard(self, param_name):
         return self.get_var_rank(param_name) == self.local_rank
 
-    # NOTE the follwo logic is designed for supporting AMP O1 when
-    # the param would be cast to fp16 before used for caculation.
+    # NOTE the follow logic is designed for supporting AMP O1 when
+    # the param would be cast to fp16 before used for calculation.
     # and sharding should only broadcast the casted fp16 param
     # instead of the origin fp32 version param.
     def get_broadcast_vars_and_param_usage(self, block):
@@ -1879,7 +1879,7 @@ def get_param_grad(self, param_name):
 
 class VarGroup:
     def __init__(self, max_size):
-        self.max_siez = max_size
+        self.max_size = max_size
         self.dtype = None
         self.rank = -1
         self.numel = 0
@@ -1899,7 +1899,7 @@ def acceptable(self, param, rank):
                 return False
             if rank != self.rank:
                 return False
-            if self.numel + get_var_numel(param) > self.max_siez:
+            if self.numel + get_var_numel(param) > self.max_size:
                 return False
             return True
 
diff --git a/python/paddle/distributed/passes/auto_parallel_supplement_explicit_dependencies.py b/python/paddle/distributed/passes/auto_parallel_supplement_explicit_dependencies.py
@@ -42,15 +42,15 @@ def _sharding_pass_applied(pass_ctx):
 class AutoParalSupplementDepPass(PassBase):
     """
     Functional Concern.
-    for strategies like amp & global norm, there is a collective communication to sync gradient inforation in every rank.
+    for strategies like amp & global norm, there is a collective communication to sync gradient information in every rank.
     after partition the gradients to each rank, the order of that collective communication is different in each rank
     and might cause hang problem in graph based random order executor. here supplement explicit dependencies for those cases.
 
     TODO Performance Concern.
     global collective will introduce global synchronization which forces the fast workers to wait for slow ones.
     therefore we should conduct this collective when all the ranks reach a same stage.
     BUT the depend API offered by executor could only ensure "conduct-not-before" but not "conduct-right-after".
-    Some ranks might call the colletives first than other ranks while they still some local could be performed to wait for slow peers.
+    Some ranks might call the collectives first than other ranks while they still some local could be performed to wait for slow peers.
     IR Pass currently could not have the fully control of time the to perform these global collectives.
     """
 
diff --git a/python/paddle/distributed/passes/pass_utils.py b/python/paddle/distributed/passes/pass_utils.py
@@ -278,7 +278,7 @@ def set_skip_gc_vars(num_micro_batches, job_types, sub_programs, jobs):
         if job_type == "backward":
             assert (
                 len(skip_gc_vars) == 0
-            ), f"When enabling pipeline parallelism stategy, the skip_gc_vars for backward subprogram must be empty, but it is {skip_gc_vars}."
+            ), f"When enabling pipeline parallelism strategy, the skip_gc_vars for backward subprogram must be empty, but it is {skip_gc_vars}."
 
         job.set_skip_gc_vars(skip_gc_vars)
         suffixed_required_vars[micro_batch_id] |= required_vars
@@ -294,7 +294,7 @@ def shadow_var_between_sub_programs(sub_programs):
         set()
     )  # arg_names that are required in later sub_programs
     for sub_program in reversed(sub_programs):
-        # step 1: parse shadow arguements
+        # step 1: parse shadow arguments
         block = sub_program.global_block()
         input_arg_names = set()
         output_arg_names = set()
@@ -305,7 +305,7 @@ def shadow_var_between_sub_programs(sub_programs):
                     input_arg_names.add(input_arg_name)
                     # NOTE(Ruibiao): When translating these codes to pir, we can simplely set
                     # `shadow_arg_names=input_arg_names-output_arg_names` since the program
-                    # in pir satifies SSA form.
+                    # in pir satisfies SSA form.
                     if input_arg_name not in output_arg_names:
                         shadow_arg_names.add(input_arg_name)
             for output_arg_name in op.output_arg_names:
@@ -546,7 +546,7 @@ def _overlap_send_recv(program):
     The finally target of this function is as follows:
         1. no need to insert the 'c_sync_calc' and 'c_sync_calc' operators
         2. 'send_v2' operator uses 'dist_attr.execution_stream' to set stream of its own.
-        3. 'recv_v2' opeator uses 'dist_attr.execution_stream' to set stream of its own.
+        3. 'recv_v2' operator uses 'dist_attr.execution_stream' to set stream of its own.
     """
     for block in program.blocks:
         for op in block.ops:
@@ -772,7 +772,7 @@ def _split_ops(block):
 
 def _add_event_dependency(recorder_op, waiter_op):
     '''
-    Add the extra event dependcy of the two operators.
+    Add the extra event dependency of the two operators.
     This function mainly aims for the cross-programs in pipeline parallelism,
     especial for the 'send_v2' 'recv_v2' etc.
     '''
diff --git a/python/paddle/distributed/ps/utils/public.py b/python/paddle/distributed/ps/utils/public.py
