Fix some typos (distributd, _valide_env, etc.) (#61741)

co63oc · web-flow · commit 017ffb0425a1 · 2024-02-19T15:56:43.000+08:00
* Fix

* ci
diff --git a/python/paddle/distributed/launch/controllers/ipu_controller.py b/python/paddle/distributed/launch/controllers/ipu_controller.py
@@ -34,7 +34,7 @@ def enable(cls, ctx):
     def parse_ipu_args(self, args_list):
         parser = argparse.ArgumentParser()
         parser.add_argument(
-            "--hosts", type=str, help="The hosts for IPU distributd training."
+            "--hosts", type=str, help="The hosts for IPU distributed training."
         )
         parser.add_argument(
             "--nproc_per_host",
diff --git a/python/paddle/distributed/launch/job/container.py b/python/paddle/distributed/launch/job/container.py
@@ -92,7 +92,7 @@ def update_env(self, env={}, **kwargs):
         kwargs = {k: v for k, v in kwargs.items() if isinstance(v, str)}
         self._env.update(kwargs)
 
-    def _valide_env(self):
+    def _validate_env(self):
         for k, v in self._env.items():
             assert isinstance(k, str) and isinstance(
                 v, str
@@ -114,7 +114,7 @@ def start(self):
         if self._proc and self._proc.alive():
             return True
 
-        self._valide_env()
+        self._validate_env()
 
         self._stdout = self._get_fd(self._out) or sys.stdout
         if self._out == self._err:
diff --git a/python/paddle/distributed/launch/job/pod.py b/python/paddle/distributed/launch/job/pod.py
@@ -21,7 +21,7 @@
 from .status import Status
 
 
-class PodSepc:
+class PodSpec:
     def __init__(self):
         self._name = ''.join(
             random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6)
@@ -41,7 +41,7 @@ def __init__(self):
         self._exit_code = 0
 
 
-class Pod(PodSepc):
+class Pod(PodSpec):
     def __init__(self):
         super().__init__()
 
diff --git a/python/paddle/distributed/launch/main.py b/python/paddle/distributed/launch/main.py
@@ -94,13 +94,13 @@ def launch():
         - ``--elastic_timeout``: Seconds to wait before elastic job begin to train. Default ``--elastic_timeout=30``.
 
     IPU Parameters:
-        IPU distributed launch only requires and allowes three arguments ``--devices``, ``training_script`` and ``training_script_args``.
+        IPU distributed launch only requires and allows three arguments ``--devices``, ``training_script`` and ``training_script_args``.
         The ``--devices`` is the number of IPU devices. e.g., ``--devices=4`` will launch the training program with four IPU devices.
         The ``training_script`` is only allowed to set as ``ipu``.
         The ``training_script_args`` includes arguments required by IPU distributed launch and illustrated as below.
         ``Examples 10`` has provided a example of paddle.distributed.launch with IPUs.
 
-        - ``--hosts``: The hosts for IPU distributd training. Each host is able to include multiple processes.
+        - ``--hosts``: The hosts for IPU distributed training. Each host is able to include multiple processes.
 
         - ``--nproc_per_host``: The number of processes launched per host. Each process is able to include multiple replicas.
 
@@ -498,8 +498,8 @@ def launch():
                 c.run()
 
                 # process generated result
-                # TODO diffentiate out of memory and no loss(maybe over time)
-                # TODO integragte memory and metric read
+                # TODO differentiate out of memory and no loss(maybe over time)
+                # TODO integrate memory and metric read
                 metric, mem, err = read_log(
                     path=ctx.args.log_dir,
                     metric_file="workerlog.0",
@@ -578,10 +578,10 @@ def launch():
 
             end_time = time.time()
             ctx.logger.info(
-                f"AtuoTuner for GBS search ends in {end_time - start_time}s."
+                f"AutoTuner for GBS search ends in {end_time - start_time}s."
             )
             logger.info(
-                f"AtuoTuner for GBS search ends in {end_time - start_time}s."
+                f"AutoTuner for GBS search ends in {end_time - start_time}s."
             )
 
         # build AutoTuner to get new config
@@ -1027,7 +1027,7 @@ def launch():
                         comm_time = model_size_b * (4 + 2) / bw
                     else:
                         comm_time = model_size_b * 4 / bw
-                    multi_dp_performace = (
+                    multi_dp_performance = (
                         round(
                             step_time
                             / (step_time + comm_time)
@@ -1039,16 +1039,16 @@ def launch():
                     )
                     cur_cfg[
                         f"bw_{bw}_{tuner_cfg['metric_cfg']['name']}"
-                    ] = multi_dp_performace
+                    ] = multi_dp_performance
                     cur_cfg[
                         f"unified_bw_{bw}_{tuner_cfg['metric_cfg']['name']}"
                     ] = (
-                        round(multi_dp_performace / num_gpus, 2)
-                        if multi_dp_performace
+                        round(multi_dp_performance / num_gpus, 2)
+                        if multi_dp_performance
                         and tuner_cfg["search_algo"]["conversion"].get(
                             "need_unify", False
                         )
-                        else multi_dp_performace
+                        else multi_dp_performance
                     )
                     if recorder.additional_metric_key is None:
                         recorder.additional_metric_key = (
diff --git a/python/paddle/distributed/passes/allreduce_matmul_grad_overlapping.py b/python/paddle/distributed/passes/allreduce_matmul_grad_overlapping.py
@@ -29,7 +29,7 @@
 #   dX, dY = matmul_grad(X, Y, dOut)
 #   dX = c_allreduce_sum(dX)
 # Split matmul_grad to 2 matmul:
-#   dX = mutmul(dOut, Y^T)
+#   dX = matmul(dOut, Y^T)
 #   dX = c_allreduce_sum(dX)
 #   dY = matmul(X^T, dOut)
 #
@@ -154,7 +154,7 @@ def _split_matmul_grad_and_multi_streaming_allreduce(
             y_grad = matmul_grad_op.output("Y@GRAD")
             op_role = matmul_grad_op.attr("op_role")
 
-            # NOTE(Ruibiao): Required OP scheduling order: mutmul(dOut, Y^T) -> c_allreduce_sum(dX) -> matmul(X^T, dOut).
+            # NOTE(Ruibiao): Required OP scheduling order: matmul(dOut, Y^T) -> c_allreduce_sum(dX) -> matmul(X^T, dOut).
             # c_allreduce_sum(dX) and matmul(X^T, dOut) cannot be swapped. Otherwise, after buffer_shared_inplace_pass
             # adding share_buffer OP before c_allreduce_sum, c_allreduce_sum will synchronous with comp-stream, and then
             # the matmul op before it cannot be overlapped.
diff --git a/python/paddle/distributed/passes/auto_parallel_amp.py b/python/paddle/distributed/passes/auto_parallel_amp.py
@@ -737,7 +737,7 @@ def _apply_single_impl(self, main_program, startup_program, context):
     def _update_backward_cast_ops(self):
         """
         move param grad cast to the end of backward segment
-        in order to enabel fp16 allreduce
+        in order to enable fp16 allreduce
         """
         # TODO filter optimize ops in future
 
diff --git a/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py b/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py
@@ -306,11 +306,11 @@ def _calc_wait_comms(self):
 
         block = default_main_program().global_block()
 
-        # NOTE the naive overlap implement in static hybird parallel only sync comm stream
+        # NOTE the naive overlap implement in static hybrid parallel only sync comm stream
         # at the end of Backward phase, based on a strong constraint that
         # all communicating gradient would NOT be used after communication in Backward phase.
         # BUT this constraint will fail for scenario like Weight-Sharing and Higher-Order Differentiation,
-        # where gradient will be involved in other calculation between data-parallel allreduce kernel submmited
+        # where gradient will be involved in other calculation between data-parallel allreduce kernel submitted
         # into comm streams and the synchronization of comm stream at the end of Backward phase.
         # synchronization of  comm stream should add according to the usage of communicating gradients
         # to support Overlapping for Weight-Sharing and Higher-Order Differentiation.
@@ -518,7 +518,7 @@ def _update_program(self, grad_groups):
                 new_out_name, out_dist_attr
             )
 
-            # remvoe un-used op
+            # remove un-used op
             remove_op_indices = (
                 group.remove_wait_op_indices
                 + group.remove_allreduce_op_indices
diff --git a/python/paddle/distributed/passes/auto_parallel_grad_clip.py b/python/paddle/distributed/passes/auto_parallel_grad_clip.py
@@ -286,7 +286,7 @@ def _partition_parameters(self, params):
 
 
 @register_pass("auto_parallel_grad_clip")
-class ClipGradByGloblNormPass(PassBase):
+class ClipGradByGlobalNormPass(PassBase):
     """
     1. Remove norm-compute op and grad-scale op when the grad is not in current rank
        or is independent of the calculation of norm.
diff --git a/python/paddle/distributed/passes/auto_parallel_quantization.py b/python/paddle/distributed/passes/auto_parallel_quantization.py
@@ -221,7 +221,7 @@ def move_persist_var_to_global_block(self, program):
         return program
 
     def reset_scope_var(self, quant_program, dist_context, scope, place):
-        # The var_value, created by qatization_passes, should has same shape with the value after parallel.
+        # The var_value, created by quantization_passes, should has same shape with the value after parallel.
         for var in quant_program.list_vars():
             scope_var = scope.find_var(var.name)
             if not (scope_var and scope_var.get_tensor()._is_initialized()):
diff --git a/python/paddle/distributed/passes/auto_parallel_recompute.py b/python/paddle/distributed/passes/auto_parallel_recompute.py
@@ -119,7 +119,7 @@ def get_recompute_segments(self, no_recompute_segments=[]):
 
     def modify_forward_desc_for_recompute(self, dist_context):
         """
-        If program's foward part has 'dropout' op, this function will insert
+        If program's forward part has 'dropout' op, this function will insert
         a seed op before it to guarantee that two dropout op have the same outputs.
         """
         op_types = [op.type for op in self.ops]
@@ -150,7 +150,7 @@ def modify_forward_desc_for_recompute(self, dist_context):
             cur_op_dist_attr = dist_context.get_op_dist_attr_for_program(cur_op)
             # insert seed op to guarantee that two dropout op have the same outputs
             # NOTE Hack for adopt recompute for random control, for more info see dist_dropout.py
-            # new seed added by recompute should have a prefix to distinguish with seed added by user or other moudule.
+            # new seed added by recompute should have a prefix to distinguish with seed added by user or other module.
             op_unique_name = unique_name.generate("rc_seed")
             var_unique_name = unique_name.generate_with_ignorable_key(
                 ".".join([op_unique_name, 'tmp'])
@@ -298,7 +298,7 @@ def get_ops_per_device(self, ops, all_ops_process_meshs, sr=0):
         Get ops and op_names of each process mesh excluding ops within the first "sr" chunks
         """
 
-        def reset_recomupte_op(op):
+        def reset_recompute_op(op):
             if is_recompute_op(op) or is_recompute_exclude_op(op):
                 op._set_attr("op_namescope", "")
 
@@ -311,7 +311,7 @@ def reset_recomupte_op(op):
         for op_id, op in enumerate(ops):
             if chunk_id // all_process_meshes_count < sr:
                 reset_ops_count += 1
-                reset_recomupte_op(op)
+                reset_recompute_op(op)
             if (
                 op_id < len(ops) - 1
                 and op.dist_attr.process_mesh
@@ -328,7 +328,7 @@ def reset_recomupte_op(op):
                     op_names_of_stages[id].append(op.type)
         assert (
             len(ops) == reset_ops_count + pushed_ops_count
-        ), "The sum of pushed_ops_count and reset_ops_count must be the same as lenght of ops, but the sum is {} while lenght of ops is {}".format(
+        ), "The sum of pushed_ops_count and reset_ops_count must be the same as length of ops, but the sum is {} while length of ops is {}".format(
             reset_ops_count + pushed_ops_count, len(ops)
         )
         return ops_of_stages, op_names_of_stages
@@ -345,7 +345,7 @@ def _apply_single_impl(self, main_program, startup_program, context):
         main_block = main_program.global_block()
         op_path = _find_op_path(main_program, loss, no_grad_set)
 
-        # 1. mark exclude ops for refined-reompute according to ops-patterns(mainly linear and flash_attn)
+        # 1. mark exclude ops for refined-recompute according to ops-patterns(mainly linear and flash_attn)
         # 1.1 get all process_meshs in op_path
         all_ops_process_meshs = []
         for op in op_path:
@@ -358,7 +358,7 @@ def _apply_single_impl(self, main_program, startup_program, context):
         )
         all_ops_len = len(op_path)
         all_exclude_ops_ids = [[] for _ in op_names_devices]
-        # 1.3 find exclude ops for refined-reompute according to ops-patterns
+        # 1.3 find exclude ops for refined-recompute according to ops-patterns
         for refined_ops_pattern in self._refined_ops_patterns:
             num = refined_ops_pattern['num']
             num = (
@@ -583,8 +583,8 @@ def _apply_single_impl(self, main_program, startup_program, context):
                                 posterior_op
                             ).process_mesh
                         )
-                        # NOTE if two recompute segements across two pipeline stages
-                        # not need dependecies for it
+                        # NOTE if two recompute segments across two pipeline stages
+                        # not need dependencies for it
                         if prior_mesh == posterior_mesh:
                             insert_dependencies_for_two_ops(
                                 main_block,
diff --git a/python/paddle/distributed/passes/pipeline_scheduler_pass.py b/python/paddle/distributed/passes/pipeline_scheduler_pass.py
@@ -262,10 +262,10 @@ def _multistreaming_for_overlapping(self, programs, job_type):
         for program_id, program in enumerate(programs):
             last_op = program.global_block().ops[-1]
             if self.is_comm_op_valid_to_overlap(last_op):
-                # TODO(Ruibiao): Assign different stream to FORWAD and BACKWARD CommOps,
+                # TODO(Ruibiao): Assign different stream to FORWARD and BACKWARD CommOps,
                 # and set a lower priority for FORWARD Comm stream. It can reduce the
-                # impact of FORWARD Comm on BACKWARD Comp. Now the defalut stream prirotiy
-                # in standalone executor is already the lowest priority (correspongding to
+                # impact of FORWARD Comm on BACKWARD Comp. Now the default stream priority
+                # in standalone executor is already the lowest priority (corresponding to
                 # 0 in V100), cannot set a lower one. Maybe we need to support setting
                 # default stream for executor.
                 last_op.dist_attr.execution_stream = (
@@ -297,7 +297,7 @@ def _multistreaming_for_overlapping(self, programs, job_type):
                             _add_event_dependency(last_op, posterior_op)
 
     # TODO(Ruibiao): The cost here is just the experience value for a specific task (GPT-3-6.7B-MP2-PP4).
-    # A more genereal cost estimation scheme is required.
+    # A more general cost estimation scheme is required.
     def _op_cost(self, op):
         handwritten_cost_map = {
             "c_allreduce_sum": 0,
@@ -565,7 +565,7 @@ def apply_pass(main_program, startup_program, pass_name, pass_attr={}):
         "1F1B",
         "Eager1F1B",
         "VPP",
-    ], f"pipeline scheduler only support FThenB, 1F1B and Eager1F1B, but recieve {pass_name}"
+    ], f"pipeline scheduler only support FThenB, 1F1B and Eager1F1B, but receive {pass_name}"
 
     if pass_name == "1F1B":
         # TODO(Ruibiao): Move FLAGS_1f1b_backward_forward_overlap and
diff --git a/python/paddle/distributed/passes/ps_server_pass.py b/python/paddle/distributed/passes/ps_server_pass.py
@@ -67,7 +67,7 @@ def _add_tensor_table(
         attrs['tensor_table'] = tensor_table_dict
 
     def _get_lr_scheduler_program(self, lr_scheduler, lr_decay_steps):
-        schedler_decay = [
+        scheduler_decay = [
             'NoamDecay',
             'NaturalExpDecay',
             'InverseTimeDecay',
@@ -138,7 +138,7 @@ def _get_lr_scheduler_program(self, lr_scheduler, lr_decay_steps):
         else:
             raise ValueError(
                 "Not supported current LearningRate strategy, please use follow decay strategy: {}".format(
-                    schedler_decay
+                    scheduler_decay
                 )
             )
 

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ def enable(cls, ctx):`
`34`	`34`	`def parse_ipu_args(self, args_list):`
`35`	`35`	`parser = argparse.ArgumentParser()`
`36`	`36`	`parser.add_argument(`
`37`		`- "--hosts", type=str, help="The hosts for IPU distributd training."`
	`37`	`+ "--hosts", type=str, help="The hosts for IPU distributed training."`
`38`	`38`	`)`
`39`	`39`	`parser.add_argument(`
`40`	`40`	`"--nproc_per_host",`