Fix some typos (parallism, neccessary, etc.) (#61764)

co63oc · web-flow · commit cacfd93474cb · 2024-02-19T16:32:56.000+08:00
diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py
@@ -60,7 +60,7 @@
 
 class Engine:
     """
-    An High-Level API for auto parallel, which could be used for distributed Training (engine.fit) and Inferenced (engine.predict).
+    An High-Level API for auto parallel, which could be used for distributed Training (engine.fit) and Inference (engine.predict).
     Static graph mode is supported natively, Dynamic graph mode is also supported under `@to_static <https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/api/paddle/jit/to_static_cn.html#to-static>`_ .
 
     Args:
@@ -621,7 +621,7 @@ def _prepare_program(self, mode, init_parameters=True):
         self._init_comm()
         # startup program
         self._initialize(mode, init_parameters)
-        # mark main program for futher decompose
+        # mark main program for further decompose
         self._mark_prim(mode)
         self._has_prepared[mode] = True
 
diff --git a/python/paddle/distributed/auto_parallel/static/planner_v2.py b/python/paddle/distributed/auto_parallel/static/planner_v2.py
@@ -49,10 +49,10 @@ def __init__(self, mode, dist_context):
         self._dist_context._dist_op_context = default_ctx.dist_op_context
         self._dist_context.data_parallel = default_ctx.data_parallel
         if not is_naive_data_parallel(self._dist_context):
-            # Use SSA graph for complex parallism
+            # Use SSA graph for complex parallelism
             self._dist_context.initialize(with_graph=True)
         else:
-            # Use program for data parallel parallism
+            # Use program for data parallel parallelism
             self._dist_context.initialize(with_graph=False)
 
         self._completer = Completer(self._dist_context)
diff --git a/python/paddle/distributed/auto_parallel/static/tuner/tunable_variable.py b/python/paddle/distributed/auto_parallel/static/tuner/tunable_variable.py
@@ -20,7 +20,7 @@
 
 class TunableVariable:
     """
-    Tunablevariable base class.
+    TunableVariable base class.
     """
 
     def __init__(self, name, default=None):
diff --git a/python/paddle/distributed/auto_tuner/search.py b/python/paddle/distributed/auto_tuner/search.py
@@ -72,9 +72,9 @@ def search_once(self, history_cfgs):
                         key=performance_sort,
                     )
                     if self.tuner_cfg.get("schedule_prior", False):
-                        from .utils import sort_by_sepecial
+                        from .utils import sort_by_special
 
-                        self.all_tasks[self.idx :] = sort_by_sepecial(
+                        self.all_tasks[self.idx :] = sort_by_special(
                             self.all_tasks[self.idx :], self.tuner_cfg
                         )
         while not stop:
@@ -146,7 +146,7 @@ def __init__(self, tuner_cfg):
         self.configs_csv = tuner_cfg.get("configs_csv", None)
         assert os.path.exists(
             self.configs_csv
-        ), "configs_csv file is neccessary in CustomizeSearch mode."
+        ), "configs_csv file is necessary in CustomizeSearch mode."
         self.all_tasks = load_configs_from_csv(self.configs_csv)
 
     def search_once(self, history_cfgs):
diff --git a/python/paddle/distributed/auto_tuner/utils.py b/python/paddle/distributed/auto_tuner/utils.py
@@ -417,11 +417,11 @@ def search_all(tuner_cfg):
         f"{search_space_size_before_prune - search_space_size_after_prune} tasks are pruned before launching."
     )
     if tuner_cfg.get("schedule_prior", False):
-        pruned_all_cfgs = sort_by_sepecial(pruned_all_cfgs, tuner_cfg)
+        pruned_all_cfgs = sort_by_special(pruned_all_cfgs, tuner_cfg)
     return pruned_all_cfgs
 
 
-def sort_by_sepecial(cfgs, tuner_cfg):
+def sort_by_special(cfgs, tuner_cfg):
     assert tuner_cfg.get("schedule_prior", False)
     prior_strategy = tuner_cfg["schedule_prior"]
     prior_strategy.sort(reverse=True)
@@ -613,7 +613,7 @@ def search_by_dp_estimation(tuner_cfg):
         if task not in new_all_cfgs and task["nodes"] <= tuner_cfg["nodes"]:
             new_all_cfgs.append(task)
 
-    # expanding sharding degree to run overlap and nonoverlap to calculate overlap benefits
+    # expanding sharding degree to run overlap and non-overlap to calculate overlap benefits
     sharding_all_cfgs = []
     if tuner_cfg["search_algo"].get("sharding_overlap", None):
         for task in new_all_cfgs:
@@ -1678,11 +1678,11 @@ def gbs_default_candidates(tuner_cfg):
         dp_candidate, mp_candidate, pp_candidate = gbs_dp_mp_pp_candidates(
             tuner_cfg, num_gpus, num_nodes
         )
-        sharding_dgree_candidate = dp_candidate
+        sharding_degree_candidate = dp_candidate
         candidates["dp_degree"] = [1]
         candidates["mp_degree"] = [mp_candidate]
         candidates["pp_degree"] = [pp_candidate]
-        candidates["sharding_degree"] = [sharding_dgree_candidate]
+        candidates["sharding_degree"] = [sharding_degree_candidate]
         candidates["sharding_stage"] = [1]
         candidates["use_recompute"] = [False]
         candidates["recompute_granularity"] = [None]
diff --git a/python/paddle/distributed/communication/all_gather.py b/python/paddle/distributed/communication/all_gather.py
@@ -71,7 +71,7 @@ def all_gather(tensor_list, tensor, group=None, sync_op=True):
 def all_gather_object(object_list, obj, group=None):
     """
 
-    Gather picklable objects from all participators and all get the result. Similiar to all_gather(), but python object can be passed in.
+    Gather picklable objects from all participators and all get the result. Similar to all_gather(), but python object can be passed in.
 
     Args:
         object_list (list): A list of output object. The datatype of every element in the list is same as the input obj.
diff --git a/python/paddle/distributed/communication/stream/reduce.py b/python/paddle/distributed/communication/stream/reduce.py
@@ -85,7 +85,7 @@ def reduce(
 ):
     """
 
-    Perform specific reduction (for example, sum, max) on a tensor across devices and send to the destintion device.
+    Perform specific reduction (for example, sum, max) on a tensor across devices and send to the destination device.
 
     Args:
         tensor (Tensor): The input tensor on each rank. The result will overwrite this tenor after communication. Support
diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py
@@ -63,11 +63,11 @@ def get_msg_dict(msg):
     fields = msg.DESCRIPTOR.fields
     for f in fields:
         v = getattr(msg, f.name)
-        # NOTE(zhiqiu): convert repeated filed to list to
+        # NOTE(zhiqiu): convert repeated field to list to
         # avoid segment fault when the process exit?
         # WHY?
         # I guess the type or value of protobuf item is NULL when
-        # dealloc.
+        # deallocated.
         if f.label == google.protobuf.descriptor.FieldDescriptor.LABEL_REPEATED:
             if (
                 f.type
@@ -384,9 +384,9 @@ def gradient_scale_configs(self, config):
     def a_sync(self):
         """
 
-        Indicating whether we are using asynchronous stocastic gradient descent updates
+        Indicating whether we are using asynchronous stochastic gradient descent updates
         for training. This property is valid when we are using parameter server training,
-        which is implied by setting approperate RoleMaker
+        which is implied by setting appropriate RoleMaker
         Default value: True
 
         Examples:
@@ -423,7 +423,7 @@ def a_sync_configs(self):
         """
 
         Set a_sync update configurations. In general, asynchronous parameter server
-        training has serveral configurable settings that can be configured through
+        training has several configurable settings that can be configured through
         a dict.
 
         **Notes**:
@@ -982,7 +982,7 @@ def amp(self, flag):
     def amp_configs(self):
         """
 
-        Set automatic mixed precision training configurations. In general, amp has serveral configurable
+        Set automatic mixed precision training configurations. In general, amp has several configurable
         settings that can be configured through a dict.
 
         **Notes**:
@@ -1002,7 +1002,7 @@ def amp_configs(self):
 
             custom_black_list(list[str]): Users' custom black list which forbidden execution fp16.
 
-            custom_black_varnames(list[str]): Users' custom black varibles' names.
+            custom_black_varnames(list[str]): Users' custom black variables' names.
 
             use_pure_fp16(bool): Whether to use the pure fp16 training. Default False.
 
@@ -1095,7 +1095,7 @@ def qat(self, flag):
     @property
     def qat_configs(self):
         """
-        Set quantization training configurations. In general, qat has serveral configurable
+        Set quantization training configurations. In general, qat has several configurable
         settings that can be configured through a dict.
         **Notes**:
             channel_wise_abs_max(bool): Whether to use `per_channel` quantization training. Default is True.
@@ -1494,7 +1494,7 @@ def sharding_configs(self):
             communication. Default is segment_broadcast_MB.
 
             segment_broadcast_MB(float, optional): segment by the parameters broadcast volume. sharding will introduce parameter broadcast operations into program, and
-            after every segment_broadcast_MB size parameter being broadcasted, the program will be cutted into one segment.
+            after every segment_broadcast_MB size parameter being broadcasted, the program will be cut into one segment.
             This configuration will affect the communication speed in sharding training, and should be an empirical value decided by your model size and network topology.
             Only enable when sharding_segment_strategy = segment_broadcast_MB. Default is 32.0 .
 
@@ -1506,17 +1506,17 @@ def sharding_configs(self):
             gradient_merge_acc_step(int, optional): specific the accumulation steps in gradient merge; and gradient merge will be turn off if gradient_merge_acc_step=1.  Default is 1.
 
             optimize_offload(bool, optional): enable the optimizer offload which will offload the moment vars to Host memory in order to saving GPU memory for fitting larger model.
-            the moment var will be prefetch from and offloaded to Host memory during update stage. it is a stragtegy that trades off between training speed and GPU memory, and is recommened to be turn on only when gradient_merge_acc_step large, where
+            the moment var will be prefetch from and offloaded to Host memory during update stage. it is a strategy that trades off between training speed and GPU memory, and is recommended to be turn on only when gradient_merge_acc_step large, where
             the number of time of update stage will be relatively small compared with forward&backward's.  Default is False.
 
-            dp_degree(int, optional): specific the number of data parallelism group; when dp_degree >= 2, it will introduce dp_degree ways data parallelism as the outer parallelsim for the inner parallelsim. User is responsible to ensure global_world_size = mp_degree * sharding_degree * pp_degree * dp_degree. Default is 1.
+            dp_degree(int, optional): specific the number of data parallelism group; when dp_degree >= 2, it will introduce dp_degree ways data parallelism as the outer parallelism for the inner parallelism. User is responsible to ensure global_world_size = mp_degree * sharding_degree * pp_degree * dp_degree. Default is 1.
 
             mp_degree(int, optional): [Hybrid parallelism ONLY] specific the number of gpus within each megatron parallelism group; and megatron parallelism will turn be off if mp_degree=1.  Default is 1.
 
             pp_degree(int, optional): [Hybrid parallelism ONLY] specific the number of gpus within each pipeline parallelism group; and pipeline parallelism will turn be off if pp_degree=1.  Default is 1.
 
-            pp_allreduce_in_optimize(bool, optional): [Hybrid parallelism ONLY] move the allreduce operations from backward stage to update(optimize) stage when pipeline parallelsim is on.
-            This configuration will affect the communication speed of Hybrid parallelism training depeneded on network topology. this strategy is experimental by now..  Default is False.
+            pp_allreduce_in_optimize(bool, optional): [Hybrid parallelism ONLY] move the allreduce operations from backward stage to update(optimize) stage when pipeline parallelism is on.
+            This configuration will affect the communication speed of Hybrid parallelism training depended on network topology. this strategy is experimental by now..  Default is False.
 
             optimize_cast(bool, optional): [Hybrid parallelism ONLY] Move the cast op of AMP which cast fp32 param to fp16 param to optimizer. optimize_cast will persist fp16 param, it
             will take more memory, but will be faster, trade space for time. Recommend to turn on only when using pipeline or gradient_merge_acc_step large.
@@ -1714,7 +1714,7 @@ def pipeline_configs(self):
         different parts of neural networks are running on different GPUS.
         There are Tensor queue buffer between each pair of neighborhood GPUS
         that are responsible for synchronizing hidden Tensor results between
-        GPUs. Pipeline parallelism consists of serveral producer-consumer style
+        GPUs. Pipeline parallelism consists of several producer-consumer style
         hardware pairs, such as GPU-GPU, CPU-GPU, GPU-XPU. The best way to speedup
         pipeline parallelism is to make the size of Tensor in Tensor queue smaller,
         so that we will have a faster producer for downstream consumers.
@@ -2021,7 +2021,7 @@ def dgc(self, flag):
     def dgc_configs(self):
         r"""
 
-        Set Deep Gradient Compression training configurations. In general, dgc has serveral configurable
+        Set Deep Gradient Compression training configurations. In general, dgc has several configurable
         settings that can be configured through a dict.
 
         **Notes**:
@@ -2177,7 +2177,7 @@ def lars_configs(self):
         **Notes**:
         **lars_coeff (float)**: trust ratio in lars formula.
         **lars_weight_decay** (float): weight decay coefficient in lars formula.
-        **epsilon (float)**: argument is used to avoid potential devision-by-zero
+        **epsilon (float)**: argument is used to avoid potential division-by-zero
         when compute the local lr;
         **exclude_from_weight_decay ([string])**: is a list of name strings of layers which
         will be exclude from weight decay in lars formula.
@@ -2431,7 +2431,7 @@ def qat(self, flag):
     def qat_configs(self):
         """
 
-        Set quantization training configurations. In general, qat has serveral configurable
+        Set quantization training configurations. In general, qat has several configurable
         settings that can be configured through a dict.
 
         **Notes**:
@@ -2446,7 +2446,7 @@ def qat_configs(self):
 
             algo(str): Other quantization training algorithm.
 
-        Exampless:
+        Examples:
             .. code-block:: python
 
                 >>> import paddle.distributed.fleet as fleet
@@ -2506,7 +2506,7 @@ def cudnn_exhaustive_search(self):
 
         Indicating whether to use exhaustive search method to choose convolution algorithms.
         Exhaustive search attempts all cuDNN algorithms to choose the fastest algorithm.
-        This method is time-consuming, the choosed algorithm will be cached for the given layer specifications.
+        This method is time-consuming, the chosen algorithm will be cached for the given layer specifications.
         Once the layer specifications (like batch size, feature map size) are changed, it will search again.
         Default Value: True
 
diff --git a/python/paddle/distributed/fleet/base/orthogonal_strategy.py b/python/paddle/distributed/fleet/base/orthogonal_strategy.py
@@ -23,10 +23,10 @@
 class OrthogonalStrategy:
     """
     A hybrid of multiple distributed strategies. Strategies need to be orthogonal, means the ranks are organized like
-    a square if there are two strategies, a cube if there aree three strategies, etc.
+    a square if there are two strategies, a cube if there are three strategies, etc.
 
     Args:
-        list_of_strategy(list): Stategy in the list should be represented as tuple, format as (strategy_name, degree, strategy_class).
+        list_of_strategy(list): Strategy in the list should be represented as tuple, format as (strategy_name, degree, strategy_class).
         fused_strategy_dict(dict, optional): Exist strategies can be fused to new strategy. Use the name of new strategy as key, a list of
             strategy names you want to fuse as value.
 
diff --git a/python/paddle/distributed/fleet/base/role_maker.py b/python/paddle/distributed/fleet/base/role_maker.py
@@ -699,7 +699,7 @@ def _role_id(self):
 
     def _worker_num(self):
         """
-        retrun the current number of worker
+        return the current number of worker
         """
         if not self._role_is_generated:
             self._generate_role()
diff --git a/python/paddle/distributed/fleet/base/strategy_compiler.py b/python/paddle/distributed/fleet/base/strategy_compiler.py
@@ -112,8 +112,8 @@ def __init__(self):
 class StrategyCompiler(StrategyCompilerBase):
     """
     StrategyCompiler is responsible for meta optimizers combination
-    Generally, a user can define serveral distributed strategies that
-    can generate serveral meta optimizer. The combination of these
+    Generally, a user can define several distributed strategies that
+    can generate several meta optimizer. The combination of these
     meta optimizers should have the right order to apply the optimizers'
     minimize function.
     This class is responsible for the executable distributed optimizer
@@ -162,7 +162,7 @@ def _get_valid_strategy(self, dist_strategy, can_not_apply_optimizer_list):
                            results will be splitted in async, sync, pipeline
     Meta Optimizer Type B: rewrite forward,
                            e.g. AMP and the corresponding backward is generated by rewritten forward
-    Meta Opitmizer Type B: rewrite backward. e.g. gradient fusion
+    Meta Optimizer Type B: rewrite backward. e.g. gradient fusion
     Meta Optimizer Type D: rewrite optimize. e.g. lars, lamb, localsgd, gradient merge, dgc
     Meta Optimizer Type E: only transpile to Graph structure for runtime,
                            currently, grad fusion and kernel fusion, sync batch-norm included.
diff --git a/python/paddle/distributed/fleet/base/util_factory.py b/python/paddle/distributed/fleet/base/util_factory.py
@@ -70,7 +70,7 @@ def all_reduce(self, input, mode="sum", comm_world="worker"):
         Args:
             input (list|tuple|numpy.array): The input variable to do all_reduce between specified collection.
             mode (str): "sum" or "min" or "max".
-            comm_world (str, optional): Collection used to execute all_reduce operation. Supported collections incude `worker` , `server` and `all` . The default is `worker` .
+            comm_world (str, optional): Collection used to execute all_reduce operation. Supported collections include `worker` , `server` and `all` . The default is `worker` .
 
         Returns:
             output(Numpy.array|None): A numpy array with the same shape as the `input` .
@@ -118,7 +118,7 @@ def barrier(self, comm_world="worker"):
         Barrier between specified collection.
 
         Args:
-            comm_world (str, optional): Collection used to execute barrier operation. Supported collections incude `worker` , `server` and `all` . The default is `worker` .
+            comm_world (str, optional): Collection used to execute barrier operation. Supported collections include `worker` , `server` and `all` . The default is `worker` .
 
         Examples:
 
@@ -160,7 +160,7 @@ def all_gather(self, input, comm_world="worker"):
 
         Args:
             input (Int|Float): The input variable to do all_gather between specified collection.
-            comm_world (str, optional): Collection used to execute all_reduce operation. Supported collections incude `worker` , `server` and `all` . The default is `worker` .
+            comm_world (str, optional): Collection used to execute all_reduce operation. Supported collections include `worker` , `server` and `all` . The default is `worker` .
 
         Returns:
             output (List): A list of gathered values.
@@ -289,7 +289,7 @@ def get_file_shard(self, files):
 
     def print_on_rank(self, message, rank_id):
         """
-        Woker of rank `rank_id` print some message.
+        Worker of rank `rank_id` print some message.
 
         Args:
             message(str): Log to be printed.
