Del distutils.util (#65635)

tianshuo78520a · SigureMo · web-flow · commit 04ea70f13e02 · 2024-07-03T10:41:41.000+08:00
* Del distutils.util

* move strtobool to `paddle.utils`

---------

Co-authored-by: SigureMo &lt;sigure.qaq@gmail.com&gt;
diff --git a/paddle/fluid/pir/dialect/op_generator/op_gen.py b/paddle/fluid/pir/dialect/op_generator/op_gen.py
@@ -18,7 +18,6 @@
 import os
 import pathlib
 import sys
-from distutils.util import strtobool
 
 import yaml
 from decomp_interface_gen_op_list import (
@@ -2421,6 +2420,16 @@ def OpGenerator(
             f.write(vjp_source_file_str)
 
 
+def strtobool(val):
+    val = val.lower()
+    if val in ['y', 'yes', 't', 'true', 'on', '1']:
+        return True
+    elif val in ['n', 'no', 'f', 'false', 'off', '0']:
+        return False
+    else:
+        raise ValueError(f"Invalid truth value {val!r}")
+
+
 # =====================================
 # Script parameter parsing
 # =====================================
diff --git a/python/paddle/distributed/fleet/launch_utils.py b/python/paddle/distributed/fleet/launch_utils.py
@@ -26,10 +26,10 @@
 import tempfile
 import time
 from contextlib import closing
-from distutils.util import strtobool
 
 import paddle.utils.cpp_extension.extension_utils as utils
 from paddle import framework
+from paddle.utils import strtobool
 
 logger = logging.getLogger("root")
 logger.propagate = False
@@ -1855,11 +1855,11 @@ def start_pod_heter_worker(self, args, pod):
             proc_env = {
                 "PADDLE_PSERVERS_IP_PORT_LIST": self.server_endpoints,
                 "PADDLE_TRAINER_ENDPOINTS": self.worker_endpoints,
-                "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST": self.stage_heter_map[
-                    stage_id + 1
-                ]
-                if stage_id <= self.stage_num - 1
-                else "",
+                "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST": (
+                    self.stage_heter_map[stage_id + 1]
+                    if stage_id <= self.stage_num - 1
+                    else ""
+                ),
                 "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST": self.stage_heter_map[
                     stage_id - 1
                 ],
diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py
@@ -12,10 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-######
+
 import os
 import warnings
-from distutils.util import strtobool
 from functools import reduce
 
 import paddle
@@ -27,6 +26,7 @@
     ReduceOp,
     is_avg_reduce_op_supported,
 )
+from paddle.utils import strtobool
 
 from ...utils.log_util import logger
 from ...utils.tensor_fusion_helper import (
diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import distutils.util
 import os
 
 import numpy as np
@@ -25,6 +24,7 @@
     _get_global_group,
     _warn_cur_rank_not_in_group,
 )
+from paddle.utils import strtobool
 
 from ...utils import timer_helper as timer
 from .utils import number_2_dtype, paddle_2_number
@@ -430,9 +430,7 @@ def _batched_p2p_ops(
 
     if len(ops) > 0:
         batch_send_recv_on_calc_stream(ops)
-        if distutils.util.strtobool(
-            os.getenv('FLAGS_p2p_device_synchronize', '0')
-        ):
+        if strtobool(os.getenv('FLAGS_p2p_device_synchronize', '0')):
             paddle.device.cuda.synchronize()
 
     tensors_for_all_gather = []
diff --git a/python/paddle/distributed/launch/context/args_envs.py b/python/paddle/distributed/launch/context/args_envs.py
@@ -14,7 +14,8 @@
 
 import os
 from argparse import REMAINDER, ArgumentParser
-from distutils.util import strtobool
+
+from paddle.utils import strtobool
 
 env_args_mapping = {
     'POD_IP': ('host', str),
diff --git a/python/paddle/distributed/utils/launch_utils.py b/python/paddle/distributed/utils/launch_utils.py
@@ -20,9 +20,9 @@
 import sys
 import time
 from contextlib import closing
-from distutils.util import strtobool
 
 from paddle.distributed.fleet.launch_utils import get_backend_by_compile_flag
+from paddle.utils import strtobool
 
 from ..utils.log_utils import get_logger
 
diff --git a/python/paddle/utils/__init__.py b/python/paddle/utils/__init__.py
@@ -22,6 +22,7 @@
     unique_name,
 )
 from .deprecated import deprecated
+from .environments import strtobool as strtobool
 from .install_check import run_check
 from .layers_utils import (  # noqa: F401
     _contain_var,
diff --git a/python/paddle/utils/environments.py b/python/paddle/utils/environments.py
@@ -22,6 +22,16 @@
 T = TypeVar("T")
 
 
+def strtobool(val):
+    val = val.lower()
+    if val in ['y', 'yes', 't', 'true', 'on', '1']:
+        return True
+    elif val in ['n', 'no', 'f', 'false', 'off', '0']:
+        return False
+    else:
+        raise ValueError(f"Invalid truth value {val!r}")
+
+
 class EnvironmentVariable(Generic[T]):
     name: str
     default: T
@@ -57,16 +67,14 @@ def set(self, value: str) -> None:
 
 
 class BooleanEnvironmentVariable(EnvironmentVariable[bool]):
-    BOOLEAN_IS_SET = ("y", "yes", "t", "true", "on", "1")
-
     def __init__(self, name: str, default: bool):
         super().__init__(name, default)
         assert isinstance(default, bool), "default must be a boolean"
 
     def get(self) -> bool:
-        default = str(self.default).lower()
-        env_str = os.getenv(self.name, default).lower()
-        return env_str in BooleanEnvironmentVariable.BOOLEAN_IS_SET
+        default = str(self.default)
+        env_str = os.getenv(self.name, default)
+        return strtobool(env_str)
 
     def set(self, value: bool) -> None:
         assert isinstance(value, bool), "value must be a boolean"
diff --git a/test/legacy_test/distributed_fused_lamb_test_base.py b/test/legacy_test/distributed_fused_lamb_test_base.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import distutils
 import os
 import unittest
 
@@ -24,6 +23,7 @@
 from paddle.distributed.fleet.meta_optimizers.common import CollectiveHelper
 from paddle.incubate import DistributedFusedLamb
 from paddle.nn.clip import ClipGradBase, _clip_by_global_norm_using_mp_type
+from paddle.utils import strtobool
 from paddle.vision.models import resnet18 as resnet
 
 
@@ -280,7 +280,7 @@ def setUpClass(cls):
 
     def config(self):
         clip_after_allreduce = bool(
-            distutils.util.strtobool(os.getenv('CLIP_AFTER_ALLREDUCE', 'True'))
+            strtobool(os.getenv('CLIP_AFTER_ALLREDUCE', 'True'))
         )
         max_global_norm = float(os.getenv('MAX_GLOBAL_NORM', -1.0))
         gm_steps = int(os.getenv('GRADIENT_MERGE_STEPS', 1))
@@ -291,9 +291,11 @@ def config(self):
         return {
             'clip_after_allreduce': clip_after_allreduce,
             'gradient_accumulation_steps': gm_steps,
-            'grad_clip': paddle.nn.ClipGradByGlobalNorm(max_global_norm)
-            if max_global_norm > 0
-            else None,
+            'grad_clip': (
+                paddle.nn.ClipGradByGlobalNorm(max_global_norm)
+                if max_global_norm > 0
+                else None
+            ),
             'use_master_acc_grad': use_master_acc_grad,
         }
 
diff --git a/tools/dockerfile/build_scripts/manylinux1-check.py b/tools/dockerfile/build_scripts/manylinux1-check.py
@@ -17,9 +17,12 @@
 
 def is_manylinux1_compatible():
     # Only Linux, and only x86-64 / i686
-    from distutils.util import get_platform
+    import platform
 
-    if get_platform() not in ["linux-x86_64", "linux-i686"]:
+    if platform.system() != 'Linux' and platform.machine() not in [
+        "linux-x86_64",
+        "linux-i686",
+    ]:
         return False
 
     # Check for presence of _manylinux module

Original file line number	Diff line number	Diff line change
`@@ -22,6 +22,7 @@`
`22`	`22`	`unique_name,`
`23`	`23`	`)`
`24`	`24`	`from .deprecated import deprecated`
	`25`	`+from .environments import strtobool as strtobool`
`25`	`26`	`from .install_check import run_check`
`26`	`27`	`from .layers_utils import ( # noqa: F401`
`27`	`28`	`_contain_var,`