make some auto_parallel uts deprecated (#67862)

ooooo-create · web-flow · commit 4b1516d2be3d · 2024-09-04T15:47:10.000+08:00
* make some auto_parallel uts deprecated

* fix
diff --git a/test/auto_parallel/CMakeLists.txt b/test/auto_parallel/CMakeLists.txt
@@ -17,9 +17,6 @@ if(WITH_DISTRIBUTE AND WITH_GPU)
                   test_mp_allreduce_matmul_grad_overlapping)
   set_tests_properties(test_mp_allreduce_matmul_grad_overlapping
                        PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 120)
-  py_test_modules(test_relaunch_with_planner MODULES test_relaunch_with_planner)
-  set_tests_properties(test_relaunch_with_planner
-                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 120)
   py_test_modules(test_relaunch_with_gpt_planner MODULES
                   test_relaunch_with_gpt_planner)
   set_tests_properties(test_relaunch_with_gpt_planner
@@ -46,9 +43,6 @@ if(WITH_DISTRIBUTE AND WITH_GPU)
   py_test_modules(test_pass_grad_clip MODULES test_pass_grad_clip)
   set_tests_properties(test_pass_grad_clip
                        PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 50)
-  py_test_modules(test_random_ctrl MODULES test_random_ctrl)
-  set_tests_properties(test_random_ctrl PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE"
-                                                   TIMEOUT 50)
   py_test_modules(test_pass_gradient_merge MODULES test_pass_gradient_merge)
   set_tests_properties(test_pass_gradient_merge
                        PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 50)
@@ -83,13 +77,6 @@ if(WITH_DISTRIBUTE AND WITH_GPU)
   py_test_modules(test_auto_tuner MODULES test_auto_tuner)
   set_tests_properties(test_auto_tuner PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE"
                                                   TIMEOUT 100)
-  py_test_modules(test_pipeline_scheduler_vpp MODULES
-                  test_pipeline_scheduler_vpp)
-  set_tests_properties(test_pipeline_scheduler_vpp
-                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 50)
-  py_test_modules(test_pipeline_scheduler_zb MODULES test_pipeline_scheduler_zb)
-  set_tests_properties(test_pipeline_scheduler_zb
-                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 50)
   py_test_modules(test_pipeline_scheduler_zb_vpp MODULES
                   test_pipeline_scheduler_zb_vpp)
   set_tests_properties(test_pipeline_scheduler_zb_vpp
@@ -234,8 +221,6 @@ if(WITH_DISTRIBUTE AND WITH_GPU)
   py_test_modules(test_parallel_tuner_predict MODULES
                   test_parallel_tuner_predict)
   set_tests_properties(test_parallel_tuner_predict PROPERTIES TIMEOUT 120)
-  py_test_modules(test_selective_recompute MODULES test_selective_recompute)
-  set_tests_properties(test_selective_recompute PROPERTIES TIMEOUT 50)
   py_test_modules(test_tuning_recompute MODULES test_tuning_recompute)
   set_tests_properties(test_tuning_recompute PROPERTIES TIMEOUT 300)
   py_test_modules(test_tuning_recompute_with_amp MODULES
@@ -257,8 +242,6 @@ if(WITH_DISTRIBUTE AND WITH_GPU)
 
   # NOTE(zyl): unittests WITH single card and WITHOUT timeout
   py_test_modules(test_align_mode MODULES test_align_mode)
-  py_test_modules(test_while_op_completion MODULES test_while_op_completion)
-  py_test_modules(test_while_op_partition MODULES test_while_op_partition)
   py_test_modules(test_tunable_variable MODULES test_tunable_variable)
   py_test_modules(test_tunable_space MODULES test_tunable_space)
   py_test_modules(test_recorder MODULES test_recorder)
@@ -292,19 +275,15 @@ if(WITH_DISTRIBUTE AND WITH_GPU)
   py_test_modules(test_engine_api_error MODULES test_engine_api_error)
   py_test_modules(test_fp16_assign MODULES test_fp16_assign)
   py_test_modules(test_group_operators MODULES test_group_operators)
-  py_test_modules(test_pattern MODULES test_pattern)
-  py_test_modules(test_pattern_match MODULES test_pattern_match)
   py_test_modules(test_cluster_partition MODULES test_cluster_partition)
   py_test_modules(test_convert_to_process_meshes MODULES
                   test_convert_to_process_meshes)
   py_test_modules(test_pass_bf16 MODULES test_pass_bf16)
   py_test_modules(test_dist_saver MODULES test_dist_saver)
   py_test_modules(test_engine_save_load MODULES test_engine_save_load)
-  py_test_modules(test_rule_based_tuner MODULES test_rule_based_tuner)
   py_test_modules(test_dist_tensor MODULES test_dist_tensor)
   py_test_modules(test_api_dist_branch MODULES test_api_dist_branch)
   py_test_modules(test_shard_tensor_api MODULES test_shard_tensor_api)
-  py_test_modules(test_shard_layer_api MODULES test_shard_layer_api)
   py_test_modules(test_cost_interface MODULES test_cost_interface)
   py_test_modules(test_auto_conditional_block MODULES
                   test_auto_conditional_block)
diff --git a/test/auto_parallel/get_gpt_model.py b/test/auto_parallel/get_gpt_model.py
@@ -12,10 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import random
 import sys
 
-sys.path.append("../legacy_test")
+cur_path = os.path.dirname(__file__)
+sys.path.append(cur_path + "/../legacy_test")
 import auto_parallel_gpt_model as modeling
 import numpy as np
 from auto_parallel_gpt_model import (
diff --git a/test/deprecated/CMakeLists.txt b/test/deprecated/CMakeLists.txt
@@ -158,6 +158,7 @@ if(WITH_TESTING)
   add_subdirectory(sequence)
 
   if(WITH_DISTRIBUTE)
+    add_subdirectory(auto_parallel)
     add_subdirectory(collective)
     add_subdirectory(distributed_passes)
   endif()
diff --git a/test/deprecated/auto_parallel/CMakeLists.txt b/test/deprecated/auto_parallel/CMakeLists.txt
@@ -0,0 +1,46 @@
+# file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
+# string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
+
+# add_subdirectory(spmd_rules)
+# add_subdirectory(hybrid_strategy)
+# add_subdirectory(custom_op)
+# add_subdirectory(pir)
+
+if(WITH_DISTRIBUTE AND WITH_GPU)
+
+  py_test_modules(test_relaunch_with_planner_deprecated MODULES
+                  test_relaunch_with_planner_deprecated)
+  set_tests_properties(test_relaunch_with_planner_deprecated
+                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 120)
+  py_test_modules(test_random_ctrl_deprecated MODULES
+                  test_random_ctrl_deprecated)
+  set_tests_properties(test_random_ctrl_deprecated
+                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 50)
+  py_test_modules(test_pipeline_scheduler_vpp_deprecated MODULES
+                  test_pipeline_scheduler_vpp_deprecated)
+  set_tests_properties(test_pipeline_scheduler_vpp_deprecated
+                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 50)
+  py_test_modules(test_pipeline_scheduler_zb_deprecated MODULES
+                  test_pipeline_scheduler_zb_deprecated)
+  set_tests_properties(test_pipeline_scheduler_zb_deprecated
+                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 50)
+  py_test_modules(test_selective_recompute_deprecated MODULES
+                  test_selective_recompute_deprecated)
+  set_tests_properties(test_selective_recompute_deprecated PROPERTIES TIMEOUT
+                                                                      50)
+  py_test_modules(test_while_op_completion_deprecated MODULES
+                  test_while_op_completion_deprecated)
+  py_test_modules(test_while_op_partition_deprecated MODULES
+                  test_while_op_partition_deprecated)
+  py_test_modules(test_pattern_deprecated MODULES test_pattern_deprecated)
+  py_test_modules(test_pattern_match_deprecated MODULES
+                  test_pattern_match_deprecated)
+  py_test_modules(test_rule_based_tuner_deprecated MODULES
+                  test_rule_based_tuner_deprecated)
+  py_test_modules(test_shard_layer_api_deprecated MODULES
+                  test_shard_layer_api_deprecated)
+  # End of unittests WITH single card WITHOUT timeout
+
+endif()
+
+set_pir_tests_properties()
diff --git a/test/deprecated/auto_parallel/auto_parallel_relaunch_with_planner_deprecated.py b/test/deprecated/auto_parallel/auto_parallel_relaunch_with_planner_deprecated.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
+
 import paddle
 from paddle import static
 from paddle.distributed import fleet
@@ -21,6 +23,8 @@
     get_default_distributed_context,
 )
 
+sys.path.append("../../auto_parallel")
+
 
 def train():
     from auto_parallel_relaunch_model import (
diff --git a/test/deprecated/auto_parallel/pipeline_scheduler_vpp_unittest_deprecated.py b/test/deprecated/auto_parallel/pipeline_scheduler_vpp_unittest_deprecated.py
diff --git a/test/deprecated/auto_parallel/pipeline_scheduler_zb_unittest_deprecated.py b/test/deprecated/auto_parallel/pipeline_scheduler_zb_unittest_deprecated.py
@@ -13,9 +13,12 @@
 # limitations under the License.
 
 import random
+import sys
 import unittest
 
 import numpy as np
+
+sys.path.append("../../auto_parallel")
 from get_gpt_model import FakeDataset, generate_model
 
 import paddle
diff --git a/test/deprecated/auto_parallel/random_control_unittest_deprecated.py b/test/deprecated/auto_parallel/random_control_unittest_deprecated.py
@@ -13,9 +13,13 @@
 # limitations under the License.
 
 import random
+import sys
 import unittest
 
 import numpy as np
+
+sys.path.append("../../auto_parallel")
+
 from get_gpt_model import FakeDataset, generate_model
 
 import paddle
diff --git a/test/deprecated/auto_parallel/test_pattern_deprecated.py b/test/deprecated/auto_parallel/test_pattern_deprecated.py
@@ -13,8 +13,10 @@
 # limitations under the License.
 
 
+import sys
 import unittest
 
+sys.path.append("../..")
 import auto_parallel_gpt_model as modeling
 import numpy as np
 from auto_parallel_gpt_model import (
diff --git a/test/deprecated/auto_parallel/test_pattern_match_deprecated.py b/test/deprecated/auto_parallel/test_pattern_match_deprecated.py
@@ -13,8 +13,10 @@
 # limitations under the License.
 
 
+import sys
 import unittest
 
+sys.path.append("../..")
 import auto_parallel_gpt_model as modeling
 import numpy as np
 from auto_parallel_gpt_model import (
diff --git a/test/deprecated/auto_parallel/test_pipeline_scheduler_vpp_deprecated.py b/test/deprecated/auto_parallel/test_pipeline_scheduler_vpp_deprecated.py
@@ -23,7 +23,7 @@ class TestVPPPass(unittest.TestCase):
     def test_pp2(self):
         file_dir = os.path.dirname(os.path.abspath(__file__))
         launch_model_path = os.path.join(
-            file_dir, "pipeline_scheduler_vpp_unittest.py"
+            file_dir, "pipeline_scheduler_vpp_unittest_deprecated.py"
         )
 
         if os.environ.get("WITH_COVERAGE", "OFF") == "ON":
diff --git a/test/deprecated/auto_parallel/test_pipeline_scheduler_zb_deprecated.py b/test/deprecated/auto_parallel/test_pipeline_scheduler_zb_deprecated.py
@@ -23,7 +23,7 @@ class TestZBH1Pass(unittest.TestCase):
     def test_pp2(self):
         file_dir = os.path.dirname(os.path.abspath(__file__))
         launch_model_path = os.path.join(
-            file_dir, "pipeline_scheduler_zb_unittest.py"
+            file_dir, "pipeline_scheduler_zb_unittest_deprecated.py"
         )
 
         if os.environ.get("WITH_COVERAGE", "OFF") == "ON":
diff --git a/test/deprecated/auto_parallel/test_random_ctrl_deprecated.py b/test/deprecated/auto_parallel/test_random_ctrl_deprecated.py
@@ -22,7 +22,9 @@
 class TestRandomCtrlPass(unittest.TestCase):
     def test_mp2_with_recompute(self):
         file_dir = os.path.dirname(os.path.abspath(__file__))
-        launch_model_path = os.path.join(file_dir, "random_control_unittest.py")
+        launch_model_path = os.path.join(
+            file_dir, "random_control_unittest_deprecated.py"
+        )
 
         if os.environ.get("WITH_COVERAGE", "OFF") == "ON":
             coverage_args = ["-m", "coverage", "run", "--branch", "-p"]
diff --git a/test/deprecated/auto_parallel/test_relaunch_with_planner_deprecated.py b/test/deprecated/auto_parallel/test_relaunch_with_planner_deprecated.py
@@ -19,6 +19,8 @@
 import tempfile
 import unittest
 
+sys.path.append("../../auto_parallel")
+
 
 class TestPlannerReLaunch(unittest.TestCase):
     def setUp(self):
@@ -47,7 +49,7 @@ def test_relaunch_with_planner(self):
 
         file_dir = os.path.dirname(os.path.abspath(__file__))
         launch_model_path = os.path.join(
-            file_dir, "auto_parallel_relaunch_with_planner.py"
+            file_dir, "auto_parallel_relaunch_with_planner_deprecated.py"
         )
 
         if os.environ.get("WITH_COVERAGE", "OFF") == "ON":
diff --git a/test/deprecated/auto_parallel/test_rule_based_tuner_deprecated.py b/test/deprecated/auto_parallel/test_rule_based_tuner_deprecated.py
@@ -13,8 +13,10 @@
 # limitations under the License.
 
 
+import sys
 import unittest
 
+sys.path.append("../..")
 import auto_parallel_gpt_model as modeling
 import numpy as np
 from auto_parallel_gpt_model import (
diff --git a/test/deprecated/auto_parallel/test_selective_recompute_deprecated.py b/test/deprecated/auto_parallel/test_selective_recompute_deprecated.py
@@ -17,12 +17,14 @@
 import unittest
 
 import numpy as np
+
+sys.path.append("../../auto_parallel")
 from get_gpt_model import FakeDataset
 
 import paddle
 from paddle.distributed.fleet import auto
 
-sys.path.append("..")
+sys.path.append("../..")
 import auto_parallel_gpt_model as modeling
 from auto_parallel_gpt_model import (
     GPTForPretraining,
diff --git a/test/deprecated/auto_parallel/test_shard_layer_api_deprecated.py b/test/deprecated/auto_parallel/test_shard_layer_api_deprecated.py
diff --git a/test/deprecated/auto_parallel/test_while_op_completion_deprecated.py b/test/deprecated/auto_parallel/test_while_op_completion_deprecated.py
diff --git a/test/deprecated/auto_parallel/test_while_op_partition_deprecated.py b/test/deprecated/auto_parallel/test_while_op_partition_deprecated.py
