[XPU] update xccl to 1.2.3 (#65533)

lj970926 · web-flow · commit 1749a8783e2d · 2024-07-05T11:12:00.000+08:00
* fix comm bugs

* refine code

* bug fix

* update xccl version

* fix ci bugs

* fix ci bugs

* fix typo

* change filter regex
diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake
@@ -32,7 +32,7 @@ endif()
 if(NOT DEFINED XPU_XHPC_BASE_DATE)
   set(XPU_XHPC_BASE_DATE "20240626")
 endif()
-set(XPU_XCCL_BASE_VERSION "1.2.1.2")
+set(XPU_XCCL_BASE_VERSION "1.2.3")
 if(NOT DEFINED XPU_XFT_BASE_VERSION)
   set(XPU_XFT_BASE_VERSION "20230602")
 endif()
@@ -43,7 +43,7 @@ set(XPU_XRE_BASE_URL
 )
 
 set(XPU_XCCL_BASE_URL
-    "https://klx-sdk-release-public.su.bcebos.com/xccl/release/${XPU_XCCL_BASE_VERSION}"
+    "https://klx-sdk-release-public.su.bcebos.com/xccl/release/${XPU_XCCL_BASE_VERSION}.1"
 )
 
 if(NOT XPU_XFT_BASE_URL)
@@ -64,17 +64,24 @@ if(WITH_XPU_XRE5)
   set(XPU_XRE_BASE_URL
       "https://klx-sdk-release-public.su.bcebos.com/xre/kl3-release/${XPU_XRE_BASE_VERSION}"
   )
+  set(XPU_XCCL_BASE_URL
+      "https://klx-sdk-release-public.su.bcebos.com/xccl/release/${XPU_XCCL_BASE_VERSION}.2"
+  )
 endif()
 
 if(WITH_XCCL_RDMA)
-  set(XPU_XCCL_PREFIX "xccl_rdma")
+  set(XPU_XCCL_PREFIX "xccl_rdma-")
 else()
-  set(XPU_XCCL_PREFIX "xccl_socket")
+  set(XPU_XCCL_PREFIX "xccl_socket-")
+  # NOTE(lijin23): socket has not been supported for XPU3, so the xccl output name was changed.
+  if(WITH_XPU_XRE5)
+    set(XPU_XCCL_PREFIX "xccl_")
+  endif()
 endif()
 
 if(WITH_AARCH64)
   set(XPU_XRE_DIR_NAME "xre-kylin_aarch64")
-  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}-kylin_aarch64")
+  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}kylin_aarch64")
   set(XPU_XFT_DIR_NAME "") # TODO: xft has no kylin output at now.
 elseif(WITH_SUNWAY)
   set(XPU_XRE_DIR_NAME "xre-deepin_sw6_64")
@@ -87,11 +94,11 @@ elseif(WITH_BDCENTOS)
   else()
     set(XPU_XRE_DIR_NAME "xre-bdcentos_x86_64")
   endif()
-  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}-bdcentos_x86_64")
+  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}bdcentos_x86_64")
   set(XPU_XFT_DIR_NAME "xft_bdcentos6u3_x86_64_gcc82")
 elseif(WITH_CENTOS)
   set(XPU_XRE_DIR_NAME "xre-centos7_x86_64")
-  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}-bdcentos_x86_64")
+  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}bdcentos_x86_64")
   set(XPU_XFT_DIR_NAME "xft_bdcentos6u3_x86_64_gcc82")
 else()
   # Ubuntu as default
@@ -102,7 +109,7 @@ else()
     set(XPU_XRE_DIR_NAME "xre-ubuntu_1604_x86_64")
     set(XPU_XHPC_DIR_NAME "xhpc-ubuntu1604_x86_64")
   endif()
-  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}-ubuntu_x86_64")
+  set(XPU_XCCL_DIR_NAME "${XPU_XCCL_PREFIX}ubuntu_x86_64")
   set(XPU_XFT_DIR_NAME "xft_ubuntu1604_x86_64")
 endif()
 
diff --git a/paddle/fluid/distributed/collective/process_group_bkcl.cc b/paddle/fluid/distributed/collective/process_group_bkcl.cc
@@ -48,17 +48,8 @@ bool ProcessGroupBKCL::BKCLTask::IsCompleted() {
 
 // TODO(sheniang03): Add timeout for wait, now timeout unused
 bool ProcessGroupBKCL::BKCLTask::Wait(std::chrono::milliseconds timeout) {
-  // Warning here when use calc stream but also invoke waiting explicitly.
-  if (UseCalcStream()) {
-    VLOG(3) << "Warning: The communication is on calc stream, wait here is "
-               "useless.";
-    return true;
-  }
-
   const auto* calc_ctx = static_cast<XPUContext*>(
       platform::DeviceContextPool::Instance().Get(place_));
-  comm_event_->Block(*calc_ctx);
-
   if (barrier_) {
     // If we use the work to do barrier, we should block cpu
 
@@ -70,6 +61,15 @@ bool ProcessGroupBKCL::BKCLTask::Wait(std::chrono::milliseconds timeout) {
     xpu_wait();
     calc_ctx->Wait();
   }
+  // Warning here when use calc stream but also invoke waiting explicitly.
+  if (UseCalcStream()) {
+    VLOG(3) << "Warning: The communication is on calc stream, wait here is "
+               "useless.";
+    return true;
+  }
+
+  comm_event_->Block(*calc_ctx);
+
   return true;
 }
 
@@ -318,7 +318,6 @@ std::shared_ptr<ProcessGroup::Task> ProcessGroupBKCL::Point2Point(
   auto bkcl_stream = use_calc_stream ? calc_ctx->stream() : comm_ctx->stream();
 
   auto bkcl_comm_ctx = this->GetCommContext();
-
   fn(bkcl_comm_ctx, bkcl_stream, p2p_target_rank);
 
   if (!use_calc_stream) {
diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
@@ -2369,6 +2369,9 @@ set +x
         ut_startTime_s=`date +%s`
         test_cases=$(ctest -N -V -LE "(RUN_TYPE=DIST_KUNLUN)" | grep "_xpu" )        # cases list which would be run exclusively
         get_quickly_disable_ut||disable_ut_quickly='disable_ut'   # indicate whether the case was in quickly disable list
+        # XPU has extra log to stdout when import paddle now, so we filter them out.
+        # To be removed after the version update of bkcl.
+        disable_ut_quickly="${disable_ut_quickly%%[*}"
         while read -r line; do
             if [[ "$line" == "" ]]; then
                 continue
