[CI] Add Auto-Parallel (#72744)

swgu98 · web-flow · commit 5bc4dede0ba7 · 2025-06-24T11:27:37.000+08:00
* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* old image

* old image

* old image

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* 2 exec

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix final

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* name test=document_fix

* test=document_fix

* test=document_fix

* fix

* test=document_fix

* test=document_fix

* test=document_fix
diff --git a/.github/workflows/CI-Build.yml b/.github/workflows/CI-Build.yml
@@ -0,0 +1,87 @@
+name: CI-Build
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    branches: [develop, release/**]
+
+permissions: read-all
+
+concurrency:
+  group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  PR_ID: ${{ github.event.pull_request.number }}
+  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
+
+jobs:
+  clone:
+    name: Clone-linux
+    uses: ./.github/workflows/_Clone-linux.yml
+    with:
+      bos_dir: Paddle-build
+
+  build-docker:
+    name: build docker images
+    needs: clone
+    uses: ./.github/workflows/docker.yml
+    with:
+      bos_dir: Paddle-build
+      task: build
+
+  build:
+    name: Linux-build
+    uses: ./.github/workflows/_Linux-build.yml
+    needs: build-docker
+    with:
+      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+
+  static-check:
+    name: Static-Check
+    uses: ./.github/workflows/_Static-Check.yml
+    needs: [build-docker, build]
+    with:
+      can-skip: ${{ needs.build.outputs.can-skip }}
+      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+
+  ce-framework:
+    name: CE-Framework
+    uses: ./.github/workflows/_CE-Framework.yml
+    needs: [build-docker, build]
+    with:
+      can-skip: ${{ needs.build.outputs.can-skip }}
+      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+
+  ce-cinn-framework:
+    name: CE-CINN-Framework
+    uses: ./.github/workflows/_CE-CINN-Framework.yml
+    needs: [build-docker, build]
+    with:
+      can-skip: ${{ needs.build.outputs.can-skip }}
+      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+
+  api-benchmark:
+    name: Api-Benchmark
+    uses: ./.github/workflows/_Api-Benchmark.yml
+    needs: [build-docker, build]
+    with:
+      can-skip: ${{ needs.build.outputs.can-skip }}
+      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+
+  auto-parallel:
+    name: Auto-Parallel
+    uses: ./.github/workflows/_Auto-Parallel.yml
+    needs: [build-docker, build]
+    with:
+      can-skip: ${{ needs.build.outputs.can-skip }}
+      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+
+  slice:
+    name: Slice
+    uses: ./.github/workflows/_Slice.yml
+    needs: [clone, build-docker, build]
+    with:
+      can-skip: ${{ needs.build.outputs.can-skip }}
+      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+      slice-check: ${{ needs.clone.outputs.slice-check }}
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -8,7 +8,7 @@ on:
 permissions: read-all
 
 concurrency:
-  group: ${{ github.event.pull_request.number }}
+  group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
   cancel-in-progress: true
 
 env:
@@ -60,58 +60,12 @@ jobs:
     name: Linux-NPU
     uses: ./.github/workflows/_Linux-NPU.yml
     needs: cpu
+    with:
+      can-skip: ${{ needs.cpu.outputs.can-skip }}
 
   distribute:
     name: Distribute-stable
     uses: ./.github/workflows/_Distribute-stable.yml
     needs: build-docker
     with:
       docker_distribute_image: ${{ needs.build-docker.outputs.docker_distribute_image }}
-
-  build:
-    name: Linux-build
-    uses: ./.github/workflows/_Linux-build.yml
-    needs: build-docker
-    with:
-      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
-
-  static-check:
-    name: Static-Check
-    uses: ./.github/workflows/_Static-Check.yml
-    needs: [build-docker, build]
-    with:
-      can-skip: ${{ needs.build.outputs.can-skip }}
-      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
-
-  ce-framework:
-    name: CE-Framework
-    uses: ./.github/workflows/_CE-Framework.yml
-    needs: [build-docker, build]
-    with:
-      can-skip: ${{ needs.build.outputs.can-skip }}
-      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
-
-  ce-cinn-framework:
-    name: CE-CINN-Framework
-    uses: ./.github/workflows/_CE-CINN-Framework.yml
-    needs: [build-docker, build]
-    with:
-      can-skip: ${{ needs.build.outputs.can-skip }}
-      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
-
-  api-benchmark:
-    name: Api-Benchmark
-    uses: ./.github/workflows/_Api-Benchmark.yml
-    needs: [build-docker, build]
-    with:
-      can-skip: ${{ needs.build.outputs.can-skip }}
-      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
-
-  slice:
-    name: Slice
-    uses: ./.github/workflows/_Slice.yml
-    needs: [clone, build-docker, build]
-    with:
-      can-skip: ${{ needs.build.outputs.can-skip }}
-      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
-      slice-check: ${{ needs.clone.outputs.slice-check }}
diff --git a/.github/workflows/Coverage.yml b/.github/workflows/Coverage.yml
@@ -41,6 +41,7 @@ jobs:
     uses: ./.github/workflows/docker.yml
     with:
       bos_dir: Paddle-coverage
+      task: coverage
 
   build:
     name: Coverage build
diff --git a/.github/workflows/_Auto-Parallel.yml b/.github/workflows/_Auto-Parallel.yml
@@ -0,0 +1,144 @@
+name: Auto-Parallel
+
+on:
+  workflow_call:
+    inputs:
+      docker_build_image:
+        type: string
+        required: true
+      can-skip:
+        type: string
+        required: false
+
+env:
+  PR_ID: ${{ github.event.pull_request.number }}
+  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
+  work_dir: /workspace/Paddle
+  PADDLE_ROOT: /workspace/Paddle
+  TASK: paddle-CI-${{ github.event.pull_request.number }}-Auto-Parallel
+  ci_scripts: /workspace/Paddle/ci
+  BRANCH: ${{ github.event.pull_request.base.ref }}
+  CI_name: auto-parallel
+  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  check-bypass:
+    name: Check bypass for inference
+    uses: ./.github/workflows/check-bypass.yml
+    with:
+      workflow-name: 'auto-parallel'
+    secrets:
+      github-token: ${{ secrets.GITHUB_TOKEN }}
+
+  parallel-test:
+    name: Parallel test
+    needs: check-bypass
+    if: ${{ inputs.can-skip != 'true'  && needs.check-bypass.outputs.can-skip != 'true' }}
+    runs-on:
+      group: Auto-Parallel
+    steps:
+      - name: Check docker image and run container
+        env:
+          FLAGS_dynamic_static_unified_comm: "True"
+          python_version: "3.10"
+          paddle_whl: https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
+        run: |
+          container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
+          echo "container_name=${container_name}" >> ${{ github.env }}
+          docker_image=${{ inputs.docker_build_image }}
+          nvidia-docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
+            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
+            -v ${{ github.workspace }}:/workspace \
+            -v /home/FleetX_CI:/gpt_data \
+            -v /home/Llm_gpt_CI:/llm_gpt_data \
+            -v /home/Llama_CI:/llama_data \
+            -v /home/.cache/pip:/home/.cache/pip \
+            -e BRANCH \
+            -e PR_ID \
+            -e COMMIT_ID \
+            -e work_dir \
+            -e PADDLE_ROOT \
+            -e ci_scripts \
+            -e no_proxy \
+            -e CI_name \
+            -e paddle_whl \
+            -e FLAGS_dynamic_static_unified_comm \
+            -e python_version \
+            -w /workspace --runtime=nvidia ${docker_image}
+
+      - name: Download Paddle
+        env:
+          work_dir: ${{ github.workspace }}
+        run: |
+          docker exec -t ${{ env.container_name }} /bin/bash -c '
+          rm -rf * .[^.]*
+          echo "Downloading build.tar.gz"
+          wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/build.tar.gz --no-check-certificate
+          echo "Extracting build.tar.gz"
+          tar --use-compress-program="pzstd" -xpf build.tar.gz
+          mv paddle Paddle
+          rm build.tar.gz
+          '
+
+      - name: Download PaddleTest
+        env:
+          work_dir: ${{ github.workspace }}
+        run: |
+          docker exec -t ${{ env.container_name }} /bin/bash -c '
+          echo "Download and extract PaddleNLP.tar.gz"
+          wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
+          tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
+          source ${{ github.workspace }}/../../../proxy
+          cd PaddleNLP
+          git pull
+          git submodule update --init --recursive --force
+          echo hh
+          python -m pip install bce-python-sdk==0.8.74
+          '
+
+      - name: Test
+        env:
+          work_dir: ${{ github.workspace }}
+        run: |
+          docker exec -t ${{ env.container_name }} /bin/bash -c '
+          ldconfig
+          ln -sf $(which python${python_version}) /usr/bin/python
+          '
+          docker exec -t ${{ env.container_name }} /bin/bash -c '
+          ldconfig
+          pip config set global.cache-dir "/home/.cache/pip"
+          set -e
+          timeout 80m bash Paddle/ci/auto_parallel/ci_auto_parallel.sh ${paddle_whl}
+          '
+
+      - name: Upload and display logs
+        if: always()
+        env:
+          home_path: ${{ github.workspace }}/..
+          bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
+        run: |
+          docker exec -t ${{ env.container_name }} /bin/bash -c '
+          export AK=paddle
+          export SK=paddle
+          if [ ! -f "${{ env.bos_file }}" ]; then
+            wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
+            mkdir ${{ env.home_path }}/bos_retry
+            tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
+          fi
+          cd /workspace/case_logs
+          for FILE in /workspace/case_logs/*; do
+            file=$(basename "$FILE")
+            python ${{ env.bos_file }} $file paddle-github-action/PR/Auto-Parallel/${PR_ID}/${COMMIT_ID}/logs
+            echo "$file: https://paddle-github-action.bj.bcebos.com/PR/Auto-Parallel/${PR_ID}/${COMMIT_ID}/logs/$file"
+          done
+          '
+
+      - name: Terminate and delete the container
+        if: always()
+        run: |
+          docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
+          docker rm -f ${{ env.container_name }}
diff --git a/.github/workflows/_Distribute-stable.yml b/.github/workflows/_Distribute-stable.yml
@@ -318,7 +318,6 @@ jobs:
           rm -rf * .[^.]*
           echo "Downloading Paddle.tar.gz"
           wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/gpups/${{ env.PR_ID }}/${{ env.COMMIT_ID }}/Paddle.tar.gz --no-check-certificate
-          # wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/gpups/72396/2b797c6b9b483cc0bd8433fb9d97d06b7ce137c1/Paddle.tar.gz --no-check-certificate
           echo "Extracting Paddle.tar.gz"
           tar --use-compress-program="pzstd" -xf Paddle.tar.gz --strip-components=1
           rm Paddle.tar.gz
diff --git a/.github/workflows/_Linux-CPU.yml b/.github/workflows/_Linux-CPU.yml
@@ -6,6 +6,10 @@ on:
       docker_cpu_image:
         type: string
         required: true
+    outputs:
+      can-skip:
+        description: "Whether to skip the job"
+        value: ${{ jobs.build-and-test.outputs.can-skip }}
 
 env:
   PR_ID: ${{ github.event.pull_request.number }}
@@ -25,6 +29,8 @@ defaults:
 jobs:
   build-and-test:
     name: Build and test
+    outputs:
+      can-skip: ${{ steps.check-bypass.outputs.can-skip }}
     runs-on:
       group: GZ_BD-CPU
 
diff --git a/.github/workflows/_Linux-NPU.yml b/.github/workflows/_Linux-NPU.yml
@@ -2,6 +2,10 @@ name: Linux-NPU
 
 on:
   workflow_call:
+    inputs:
+      can-skip:
+        type: string
+        required: false
 
 env:
   dockerfile: dockerfile
@@ -28,7 +32,7 @@ jobs:
   test:
     name: Test
     needs: check-bypass
-    if: ${{ github.repository_owner == 'PaddlePaddle' && needs.check-bypass.outputs.can-skip != 'true' }}
+    if: ${{ inputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
     runs-on:
       group: NPU
 
diff --git a/.github/workflows/_Linux-build.yml b/.github/workflows/_Linux-build.yml
@@ -3,6 +3,10 @@ name: Linux-build
 on:
   workflow_call:
     inputs:
+      bos_dir:
+        required: false
+        type: string
+        default: "Paddle"
       docker_build_image:
         type: string
         required: true
@@ -122,7 +126,7 @@ jobs:
           rm -rf * .[^.]*
           set -e
           echo "Downloading Paddle.tar.gz"
-          wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
+          wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/${{ inputs.bos_dir }}/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
           echo "Extracting Paddle.tar.gz"
           git config --global --add safe.directory ${work_dir}
           tar xf Paddle.tar.gz --strip-components=1
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
diff --git a/ci/auto_parallel/ci_auto_parallel.sh b/ci/auto_parallel/ci_auto_parallel.sh
