[CI] Add CE-CINN-Framework (#72696)

swgu98 · web-flow · commit 15d692f1d397 · 2025-05-15T19:31:21.000+08:00
* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix all

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix
diff --git a/.github/workflows/Build-develop.yml b/.github/workflows/Build-develop.yml
@@ -14,10 +14,10 @@ env:
   COMMIT_ID: ${{ github.sha }}
   work_dir: /paddle
   PADDLE_ROOT: /paddle
-  TASK: paddle-CI-bbild-develop
+  TASK: paddle-CI-build-develop-${{ github.sha }}
   ci_scripts: /paddle/ci
   BRANCH: ${{ github.base.ref }}
-  CI_name: build-develop
+  CI_name: build
   no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
 
 jobs:
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -78,6 +78,14 @@ jobs:
       can-skip: ${{ needs.build.outputs.can-skip }}
       docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
 
+  ce-cinn-framework:
+    name: CE-CINN-Framework
+    uses: ./.github/workflows/_CE-CINN-Framework.yml
+    needs: [build-docker, build]
+    with:
+      can-skip: ${{ needs.build.outputs.can-skip }}
+      docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+
   api-benchmark:
     name: Api-Benchmark
     uses: ./.github/workflows/_Api-Benchmark.yml
diff --git a/.github/workflows/_CE-CINN-Framework.yml b/.github/workflows/_CE-CINN-Framework.yml
@@ -0,0 +1,129 @@
+name: CE-CINN-Framework
+
+on:
+  workflow_call:
+    inputs:
+      docker_build_image:
+        type: string
+        required: true
+      can-skip:
+        type: string
+        required: false
+
+env:
+  PR_ID: ${{ github.event.pull_request.number }}
+  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
+  work_dir: /paddle
+  PADDLE_ROOT: /paddle
+  TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-CINN-Framework
+  ci_scripts: /paddle/ci
+  BRANCH: ${{ github.event.pull_request.base.ref }}
+  CI_name: ce-cinn-framework
+  no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  cinn:
+    name: CINN
+    if: ${{ inputs.can-skip != 'true' }}
+    runs-on:
+      group: BD_BJ-V100
+    steps:
+      - name: Check docker image and run container
+        env:
+          CACHE_DIR: /home/data/cfs/.cache
+        run: |
+          container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
+          echo "container_name=${container_name}" >> ${{ github.env }}
+          docker_image=${{ inputs.docker_build_image }}
+          docker container ls -a --filter "name=paddle-CI-*-api-benchmark-${core_index}*" --format "{{.ID}}" | xargs -r docker rm -f
+          docker container ls -a --filter "name=api_benchmark_ci_baseline_" --format "{{.ID}} {{.CreatedAt}}" | awk '$2 <= "'$(date -d '1 day ago' +'%Y-%m-%d')'" {print $1}' | xargs -r docker rm -f
+          docker run -d -t --gpus all --name ${container_name} --privileged --shm-size=128g \
+            -v "/home/data/cfs:/home/data/cfs" \
+            -v "/home/data/cfs/.cache:/root/.cache" \
+            -v "/home/data/cfs/.ccache:/root/.ccache" \
+            -v "/dev/shm:/dev/shm" \
+            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
+            -v ${{ github.workspace }}:/paddle \
+            -e python \
+            -e core_index \
+            -e BRANCH \
+            -e PR_ID \
+            -e COMMIT_ID \
+            -e work_dir \
+            -e PADDLE_ROOT \
+            -e ci_scripts \
+            -e no_proxy \
+            -e CI_name \
+            -e CACHE_DIR \
+            -e GITHUB_API_TOKEN \
+            -w /paddle --network host ${docker_image}
+
+      - name: Download Paddle and PaddleTest
+        env:
+          work_dir: ${{ github.workspace }}
+        run: |
+          docker exec -t ${{ env.container_name }} /bin/bash -c '
+          rm -rf * .[^.]*
+          echo "Downloading build.tar.gz"
+          wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/build/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
+          echo "Extracting build.tar.gz"
+          git config --global --add safe.directory ${work_dir}
+          tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
+          git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
+          git checkout test
+          rm build.tar.gz
+          cd /
+          echo "Downloading PaddleTest.tar.gz"
+          wget -q --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-check-certificate
+          echo "Extracting PaddleTest.tar.gz"
+          tar -zvxf PaddleTest.tar.gz 1>/dev/null 2>&1
+          '
+
+      - name: Check bypass
+        id: check-bypass
+        uses: ./.github/actions/check-bypass
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          workflow-name: ce-cinn
+
+      - name: Determine ci trigger
+        if: steps.check-bypass.outputs.can-skip != 'true'
+        env:
+          work_dir: ${{ github.workspace }}
+        run: |
+          docker exec -t ${{ env.container_name }} /bin/bash -c '
+          source ${{ github.workspace }}/../../../proxy
+          source ${ci_scripts}/ce_cinn_diff.sh
+          if [ ${sum_num} -eq 0 ];then
+            echo "The modified files does not affect LayerCase in CE-CINN-Framework, so skip this ci."
+            echo "skip_ci=true" >> ${{ github.env }}
+          fi
+          '
+
+      - name: Run check
+        if: ${{ steps.check-bypass.outputs.can-skip != 'true' && env.skip_ci != 'true' }}
+        env:
+          work_dir: ${{ github.workspace }}
+        run: |
+          docker exec -t ${{ env.container_name }} /bin/bash -c '
+          source ~/.bashrc
+          pip config set global.cache-dir "$CACHE_DIR/pip"
+          pip install /paddle/build/pr_whl/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
+          cd /PaddleTest/framework/e2e/PaddleLT_new
+          pip install -r requirement.txt
+          source ./scene/set_ci_dy^dy2stcinn_train^dy2stcinn_eval_inputspec_env.sh
+          python support/dict_to_yml.py --filename apibm_config.yml --data_str "$(cat $CACHE_DIR/cinn_config)"
+          set -e
+          python run.py
+          exit $(head -n 1 "exit_code.txt")
+          '
+
+      - name: Terminate and delete the container
+        if: always()
+        run: |
+          sudo rm -rf * .[^.]*
+          docker rm -f ${{ env.container_name }}
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -56,6 +56,7 @@ jobs:
           dockerfile_origin: https://raw.githubusercontent.com/PaddlePaddle/Paddle/refs/heads/develop/tools/dockerfile/Dockerfile.ubuntu20
         run: |
           set -x
+          rm -rf * .[^.]*
           if [ "${{ inputs.is_merge }}" == "true" ]; then
             mkdir -p tools/dockerfile && cd tools/dockerfile
             wget -q --no-proxy ${dockerfile_script} ${dockerfile_origin} --no-check-certificate
diff --git a/.github/workflows/re-run.yml b/.github/workflows/re-run.yml
@@ -199,6 +199,16 @@ jobs:
           REPO: ${{ github.event.repository.name }}
           JOB_NAME: 'Static-Check / Test'
 
+      - name: Rerun CE-CINN-Framework
+        if: ${{ contains(env.comment_body, 'ce-cinn') }}
+        uses: ./.github/actions/rerun-workflow
+        with:
+          PR_ID: ${{ github.event.issue.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OWNER: ${{ github.repository_owner }}
+          REPO: ${{ github.event.repository.name }}
+          JOB_NAME: 'CE-CINN-Framework / CINN'
+
       - name: Rerun Api-Benchmark
         if: ${{ contains(env.comment_body, 'api-bm') }}
         uses: ./.github/actions/rerun-workflow
diff --git a/ci/ce_cinn_diff.sh b/ci/ce_cinn_diff.sh
@@ -0,0 +1,38 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+PATH=/usr/local/bin:${PATH}
+echo "PATH=${PATH}" >> ~/.bashrc
+ln -sf $(which python3.10) /usr/local/bin/python
+ln -sf $(which pip3.10) /usr/local/bin/pip
+
+cp -r /PaddleTest/framework/e2e/PaddleLT_new/support/pr_info.py .
+python pr_info.py --pr_id ${PR_ID} --title_keyword CINN
+title_num=`grep -o '[0-9]\+' pr_title.log`
+
+git diff --numstat --diff-filter=AMRD $BRANCH | grep paddle/cinn | awk '{print $NF}' | tee pr_filelist1.log
+filelist_num1=`cat pr_filelist1.log | wc -l`
+
+git diff --numstat --diff-filter=AMRD $BRANCH | grep paddle/fluid/primitive/ | awk '{print $NF}' | tee pr_filelist2.log
+filelist_num2=`cat pr_filelist2.log | wc -l`
+
+git diff --numstat --diff-filter=AMRD $BRANCH | grep paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape/ | awk '{print $NF}' | tee pr_filelist3.log
+filelist_num3=`cat pr_filelist3.log | wc -l`
+
+git diff --numstat --diff-filter=AMRD $BRANCH | grep paddle/fluid/pir/dialect/operator/interface | awk '{print $NF}' | tee pr_filelist4.log
+filelist_num4=`cat pr_filelist4.log | wc -l`
+
+sum_num=$((title_num + filelist_num1 + filelist_num2 + filelist_num3 + filelist_num4))
+echo "sum_num: ${sum_num}"
