Test sot docker;test=document_fix (#72236)

tianshuo78520a · web-flow · commit 63cdada61fdb · 2025-04-30T18:38:09.000+08:00
* Test sot docker;test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* Fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix;docker.yml

* test=document_fix;docker.yml

* test=document_fix;docker.yml

* test=document_fix;docker.yml

* test=document_fix;docker.yml

* test=document_fix;docker.yml

* test=document_fix;docker.yml

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* Update

* test=document_fix

* test=document_fix

* Update

* Fix docker build

* test=document_fix
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -20,10 +20,17 @@ jobs:
     name: Clone-linux
     uses: ./.github/workflows/_Clone-linux.yml
 
+  build-docker:
+    name: build docker images
+    needs: clone
+    uses: ./.github/workflows/docker.yml
+
   sot:
     name: PR-CI-SOT
     uses: ./.github/workflows/_SOT.yml
-    needs: clone
+    needs: build-docker
+    with:
+      docker_cpu_image: ${{ needs.build-docker.outputs.docker_cpu_image }}
 
   xpu:
     name: Linux-XPU
@@ -33,7 +40,9 @@ jobs:
   inference:
     name: PR-CI-Inference
     uses: ./.github/workflows/_Inference.yml
-    needs: clone
+    needs: build-docker
+    with:
+      docker_inference_image: ${{ needs.build-docker.outputs.docker_inference_image }}
 
   coverage:
     name: Coverage
diff --git a/.github/workflows/_Inference.yml b/.github/workflows/_Inference.yml
@@ -2,10 +2,12 @@ name: PR-CI-Inference
 
 on:
   workflow_call:
+    inputs:
+      docker_inference_image:
+        type: string
+        required: true
 
 env:
-  dockerfile: Dockerfile.cuda11.2_cudnn8.1_trt8.4_gcc8.2_ubuntu18
-  docker_image: 0ca5c4c33a67
   PR_ID: ${{ github.event.pull_request.number }}
   COMMIT_ID: ${{ github.event.pull_request.head.sha }}
   ci_scripts: /paddle/ci
@@ -65,6 +67,8 @@ jobs:
         run: |
           container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
           echo  "container_name=${container_name}" >> ${{ github.env }}
+          docker_image=${{ inputs.docker_inference_image }}
+          docker pull $docker_image
           docker run -d -t --name ${container_name} \
           -v "/home/data/cfs:/home/data/cfs" \
           -v "/home/data/cfs/.cache:/root/.cache" \
@@ -202,6 +206,8 @@ jobs:
         run: |
           container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
           echo "container_name=${container_name}" >> ${{ github.env }}
+          docker_image=${{ inputs.docker_inference_image }}
+          docker pull $docker_image
           docker run -d -t --gpus all --name ${container_name} \
           -v "/home/data/cfs:/home/data/cfs" \
           -v "/home/data/cfs/.cache:/root/.cache" \
diff --git a/.github/workflows/_SOT.yml b/.github/workflows/_SOT.yml
@@ -2,10 +2,12 @@ name: PR-CI-SOT
 
 on:
   workflow_call:
+    inputs:
+      docker_cpu_image:
+        type: string
+        required: true
 
 env:
-  dockerfile: Dockerfile.cuda9_cudnn7_gcc48_py35_centos6
-  docker_image: 077ca344c322
   PR_ID: ${{ github.event.pull_request.number }}
   COMMIT_ID: ${{ github.event.pull_request.head.sha }}
   TASK: paddle-CI-${{ github.event.pull_request.number }}-sot
@@ -35,7 +37,7 @@ jobs:
 
     steps:
       # sudo usermod -aG docker $USER
-      - name: Check docker image and run container
+      - name: run container
         env:
           BRANCH: ${{ github.base_ref }}
           work_dir: /paddle
@@ -58,9 +60,11 @@ jobs:
           WITH_AVX: "OFF"
           CCACHE_DIR: "/root/.ccache/sot"
         run: |
-          # bash ${{ github.workspace }}/paddle/scripts/paddle_build/check_docker_image.sh
+          set -x
           container_name=${TASK}-$(date +%s)
           echo "container_name=${container_name}" >> ${{ github.env }}
+          docker_image=${{ inputs.docker_cpu_image }}
+          docker pull $docker_image
           docker run -d -t --name ${container_name} \
             -v "/home/data/cfs:/home/data/cfs" \
             -v "/home/data/cfs/.cache:/root/.cache" \
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -0,0 +1,77 @@
+on:
+  workflow_call:
+    outputs:
+      docker_cpu_image:
+        description: "Generate images for all CI usage"
+        value: ${{ jobs.build-docker-images.outputs.docker_cpu_image }}
+
+      docker_inference_image:
+        description: "Generate images for all CI usage"
+        value: ${{ jobs.build-docker-images.outputs.docker_inference_image }}
+
+      docker_coverage_image:
+        description: "Generate images for all CI usage"
+        value: ${{ jobs.build-docker-images.outputs.docker_coverage_image }}
+
+      docker_build_image:
+        description: "Generate images for all CI usage"
+        value: ${{ jobs.build-docker-images.outputs.docker_build_image }}
+
+      docker_distribute_file:
+        description: "Generate images for all CI usage"
+        value: ${{ jobs.build-docker-images.outputs.docker_distribute_image }}
+
+jobs:
+  build-docker-images:
+    name: build docker
+    runs-on:
+      group: Docker-build
+    outputs:
+      docker_cpu_image: ${{ steps.build-docker-images.outputs.docker_cpu_image }}
+      docker_inference_image: ${{ steps.build-docker-images.outputs.docker_inference_image }}
+      docker_coverage_image: ${{ steps.build-docker-images.outputs.docker_coverage_image }}
+      docker_build_image: ${{ steps.build-docker-images.outputs.docker_build_image }}
+      docker_distribute_image: ${{ steps.build-docker-images.outputs.docker_distribute_image }}
+    steps:
+      - id: build-docker-images
+        name: build docker images
+        env:
+          PR_ID: ${{ github.event.pull_request.number }}
+          COMMIT_ID: ${{ github.event.pull_request.head.sha }}
+          docker_cpu_file: Dockerfile.cuda9_cudnn7_gcc48_py35_centos6
+          docker_inference_file: Dockerfile.cuda11.2_cudnn8.1_trt8.4_gcc8.2_ubuntu18
+          docker_coverage_file: Dockerfile.cuda117_cudnn8_gcc82_ubuntu18_coverage
+          docker_build_file: Dockerfile.cuda11.2_cudnn8_gcc82_trt8
+          docker_distribute_file: Dockerfile.cuda123_cudnn9_gcc122_ubuntu20
+        run: |
+          set -x
+          wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
+          tar xf Paddle.tar.gz --strip-components=1
+          rm Paddle.tar.gz
+          cd tools/dockerfile
+          bash ci_dockerfile.sh
+          cd ../..
+
+          # docker build images
+          declare -A docker_files=(["docker_cpu"]="$docker_cpu_file" ["docker_inference"]="$docker_inference_file" ["docker_coverage"]="$docker_coverage_file" ["docker_build"]="$docker_build_file" ["docker_distribute"]="$docker_distribute_file")
+          for name in "${!docker_files[@]}"
+          do
+            md5_value=`md5sum tools/dockerfile/${docker_files[$name]} | awk '{print $1}'`
+            docker_image="ccr-2vdh3abv-pub.cnc.bj.baidubce.com/ci/paddle:${md5_value}"
+            declare "${name}_image=${docker_image}"
+            echo "${name}_image=${docker_image}" >> $GITHUB_OUTPUT
+            set +e
+            docker pull $docker_image
+            if [ $? -eq 0 ];then
+              echo use docker cache
+            else
+              docker build -t $docker_image -f tools/dockerfile/${docker_files[$name]} .
+              docker push $docker_image
+              echo end docker build
+            fi
+            set -e
+          done
+
+          # clean workspace
+          cd ${{ github.workspace }}
+          rm -rf ./*
diff --git a/ci/check_docker_image.sh b/ci/check_docker_image.sh
diff --git a/ci/run_setup.sh b/ci/run_setup.sh
@@ -275,7 +275,7 @@ EOF
     else
       parallel_number=8
     fi
-    if [ $parallel_number_env != "" ]; then
+    if [ "$parallel_number_env" != "" ]; then
       parallel_number=$parallel_number_env
     fi
 
diff --git a/python/unittest_py/requirements.txt b/python/unittest_py/requirements.txt
@@ -5,7 +5,7 @@ gymnasium>=1.0.0a1
 hypothesis
 opencv-python>= 4.10.0.84
 visualdl==2.5.3
-paddle2onnx>=0.9.6; python_version < "3.13"
+paddle2onnx>=0.9.6; python_version < "3.12"
 scipy>=1.6, !=1.7.2, !=1.7.3 ; platform_system == "Windows"
 scipy == 1.10.1; python_version == "3.8"
 scipy > 1.13; python_version > "3.8"
