[SOT][CI] add PR-CI-SOT workflow (#57754)

SigureMo · tianshuo78520a · web-flow · commit bbdf952be51c · 2023-10-11T11:17:10.000+08:00
Co-authored-by: tianshuo78520a &lt;707759223@qq.com&gt;
diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
@@ -932,6 +932,50 @@ set -ex
     fi
 }
 
+function run_sot_test() {
+    PADDLE_SOT_ROOT=$1
+    PY_VERSION=$2
+    PYTHON_WITH_SPECIFY_VERSION=python$PY_VERSION
+    PY_VERSION_NO_DOT=`echo $PY_VERSION | sed 's/\.//g'`
+
+    export STRICT_MODE=1
+    export COST_MODEL=False
+    export MIN_GRAPH_SIZE=0
+
+    # Install PaddlePaddle
+    $PYTHON_WITH_SPECIFY_VERSION -m pip install ${PADDLE_ROOT}/dist/paddlepaddle-0.0.0-cp${PY_VERSION_NO_DOT}-cp${PY_VERSION_NO_DOT}-linux_x86_64.whl
+    # Install PaddleSOT
+    cd $PADDLE_SOT_ROOT
+    $PYTHON_WITH_SPECIFY_VERSION -m pip install -e .
+
+    # Run unittest
+    cd tests
+    failed_tests=()
+
+    for file in ./test_*.py; do
+        # check file is python file
+        if [ -f "$file" ]; then
+            echo Running: PYTHONPATH=$PYTHONPATH " STRICT_MODE=1 python " $file
+            # run unittests
+            python_output=$($PYTHON_WITH_SPECIFY_VERSION $file 2>&1)
+
+            if [ $? -ne 0 ]; then
+                echo "run $file failed"
+                failed_tests+=("$file")
+                echo -e "$python_output"
+            fi
+        fi
+    done
+
+    if [ ${#failed_tests[@]} -ne 0 ]; then
+        echo "failed tests file:"
+        for failed_test in "${failed_tests[@]}"; do
+            echo "$failed_test"
+        done
+        exit 1
+    fi
+}
+
 function get_precision_ut_mac() {
     on_precision=0
     UT_list=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d')
@@ -3289,7 +3333,7 @@ function build_pr_and_develop() {
             rm -rf ${PADDLE_ROOT}/build/Makefile ${PADDLE_ROOT}/build/CMakeCache.txt ${PADDLE_ROOT}/build/build.ninja
             rm -rf ${PADDLE_ROOT}/build/third_party
         fi
-        
+
         git checkout -b develop_base_pr upstream/$BRANCH
         git submodule update --init
         run_setup ${PYTHON_ABI:-""} "rerun-cmake bdist_wheel" ${parallel_number}
@@ -3299,7 +3343,7 @@ function build_pr_and_develop() {
         mv ${PADDLE_ROOT}/dist/*.whl ${PADDLE_ROOT}/build/python/dist/
         mkdir ${PADDLE_ROOT}/build/dev_whl && cp ${PADDLE_ROOT}/build/python/dist/*.whl ${PADDLE_ROOT}/build/dev_whl
     fi
-    
+
     generate_api_spec "$1" "DEV"
 
 }
@@ -3555,7 +3599,7 @@ EOF
     export WITH_CUDNN_FRONTEND=${WITH_CUDNN_FRONTEND:-OFF}
     export WITH_SHARED_PHI=${WITH_SHARED_PHI:-OFF}
     export WITH_NVCC_LAZY=${WITH_NVCC_LAZY:-ON}
-    
+
     if [ "$SYSTEM" == "Linux" ];then
       if [ `nproc` -gt 16 ];then
           parallel_number=$(expr `nproc` - 8)
@@ -3568,7 +3612,7 @@ EOF
     if [ "$3" != "" ]; then
       parallel_number=$3
     fi
-    
+
     # reset ccache zero stats for collect PR's actual hit rate
     if [ "${MAX_JOBS}" == "" ]; then
         export MAX_JOBS=${parallel_number}
@@ -3845,7 +3889,7 @@ EOF
     fi
     # ci will collect ccache hit rate
     collect_ccache_hits
-    
+
     if [ "$build_error" != 0 ];then
         exit 7;
     fi
@@ -4077,6 +4121,19 @@ function main() {
       test_cicheck_py37)
         run_linux_cpu_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
         ;;
+      cicheck_sot)
+        export WITH_SHARED_PHI=ON
+        PADDLE_SOT_ROOT=${PADDLE_ROOT}/sot
+        git clone https://github.com/PaddlePaddle/PaddleSOT.git ${PADDLE_SOT_ROOT}
+        PYTHON_VERSIONS=(3.8 3.9 3.10 3.11)
+        for PY_VERSION in ${PYTHON_VERSIONS[@]}; do
+            ln -sf $(which python${PY_VERSION}) /usr/local/bin/python
+            ln -sf $(which pip${PY_VERSION}) /usr/local/bin/pip
+            run_setup ${PYTHON_ABI:-""} bdist_wheel ${parallel_number}
+            run_sot_test $PADDLE_SOT_ROOT $PY_VERSION
+            rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
+        done
+        ;;
       build_gpubox)
         run_setup ${PYTHON_ABI:-""} install ${parallel_number}
         ;;
diff --git a/python/unittest_py/requirements.txt b/python/unittest_py/requirements.txt
@@ -2,8 +2,8 @@ PyGithub
 coverage==5.5
 pycrypto ; platform_system != "Windows"
 mock
-gym==0.25.2
-pygame==2.1.0
+gym==0.26.2
+pygame==2.5.2
 hypothesis
 opencv-python<=4.2.0.32
 visualdl==2.5.3
diff --git a/test/dygraph_to_static/test_reinforcement_learning.py b/test/dygraph_to_static/test_reinforcement_learning.py
@@ -64,7 +64,7 @@ def train(args, place, to_static):
     paddle.jit.enable_to_static(to_static)
 
     env = gym.make('CartPole-v0')
-    env.seed(SEED)
+    env.reset(seed=SEED)
 
     with base.dygraph.guard(place):
         paddle.seed(SEED)
@@ -169,12 +169,13 @@ def finish_episode():
         loss_data = []
         running_reward = 10
         for i_episode in itertools.count(1):
-            state, ep_reward = env.reset(), 0
+            state, _ = env.reset()
+            ep_reward = 0
             # The default loop number is 10000 is models, we changed it to 1000 for smaller test
             for t in range(1, 1000):
                 state = np.array(state).astype("float32")
                 action, loss = select_action(state)
-                state, reward, done, _ = env.step(action)
+                state, reward, done, _, _ = env.step(action)
 
                 # log loss_probs
                 loss_data.append(float(loss))
diff --git a/tools/dockerfile/Dockerfile.ubuntu20 b/tools/dockerfile/Dockerfile.ubuntu20
@@ -61,7 +61,8 @@ RUN apt-get update && \
   apt-get install -y python3.7 python3.7-dev python3.7-distutils\
   python3.8 python3.8-dev python3.8-distutils \
   python3.9 python3.9-dev python3.9-distutils \
-  python3.10 python3.10-dev python3.10-distutils && \
+  python3.10 python3.10-dev python3.10-distutils \
+  python3.11 python3.11-dev python3.11-distutils && \
   apt-get install python-is-python3 && \
   rm /usr/bin/python && ln -s /usr/bin/python3.9 /usr/bin/python && \
   rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3
@@ -72,7 +73,8 @@ WORKDIR /home/setuptools-50.3.2
 RUN python3.9 setup.py build && python3.9 setup.py install && \
   python3.8 setup.py build && python3.8 setup.py install && \
   python3.7 setup.py build && python3.7 setup.py install && \
-  python3.10 setup.py build && python3.10 setup.py install
+  python3.10 setup.py build && python3.10 setup.py install && \
+  python3.11 setup.py build && python3.11 setup.py install
 
 WORKDIR /home
 RUN wget https://files.pythonhosted.org/packages/28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz && tar -zxvf pip-20.0.1.tar.gz
@@ -81,7 +83,8 @@ WORKDIR pip-20.0.1
 RUN python3.9 setup.py install && \
   python3.8 setup.py install && \
   python3.7 setup.py install && \
-  python3.10 setup.py install
+  python3.10 setup.py install && \
+  python3.11 setup.py install
 
 WORKDIR /home
 RUN rm setuptools-50.3.2.zip pip-20.0.1.tar.gz && \
@@ -115,7 +118,8 @@ RUN localedef -i en_US -f UTF-8 en_US.UTF-8
 RUN python3.7 -m pip --no-cache-dir install --upgrade pip && \
     python3.8 -m pip --no-cache-dir install --upgrade pip && \
     python3.9 -m pip --no-cache-dir install --upgrade pip && \
-    python3.10 -m pip --no-cache-dir install --upgrade pip
+    python3.10 -m pip --no-cache-dir install --upgrade pip && \
+    python3.11 -m pip --no-cache-dir install --upgrade pip
 
 RUN rm -f /usr/local/bin/pip && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip && \
     rm -f /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip3
@@ -127,13 +131,16 @@ RUN pip3.7 --no-cache-dir install ipython==5.3.0 && \
     pip3.9 --no-cache-dir install ipython==5.3.0 && \
     pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel && \
     pip3.10 --no-cache-dir install ipython==5.3.0 && \
-    pip3.10 --no-cache-dir install ipykernel==4.6.0 wheel
+    pip3.10 --no-cache-dir install ipykernel==4.6.0 wheel && \
+    pip3.11 --no-cache-dir install ipython==5.3.0 && \
+    pip3.11 --no-cache-dir install ipykernel==4.6.0 wheel
 
 #For docstring checker
 RUN pip3.7 --no-cache-dir install pytest astroid isort && \
     pip3.8 --no-cache-dir install pytest astroid isort && \
     pip3.9 --no-cache-dir install pytest astroid isort && \
-    pip3.10 --no-cache-dir install pytest astroid isort
+    pip3.10 --no-cache-dir install pytest astroid isort && \
+    pip3.11 --no-cache-dir install pytest astroid isort
 
 RUN pip3.7 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
     pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
@@ -142,7 +149,8 @@ RUN pip3.7 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
     pip3.7 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
     pip3.8 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
     pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
-    pip3.10 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
+    pip3.10 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    pip3.11 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
 
 COPY ./python/requirements.txt /root/
 COPY ./python/unittest_py/requirements.txt /home/
@@ -154,7 +162,9 @@ RUN pip3.7 --no-cache-dir install -r /root/requirements.txt && \
     pip3.9 --no-cache-dir install -r /root/requirements.txt && \
     pip3.9 --no-cache-dir install -r /home/requirements.txt && \
     pip3.10 --no-cache-dir install -r /root/requirements.txt && \
-    pip3.10 --no-cache-dir install -r /home/requirements.txt
+    pip3.10 --no-cache-dir install -r /home/requirements.txt && \
+    pip3.11 --no-cache-dir install -r /root/requirements.txt && \
+    pip3.11 --no-cache-dir install -r /home/requirements.txt
 
 
 # ccache 4.2.0
