Add py3.10 (#55286)

tianshuo78520a · web-flow · commit 97ec1d84c975 · 2023-07-26T11:47:53.000+08:00
* Add py3.10;test=py3-ninja

* Add py3.10;test=py3-ninja

* test=py3-ninja

* test=py3-ninja

* test=py3-ninja

* test=py3-ninja

* test=py3-ninja

* Fix test error

* Fix build docker error

* Fix build docker error
diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py
@@ -16,6 +16,7 @@
 import collections
 import glob
 import hashlib
+import importlib.abc
 import importlib.util
 import json
 import logging
@@ -173,6 +174,7 @@ def custom_write_stub(resource, pyfile):
         import sys
         import types
         import paddle
+        import importlib.abc
         import importlib.util
 
         cur_dir = os.path.dirname(os.path.abspath(__file__))
diff --git a/tools/dockerfile/Dockerfile.ubuntu20 b/tools/dockerfile/Dockerfile.ubuntu20
@@ -61,41 +61,40 @@ ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH
 RUN apt-get update && \
   apt-get install -y python3.7 python3.7-dev python3.7-distutils\
   python3.8 python3.8-dev python3.8-distutils \
-  python3.9 python3.9-dev python3.9-distutils && \
+  python3.9 python3.9-dev python3.9-distutils \
+  python3.10 python3.10-dev python3.10-distutils && \
   apt-get install python-is-python3 && \
-  rm /usr/bin/python && ln -s /usr/bin/python3.7 /usr/bin/python && \
-  rm /usr/bin/python3 && ln -s /usr/bin/python3.7 /usr/bin/python3
+  rm /usr/bin/python && ln -s /usr/bin/python3.9 /usr/bin/python && \
+  rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3
 
 
 WORKDIR /home
 RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && apt-get -y install unzip && unzip setuptools-50.3.2.zip
 WORKDIR /home/setuptools-50.3.2
 RUN python3.9 setup.py build && python3.9 setup.py install && \
   python3.8 setup.py build && python3.8 setup.py install && \
-  python3.7 setup.py build && python3.7 setup.py install
+  python3.7 setup.py build && python3.7 setup.py install && \
+  python3.10 setup.py build && python3.10 setup.py install
 
 WORKDIR /home
 RUN wget https://files.pythonhosted.org/packages/28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz && tar -zxvf pip-20.0.1.tar.gz
 WORKDIR pip-20.0.1
 
 RUN python3.9 setup.py install && \
   python3.8 setup.py install && \
-  python3.7 setup.py install
+  python3.7 setup.py install && \
+  python3.10 setup.py install
 
 WORKDIR /home
 RUN rm setuptools-50.3.2.zip pip-20.0.1.tar.gz && \
     rm -r setuptools-50.3.2 pip-20.0.1
-RUN rm /usr/local/bin/pip && ln -s /usr/local/bin/pip3.7 /usr/local/bin/pip && \
-  rm /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.7 /usr/local/bin/pip3
-
 
 # remove them when apt-get support 2.27 and higher version
 RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \ 
     tar -xzf binutils-2.33.1.tar.gz && \ 
     cd binutils-2.33.1 && \
     ./configure && make -j && make install && cd .. && rm -rf binutils-2.33.1 binutils-2.33.1.tar.gz
 
-
 # Install Go and glide
 RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.17.2.linux-amd64.tar.gz | \
     tar -xz -C /usr/local && \
@@ -114,34 +113,44 @@ RUN git config --global credential.helper store
 # Fix locales to en_US.UTF-8
 RUN localedef -i en_US -f UTF-8 en_US.UTF-8
 
+#For pre-commit
+RUN python3.7 -m pip --no-cache-dir install --upgrade pip && \
+    python3.8 -m pip --no-cache-dir install --upgrade pip && \
+    python3.9 -m pip --no-cache-dir install --upgrade pip && \
+    python3.10 -m pip --no-cache-dir install --upgrade pip
+
+RUN rm -f /usr/local/bin/pip && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip && \
+    rm -f /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip3
+
 RUN pip3.7 --no-cache-dir install ipython==5.3.0 && \
     pip3.7 --no-cache-dir install ipykernel==4.6.0 wheel && \
     pip3.8 --no-cache-dir install ipython==5.3.0 && \
     pip3.8 --no-cache-dir install ipykernel==4.6.0 wheel && \
     pip3.9 --no-cache-dir install ipython==5.3.0 && \
-    pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel
+    pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel && \
+    pip3.10 --no-cache-dir install ipython==5.3.0 && \
+    pip3.10 --no-cache-dir install ipykernel==4.6.0 wheel
 
 #For docstring checker
 RUN pip3.7 --no-cache-dir install pytest astroid isort && \
     pip3.8 --no-cache-dir install pytest astroid isort && \
-    pip3.9 --no-cache-dir install pytest astroid isort
-
-#For pre-commit
-RUN pip3.7 --no-cache-dir install --upgrade pip && \
-    pip3.8 --no-cache-dir install --upgrade pip && \
-    pip3.9 --no-cache-dir install --upgrade pip
+    pip3.9 --no-cache-dir install pytest astroid isort && \
+    pip3.10 --no-cache-dir install pytest astroid isort
 
 RUN pip3.7 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
     pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
     pip3.9 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
+    pip3.10 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
     pip3.7 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
     pip3.8 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
-    pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
+    pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    pip3.10 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
 
 COPY ./python/requirements.txt /root/
 RUN pip3.7 --no-cache-dir install -r /root/requirements.txt && \
     pip3.8 --no-cache-dir install -r /root/requirements.txt && \
-    pip3.9 --no-cache-dir install -r /root/requirements.txt
+    pip3.9 --no-cache-dir install -r /root/requirements.txt && \
+    pip3.10 --no-cache-dir install -r /root/requirements.txt
 
 # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
 #RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
diff --git a/tools/dockerfile/ci_dockerfile.sh b/tools/dockerfile/ci_dockerfile.sh
@@ -14,32 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-function make_ubuntu_dockerfile(){
-  dockerfile_name="Dockerfile.cuda10_cudnn7_gcc82_ubuntu16"
-  sed "s/<baseimg>/10.1-cudnn7-devel-ubuntu16.04/g" ./Dockerfile.ubuntu >${dockerfile_name}
-  sed -i "s#liblzma-dev#liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev#g" ${dockerfile_name} 
-  dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
-  sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
-     tar -xzf     hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libcurl4-openssl-dev gettext zstd \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
-    tar -xvf git-2.17.1.tar.gz \&\& \
-    cd git-2.17.1 \&\& \
-    ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
-    make -j8 \&\& make install " ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub distro" ${dockerfile_name}
-  sed -i "s#<install_gcc>#WORKDIR /usr/bin \\
-    COPY tools/dockerfile/build_scripts /build_scripts \\
-    RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \\
-    RUN cp gcc  gcc.bak \&\& cp g++  g++.bak \&\& rm gcc \&\& rm g++ \\
-    RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \\
-    RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \\
-    RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \\
-    RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \\
-    ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name}
-  sed -i "s#bash /build_scripts/install_nccl2.sh#wget -q --no-proxy https://nccl2-deb.cdn.bcebos.com/nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb \\
-    RUN dpkg -i nccl-repo-ubuntu1604-2.7.8-ga-cuda10.1_1-1_amd64.deb \\
-    RUN apt remove -y libnccl* --allow-change-held-packages \&\&  apt-get install -y libsndfile1 libnccl2=2.7.8-1+cuda10.1 libnccl-dev=2.7.8-1+cuda10.1 zstd pigz ninja-build --allow-change-held-packages #g" ${dockerfile_name}
-}
 
 function make_ubuntu_trt7_dockerfile(){
   dockerfile_name="Dockerfile.cuda102_cudnn8_gcc82_ubuntu16"
@@ -88,9 +62,9 @@ function make_centos_dockerfile(){
   sed -i 's#RUN bash /build_scripts/install_trt.sh##g' ${dockerfile_name}
   sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
      tar -xzf     hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libsndfile1 zstd pigz libcurl4-openssl-dev gettext zstd ninja-build" ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.8 install PyGithub distro \&\& pip3.9 install PyGithub wheel \&\& pip3.10 install PyGithub distro" ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
+  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libsndfile1 zstd pigz ninja-build" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN pip install wheel PyGithub distro" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libcurl4-openssl-dev gettext pigz \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
     tar -xvf git-2.17.1.tar.gz \&\& \
     cd git-2.17.1 \&\& \
     ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
@@ -158,7 +132,7 @@ function make_unbuntu18_cu117_dockerfile(){
   dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
   sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
      tar -xzf  hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
+  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
     tar -xvf git-2.17.1.tar.gz \&\& \
     cd git-2.17.1 \&\& \
     ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
@@ -196,7 +170,6 @@ function make_ubuntu18_cu112_dockerfile(){
 }
 
 function main() {
-  make_ubuntu_dockerfile
   make_ubuntu_trt7_dockerfile
   make_centos_dockerfile
   make_cinn_dockerfile
