Add gcc-13 python13 (#69491)

tianshuo78520a · web-flow · commit 9932611863b4 · 2024-12-04T10:46:23.000+08:00
* Add gcc-13 python13

* fix gcc13

* Fix gcc

* Fix install gcc13

* Fix python3.13

* Fix pip3.13t

* Fix

* Fix pip install

* Fix

* Fix

* Fix

* Update

* Update pip

* Update

* Fix

* Fix pip

* Fix pip

* Fix pip

* Update Dockerfile.ubuntu20

fix

* Update Dockerfile.ubuntu20

test

* Update

* Update

* Fix

* Fix

* Fix

* fix test
diff --git a/python/unittest_py/requirements.txt b/python/unittest_py/requirements.txt
@@ -5,9 +5,10 @@ gymnasium>=1.0.0a1
 hypothesis
 opencv-python>= 4.10.0.84
 visualdl==2.5.3
-paddle2onnx>=0.9.6
+paddle2onnx>=0.9.6; python_version < "3.13"
 scipy>=1.6, !=1.7.2, !=1.7.3 ; platform_system == "Windows"
-scipy>=1.13 ; platform_system != "Windows"
+scipy == 1.10.1; python_version == "3.8"
+scipy > 1.13; python_version > "3.8"
 prettytable
 distro
 autograd==1.4
diff --git a/test/auto_parallel/custom_op/test_semi_auto_parallel_custom_op.py b/test/auto_parallel/custom_op/test_semi_auto_parallel_custom_op.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+import site
 import sys
 import unittest
 
@@ -31,9 +32,8 @@ def setUp(self):
         if os.name == 'nt':
             cmd = f'cd /d {cur_dir} && python custom_relu_setup.py install'
         else:
-            cmd = (
-                f'cd {cur_dir} && {sys.executable} custom_relu_setup.py install'
-            )
+            site_dir = site.getsitepackages()[0]
+            cmd = f'cd {cur_dir} && {sys.executable} custom_relu_setup.py install --install-lib={site_dir}'
         run_cmd(cmd)
 
     # test dynamic auto parallel run
diff --git a/test/cpp_extension/test_cpp_extension_setup.py b/test/cpp_extension/test_cpp_extension_setup.py
@@ -33,10 +33,12 @@ def setUp(self):
         cur_dir = os.path.dirname(os.path.abspath(__file__))
         # install general extension
         # compile, install the custom op egg into site-packages under background
+        site_dir = site.getsitepackages()[0]
         cmd = f'cd {cur_dir} && {sys.executable} cpp_extension_setup.py install'
+        if os.name != 'nt':
+            cmd += f' --install-lib={site_dir}'
         run_cmd(cmd)
 
-        site_dir = site.getsitepackages()[0]
         custom_egg_path = [
             x for x in os.listdir(site_dir) if 'custom_cpp_extension' in x
         ]
diff --git a/test/custom_op/test_custom_relu_op_setup.py b/test/custom_op/test_custom_relu_op_setup.py
@@ -149,9 +149,8 @@ def setUp(self):
         if os.name == 'nt':
             cmd = f'cd /d {cur_dir} && python custom_relu_setup.py install'
         else:
-            cmd = (
-                f'cd {cur_dir} && {sys.executable} custom_relu_setup.py install'
-            )
+            site_dir = site.getsitepackages()[0]
+            cmd = f'cd {cur_dir} && {sys.executable} custom_relu_setup.py install --install-lib={site_dir}'
         run_cmd(cmd)
 
         # NOTE(Aurelius84): Normally, it's no need to add following codes for users.
diff --git a/tools/dockerfile/Dockerfile.ubuntu20 b/tools/dockerfile/Dockerfile.ubuntu20
@@ -63,23 +63,12 @@ RUN apt-get update && \
   python3.9 python3.9-dev python3.9-distutils \
   python3.10 python3.10-dev python3.10-distutils \
   python3.11 python3.11-dev python3.11-distutils \
-  python3.12 python3.12-dev python3.12-distutils && \
+  python3.12 python3.12-dev python3.12-distutils \
+  python3.13 python3.13-dev python3.13-nogil && \
   apt-get install python-is-python3 && \
   rm /usr/bin/python && ln -s /usr/bin/python3.9 /usr/bin/python && \
   rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3
 
-WORKDIR /home
-RUN wget -q https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && unzip setuptools-50.3.2.zip
-WORKDIR /home/setuptools-50.3.2
-RUN python3.9 setup.py build && python3.9 setup.py install && \
-  python3.8 setup.py build && python3.8 setup.py install
-
-WORKDIR /home
-RUN wget -q https://files.pythonhosted.org/packages/ef/cc/93f7213b2ab5ed383f98ce8020e632ef256b406b8569606c3f160ed8e1c9/setuptools-68.2.2.tar.gz && tar xf setuptools-68.2.2.tar.gz
-WORKDIR /home/setuptools-68.2.2
-RUN python3.10 setup.py build && python3.10 setup.py install && \
-  python3.11 setup.py build && python3.11 setup.py install && \
-  python3.12 setup.py build && python3.12 setup.py install
 
 WORKDIR /home
 RUN wget -q https://bootstrap.pypa.io/get-pip.py
@@ -89,10 +78,19 @@ RUN python3.9 get-pip.py && \
   python3.11 get-pip.py && \
   python3.12 get-pip.py
 
-WORKDIR /home
-RUN rm setuptools-50.3.2.zip setuptools-68.2.2.tar.gz && \
-    rm -r setuptools-50.3.2 setuptools-68.2.2 get-pip.py
+RUN python3.13t get-pip.py && \
+  mv /usr/local/bin/pip3.13 /usr/local/bin/pip3.13t && \
+  python3.13 get-pip.py
+
+RUN python3.8 -m pip install setuptools==69.5.1 && \
+  python3.9 -m pip install setuptools==69.5.1 && \
+  python3.10 -m pip install setuptools==69.5.1 && \
+  python3.11 -m pip install setuptools==69.5.1 && \
+  python3.12 -m pip install --break-system-packages setuptools==69.5.1 && \
+  python3.13 -m pip install setuptools==69.5.1 && \
+  python3.13t -m pip install setuptools==69.5.1
 
+WORKDIR /home
 # remove them when apt-get support 2.27 and higher version
 RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \
     tar -xzf binutils-2.33.1.tar.gz && \
@@ -130,14 +128,18 @@ RUN pip3.8 --no-cache-dir install ipython==5.3.0 && \
     pip3.11 --no-cache-dir install ipython==5.3.0 && \
     pip3.11 --no-cache-dir install ipykernel==4.6.0 wheel && \
     pip3.12 --no-cache-dir install ipython==5.3.0 && \
-    pip3.12 --no-cache-dir install ipykernel==4.6.0 wheel
+    pip3.12 --no-cache-dir install ipykernel==4.6.0 wheel && \
+    pip3.13 --no-cache-dir install ipython==5.3.0 && \
+    pip3.13 --no-cache-dir install ipykernel==4.6.0 wheel && \
+    pip3.13t --no-cache-dir install wheel
 
 # For PaddleTest CE
 RUN pip3.8 --no-cache-dir install pytest && \
     pip3.9 --no-cache-dir install pytest && \
     pip3.10 --no-cache-dir install pytest && \
     pip3.11 --no-cache-dir install pytest && \
-    pip3.12 --no-cache-dir install pytest
+    pip3.12 --no-cache-dir install pytest && \
+    pip3.13 --no-cache-dir install pytest
 
 RUN pip3.8 --no-cache-dir install pre-commit==2.17.0 && \
     pip3.9 --no-cache-dir install pre-commit==2.17.0 && \
@@ -146,7 +148,8 @@ RUN pip3.8 --no-cache-dir install pre-commit==2.17.0 && \
     pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
     pip3.10 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
     pip3.11 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
-    pip3.12 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
+    pip3.12 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    pip3.13 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
 
 COPY ./python/requirements.txt /root/
 COPY ./python/unittest_py/requirements.txt /home/
@@ -158,7 +161,8 @@ RUN pip3.8 --no-cache-dir install -r  /root/requirements.txt && \
     pip3.11 --no-cache-dir install -r /root/requirements.txt && \
     pip3.11 --no-cache-dir install -r /home/requirements.txt && \
     pip3.12 --no-cache-dir install -r /root/requirements.txt && \
-    pip3.12 --no-cache-dir install -r /home/requirements.txt
+    pip3.12 --no-cache-dir install -r /home/requirements.txt && \
+    pip3.13 --no-cache-dir install -r /root/requirements.txt
 
 
 # ccache 4.2.0
diff --git a/tools/dockerfile/ci_dockerfile.sh b/tools/dockerfile/ci_dockerfile.sh
@@ -44,6 +44,10 @@ function make_cpu_dockerfile(){
     ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
     make -j8 \&\& make install " ${dockerfile_name}
   sed -i 's#<install_cpu_package>#RUN apt-get install -y gcc g++ make#g' ${dockerfile_name}
+  sed -i 's#RUN bash /build_scripts/install_gcc.sh gcc121#RUN add-apt-repository ppa:ubuntu-toolchain-r/test \&\& apt-get update \&\& apt-get install -y gcc-13 g++-13#g' ${dockerfile_name}
+  sed -i 's#/usr/local/gcc-12.1/bin/gcc#/usr/bin/gcc-13#g' ${dockerfile_name}
+  sed -i 's#/usr/local/gcc-12.1/bin/g++#/usr/bin/g++-13#g' ${dockerfile_name}
+  sed -i 's#ENV PATH=/usr/local/gcc-12.1/bin:$PATH##g' ${dockerfile_name}
 }
 
 
@@ -79,7 +83,7 @@ function make_ubuntu20_cu12_dockerfile(){
   dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
   sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
      tar -xzf  hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
-  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
+  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd ninja-build  \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
     tar -xvf git-2.17.1.tar.gz \&\& \
     cd git-2.17.1 \&\& \
     ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
