Add 12.4 (#66560)

tianshuo78520a · web-flow · commit b8df23b6b764 · 2024-10-14T16:02:21.000+08:00
* Add cuda12.4

* test=document_fix

* test=document_fix

* test=document_fix

* Fix

* test=document_fix

* test=document_fix

* fix openssl

* fix;test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* Fix 12.4

* Fix 12.4

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* test=document_fix

* Add 12.6 docker

* Update pip

* Fix pip dir

* Fix pip dir

* Fix
diff --git a/tools/dockerfile/Dockerfile.centos b/tools/dockerfile/Dockerfile.centos
@@ -13,7 +13,7 @@ ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH
 ENV LD_LIBRARY_PATH /usr/local/ssl/lib:/opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH}
 ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
 
-RUN yum install -y bzip2 gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel patch
+RUN yum update -y && yum install -y bzip2 gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel libtool xz graphviz wget curl-devel patch perl swig
 COPY build_scripts /build_scripts
 RUN bash build_scripts/build.sh
 #RUN bash build_scripts/install_nccl2.sh
@@ -25,7 +25,7 @@ RUN ln -s /usr/local/ssl/include/openssl /usr/include
 RUN wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz && \
   tar -xvf git-2.17.1.tar.gz && \
   cd git-2.17.1 && \
-  ./configure --with-openssl --prefix=/usr/local && \
+  ./configure --with-openssl CFLAGS="-Dsocklen_t=uint32_t" --prefix=/usr/local && \
   make -j8 && make install
 
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem
@@ -42,11 +42,6 @@ RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.15.12.l
     mkdir /root/gopath/src
 
 
-# protobuf 3.6.1
-RUN cd /opt && wget -q --no-check-certificate https://paddle-ci.cdn.bcebos.com/protobuf-cpp-3.6.1.tar.gz && \
-    tar xzf protobuf-cpp-3.6.1.tar.gz && \
-    cd protobuf-3.6.1 && ./configure && make -j4 && make install && cd .. && rm -f protobuf-cpp-3.6.1.tar.gz
-
 RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt
 
 
@@ -70,14 +65,14 @@ RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} /opt/_i
     LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \
     LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install pre-commit 'ipython==5.3.0'
 
-RUN wget -O /opt/swig-2.0.12.tar.gz --no-check-certificate https://sourceforge.net/projects/swig/files/swig/swig-2.0.12/swig-2.0.12.tar.gz/download && \
-    cd /opt && tar xzf swig-2.0.12.tar.gz && cd /opt/swig-2.0.12 && ./configure && make && make install && cd /opt && rm swig-2.0.12.tar.gz
 
-# ccache 3.7.9
-RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \
-    tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \
-    ./configure -prefix=/usr/local/ccache-3.7.9 && \
+# ccache 4.8.2
+RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
+    tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
     make -j8 && make install && \
-    ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache
+    ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
+    cd ../../ && rm -rf ccache-4.8.2.tar.gz
 
 CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"]
diff --git a/tools/dockerfile/Dockerfile.ubuntu20 b/tools/dockerfile/Dockerfile.ubuntu20
@@ -69,30 +69,29 @@ RUN apt-get update && \
   rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3
 
 WORKDIR /home
-RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && unzip setuptools-50.3.2.zip
+RUN wget -q https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && unzip setuptools-50.3.2.zip
 WORKDIR /home/setuptools-50.3.2
 RUN python3.9 setup.py build && python3.9 setup.py install && \
   python3.8 setup.py build && python3.8 setup.py install
 
 WORKDIR /home
-RUN wget https://files.pythonhosted.org/packages/ef/cc/93f7213b2ab5ed383f98ce8020e632ef256b406b8569606c3f160ed8e1c9/setuptools-68.2.2.tar.gz && tar xf setuptools-68.2.2.tar.gz
+RUN wget -q https://files.pythonhosted.org/packages/ef/cc/93f7213b2ab5ed383f98ce8020e632ef256b406b8569606c3f160ed8e1c9/setuptools-68.2.2.tar.gz && tar xf setuptools-68.2.2.tar.gz
 WORKDIR /home/setuptools-68.2.2
 RUN python3.10 setup.py build && python3.10 setup.py install && \
   python3.11 setup.py build && python3.11 setup.py install && \
   python3.12 setup.py build && python3.12 setup.py install
 
 WORKDIR /home
-RUN wget https://files.pythonhosted.org/packages/1f/7f/4da15e07ccd11c84c1ccc8f6e24288d5e76c99441bf80e315b33542db951/pip-23.3.1.tar.gz && tar -zxf pip-23.3.1.tar.gz
-WORKDIR pip-23.3.1
-RUN python3.9 setup.py install && \
-  python3.8 setup.py install  && \
-  python3.10 setup.py install && \
-  python3.11 setup.py install && \
-  python3.12 setup.py install
+RUN wget -q https://bootstrap.pypa.io/get-pip.py
+RUN python3.9 get-pip.py && \
+  python3.8 get-pip.py && \
+  python3.10 get-pip.py && \
+  python3.11 get-pip.py && \
+  python3.12 get-pip.py
 
 WORKDIR /home
-RUN rm setuptools-50.3.2.zip setuptools-68.2.2.tar.gz pip-23.3.1.tar.gz && \
-    rm -r setuptools-50.3.2 setuptools-68.2.2 pip-23.3.1
+RUN rm setuptools-50.3.2.zip setuptools-68.2.2.tar.gz && \
+    rm -r setuptools-50.3.2 setuptools-68.2.2 get-pip.py
 
 # remove them when apt-get support 2.27 and higher version
 RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \
@@ -151,7 +150,6 @@ RUN pip3.8 --no-cache-dir install pre-commit==2.17.0 && \
 
 COPY ./python/requirements.txt /root/
 COPY ./python/unittest_py/requirements.txt /home/
-
 RUN pip3.8 --no-cache-dir install -r  /root/requirements.txt && \
     pip3.9 --no-cache-dir install -r  /root/requirements.txt && \
     pip3.9 --no-cache-dir install -r  /home/requirements.txt && \
diff --git a/tools/dockerfile/build_scripts/build.sh b/tools/dockerfile/build_scripts/build.sh
@@ -36,7 +36,7 @@ AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969
 
 # Dependencies for compiling Python that we want to remove from
 # the final image after compiling Python
-PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel libffi-devel"
+PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel xz-devel libffi-devel"
 
 # Libraries that are allowed as part of the manylinux1 profile
 MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel  mesa-libGL-devel libICE-devel libSM-devel ncurses-devel freetype-devel libpng-devel"
@@ -49,11 +49,8 @@ source $MY_DIR/build_utils.sh
 yum -y install wget curl epel-release
 
 # Development tools and libraries
-yum -y install bzip2 make git patch unzip bison yasm diffutils \
+yum -y install bzip2 make git patch unzip bison diffutils \
     automake which file \
-    kernel-devel-`uname -r` \
-    devtoolset-2-binutils devtoolset-2-gcc \
-    devtoolset-2-gcc-c++ devtoolset-2-gcc-gfortran \
     ${PYTHON_COMPILE_DEPS}
 
 # Install more recent version of cmake
diff --git a/tools/dockerfile/build_scripts/build_utils.sh b/tools/dockerfile/build_scripts/build_utils.sh
@@ -144,7 +144,7 @@ function do_openssl_build {
     ./config -fPIC --prefix=/usr/local/ssl > /dev/null
     make > /dev/null
     make install > /dev/null
-
+    ln -sf /usr/lib64/libcrypto.so.1.1 /usr/local/ssl/lib/libcrypto.so.1.1
 }
 
 
diff --git a/tools/dockerfile/build_scripts/install_cudnn.sh b/tools/dockerfile/build_scripts/install_cudnn.sh
@@ -96,4 +96,14 @@ elif [[ "$1" == "cudnn900" ]]; then
   cp -r lib /usr && cd ../ && \
   rm -f cudnn-linux-x86_64-9.0.0.312_cuda12-archive.tar.xz && \
   rm -rf cudnn-linux-x86_64-9.0.0.312_cuda12-archive
+elif [[ "$1" == "cudnn911" ]]; then
+  wget -q https://paddle-ci.gz.bcebos.com/cudnn/cudnn-linux-x86_64-9.1.1.17_cuda12-archive.tar.xz --no-check-certificate
+  tar xJvf cudnn-linux-x86_64-9.1.1.17_cuda12-archive.tar.xz && \
+  cd cudnn-linux-x86_64-9.1.1.17_cuda12-archive && \
+  cp -r include /usr && \
+  mkdir -p /usr/lib/x86_64-linux-gnu && \
+  cp -r lib/libcudnn* /usr/lib/x86_64-linux-gnu && \
+  cp -r lib /usr && cd ../ && \
+  rm -f cudnn-linux-x86_64-9.1.1.17_cuda12-archive.tar.xz && \
+  rm -rf cudnn-linux-x86_64-9.1.1.17_cuda12-archive
 fi
diff --git a/tools/dockerfile/centos7_manylinux.sh b/tools/dockerfile/centos7_manylinux.sh
@@ -60,6 +60,22 @@ function make_cuda123cudnn900trt8616() {
   sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
 }
 
+function make_cuda124cudnn911trt8616() {
+  sed 's/<baseimg>/12.4.1-cudnn-devel-rockylinux8/g' Dockerfile.rockylinux8 >Dockerfile.tmp
+  sed -i "s#<install_gcc>#RUN dnf install -y gcc-toolset-12-gcc* \&\& source /opt/rh/gcc-toolset-12/enable \&\& echo 'source /opt/rh/gcc-toolset-12/enable' >>~/.bashrc \nENV PATH=/opt/rh/gcc-toolset-12/root/usr/bin:/usr/share/Modules/bin:\$PATH \nENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:\$LD_LIBRARY_PATH #g" Dockerfile.tmp
+  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_cudnn.sh cudnn911 \nENV CUDNN_VERSION=9.1.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
+  sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8616#g" Dockerfile.tmp
+  sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
+}
+
+function make_cuda125cudnn911trt8616() {
+  sed 's/<baseimg>/12.5.1-cudnn-devel-rockylinux8/g' Dockerfile.rockylinux8 >Dockerfile.tmp
+  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc122 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-12.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-12.2/bin:\$PATH \nRUN bash build_scripts/install_cudnn.sh cudnn911 \nENV CUDNN_VERSION=9.1.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
+  sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8616#g" Dockerfile.tmp
+  sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
+}
+
+
 function main() {
   local CMD=$1
   case $CMD in
@@ -81,6 +97,12 @@ function main() {
     cuda123cudnn900trt8616)
      make_cuda123cudnn900trt8616
      ;;
+    cuda124cudnn911trt8616)
+     make_cuda124cudnn911trt8616
+     ;;
+    cuda125cudnn911trt8616)
+     make_cuda125cudnn911trt8616
+     ;;
     *)
       echo "Make dockerfile error, Without this paramet."
       exit 1
diff --git a/tools/dockerfile/ubuntu20_dev.sh b/tools/dockerfile/ubuntu20_dev.sh
@@ -86,6 +86,24 @@ function base_image(){
     sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name}
     sed -i 's#cudnn841#cudnn900#g' ${dockerfile_name}
     sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=9.0.0#g' ${dockerfile_name}
+  elif [[ ${ref_CUDA_MAJOR} == "12.4" ]];then
+    dockerfile_name="Dockerfile-124"
+    sed "s#<baseimg>#nvidia/cuda:12.4.1-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
+    sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.4/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+    sed -i "s#<install_gcc>#WORKDIR /usr/bin ENV PATH=/usr/local/gcc-12.0/bin:\$PATH #g" ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name}
+    sed -i 's#cudnn841#cudnn911#g' ${dockerfile_name}
+    sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=9.1.1#g' ${dockerfile_name}
+  elif [[ ${ref_CUDA_MAJOR} == "12.6" ]];then
+    dockerfile_name="Dockerfile-126"
+    sed "s#<baseimg>#nvidia/cuda:12.6.0-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
+    sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.6/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+    sed -i "s#<install_gcc>#WORKDIR /usr/bin ENV PATH=/usr/local/gcc-12.0/bin:\$PATH #g" ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name}
+    sed -i 's#cudnn841#cudnn911#g' ${dockerfile_name}
+    sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=9.1.1#g' ${dockerfile_name}
   else
     echo "Dockerfile ERROR!!!"
     exit 1
@@ -108,3 +126,7 @@ export ref_CUDA_MAJOR=12.0
 base_image
 export ref_CUDA_MAJOR=12.3
 base_image
+export ref_CUDA_MAJOR=12.4
+base_image
+export ref_CUDA_MAJOR=12.6
+base_image

Original file line number	Diff line number	Diff line change
`@@ -144,7 +144,7 @@ function do_openssl_build {`
`144`	`144`	`./config -fPIC --prefix=/usr/local/ssl > /dev/null`
`145`	`145`	`make > /dev/null`
`146`	`146`	`make install > /dev/null`
`147`		`-`
	`147`	`+ ln -sf /usr/lib64/libcrypto.so.1.1 /usr/local/ssl/lib/libcrypto.so.1.1`
`148`	`148`	`}`
`149`	`149`
`150`	`150`