Skip to content

Commit b8df23b

Browse files
tianshuo78520atianshuo78520a
authored
Add 12.4 (#66560)
* Add cuda12.4 * test=document_fix * test=document_fix * test=document_fix * Fix * test=document_fix * test=document_fix * fix openssl * fix;test=document_fix * test=document_fix * test=document_fix * test=document_fix * Fix 12.4 * Fix 12.4 * test=document_fix * test=document_fix * test=document_fix * test=document_fix * test=document_fix * test=document_fix * test=document_fix * Add 12.6 docker * Update pip * Fix pip dir * Fix pip dir * Fix
1 parent 923cc24 commit b8df23b

File tree

7 files changed

+76
-32
lines changed

7 files changed

+76
-32
lines changed

‎tools/dockerfile/Dockerfile.centos

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH
1313
ENV LD_LIBRARY_PATH /usr/local/ssl/lib:/opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH}
1414
ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
1515

16-
RUN yum install -y bzip2 gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel patch
16+
RUN yum update -y && yum install -y bzip2 gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel libtool xz graphviz wget curl-devel patch perl swig
1717
COPY build_scripts /build_scripts
1818
RUN bash build_scripts/build.sh
1919
#RUN bash build_scripts/install_nccl2.sh
@@ -25,7 +25,7 @@ RUN ln -s /usr/local/ssl/include/openssl /usr/include
2525
RUN wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz && \
2626
tar -xvf git-2.17.1.tar.gz && \
2727
cd git-2.17.1 && \
28-
./configure --with-openssl --prefix=/usr/local && \
28+
./configure --with-openssl CFLAGS="-Dsocklen_t=uint32_t" --prefix=/usr/local && \
2929
make -j8 && make install
3030

3131
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
@@ -42,11 +42,6 @@ RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.15.12.l
4242
mkdir /root/gopath/src
4343

4444

45-
# protobuf 3.6.1
46-
RUN cd /opt && wget -q --no-check-certificate https://paddle-ci.cdn.bcebos.com/protobuf-cpp-3.6.1.tar.gz && \
47-
tar xzf protobuf-cpp-3.6.1.tar.gz && \
48-
cd protobuf-3.6.1 && ./configure && make -j4 && make install && cd .. && rm -f protobuf-cpp-3.6.1.tar.gz
49-
5045
RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt
5146

5247

@@ -70,14 +65,14 @@ RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} /opt/_i
7065
LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \
7166
LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install pre-commit 'ipython==5.3.0'
7267

73-
RUN wget -O /opt/swig-2.0.12.tar.gz --no-check-certificate https://sourceforge.net/projects/swig/files/swig/swig-2.0.12/swig-2.0.12.tar.gz/download && \
74-
cd /opt && tar xzf swig-2.0.12.tar.gz && cd /opt/swig-2.0.12 && ./configure && make && make install && cd /opt && rm swig-2.0.12.tar.gz
7568

76-
# ccache 3.7.9
77-
RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \
78-
tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \
79-
./configure -prefix=/usr/local/ccache-3.7.9 && \
69+
# ccache 4.8.2
70+
RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
71+
tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
72+
mkdir build && cd build && \
73+
cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
8074
make -j8 && make install && \
81-
ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache
75+
ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
76+
cd ../../ && rm -rf ccache-4.8.2.tar.gz
8277

8378
CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"]

‎tools/dockerfile/Dockerfile.ubuntu20

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -69,30 +69,29 @@ RUN apt-get update && \
6969
rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3
7070

7171
WORKDIR /home
72-
RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && unzip setuptools-50.3.2.zip
72+
RUN wget -q https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && unzip setuptools-50.3.2.zip
7373
WORKDIR /home/setuptools-50.3.2
7474
RUN python3.9 setup.py build && python3.9 setup.py install && \
7575
python3.8 setup.py build && python3.8 setup.py install
7676

7777
WORKDIR /home
78-
RUN wget https://files.pythonhosted.org/packages/ef/cc/93f7213b2ab5ed383f98ce8020e632ef256b406b8569606c3f160ed8e1c9/setuptools-68.2.2.tar.gz && tar xf setuptools-68.2.2.tar.gz
78+
RUN wget -q https://files.pythonhosted.org/packages/ef/cc/93f7213b2ab5ed383f98ce8020e632ef256b406b8569606c3f160ed8e1c9/setuptools-68.2.2.tar.gz && tar xf setuptools-68.2.2.tar.gz
7979
WORKDIR /home/setuptools-68.2.2
8080
RUN python3.10 setup.py build && python3.10 setup.py install && \
8181
python3.11 setup.py build && python3.11 setup.py install && \
8282
python3.12 setup.py build && python3.12 setup.py install
8383

8484
WORKDIR /home
85-
RUN wget https://files.pythonhosted.org/packages/1f/7f/4da15e07ccd11c84c1ccc8f6e24288d5e76c99441bf80e315b33542db951/pip-23.3.1.tar.gz && tar -zxf pip-23.3.1.tar.gz
86-
WORKDIR pip-23.3.1
87-
RUN python3.9 setup.py install && \
88-
python3.8 setup.py install && \
89-
python3.10 setup.py install && \
90-
python3.11 setup.py install && \
91-
python3.12 setup.py install
85+
RUN wget -q https://bootstrap.pypa.io/get-pip.py
86+
RUN python3.9 get-pip.py && \
87+
python3.8 get-pip.py && \
88+
python3.10 get-pip.py && \
89+
python3.11 get-pip.py && \
90+
python3.12 get-pip.py
9291

9392
WORKDIR /home
94-
RUN rm setuptools-50.3.2.zip setuptools-68.2.2.tar.gz pip-23.3.1.tar.gz && \
95-
rm -r setuptools-50.3.2 setuptools-68.2.2 pip-23.3.1
93+
RUN rm setuptools-50.3.2.zip setuptools-68.2.2.tar.gz && \
94+
rm -r setuptools-50.3.2 setuptools-68.2.2 get-pip.py
9695

9796
# remove them when apt-get support 2.27 and higher version
9897
RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \
@@ -151,7 +150,6 @@ RUN pip3.8 --no-cache-dir install pre-commit==2.17.0 && \
151150

152151
COPY ./python/requirements.txt /root/
153152
COPY ./python/unittest_py/requirements.txt /home/
154-
155153
RUN pip3.8 --no-cache-dir install -r /root/requirements.txt && \
156154
pip3.9 --no-cache-dir install -r /root/requirements.txt && \
157155
pip3.9 --no-cache-dir install -r /home/requirements.txt && \

‎tools/dockerfile/build_scripts/build.sh

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969
3636

3737
# Dependencies for compiling Python that we want to remove from
3838
# the final image after compiling Python
39-
PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel libffi-devel"
39+
PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel xz-devel libffi-devel"
4040

4141
# Libraries that are allowed as part of the manylinux1 profile
4242
MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel ncurses-devel freetype-devel libpng-devel"
@@ -49,11 +49,8 @@ source $MY_DIR/build_utils.sh
4949
yum -y install wget curl epel-release
5050

5151
# Development tools and libraries
52-
yum -y install bzip2 make git patch unzip bison yasm diffutils \
52+
yum -y install bzip2 make git patch unzip bison diffutils \
5353
automake which file \
54-
kernel-devel-`uname -r` \
55-
devtoolset-2-binutils devtoolset-2-gcc \
56-
devtoolset-2-gcc-c++ devtoolset-2-gcc-gfortran \
5754
${PYTHON_COMPILE_DEPS}
5855

5956
# Install more recent version of cmake

‎tools/dockerfile/build_scripts/build_utils.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ function do_openssl_build {
144144
./config -fPIC --prefix=/usr/local/ssl > /dev/null
145145
make > /dev/null
146146
make install > /dev/null
147-
147+
ln -sf /usr/lib64/libcrypto.so.1.1 /usr/local/ssl/lib/libcrypto.so.1.1
148148
}
149149

150150

‎tools/dockerfile/build_scripts/install_cudnn.sh

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,4 +96,14 @@ elif [[ "$1" == "cudnn900" ]]; then
9696
cp -r lib /usr && cd ../ && \
9797
rm -f cudnn-linux-x86_64-9.0.0.312_cuda12-archive.tar.xz && \
9898
rm -rf cudnn-linux-x86_64-9.0.0.312_cuda12-archive
99+
elif [[ "$1" == "cudnn911" ]]; then
100+
wget -q https://paddle-ci.gz.bcebos.com/cudnn/cudnn-linux-x86_64-9.1.1.17_cuda12-archive.tar.xz --no-check-certificate
101+
tar xJvf cudnn-linux-x86_64-9.1.1.17_cuda12-archive.tar.xz && \
102+
cd cudnn-linux-x86_64-9.1.1.17_cuda12-archive && \
103+
cp -r include /usr && \
104+
mkdir -p /usr/lib/x86_64-linux-gnu && \
105+
cp -r lib/libcudnn* /usr/lib/x86_64-linux-gnu && \
106+
cp -r lib /usr && cd ../ && \
107+
rm -f cudnn-linux-x86_64-9.1.1.17_cuda12-archive.tar.xz && \
108+
rm -rf cudnn-linux-x86_64-9.1.1.17_cuda12-archive
99109
fi

‎tools/dockerfile/centos7_manylinux.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,22 @@ function make_cuda123cudnn900trt8616() {
6060
sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
6161
}
6262

63+
function make_cuda124cudnn911trt8616() {
64+
sed 's/<baseimg>/12.4.1-cudnn-devel-rockylinux8/g' Dockerfile.rockylinux8 >Dockerfile.tmp
65+
sed -i "s#<install_gcc>#RUN dnf install -y gcc-toolset-12-gcc* \&\& source /opt/rh/gcc-toolset-12/enable \&\& echo 'source /opt/rh/gcc-toolset-12/enable' >>~/.bashrc \nENV PATH=/opt/rh/gcc-toolset-12/root/usr/bin:/usr/share/Modules/bin:\$PATH \nENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:\$LD_LIBRARY_PATH #g" Dockerfile.tmp
66+
sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_cudnn.sh cudnn911 \nENV CUDNN_VERSION=9.1.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
67+
sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8616#g" Dockerfile.tmp
68+
sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
69+
}
70+
71+
function make_cuda125cudnn911trt8616() {
72+
sed 's/<baseimg>/12.5.1-cudnn-devel-rockylinux8/g' Dockerfile.rockylinux8 >Dockerfile.tmp
73+
sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc122 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-12.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-12.2/bin:\$PATH \nRUN bash build_scripts/install_cudnn.sh cudnn911 \nENV CUDNN_VERSION=9.1.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
74+
sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8616#g" Dockerfile.tmp
75+
sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
76+
}
77+
78+
6379
function main() {
6480
local CMD=$1
6581
case $CMD in
@@ -81,6 +97,12 @@ function main() {
8197
cuda123cudnn900trt8616)
8298
make_cuda123cudnn900trt8616
8399
;;
100+
cuda124cudnn911trt8616)
101+
make_cuda124cudnn911trt8616
102+
;;
103+
cuda125cudnn911trt8616)
104+
make_cuda125cudnn911trt8616
105+
;;
84106
*)
85107
echo "Make dockerfile error, Without this paramet."
86108
exit 1

‎tools/dockerfile/ubuntu20_dev.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,24 @@ function base_image(){
8686
sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name}
8787
sed -i 's#cudnn841#cudnn900#g' ${dockerfile_name}
8888
sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=9.0.0#g' ${dockerfile_name}
89+
elif [[ ${ref_CUDA_MAJOR} == "12.4" ]];then
90+
dockerfile_name="Dockerfile-124"
91+
sed "s#<baseimg>#nvidia/cuda:12.4.1-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
92+
sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.4/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
93+
sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
94+
sed -i "s#<install_gcc>#WORKDIR /usr/bin ENV PATH=/usr/local/gcc-12.0/bin:\$PATH #g" ${dockerfile_name}
95+
sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name}
96+
sed -i 's#cudnn841#cudnn911#g' ${dockerfile_name}
97+
sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=9.1.1#g' ${dockerfile_name}
98+
elif [[ ${ref_CUDA_MAJOR} == "12.6" ]];then
99+
dockerfile_name="Dockerfile-126"
100+
sed "s#<baseimg>#nvidia/cuda:12.6.0-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
101+
sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.6/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
102+
sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
103+
sed -i "s#<install_gcc>#WORKDIR /usr/bin ENV PATH=/usr/local/gcc-12.0/bin:\$PATH #g" ${dockerfile_name}
104+
sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name}
105+
sed -i 's#cudnn841#cudnn911#g' ${dockerfile_name}
106+
sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=9.1.1#g' ${dockerfile_name}
89107
else
90108
echo "Dockerfile ERROR!!!"
91109
exit 1
@@ -108,3 +126,7 @@ export ref_CUDA_MAJOR=12.0
108126
base_image
109127
export ref_CUDA_MAJOR=12.3
110128
base_image
129+
export ref_CUDA_MAJOR=12.4
130+
base_image
131+
export ref_CUDA_MAJOR=12.6
132+
base_image

0 commit comments

Comments
 (0)