Skip to content

Commit 1c0b2ed

Browse files
GauthamBanasandraGauthamBanasandra
authored
HADOOP-17727. Modularize docker images (#3043)
Reviewed-by: Inigo Goiri <inigoiri@apache.org>
1 parent 57a3613 commit 1c0b2ed

22 files changed

+1190
-459
lines changed

‎dev-support/bin/create-release‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -514,7 +514,7 @@ function dockermode
514514

515515
echo "USER ${user_name}"
516516
printf "\n\n"
517-
) | docker build -t "${imgname}" -
517+
) | docker build -t "${imgname}" -f - "${BASEDIR}"/dev-support/docker/
518518

519519
run docker run -i -t \
520520
--privileged \

‎dev-support/docker/Dockerfile‎

Lines changed: 25 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -32,56 +32,20 @@ RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
3232
ENV DEBIAN_FRONTEND noninteractive
3333
ENV DEBCONF_TERSE true
3434

35-
# hadolint ignore=DL3008
35+
######
36+
# Platform package dependency resolver
37+
######
38+
COPY pkg-resolver pkg-resolver
39+
RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
40+
&& chmod a+r pkg-resolver/*.json
41+
42+
######
43+
# Install packages from apt
44+
######
45+
# hadolint ignore=DL3008,SC2046
3646
RUN apt-get -q update \
37-
&& apt-get -q install -y --no-install-recommends \
38-
ant \
39-
apt-utils \
40-
bats \
41-
build-essential \
42-
bzip2 \
43-
clang \
44-
cmake \
45-
curl \
46-
doxygen \
47-
fuse \
48-
g++ \
49-
gcc \
50-
git \
51-
gnupg-agent \
52-
hugo \
53-
libbcprov-java \
54-
libbz2-dev \
55-
libcurl4-openssl-dev \
56-
libfuse-dev \
57-
libprotobuf-dev \
58-
libprotoc-dev \
59-
libsasl2-dev \
60-
libsnappy-dev \
61-
libssl-dev \
62-
libtool \
63-
libzstd-dev \
64-
locales \
65-
make \
66-
maven \
67-
nodejs \
68-
node-yarn \
69-
npm \
70-
openjdk-11-jdk \
71-
openjdk-8-jdk \
72-
pinentry-curses \
73-
pkg-config \
74-
python3 \
75-
python3-pip \
76-
python3-pkg-resources \
77-
python3-setuptools \
78-
python3-wheel \
79-
rsync \
80-
shellcheck \
81-
software-properties-common \
82-
sudo \
83-
valgrind \
84-
zlib1g-dev \
47+
&& apt-get -q install -y --no-install-recommends python3 \
48+
&& apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \
8549
&& apt-get clean \
8650
&& rm -rf /var/lib/apt/lists/*
8751

@@ -97,89 +61,16 @@ ENV MAVEN_HOME /usr
9761
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
9862

9963
#######
100-
# Install SpotBugs 4.2.2
64+
# Set env vars for SpotBugs 4.2.2
10165
#######
102-
RUN mkdir -p /opt/spotbugs \
103-
&& curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
104-
-o /opt/spotbugs.tgz \
105-
&& tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
106-
&& chmod +x /opt/spotbugs/bin/*
10766
ENV SPOTBUGS_HOME /opt/spotbugs
10867

10968
#######
110-
# Install Boost 1.72 (1.71 ships with Focal)
69+
# Set env vars for Google Protobuf 3.7.1
11170
#######
112-
# hadolint ignore=DL3003
113-
RUN mkdir -p /opt/boost-library \
114-
&& curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
115-
&& mv boost_1_72_0.tar.bz2 /opt/boost-library \
116-
&& cd /opt/boost-library \
117-
&& tar --bzip2 -xf boost_1_72_0.tar.bz2 \
118-
&& cd /opt/boost-library/boost_1_72_0 \
119-
&& ./bootstrap.sh --prefix=/usr/ \
120-
&& ./b2 --without-python install \
121-
&& cd /root \
122-
&& rm -rf /opt/boost-library
123-
124-
######
125-
# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
126-
######
127-
# hadolint ignore=DL3003
128-
RUN mkdir -p /opt/protobuf-src \
129-
&& curl -L -s -S \
130-
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
131-
-o /opt/protobuf.tar.gz \
132-
&& tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
133-
&& cd /opt/protobuf-src \
134-
&& ./configure --prefix=/opt/protobuf \
135-
&& make "-j$(nproc)" \
136-
&& make install \
137-
&& cd /root \
138-
&& rm -rf /opt/protobuf-src
13971
ENV PROTOBUF_HOME /opt/protobuf
14072
ENV PATH "${PATH}:/opt/protobuf/bin"
14173

142-
####
143-
# Install pylint and python-dateutil
144-
####
145-
RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
146-
147-
####
148-
# Install bower
149-
####
150-
# hadolint ignore=DL3008
151-
RUN npm install -g bower@1.8.8
152-
153-
###
154-
# Install hadolint
155-
####
156-
RUN curl -L -s -S \
157-
https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
158-
-o /bin/hadolint \
159-
&& chmod a+rx /bin/hadolint \
160-
&& shasum -a 512 /bin/hadolint | \
161-
awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'
162-
163-
######
164-
# Intel ISA-L 2.29.0
165-
######
166-
# hadolint ignore=DL3003,DL3008
167-
RUN mkdir -p /opt/isa-l-src \
168-
&& apt-get -q update \
169-
&& apt-get install -y --no-install-recommends automake yasm \
170-
&& apt-get clean \
171-
&& curl -L -s -S \
172-
https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \
173-
-o /opt/isa-l.tar.gz \
174-
&& tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \
175-
&& cd /opt/isa-l-src \
176-
&& ./autogen.sh \
177-
&& ./configure \
178-
&& make "-j$(nproc)" \
179-
&& make install \
180-
&& cd /root \
181-
&& rm -rf /opt/isa-l-src
182-
18374
###
18475
# Avoid out of memory errors in builds
18576
###
@@ -188,6 +79,16 @@ ENV MAVEN_OPTS -Xms256m -Xmx3072m
18879
# Skip gpg verification when downloading Yetus via yetus-wrapper
18980
ENV HADOOP_SKIP_YETUS_VERIFICATION true
19081

82+
####
83+
# Install packages
84+
####
85+
RUN pkg-resolver/install-common-pkgs.sh
86+
RUN pkg-resolver/install-spotbugs.sh ubuntu:focal
87+
RUN pkg-resolver/install-boost.sh ubuntu:focal
88+
RUN pkg-resolver/install-protobuf.sh ubuntu:focal
89+
RUN pkg-resolver/install-hadolint.sh ubuntu:focal
90+
RUN pkg-resolver/install-intel-isa-l.sh ubuntu:focal
91+
19192
###
19293
# Everything past this point is either not needed for testing or breaks Yetus.
19394
# So tell Yetus not to read the rest of the file:

‎dev-support/docker/Dockerfile_aarch64‎

Lines changed: 21 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -33,61 +33,19 @@ ENV DEBIAN_FRONTEND noninteractive
3333
ENV DEBCONF_TERSE true
3434

3535
######
36-
# Install common dependencies from packages. Versions here are either
37-
# sufficient or irrelevant.
36+
# Platform package dependency resolver
3837
######
39-
# hadolint ignore=DL3008
38+
COPY pkg-resolver pkg-resolver
39+
RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
40+
&& chmod a+r pkg-resolver/*.json
41+
42+
######
43+
# Install packages from apt
44+
######
45+
# hadolint ignore=DL3008,SC2046
4046
RUN apt-get -q update \
41-
&& apt-get -q install -y --no-install-recommends \
42-
ant \
43-
apt-utils \
44-
bats \
45-
build-essential \
46-
bzip2 \
47-
clang \
48-
cmake \
49-
curl \
50-
doxygen \
51-
fuse \
52-
g++ \
53-
gcc \
54-
git \
55-
gnupg-agent \
56-
hugo \
57-
libbcprov-java \
58-
libbz2-dev \
59-
libcurl4-openssl-dev \
60-
libfuse-dev \
61-
libprotobuf-dev \
62-
libprotoc-dev \
63-
libsasl2-dev \
64-
libsnappy-dev \
65-
libssl-dev \
66-
libtool \
67-
libzstd-dev \
68-
locales \
69-
make \
70-
maven \
71-
nodejs \
72-
node-yarn \
73-
npm \
74-
openjdk-11-jdk \
75-
openjdk-8-jdk \
76-
phantomjs \
77-
pinentry-curses \
78-
pkg-config \
79-
python2.7 \
80-
python3 \
81-
python3-pip \
82-
python3-pkg-resources \
83-
python3-setuptools \
84-
python3-wheel \
85-
rsync \
86-
shellcheck \
87-
software-properties-common \
88-
sudo \
89-
valgrind \
90-
zlib1g-dev \
47+
&& apt-get -q install -y --no-install-recommends python3 \
48+
&& apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \
9149
&& apt-get clean \
9250
&& rm -rf /var/lib/apt/lists/*
9351

@@ -103,59 +61,16 @@ ENV MAVEN_HOME /usr
10361
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
10462

10563
#######
106-
# Install SpotBugs 4.2.2
64+
# Set env vars for SpotBugs 4.2.2
10765
#######
108-
RUN mkdir -p /opt/spotbugs \
109-
&& curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
110-
-o /opt/spotbugs.tgz \
111-
&& tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
112-
&& chmod +x /opt/spotbugs/bin/*
11366
ENV SPOTBUGS_HOME /opt/spotbugs
11467

11568
#######
116-
# Install Boost 1.72 (1.71 ships with Focal)
69+
# Set env vars for Google Protobuf 3.7.1
11770
#######
118-
# hadolint ignore=DL3003
119-
RUN mkdir -p /opt/boost-library \
120-
&& curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
121-
&& mv boost_1_72_0.tar.bz2 /opt/boost-library \
122-
&& cd /opt/boost-library \
123-
&& tar --bzip2 -xf boost_1_72_0.tar.bz2 \
124-
&& cd /opt/boost-library/boost_1_72_0 \
125-
&& ./bootstrap.sh --prefix=/usr/ \
126-
&& ./b2 --without-python install \
127-
&& cd /root \
128-
&& rm -rf /opt/boost-library
129-
130-
######
131-
# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
132-
######
133-
# hadolint ignore=DL3003
134-
RUN mkdir -p /opt/protobuf-src \
135-
&& curl -L -s -S \
136-
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
137-
-o /opt/protobuf.tar.gz \
138-
&& tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
139-
&& cd /opt/protobuf-src \
140-
&& ./configure --prefix=/opt/protobuf \
141-
&& make "-j$(nproc)" \
142-
&& make install \
143-
&& cd /root \
144-
&& rm -rf /opt/protobuf-src
14571
ENV PROTOBUF_HOME /opt/protobuf
14672
ENV PATH "${PATH}:/opt/protobuf/bin"
14773

148-
####
149-
# Install pylint and python-dateutil
150-
####
151-
RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
152-
153-
####
154-
# Install bower
155-
####
156-
# hadolint ignore=DL3008
157-
RUN npm install -g bower@1.8.8
158-
15974
###
16075
# Avoid out of memory errors in builds
16176
###
@@ -167,6 +82,14 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true
16782
# Force PhantomJS to be in 'headless' mode, do not connect to Xwindow
16883
ENV QT_QPA_PLATFORM offscreen
16984

85+
####
86+
# Install packages
87+
####
88+
RUN pkg-resolver/install-common-pkgs.sh
89+
RUN pkg-resolver/install-spotbugs.sh ubuntu:focal::arch64
90+
RUN pkg-resolver/install-boost.sh ubuntu:focal::arch64
91+
RUN pkg-resolver/install-protobuf.sh ubuntu:focal::arch64
92+
17093
###
17194
# Everything past this point is either not needed for testing or breaks Yetus.
17295
# So tell Yetus not to read the rest of the file:

0 commit comments

Comments
 (0)