HADOOP-17727. Modularize docker images

dev-support/bin/create-release

@@ -514,7 +514,7 @@ function dockermode
 
     echo "USER ${user_name}"
     printf "\n\n"
-  ) | docker build -t "${imgname}" -
+  ) | docker build -t "${imgname}" -f - "${BASEDIR}"/dev-support/docker/
 
   run docker run -i -t \
     --privileged \

dev-support/docker/Dockerfile

@@ -32,56 +32,20 @@ RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
 ENV DEBIAN_FRONTEND noninteractive
 ENV DEBCONF_TERSE true
 
-# hadolint ignore=DL3008
+######
+# Platform package dependency resolver
+######
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+    && chmod a+r pkg-resolver/*.json
+
+######
+# Install packages from apt
+######
+# hadolint ignore=DL3008,SC2046
 RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends \
-        ant \
-        apt-utils \
-        bats \
-        build-essential \
-        bzip2 \
-        clang \
-        cmake \
-        curl \
-        doxygen \
-        fuse \
-        g++ \
-        gcc \
-        git \
-        gnupg-agent \
-        hugo \
-        libbcprov-java \
-        libbz2-dev \
-        libcurl4-openssl-dev \
-        libfuse-dev \
-        libprotobuf-dev \
-        libprotoc-dev \
-        libsasl2-dev \
-        libsnappy-dev \
-        libssl-dev \
-        libtool \
-        libzstd-dev \
-        locales \
-        make \
-        maven \
-        nodejs \
-        node-yarn \
-        npm \
-        openjdk-11-jdk \
-        openjdk-8-jdk \
-        pinentry-curses \
-        pkg-config \
-        python3 \
-        python3-pip \
-        python3-pkg-resources \
-        python3-setuptools \
-        python3-wheel \
-        rsync \
-        shellcheck \
-        software-properties-common \
-        sudo \
-        valgrind \
-        zlib1g-dev \
+    && apt-get -q install -y --no-install-recommends python3 \
+    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
@@ -97,89 +61,16 @@ ENV MAVEN_HOME /usr
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
 
 #######
-# Install SpotBugs 4.2.2
+# Set env vars for SpotBugs 4.2.2
 #######
-RUN mkdir -p /opt/spotbugs \
-    && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-      -o /opt/spotbugs.tgz \
-    && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
-    && chmod +x /opt/spotbugs/bin/*
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Install Boost 1.72 (1.71 ships with Focal)
+# Set env vars for Google Protobuf 3.7.1
 #######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/boost-library \
-    && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
-    && mv boost_1_72_0.tar.bz2 /opt/boost-library \
-    && cd /opt/boost-library \
-    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
-    && cd /opt/boost-library/boost_1_72_0 \
-    && ./bootstrap.sh --prefix=/usr/ \
-    && ./b2 --without-python install \
-    && cd /root \
-    && rm -rf /opt/boost-library
-
-######
-# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
-######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/protobuf-src \
-    && curl -L -s -S \
-      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-      -o /opt/protobuf.tar.gz \
-    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
-    && cd /opt/protobuf-src \
-    && ./configure --prefix=/opt/protobuf \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root \
-    && rm -rf /opt/protobuf-src
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
-####
-# Install pylint and python-dateutil
-####
-RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
-
-####
-# Install bower
-####
-# hadolint ignore=DL3008
-RUN npm install -g bower@1.8.8
-
-###
-# Install hadolint
-####
-RUN curl -L -s -S \
-        https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
-        -o /bin/hadolint \
-   && chmod a+rx /bin/hadolint \
-   && shasum -a 512 /bin/hadolint | \
-        awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'
-
-######
-# Intel ISA-L 2.29.0
-######
-# hadolint ignore=DL3003,DL3008
-RUN mkdir -p /opt/isa-l-src \
-    && apt-get -q update \
-    && apt-get install -y --no-install-recommends automake yasm \
-    && apt-get clean \
-    && curl -L -s -S \
-      https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \
-      -o /opt/isa-l.tar.gz \
-    && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \
-    && cd /opt/isa-l-src \
-    && ./autogen.sh \
-    && ./configure \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root \
-    && rm -rf /opt/isa-l-src
-
 ###
 # Avoid out of memory errors in builds
 ###
@@ -188,6 +79,16 @@ ENV MAVEN_OPTS -Xms256m -Xmx3072m
 # Skip gpg verification when downloading Yetus via yetus-wrapper
 ENV HADOOP_SKIP_YETUS_VERIFICATION true
 
+####
+# Install packages
+####
+RUN pkg-resolver/install-common-pkgs.sh
+RUN pkg-resolver/install-spotbugs.sh ubuntu:focal
+RUN pkg-resolver/install-boost.sh ubuntu:focal
+RUN pkg-resolver/install-protobuf.sh ubuntu:focal
+RUN pkg-resolver/install-hadolint.sh ubuntu:focal
+RUN pkg-resolver/install-intel-isa-l.sh ubuntu:focal
+
 ###
 # Everything past this point is either not needed for testing or breaks Yetus.
 # So tell Yetus not to read the rest of the file:

dev-support/docker/Dockerfile_aarch64

@@ -33,61 +33,19 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV DEBCONF_TERSE true
 
 ######
-# Install common dependencies from packages. Versions here are either
-# sufficient or irrelevant.
+# Platform package dependency resolver
 ######
-# hadolint ignore=DL3008
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+    && chmod a+r pkg-resolver/*.json
+
+######
+# Install packages from apt
+######
+# hadolint ignore=DL3008,SC2046
 RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends \
-        ant \
-        apt-utils \
-        bats \
-        build-essential \
-        bzip2 \
-        clang \
-        cmake \
-        curl \
-        doxygen \
-        fuse \
-        g++ \
-        gcc \
-        git \
-        gnupg-agent \
-        hugo \
-        libbcprov-java \
-        libbz2-dev \
-        libcurl4-openssl-dev \
-        libfuse-dev \
-        libprotobuf-dev \
-        libprotoc-dev \
-        libsasl2-dev \
-        libsnappy-dev \
-        libssl-dev \
-        libtool \
-        libzstd-dev \
-        locales \
-        make \
-        maven \
-        nodejs \
-        node-yarn \
-        npm \
-        openjdk-11-jdk \
-        openjdk-8-jdk \
-        phantomjs \
-        pinentry-curses \
-        pkg-config \
-        python2.7 \
-        python3 \
-        python3-pip \
-        python3-pkg-resources \
-        python3-setuptools \
-        python3-wheel \
-        rsync \
-        shellcheck \
-        software-properties-common \
-        sudo \
-        valgrind \
-        zlib1g-dev \
+    && apt-get -q install -y --no-install-recommends python3 \
+    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
@@ -103,59 +61,16 @@ ENV MAVEN_HOME /usr
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
 
 #######
-# Install SpotBugs 4.2.2
+# Set env vars for SpotBugs 4.2.2
 #######
-RUN mkdir -p /opt/spotbugs \
-    && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-      -o /opt/spotbugs.tgz \
-    && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
-    && chmod +x /opt/spotbugs/bin/*
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Install Boost 1.72 (1.71 ships with Focal)
+# Set env vars for Google Protobuf 3.7.1
 #######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/boost-library \
-    && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
-    && mv boost_1_72_0.tar.bz2 /opt/boost-library \
-    && cd /opt/boost-library \
-    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
-    && cd /opt/boost-library/boost_1_72_0 \
-    && ./bootstrap.sh --prefix=/usr/ \
-    && ./b2 --without-python install \
-    && cd /root \
-    && rm -rf /opt/boost-library
-
-######
-# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
-######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/protobuf-src \
-    && curl -L -s -S \
-      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-      -o /opt/protobuf.tar.gz \
-    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
-    && cd /opt/protobuf-src \
-    && ./configure --prefix=/opt/protobuf \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root \
-    && rm -rf /opt/protobuf-src
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
-####
-# Install pylint and python-dateutil
-####
-RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
-
-####
-# Install bower
-####
-# hadolint ignore=DL3008
-RUN npm install -g bower@1.8.8
-
 ###
 # Avoid out of memory errors in builds
 ###
@@ -167,6 +82,14 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true
 # Force PhantomJS to be in 'headless' mode, do not connect to Xwindow
 ENV QT_QPA_PLATFORM offscreen
 
+####
+# Install packages
+####
+RUN pkg-resolver/install-common-pkgs.sh
+RUN pkg-resolver/install-spotbugs.sh ubuntu:focal::arch64
+RUN pkg-resolver/install-boost.sh ubuntu:focal::arch64
+RUN pkg-resolver/install-protobuf.sh ubuntu:focal::arch64
+
 ###
 # Everything past this point is either not needed for testing or breaks Yetus.
 # So tell Yetus not to read the rest of the file: