base: manually compile OpenMPI

Problem: the OpenMPI packaged with centos 7 does not include the PMI
plugin, which is required to boot under Flux. The OpenMPI version
packaged with centos 8 (4.0.5) has a bug that causes a segfault on
finalization, which is fixed in the next patch version.
PR that fixes the bug: (open-mpi/ompi#8380)

Solution: for centos 7, hand compile the same version of OpenMPI (1.10),
but with the `--with-pmi` flag.  For centos 8, hand compile the version
of OpenMPI with the patch (4.0.6).  Since this is an involved process,
add a helper script that will also make adding other MPIs much easier.

Author: SteVwonder

.github/workflows/ci.yml

@@ -21,7 +21,8 @@ jobs:
         docker build \
           -t exaworks/sdk-base:${{ matrix.dockerbase }} \
           --cache-from exaworks/sdk-base:${{ matrix.dockerbase }} \
-          docker/base/${{ matrix.dockerbase }}
+          -f docker/base/${{ matrix.dockerbase }}/Dockerfile \
+          docker/base/
         docker build \
           -t rp:${{ matrix.dockerbase }} \
           --build-arg BASE_IMAGE=exaworks/sdk-base:${{ matrix.dockerbase }} \

.github/workflows/deploy.yml

@@ -37,7 +37,8 @@ jobs:
       id: docker_build
       uses: docker/build-push-action@v2
       with:
-        context: ./docker/base/${{ matrix.dockerbase }}
+        context: ./docker/base/
+        file: ./docker/base/${{ matrix.dockerbase }}/Dockerfile
         push: true
         tags: |
           exaworks/sdk-base:${{ matrix.dockerbase }}

docker/base/centos7/Dockerfile

@@ -1,9 +1,9 @@
 FROM centos:7
 
 ARG MPI=openmpi-devel
-ARG MPI_PREFIX=/usr/lib64/openmpi/
+ARG MPI_PREFIX=/usr/local/
 ENV VIRTUAL_ENV=/ve_exaworks
-ENV PATH="$VIRTUAL_ENV/bin:$MPI_PREFIX/bin:$PATH"
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 ENV MPI_PREFIX=${MPI_PREFIX}
 ENV MPICC=$MPI_PREFIX/bin/mpicc
 
@@ -70,8 +70,6 @@ RUN yum -y update \
     # Radical Pilot Deps
     python3-pip \
     mongodb-org \
-    # MPI Dep
-    ${MPI} \
  && yum clean all \
  && python3 -m venv ${VIRTUAL_ENV} \
  && pip install --upgrade pip setuptools pytest \
@@ -86,3 +84,5 @@ RUN wget https://archive.apache.org/dist/ant/binaries/apache-ant-1.9.15-bin.tar.
     && ln -s /opt/apache-ant-1.9.15 /opt/ant \
     && sudo ln -s /opt/ant/bin/ant /usr/bin/ant
 
+COPY ./scripts/install-mpi.sh /install-mpi.sh
+RUN bash install-mpi.sh ${MPI}

docker/base/centos8/Dockerfile

@@ -1,7 +1,7 @@
 FROM centos:8
 
 ARG MPI=openmpi-devel
-ARG MPI_PREFIX=/usr/lib64/openmpi/
+ARG MPI_PREFIX=/usr/local/
 ENV VIRTUAL_ENV=/ve_exaworks
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 ENV MPI_PREFIX=${MPI_PREFIX}
@@ -71,8 +71,6 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.n
     # RADICAL-Pilot Dependencies
     python3-pip \
     mongodb-org \
-    # MPI Dep
-    ${MPI} \
  && dnf clean all \
  && python3 -m venv ${VIRTUAL_ENV} \
  && pip install --upgrade pip setuptools pytest \
@@ -87,3 +85,5 @@ RUN wget https://archive.apache.org/dist/ant/binaries/apache-ant-1.9.15-bin.tar.
     && ln -s /opt/apache-ant-1.9.15 /opt/ant \
     && sudo ln -s /opt/ant/bin/ant /usr/bin/ant
 
+COPY ./scripts/install-mpi.sh /install-mpi.sh
+RUN bash install-mpi.sh ${MPI}

docker/base/scripts/install-mpi.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+if [[ -z $1 ]]; then
+    echo "Must provide MPI that you want to install" 1>&2
+    exit 1
+fi
+
+function centos_major_version() {
+    cat /etc/centos-release | cut -f 4 -d " " | cut -f 1 -d .
+}
+
+# Install only the dependencies for a given package
+# Source: https://serverfault.com/questions/429123/howto-get-yum-to-install-only-dependencies-for-a-given-pakage
+yum_install_only_deps () {
+    if [[ -z "$1" ]]; then
+        echo "Package required for installing deps" 1>&2
+        exit 1
+    fi
+    yum deplist $1 | grep provider | awk '{print $2}' | sort | uniq | grep -v $PACKAGE | sed ':a;N;$!ba;s/\n/ /g' | xargs yum -y install
+}
+
+if [[ "$1" == "openmpi-devel" ]]; then
+    if [[ $(centos_major_version) == "7" ]]; then
+        yum install -y slurm-pmi-devel
+        MAJOR_MINOR=1.10
+        PATCH=7
+        CONFIGURE_ARGS="--with-pmi --with-pmi-libdir=/usr/lib64"
+    elif [[ $(centos_major_version) == "8" ]]; then
+        yum_install_only_deps $1
+        MAJOR_MINOR=4.0
+        PATCH=6
+        CONFIGURE_ARGS=""
+    else
+        echo "Unknown CentOS version. Exiting" 1>&2
+        exit 1
+    fi
+
+    OPENMPI=openmpi-${MAJOR_MINOR}.${PATCH}
+    
+    wget https://download.open-mpi.org/release/open-mpi/v${MAJOR_MINOR}/${OPENMPI}.tar.gz
+    tar -xf ./${OPENMPI}.tar.gz
+    rm ./${OPENMPI}.tar.gz
+
+    cd ${OPENMPI}
+    ./configure ${CONFIGURE_ARGS}
+    make -j 4
+    make install
+    cd ..
+    rm -rf ${OPENMPI}
+
+    if [[ $(centos_major_version) == "7" ]]; then
+        yum remove -y slurm-pmi-devel
+        yum autoremove -y
+        yum clean all
+    fi
+else
+    echo "Unknown/unsupported MPI. Existing without installing."
+    exit 1
+fi