Daint / Sarus CI

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "libs/options"]
 	path = libs/options
 	url = https://github.com/kabicm/options
+[submodule "ci"]
+	path = ci
+	url = https://gitlab.com/cscs-ci/eth-cscs/cosma-ci.git

CMakeLists.txt

@@ -52,6 +52,24 @@ endif()
 
 project(cosma VERSION 0.1 LANGUAGES CXX)
 
+# preserve rpaths when installing and make the install folder relocatable
+# use `CMAKE_SKIP_INSTALL_RPATH` to skip this
+# https://spack.readthedocs.io/en/latest/workflows.html#write-the-cmake-build
+list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES 
+          "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}" isSystemDir)
+# skip RPATH if SIRIUS is installed to system directories
+if(isSystemDir STREQUAL "-1")
+  set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+  if(APPLE)
+    set(basePoint @loader_path)
+  else()
+    set(basePoint $ORIGIN)
+  endif()
+  file(RELATIVE_PATH relDir ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}
+                            ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
+  set(CMAKE_INSTALL_RPATH ${basePoint} ${basePoint}/${relDir})
+endif()
+
 # Dependencies
 #
 find_package(MPI REQUIRED)

README.md

@@ -1,3 +1,5 @@
+[![pipeline status](https://gitlab.com/cscs-ci/eth-cscs/COSMA/badges/master/pipeline.svg)](https://gitlab.com/cscs-ci/eth-cscs/COSMA/-/commits/master)
+
 <p align="center"><img src="./docs/cosma-logo.svg" width="70%"></p>
 
 ## Table of Contents

benchmarks/CMakeLists.txt

@@ -16,7 +16,7 @@ endif()
 
 foreach(exec ${executables})
     add_executable(${exec} "${exec}.cpp")
-    target_link_libraries(${exec} cosma ${cosma_deps})
+    target_link_libraries(${exec} cosma ${BLAS_TARGET})
 endforeach()
 
 if (${COSMA_BLAS} STREQUAL "CUDA")
@@ -26,7 +26,7 @@ if (${COSMA_BLAS} STREQUAL "CUDA")
     if (CUDA_TOOLKIT_MAJOR_VERSION GREATER 10 OR 
             (CUDA_TOOLKIT_MAJOR_VERSION EQUAL 10 AND CUDA_TOOLKIT_MINOR_VERSION GREATER_EQUAL 1))
         add_executable(gpu_gemm_cublas "gpu_gemm_cublas.cpp")
-        target_link_libraries(gpu_gemm_cublas cosma ${cosma_deps} ${BLAS_TARGET} cublasLt cublas)
+        target_link_libraries(gpu_gemm_cublas cosma ${BLAS_TARGET} cublasLt cublas)
         target_compile_definitions(gpu_gemm_cublas PRIVATE ${BLAS_DEF})
     endif()
 endif()

bors.toml

@@ -0,0 +1,4 @@
+status = [
+  "ci/gitlab/%",
+]
+delete_merged_branches = true

docker/build-env/Dockerfile

@@ -0,0 +1,38 @@
+FROM nvidia/cuda:10.1-devel-ubuntu18.04
+
+WORKDIR /root
+
+ARG MKL_VERSION=2020.0-088
+ARG MPICH_VERSION=3.1.4
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV MKLROOT=/opt/intel/compilers_and_libraries/linux/mkl
+ENV FORCE_UNSAFE_CONFIGURE 1
+ENV MPICH_VERSION ${MPICH_VERSION}
+ENV MKL_VERSION ${MKL_VERSION}
+
+# Install basic tools
+RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \
+    software-properties-common \
+    build-essential \
+    git tar wget curl && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install cmake
+RUN wget -qO- "https://cmake.org/files/v3.17/cmake-3.17.0-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local
+
+# Install MPICH ABI compatible with Cray's lib on Piz Daint
+RUN wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz && \
+    tar -xzf mpich-${MPICH_VERSION}.tar.gz && \
+    cd mpich-${MPICH_VERSION} && \
+    ./configure --disable-fortran && \
+    make install -j$(nproc) && \
+    rm -rf /root/mpich-${MPICH_VERSION}.tar.gz /root/mpich-${MPICH_VERSION}
+
+# Install MKL
+RUN wget -qO - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB 2>/dev/null | apt-key add - && \
+    apt-add-repository 'deb https://apt.repos.intel.com/mkl all main' && \
+    apt-get install -y -qq --no-install-recommends intel-mkl-64bit-${MKL_VERSION} && \
+    rm -rf /var/lib/apt/lists/* && \
+    echo "/opt/intel/lib/intel64\n/opt/intel/compilers_and_libraries/linux/mkl/lib/intel64" >> /etc/ld.so.conf.d/intel.conf && \
+    ldconfig

docker/deploy/Dockerfile

@@ -0,0 +1,73 @@
+ARG BUILD_ENV
+
+FROM $BUILD_ENV as builder
+
+ARG BLAS
+
+# Add deployment tooling
+RUN wget -q https://github.com/haampie/bundler/releases/download/v0.1.5/bundler_x86_64.tar.gz && \
+    tar -xzf bundler_x86_64.tar.gz && \
+    rm bundler_x86_64.tar.gz
+
+# Build COSMA
+COPY . /COSMA
+
+RUN COMPILERVARS_ARCHITECTURE=intel64 /opt/intel/bin/compilervars.sh intel64 && \
+    mkdir /COSMA/build && cd /COSMA/build && \
+    CC=mpicc CXX=mpicxx cmake .. \
+      -DCOSMA_WITH_TESTS=ON \
+      -DCUDA_PATH=/usr/local/cuda \
+      -DCOSMA_BLAS=${BLAS} \
+      -DCOSMA_SCALAPACK=MKL \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_INSTALL_PREFIX=/usr && \
+      make -j$(nproc) && \
+      make DESTDIR=/root/COSMA-build install && \
+      rm -rf /COSMA
+
+ENV MKL_LIB=/opt/intel/compilers_and_libraries/linux/mkl/lib/intel64
+
+# Run linuxdeploy, and add a bunch of libs that are dlopen'ed by mkl
+RUN /root/bundler/bundler -d /root/COSMA.bundle/ \
+      -e /root/COSMA-build/usr/bin/test.cosma \
+      -e /root/COSMA-build/usr/bin/test.pdgemm \
+      -e /root/COSMA-build/usr/bin/test.multiply \
+      -e /root/COSMA-build/usr/bin/test.mapper \
+      -e /root/COSMA-build/usr/bin/test.scalar_matmul \
+      -e /root/COSMA-build/usr/bin/test.multiply_using_layout \
+      # MKL dlopen's some of their libs, so we have to explicitly copy them over
+      -l ${MKL_LIB}/libmkl_tbb_thread.so \
+      -l ${MKL_LIB}/libmkl_intel_thread.so \
+      -l ${MKL_LIB}/libmkl_sequential.so \
+      -l ${MKL_LIB}/libmkl_def.so \
+      -l ${MKL_LIB}/libmkl_vml_def.so \
+      -l ${MKL_LIB}/libmkl_vml_cmpt.so \
+      -l ${MKL_LIB}/libmkl_mc.so \
+      -l ${MKL_LIB}/libmkl_vml_mc.so \
+      -l ${MKL_LIB}/libmkl_mc3.so \
+      -l ${MKL_LIB}/libmkl_vml_mc3.so \
+      -l ${MKL_LIB}/libmkl_avx.so \
+      -l ${MKL_LIB}/libmkl_vml_avx.so \
+      -l ${MKL_LIB}/libmkl_avx2.so \
+      -l ${MKL_LIB}/libmkl_vml_avx2.so \
+      -l ${MKL_LIB}/libmkl_avx512_mic.so \
+      -l ${MKL_LIB}/libmkl_vml_avx512_mic.so \
+      -l ${MKL_LIB}/libmkl_avx512.so \
+      -l ${MKL_LIB}/libmkl_vml_avx512.so \
+      -l ${MKL_LIB}/libmkl_core.so 
+
+FROM ubuntu:18.04
+
+# This is the only thing necessary really from nvidia/cuda's ubuntu18.04 runtime image
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411"
+
+COPY --from=builder /root/COSMA.bundle /root/COSMA.bundle
+
+# Make it easy to call our binaries.
+ENV PATH="/root/COSMA.bundle/usr/bin:$PATH"
+
+RUN echo "/root/COSMA.bundle/usr/lib/" > /etc/ld.so.conf.d/cosma.conf && ldconfig
+
+WORKDIR /root/COSMA.bundle/usr/bin

tests/CMakeLists.txt

@@ -20,12 +20,14 @@ function(add_cosma_mpi_test test_name num_procs)
              COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs}
                      ${MPIEXEC_PREFLAGS} ./${tgt_} ${MPIEXEC_POSTFLAGS})
     add_dependencies(tests ${tgt_})
+    install(TARGETS ${tgt_} RUNTIME DESTINATION "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}")
 endfunction()
 
 add_executable(test.mapper mapper.cpp)
 target_link_libraries(test.mapper PRIVATE main_gtest gtest cosma)
 add_test(NAME test.mapper COMMAND test.mapper)
 add_dependencies(tests test.mapper)
+install(TARGETS test.mapper RUNTIME DESTINATION "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}")
 
 add_cosma_mpi_test(multiply_using_layout 4 cosma)
 add_cosma_mpi_test(multiply 16 cosma)
@@ -39,8 +41,10 @@ if(COSMA_SCALAPACK)
     add_executable(test.cosma_pdgemm cosma_pdgemm_test.cpp)
     target_link_libraries(test.cosma_pdgemm PRIVATE cosma_pxgemm_cpp options)
     target_include_directories(test.cosma_pdgemm PRIVATE .)
+    install(TARGETS test.cosma_pdgemm RUNTIME DESTINATION "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}")
 endif()
 
 add_executable(test.cosma cosma_test.cpp)
 target_link_libraries(test.cosma PRIVATE cosma options)
 target_include_directories(test.cosma PRIVATE .)
+install(TARGETS test.cosma RUNTIME DESTINATION "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}")