Initial stab at adding ability to access container-recipes docker reg…

…istry from builders, and some preliminary work to setup a docker builder backend.

Builder/src/main.cpp

@@ -75,7 +75,7 @@ void read_file(websocket::stream<tcp::socket>& client_stream,
 }
 
 std::string build_command(const ClientData& client_data) {
-    if (client_data.arch == Architecture::ppc64le) {
+    if (client_data.arch == ArchType::ppc64le && client_data.backend == BackendType::singularity) {
         // A dirty hack but the ppc64le qemu executable must be in the place the kernel expects it
         // Modify the definition to copy this executable in during %setup
         // NOTE: singularity currently doesn't have a way to inject this file in before bootstrap
@@ -85,15 +85,16 @@ std::string build_command(const ClientData& client_data) {
                 "\n%setup\ncp /usr/bin/qemu-ppc64le  ${SINGULARITY_ROOTFS}/usr/bin/qemu-ppc64le");
         def << copy_qemu;
     }
-    std::string build_command("/usr/bin/sudo ");
-    // If the client is called from a TTY we use "unbuffer" to fake the build into thinking it has a real TTY
-    // This allows utilities like wget and color ls to work nicely
-    if (client_data.tty) {
-        build_command += "/usr/bin/unbuffer ";
+
+    std::string build_command;
+    if (client_data.backend == BackendType::singularity) {
+        std::string build_command = "/usr/bin/sudo SingularityBuilderBackend";
+    } else if(client_data.backend == BackendType::docker) {
+        std::string build_command = "/usr/bin/sudo DockerBuilderBackend";
     }
-    build_command += "/usr/local/bin/singularity build ./container.img ./container.def";
 
     Logger::info("Build command prepared: " + build_command);
+
     return build_command;
 }
 
@@ -204,7 +205,7 @@ int main(int argc, char *argv[]) {
         stream_build(client_stream, build_string);
 
         Logger::info("Writing the finished container to the client");
-        write_file(client_stream, "container.img");
+        write_file(client_stream, "container.simg");
 
     } catch (const boost::exception &ex) {
         auto diagnostics = diagnostic_information(ex);

CMakeLists.txt

@@ -40,7 +40,9 @@ set(SOURCE_FILES_CLIENT
 # Scripts required for runtime
 set(SCRIPTS
         Scripts/CreateBuilder
-        Scripts/DestroyBuilder)
+        Scripts/DestroyBuilder
+        Scripts/SingularityBuilderBackend
+        Scripts/DockerBuilderBackend)
 
 # Create executables
 add_executable(builder_queue ${SOURCE_FILES_QUEUE})

Client/src/main.cpp

@@ -113,9 +113,11 @@ void parse_arguments(ClientData &client_data, int argc, char **argv) {
             ("help", "produce help message")
             ("debug", po::bool_switch(), "enable debug information")
             ("arch", po::value<std::string>()->default_value("x86_64"),
-             "select architecture, valid options are x86_64 and ppc64le")
+                 "select architecture, valid options are x86_64 and ppc64le")
+            ("backend", po::value<std::string>()->default_value("singularity"),
+                "select the builder backend to use, valid options are singularity and docker")
             ("tty", po::value<bool>()->default_value(isatty(fileno(stdout))),
-             "true if the data should be presented as if a tty is present")
+                 "true if the data should be presented as if a tty is present")
             ("container", po::value<std::string>()->required(), "(required) the container name")
             ("definition", po::value<std::string>()->required(), "(required) the definition file");
 
@@ -139,6 +141,7 @@ void parse_arguments(ClientData &client_data, int argc, char **argv) {
     client_data.container_path = vm["container"].as<std::string>();
     client_data.tty = vm["tty"].as<bool>();
     client_data.arch = Arch::to_arch(vm["arch"].as<std::string>());
+    client_data.backend = Backend::to_backend(vm["backend"].as<std::string>());
 
     // Enable debug information
     if(vm["debug"].as<bool>()) {

Common/include/ClientData.h

@@ -7,27 +7,44 @@
 #include <boost/serialization/string.hpp>
 #include <boost/core/ignore_unused.hpp>
 
-enum class Architecture {
+enum class ArchType {
     x86_64,
     ppc64le
 };
 
+enum class BackendType {
+    singularity,
+    docker
+};
+
+namespace Backend {
+    static BackendType to_backend(const std::string& backend_string) {
+        if (backend_string == "singularity")
+            return BackendType::singularity;
+        else if (backend_string == "docker")
+            return BackendType::docker;
+        else
+            throw std::runtime_error("Incorrect BackendType provided");
+    }
+}
+
 namespace Arch {
-    static Architecture to_arch(const std::string& arch_string) {
+    static ArchType to_arch(const std::string& arch_string) {
         if (arch_string == "x86_64")
-            return Architecture::x86_64;
+            return ArchType::x86_64;
         else if (arch_string == "ppc64le")
-            return Architecture::ppc64le;
+            return ArchType::ppc64le;
         else
-            throw std::runtime_error("Incorrect architecture supported");
+            throw std::runtime_error("Incorrect ArchType provided");
     }
 };
 
 class ClientData {
 public:
     std::string user_id;
     bool tty;
-    Architecture arch;
+    ArchType arch;
+    BackendType backend;
     std::string container_path;
     std::string definition_path;
     std::string queue_host;
@@ -42,6 +59,7 @@ namespace boost {
             ar & client_data.user_id;
             ar & client_data.tty;
             ar & client_data.arch;
+            ar & client_data.backend;
             ar & client_data.container_path;
             ar & client_data.definition_path;
             ar & client_data.queue_host;

Scripts/BringUpQueue

@@ -50,6 +50,7 @@ echo "Provisioning the queue"
 ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/ProvisionQueue
 
 # Copy OpenStack credentials to VM and then move to correct directory
+# These credentials are available as environment variables to the runners
 unset OS_CACERT
 printenv | grep ^OS_ > ${SCRIPT_DIR}/openrc.sh # "Reconstruct" openrc.sh
 awk '{print "export "$0}' ${SCRIPT_DIR}/openrc.sh > tmp_awk && mv tmp_awk ${SCRIPT_DIR}/openrc.sh

Scripts/CreateBuilderImage

@@ -7,7 +7,6 @@ export OS_CACERT=`pwd`/OpenStack.cer
 echo "using OS_CACERT="${OS_CACERT}
 
 # OpenStack credentials will be sourced by the gitlab runners
-#source ./openrc.sh
 
 # Destroy any existing builder if one exists
 ./TearDownQueue --no_source
@@ -16,8 +15,9 @@ echo "using OS_CACERT="${OS_CACERT}
 # Get script directory
 SCRIPT_DIR=$(dirname $0)
 
-# Create ContainerBuilder security group allowing tcp access to port 8080 and 22
+# Create ContainerBuilder security group allowing tcp access to port 5000, 8080 and 22
 openstack security group create container_builder --description "Allow ContainerBuilder communication"           &> /dev/null
+openstack security group rule create container_builder --protocol tcp --dst-port 5000:5000 --remote-ip 0.0.0.0/0 &> /dev/null
 openstack security group rule create container_builder --protocol tcp --dst-port 22:22 --remote-ip 0.0.0.0/0     &> /dev/null
 openstack security group rule create container_builder --protocol tcp --dst-port 8080:8080 --remote-ip 0.0.0.0/0 &> /dev/null
 
@@ -75,8 +75,11 @@ ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mkdir -p /IB
 ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo chmod 777 /IBM'
 scp -o StrictHostKeyChecking=no -i ${KEY_FILE} /sw/summitdev/spectrum_mpi/10.1.0.4-20170915/rpms/*.rpm cades@${VM_IP}:/IBM
 
-echo "Build Titan and Summit base containers inside of builder image"
-ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/CreateBaseContainers
+# Copy Gitlab docker registry access token to VM and then move to correct directory
+# This credentials are available as environment variables to the runners
+echo ${GL_TOKEN} > ${SCRIPT_DIR}/container-registry-token
+scp -o StrictHostKeyChecking=no -i ${KEY_FILE} ${SCRIPT_DIR}/container-registry-token cades@${VM_IP}:/home/cades/container-registry-token
+ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mv /home/cades/container-registry-token /home/builder/container-registry-token'
 
 echo "Reboot the server to ensure its in a clean state before creating the snapshot"
 openstack server reboot --wait ${VM_UUID}

Scripts/DockerBuilderBackend

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# Provide readonly access to the private gitlab docker repository
+docker login code.ornl.gov:4567 -u atj -p $(cat /home/builder/container-registry-token)
+
+# Spin up local registry
+docker run -d -p 5000:5000 --restart=always --name registry registry:2
+
+# Build the Dockerfile docker image in the current directory
+docker build -t localhost:5000/docker_image .
+
+# Push to the local registry
+docker push localhost:5000/docker_image
+
+# Build the singularity container from the docker image
+singularity pull --name container.simg docker://localhost:5000/docker_image

Scripts/ProvisionBuilder

@@ -7,14 +7,20 @@ set -o xtrace
 useradd --create-home --home-dir /home/builder --shell /bin/bash builder
 
 # Allow builder to run singularity as root
-echo 'builder ALL=(ALL) NOPASSWD: /usr/local/bin/singularity' > /etc/sudoers.d/builder
-echo 'builder ALL=(ALL) NOPASSWD: /usr/bin/unbuffer' >> /etc/sudoers.d/builder
+echo 'builder ALL=(ALL) NOPASSWD: /usr/local/bin/SingularityBuilderBackend' > /etc/sudoers.d/builder
+echo 'builder ALL=(ALL) NOPASSWD: /usr/local/bin/DockerBuilderBackend' >> /etc/sudoers.d/builder
 chmod 0440 /etc/sudoers.d/builder
 
 apt-get -y update
 apt-get -y install expect
 apt-get -y install yum rpm
 
+# Install docker
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+apt-get update
+apt-get install -y docker-ce
+
 #####################################
 # begin ppc64le QUEMU stuff
 ######################################

Scripts/SingularityBuilderBackend

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# Provide readonly access to the private gitlab docker repository
+docker login code.ornl.gov:4567 -u atj -p $(cat /home/builder/container-registry-token)
+
+/usr/bin/unbuffer /usr/local/bin/singularity build ./container.simg ./container.def