diff --git a/Builder/src/main.cpp b/Builder/src/main.cpp index 4426810..51e994b 100644 --- a/Builder/src/main.cpp +++ b/Builder/src/main.cpp @@ -75,7 +75,7 @@ void read_file(websocket::stream& client_stream, } std::string build_command(const ClientData& client_data) { - if (client_data.arch == Architecture::ppc64le) { + if (client_data.arch == ArchType::ppc64le && client_data.backend == BackendType::singularity) { // A dirty hack but the ppc64le qemu executable must be in the place the kernel expects it // Modify the definition to copy this executable in during %setup // NOTE: singularity currently doesn't have a way to inject this file in before bootstrap @@ -85,15 +85,16 @@ std::string build_command(const ClientData& client_data) { "\n%setup\ncp /usr/bin/qemu-ppc64le ${SINGULARITY_ROOTFS}/usr/bin/qemu-ppc64le"); def << copy_qemu; } - std::string build_command("/usr/bin/sudo "); - // If the client is called from a TTY we use "unbuffer" to fake the build into thinking it has a real TTY - // This allows utilities like wget and color ls to work nicely - if (client_data.tty) { - build_command += "/usr/bin/unbuffer "; + + std::string build_command; + if (client_data.backend == BackendType::singularity) { + std::string build_command = "/usr/bin/sudo SingularityBuilderBackend"; + } else if(client_data.backend == BackendType::docker) { + std::string build_command = "/usr/bin/sudo DockerBuilderBackend"; } - build_command += "/usr/local/bin/singularity build ./container.img ./container.def"; Logger::info("Build command prepared: " + build_command); + return build_command; } @@ -204,7 +205,7 @@ int main(int argc, char *argv[]) { stream_build(client_stream, build_string); Logger::info("Writing the finished container to the client"); - write_file(client_stream, "container.img"); + write_file(client_stream, "container.simg"); } catch (const boost::exception &ex) { auto diagnostics = diagnostic_information(ex); diff --git a/CMakeLists.txt b/CMakeLists.txt index c7643df..5dd8cdd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,7 +40,9 @@ set(SOURCE_FILES_CLIENT # Scripts required for runtime set(SCRIPTS Scripts/CreateBuilder - Scripts/DestroyBuilder) + Scripts/DestroyBuilder + Scripts/SingularityBuilderBackend + Scripts/DockerBuilderBackend) # Create executables add_executable(builder_queue ${SOURCE_FILES_QUEUE}) diff --git a/Client/src/main.cpp b/Client/src/main.cpp index 8b20710..ef0399b 100644 --- a/Client/src/main.cpp +++ b/Client/src/main.cpp @@ -113,9 +113,11 @@ void parse_arguments(ClientData &client_data, int argc, char **argv) { ("help", "produce help message") ("debug", po::bool_switch(), "enable debug information") ("arch", po::value()->default_value("x86_64"), - "select architecture, valid options are x86_64 and ppc64le") + "select architecture, valid options are x86_64 and ppc64le") + ("backend", po::value()->default_value("singularity"), + "select the builder backend to use, valid options are singularity and docker") ("tty", po::value()->default_value(isatty(fileno(stdout))), - "true if the data should be presented as if a tty is present") + "true if the data should be presented as if a tty is present") ("container", po::value()->required(), "(required) the container name") ("definition", po::value()->required(), "(required) the definition file"); @@ -139,6 +141,7 @@ void parse_arguments(ClientData &client_data, int argc, char **argv) { client_data.container_path = vm["container"].as(); client_data.tty = vm["tty"].as(); client_data.arch = Arch::to_arch(vm["arch"].as()); + client_data.backend = Backend::to_backend(vm["backend"].as()); // Enable debug information if(vm["debug"].as()) { diff --git a/Common/include/ClientData.h b/Common/include/ClientData.h index 7617dc4..aeb99d3 100644 --- a/Common/include/ClientData.h +++ b/Common/include/ClientData.h @@ -7,19 +7,35 @@ #include #include -enum class Architecture { +enum class ArchType { x86_64, ppc64le }; +enum class BackendType { + singularity, + docker +}; + +namespace Backend { + static BackendType to_backend(const std::string& backend_string) { + if (backend_string == "singularity") + return BackendType::singularity; + else if (backend_string == "docker") + return BackendType::docker; + else + throw std::runtime_error("Incorrect BackendType provided"); + } +} + namespace Arch { - static Architecture to_arch(const std::string& arch_string) { + static ArchType to_arch(const std::string& arch_string) { if (arch_string == "x86_64") - return Architecture::x86_64; + return ArchType::x86_64; else if (arch_string == "ppc64le") - return Architecture::ppc64le; + return ArchType::ppc64le; else - throw std::runtime_error("Incorrect architecture supported"); + throw std::runtime_error("Incorrect ArchType provided"); } }; @@ -27,7 +43,8 @@ class ClientData { public: std::string user_id; bool tty; - Architecture arch; + ArchType arch; + BackendType backend; std::string container_path; std::string definition_path; std::string queue_host; @@ -42,6 +59,7 @@ namespace boost { ar & client_data.user_id; ar & client_data.tty; ar & client_data.arch; + ar & client_data.backend; ar & client_data.container_path; ar & client_data.definition_path; ar & client_data.queue_host; diff --git a/Scripts/BringUpQueue b/Scripts/BringUpQueue index e324712..9d3e1a0 100755 --- a/Scripts/BringUpQueue +++ b/Scripts/BringUpQueue @@ -50,6 +50,7 @@ echo "Provisioning the queue" ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/ProvisionQueue # Copy OpenStack credentials to VM and then move to correct directory +# These credentials are available as environment variables to the runners unset OS_CACERT printenv | grep ^OS_ > ${SCRIPT_DIR}/openrc.sh # "Reconstruct" openrc.sh awk '{print "export "$0}' ${SCRIPT_DIR}/openrc.sh > tmp_awk && mv tmp_awk ${SCRIPT_DIR}/openrc.sh diff --git a/Scripts/CreateBaseContainers b/Scripts/CreateBaseContainers deleted file mode 100755 index fefe7ca..0000000 --- a/Scripts/CreateBaseContainers +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -e -set -o xtrace - -# Create base containers for use - hopefully this is replaced by a registry soon! -cd / -git clone https://github.com/olcf/SingularityTools.git -mkdir /BaseContainers -sudo singularity build /BaseContainers/titan_ubuntu17.04.img /SingularityTools/Titan/UbuntuBase/Ubuntu_17.04.def -sudo singularity build /BaseContainers/titan_centos7.img /SingularityTools/Titan/CentosBase/Centos7.def -sudo singularity build /BaseContainers/summit_centos7.img /SingularityTools/Summit/CentosBase/Centos7.def -cd / -rm -rf SingularityTools \ No newline at end of file diff --git a/Scripts/CreateBuilderImage b/Scripts/CreateBuilderImage index e043720..87d3605 100755 --- a/Scripts/CreateBuilderImage +++ b/Scripts/CreateBuilderImage @@ -7,7 +7,6 @@ export OS_CACERT=`pwd`/OpenStack.cer echo "using OS_CACERT="${OS_CACERT} # OpenStack credentials will be sourced by the gitlab runners -#source ./openrc.sh # Destroy any existing builder if one exists ./TearDownQueue --no_source @@ -16,8 +15,9 @@ echo "using OS_CACERT="${OS_CACERT} # Get script directory SCRIPT_DIR=$(dirname $0) -# Create ContainerBuilder security group allowing tcp access to port 8080 and 22 +# Create ContainerBuilder security group allowing tcp access to port 5000, 8080 and 22 openstack security group create container_builder --description "Allow ContainerBuilder communication" &> /dev/null +openstack security group rule create container_builder --protocol tcp --dst-port 5000:5000 --remote-ip 0.0.0.0/0 &> /dev/null openstack security group rule create container_builder --protocol tcp --dst-port 22:22 --remote-ip 0.0.0.0/0 &> /dev/null openstack security group rule create container_builder --protocol tcp --dst-port 8080:8080 --remote-ip 0.0.0.0/0 &> /dev/null @@ -75,8 +75,11 @@ ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mkdir -p /IB ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo chmod 777 /IBM' scp -o StrictHostKeyChecking=no -i ${KEY_FILE} /sw/summitdev/spectrum_mpi/10.1.0.4-20170915/rpms/*.rpm cades@${VM_IP}:/IBM -echo "Build Titan and Summit base containers inside of builder image" -ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/CreateBaseContainers +# Copy Gitlab docker registry access token to VM and then move to correct directory +# This credentials are available as environment variables to the runners +echo ${GL_TOKEN} > ${SCRIPT_DIR}/container-registry-token +scp -o StrictHostKeyChecking=no -i ${KEY_FILE} ${SCRIPT_DIR}/container-registry-token cades@${VM_IP}:/home/cades/container-registry-token +ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mv /home/cades/container-registry-token /home/builder/container-registry-token' echo "Reboot the server to ensure its in a clean state before creating the snapshot" openstack server reboot --wait ${VM_UUID} diff --git a/Scripts/DockerBuilderBackend b/Scripts/DockerBuilderBackend new file mode 100755 index 0000000..7dc7d3d --- /dev/null +++ b/Scripts/DockerBuilderBackend @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e +set -o xtrace + +# Provide readonly access to the private gitlab docker repository +docker login code.ornl.gov:4567 -u atj -p $(cat /home/builder/container-registry-token) + +# Spin up local registry +docker run -d -p 5000:5000 --restart=always --name registry registry:2 + +# Build the Dockerfile docker image in the current directory +docker build -t localhost:5000/docker_image . + +# Push to the local registry +docker push localhost:5000/docker_image + +# Build the singularity container from the docker image +singularity pull --name container.simg docker://localhost:5000/docker_image \ No newline at end of file diff --git a/Scripts/ProvisionBuilder b/Scripts/ProvisionBuilder index 2c6943a..64964d4 100755 --- a/Scripts/ProvisionBuilder +++ b/Scripts/ProvisionBuilder @@ -7,14 +7,20 @@ set -o xtrace useradd --create-home --home-dir /home/builder --shell /bin/bash builder # Allow builder to run singularity as root -echo 'builder ALL=(ALL) NOPASSWD: /usr/local/bin/singularity' > /etc/sudoers.d/builder -echo 'builder ALL=(ALL) NOPASSWD: /usr/bin/unbuffer' >> /etc/sudoers.d/builder +echo 'builder ALL=(ALL) NOPASSWD: /usr/local/bin/SingularityBuilderBackend' > /etc/sudoers.d/builder +echo 'builder ALL=(ALL) NOPASSWD: /usr/local/bin/DockerBuilderBackend' >> /etc/sudoers.d/builder chmod 0440 /etc/sudoers.d/builder apt-get -y update apt-get -y install expect apt-get -y install yum rpm +# Install docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y docker-ce + ##################################### # begin ppc64le QUEMU stuff ###################################### diff --git a/Scripts/SingularityBuilderBackend b/Scripts/SingularityBuilderBackend new file mode 100755 index 0000000..f851600 --- /dev/null +++ b/Scripts/SingularityBuilderBackend @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e +set -o xtrace + +# Provide readonly access to the private gitlab docker repository +docker login code.ornl.gov:4567 -u atj -p $(cat /home/builder/container-registry-token) + +/usr/bin/unbuffer /usr/local/bin/singularity build ./container.simg ./container.def \ No newline at end of file