Refactor script naming scheme to be more consistent

Builder/src/main.cpp

@@ -80,9 +80,9 @@ std::string build_command(const ClientData &client_data) {
 
     // Basic build command
     if (client_data.backend == BackendType::singularity) {
-        build_command = "/usr/bin/sudo /usr/local/bin/SingularityBuilderBackend";
+        build_command = "/usr/bin/sudo /usr/local/bin/singularity-builder-backend.sh";
     } else if (client_data.backend == BackendType::docker) {
-        build_command = "/usr/bin/sudo /usr/local/bin/DockerBuilderBackend";
+        build_command = "/usr/bin/sudo /usr/local/bin/docker-builder-backend.sh";
     } else {
         throw std::runtime_error("Invalid builder backend");
     }

BuilderQueue/include/OpenStack.h

@@ -26,7 +26,7 @@ class OpenStack : public std::enable_shared_from_this<OpenStack> {
     // If the builder wasn't created the handlers error_code will be set and a default constructed builder will be returned
     template<typename CreateHandler>
     void request_create(CreateHandler handler) {
-        std::string create_command("/usr/local/bin/CreateBuilder");
+        std::string create_command("/usr/local/bin/create-builder.sh");
 
         Logger::info("Launching command: " + create_command);
 
@@ -111,7 +111,7 @@ class OpenStack : public std::enable_shared_from_this<OpenStack> {
     // If the builder couldn't be destroyed the handlers error_code will be set
     template<typename DestroyHandler>
     void destroy(BuilderData builder, DestroyHandler handler) {
-        const std::string destroy_command("/usr/local/bin/DestroyBuilder " + builder.id);
+        const std::string destroy_command("/usr/local/bin/destroy-builder.sh " + builder.id);
 
         Logger::info("Launching command: " + destroy_command);
         std::error_code destroy_error;

CMakeLists.txt

@@ -45,9 +45,9 @@ set(SCRIPTS
         Scripts/DockerBuilderBackend)
 
 # Create executables
-add_executable(builder_queue ${SOURCE_FILES_QUEUE})
-add_executable(builder_server ${SOURCE_FILES_BUILDER})
-add_executable(container_builder ${SOURCE_FILES_CLIENT})
+add_executable(builder-queue ${SOURCE_FILES_QUEUE})
+add_executable(builder-server ${SOURCE_FILES_BUILDER})
+add_executable(container-builder ${SOURCE_FILES_CLIENT})
 
 # Ignore system boost and use module system boost
 set(Boost_NO_BOOST_CMAKE TRUE)
@@ -60,27 +60,27 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
     set(HARDENING_FLAGS "${HADENING_FLAGS} -Wl,-z,noexecstack,-z,now,-z,relro,-z,nodlopen")
 endif()
 
-set_target_properties(builder_queue PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} ${HARDENING_FLAGS}")
-set_target_properties(builder_queue PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}")
-set_target_properties(builder_server PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} ${HARDENING_FLAGS}")
-set_target_properties(builder_server PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}")
-set_target_properties(container_builder PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} ${HARDENING_FLAGS}")
-set_target_properties(container_builder PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}")
+set_target_properties(builder-queue PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} ${HARDENING_FLAGS}")
+set_target_properties(builder-queue PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}")
+set_target_properties(builder-server PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} ${HARDENING_FLAGS}")
+set_target_properties(builder-server PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}")
+set_target_properties(container-builder PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} ${HARDENING_FLAGS}")
+set_target_properties(container-builder PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}")
 
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
-target_link_libraries(builder_queue ${CMAKE_THREAD_LIBS_INIT})
-target_link_libraries(builder_server ${CMAKE_THREAD_LIBS_INIT})
-target_link_libraries(container_builder ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(builder-queue ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(builder-server ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(container-builder ${CMAKE_THREAD_LIBS_INIT})
 
 
 find_package(Boost 1.66.0 COMPONENTS system filesystem serialization regex thread program_options REQUIRED)
 include_directories(${Boost_INCLUDE_DIRS})
-target_link_libraries(builder_queue ${Boost_LIBRARIES})
-target_link_libraries(builder_server ${Boost_LIBRARIES})
-target_link_libraries(container_builder ${Boost_LIBRARIES})
+target_link_libraries(builder-queue ${Boost_LIBRARIES})
+target_link_libraries(builder-server ${Boost_LIBRARIES})
+target_link_libraries(container-builder ${Boost_LIBRARIES})
 
-install(TARGETS builder_queue builder_server container_builder RUNTIME DESTINATION bin)
+install(TARGETS builder-queue builder-server container-builder RUNTIME DESTINATION bin)
 
 # Install scripts
 install(FILES ${SCRIPTS}

README.md

@@ -1,3 +1,46 @@
-ContainerBuilder
+container-builder
 ================
+Containers traditionally require root access to build from a recipe file, as such native access to build cannot be granted on OLCF HPC resources such as Titan and Summit. 
+container-builder is an interactive container building utility that gets around this limitation by building each container on a remote ephemeral VM, streaming the output in real time to the client.
 
+Use
+=================
+Singularity recipe:
+```
+$ module load container-builder
+$ container-builder container.img singularity.recipe
+```
+Docker recipe:
+```
+$ module load container-builder
+$ container-builder --backend=docker container.img docker.recipe
+```
+
+OLCF Recipes
+==================
+container-builder has access to the private OLCF container-recipes docker registry.
+
+Implementation
+==================
+Some insight into the build process:
+
+* Client initiates build request through CLI
+* Client build request enters the queue
+* Queue creates builder
+* Builder details sent to the client
+* The client connects to the builder
+* Container recipe file is sent from client to builder
+* Build output is streamed in real time to the client
+* Container image is sent from the builder to the client
+* Client disconnects from queue
+* Queue destroys VM
+
+Deploy
+==================
+To deploy container-builder three steps are taken
+* The Builder OpenStack image must be created
+* The Queue OpenStack instance must be started
+* The client application must be built
+
+Due to the non-trivial complexity of provisioning Gitlab runners handle deployment through the CI system.
+Following `.gitlab-ci.yml` should provide insight into the provisioning process.

Scripts/BringUpQueue → Scripts/bring-up-queue.sh

@@ -44,10 +44,10 @@ while ! ssh_is_up; do
 done
 
 echo "Fixing ORNL TCP timeout issue for current session"
-ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/DisableTcpTimestamps
+ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/disable-TCP-timestamps.sh
 
 echo "Provisioning the queue"
-ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/ProvisionQueue
+ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/provision-queue.sh
 
 # Copy OpenStack credentials to VM and then move to correct directory
 # These credentials are available as environment variables to the runners
@@ -63,6 +63,6 @@ openstack server reboot --wait ${VM_UUID}
 
 echo "Started ${VM_UUID} with external IP ${VM_IP} using ${KEY_FILE}"
 
-cat << EOF > ${SCRIPT_DIR}/../artifacts/queue_host.sh
+cat << EOF > ${SCRIPT_DIR}/../artifacts/queue-host.sh
 QUEUE_HOST=${VM_IP}
 EOF

Scripts/CreateBuilderImage → Scripts/create-builder-image.sh

@@ -9,8 +9,8 @@ echo "using OS_CACERT="${OS_CACERT}
 # OpenStack credentials will be sourced by the gitlab runners
 
 # Destroy any existing builder if one exists
-./TearDownQueue --no_source
-./DestroyBuilderImage --no_source
+tear-down-queue.sh --no_source
+destroy-builder-image.sh --no_source
 
 # Get script directory
 SCRIPT_DIR=$(dirname $0)
@@ -59,10 +59,10 @@ while ! ssh_is_up; do
 done
 
 echo "Fixing ORNL TCP timeout issue for current session"
-ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/DisableTcpTimestamps
+ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/disable-TCP-timestamps.sh
 
 echo "Provisioning the builder"
-ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/ProvisionBuilder
+ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/provision-builder.sh
 
 # Copy Gitlab docker registry access token to VM and then move to correct directory
 # This credentials are available as environment variables to the runners

Scripts/DockerBuilderBackend → Scripts/docker-builder-backend.sh

@@ -9,7 +9,7 @@ case ${i} in
     shift # past argument with no value
     ;;
     *)
-      echo "unknown argument to SingularityBuilderBackend"
+      echo "unknown argument to singularity-builder-backend.sh"
       exit 1
     ;;
 esac

Scripts/ProvisionBuilder → Scripts/provision-builder.sh

@@ -7,8 +7,8 @@ set -o xtrace
 useradd --create-home --home-dir /home/builder --shell /bin/bash builder
 
 # Allow builder to run singularity as root
-echo 'builder ALL=(ALL) NOPASSWD: /usr/local/bin/SingularityBuilderBackend' > /etc/sudoers.d/builder
-echo 'builder ALL=(ALL) NOPASSWD: /usr/local/bin/DockerBuilderBackend' >> /etc/sudoers.d/builder
+echo 'builder ALL=(ALL) NOPASSWD: /usr/local/bin/singularity-builder-backend.sh' > /etc/sudoers.d/builder
+echo 'builder ALL=(ALL) NOPASSWD: /usr/local/bin/docker-builder-backend.sh' >> /etc/sudoers.d/builder
 chmod 0440 /etc/sudoers.d/builder
 
 apt-get -y update
@@ -82,8 +82,8 @@ rm -rf /boost_1_66_0
 
 # Install builder_server
 cd /
-git clone https://code.ornl.gov/olcf/ContainerBuilder.git
-cd ContainerBuilder
+git clone https://code.ornl.gov/olcf/container-builder.git
+cd container-builder
 mkdir build && cd build
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="/usr/local" ..
 make
@@ -92,21 +92,21 @@ make install
 cd /
 
 # Create systemd script and launch the Builder daemon
-cat << EOF > /etc/systemd/system/builder_server.service
+cat << EOF > /etc/systemd/system/builder-server.service
 [Unit]
-Description=builder_server daemon
+Description=builder-server daemon
 After=network.target
 
 [Service]
 Type=simple
 User=builder
 WorkingDirectory=/home/builder
 Environment="LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib"
-ExecStart=/usr/local/bin/builder_server
+ExecStart=/usr/local/bin/builder-server
 Restart=no
 
 [Install]
 WantedBy=multi-user.target
 EOF
 
-systemctl enable builder_server
+systemctl enable builder-server

Scripts/ProvisionQueue → Scripts/provision-queue.sh

@@ -26,30 +26,30 @@ rm -rf /boost_1_66_0
 
 # Install ContainerBuilder
 cd /
-git clone https://code.ornl.gov/olcf/ContainerBuilder.git
-cd ContainerBuilder
+git clone https://code.ornl.gov/olcf/container-builder.git
+cd container-builder
 mkdir build && cd build
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTAL_PREFIX="/usr/local" ..
 make
 make install
 cd /
-rm -rf /ContainerBuilder
+rm -rf /container-builder
 
 # Install OpenStack command line client
 pip install python-openstackclient
 
 # Create systemd script and launch the BuilderQueue daemon
-cat << EOF > /etc/systemd/system/builder_queue.service
+cat << EOF > /etc/systemd/system/builder-queue.service
 [Unit]
-Description=builder_queue daemon
+Description=builder-queue daemon
 After=network.target
 
 [Service]
 Type=simple
 User=queue
 Environment="LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib"
 WorkingDirectory=/home/queue
-ExecStart=/usr/local/bin/builder_queue
+ExecStart=/usr/local/bin/builder-queue
 Restart=no
 
 [Install]
@@ -58,4 +58,4 @@ EOF
 
 # There appears to be some weird issues with starting systemd services inside of a cloud-init script
 # The easiest thing to do is just reboot after enabling the service
-systemctl enable builder_queue
+systemctl enable builder-queue

Scripts/recipe_list

@@ -0,0 +1,5 @@
+#requires CLI utility httpie and jq to be installed
+
+CREDENTIAL=$(echo -n atj:knx21nws | base64)
+TOKEN=$(http 'https://code.ornl.gov/jwt/auth?service=container_registry&expires_in=9000&scope=repository:olcf/container-recipes/test:pull' Authorization:"Basic $CREDENTIAL" | jq -r '.token')
+http https://code.ornl.gov:4567/v2/olcf/container-recipes/test/manifests/latest Authorization:"Bearer $TOKEN"

Scripts/SingularityBuilderBackend → Scripts/singularity-builder-backend.sh

@@ -9,7 +9,7 @@ case ${i} in
     shift # past argument with no value
     ;;
     *)
-      echo "unknown argument to SingularityBuilderBackend"
+      echo "unknown argument to singularity-builder-backend.sh"
       exit 1
     ;;
 esac

deploy-openstack.sh

@@ -11,5 +11,5 @@ pip install python-openstackclient
 mkdir artifacts
 
 cd Scripts
-./CreateBuilderImage
-./BringUpQueue
+./create-builder-image.sh
+./bring-up-queue.sh

deploy-summitdev.sh

@@ -27,7 +27,7 @@ if [ ! -d ${SPACKROOT} ]; then
 fi
 cp spack-etc-summitdev/*.yaml ${SPACKROOT}/etc/spack
 
-${SPACKROOT}/bin/spack repo add spack-repo/containerbuilder
+${SPACKROOT}/bin/spack repo add spack-repo/container-builder
 ${SPACKROOT}/bin/spack spec -NIl "container-builder%gcc@7.1.0"
 ${SPACKROOT}/bin/spack install "container-builder%gcc@7.1.0"
 
@@ -40,12 +40,12 @@ real_mf_path="$SPACKROOT/$root/$arch/$mfname"
 MF_ROOT=/sw/summitdev/modulefiles/core/container-builder
 mkdir -p ${MF_ROOT}
 
-source artifacts/queue_host.sh
+source artifacts/queue-host.sh
 
 cat << EOF > ${MF_ROOT}/${VERSION}
 #%Module
 
 setenv QUEUE_HOST ${QUEUE_HOST}
 setenv QUEUE_PORT 8080
 module --ignore-cache load ${real_mf_path}
-EOF
+EOF