Initial stab at adding image list functionality

Builder/scripts/docker-builder-backend.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-# This script is run as root and so builder user environment variables aren't passed through
-source /home/builder/registry-credentials
-rm /home/builder/registry-credentials
-
 # Test for any arguments, such as --debug
 for i in "$@"
 do

Builder/scripts/singularity-builder-backend.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-# This script is run as root and so builder user environment variables aren't passed through
-source /home/builder/registry-credentials
-rm /home/builder/registry-credentials
-
 # Test for any arguments, such as --debug
 for i in "$@"
 do

BuilderQueue/include/BuilderQueue.h

@@ -62,6 +62,7 @@ class BuilderQueue {
 
     // Return a json formatted string representing the status of the queue
     std::string status_json();
+
 private:
     asio::io_context &io_context;
 

BuilderQueue/include/Connection.h

@@ -46,6 +46,8 @@ class Connection : public std::enable_shared_from_this<Connection> {
 
     void request_queue_stats();
 
+    void request_image_list();
+
     void builder_ready(BuilderData builder);
 
     void wait_for_close();

BuilderQueue/include/Docker.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <boost/process/child.hpp>
+#include <boost/process/async_pipe.hpp>
+#include <boost/process/group.hpp>
+#include <boost/process/io.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/read.hpp>
+#include <boost/asio/streambuf.hpp>
+#include "Logger.h"
+
+namespace asio = boost::asio;
+namespace bp = boost::process;
+
+class Docker : public std::enable_shared_from_this<Docker> {
+public:
+    explicit Docker(asio::io_context &io_context) : io_context(io_context), output_pipe(io_context) {}
+
+    // Request to get a list of docker builders
+    // The handler must have the form void(std::error_code error, std::string image_list)
+    template<typename ListHandler>
+    void request_image_list(ListHandler handler) {
+        std::string list_command("/usr/local/bin/list-images.sh");
+
+        Logger::info("Launching command: " + list_command);
+
+        std::error_code list_error;
+        process = bp::child(list_command, bp::std_in.close(), (bp::std_out & bp::std_err) > output_pipe, group,
+                            list_error);
+        if (list_error) {
+            auto error = std::error_code(list_error.value(), std::generic_category());
+            Logger::error("Error requesting image list: " + list_error.message());
+            io_context.post(std::bind(handler, error, std::string("image list error")));
+        }
+
+        // Read the list-images command output until we reach EOF, which is returned as an error
+        auto self(shared_from_this());
+        asio::async_read(output_pipe, output_buffer,
+                         [this, self, handler](boost::system::error_code error, std::size_t) {
+                             if (error != asio::error::eof) {
+                                 auto read_error = std::error_code(error.value(), std::generic_category());
+                                 Logger::error("Error reading image list output: " + read_error.message());
+                                 std::string no_list("Error reading image list output");
+                                 io_context.post(std::bind(handler, read_error, no_list));
+                             } else {
+                                 // Complete the handler
+                                 std::string image_list((std::istreambuf_iterator<char>(&output_buffer)), std::istreambuf_iterator<char>());
+                                 io_context.post(std::bind(handler, std::error_code(), image_list));
+                                 Logger::info("image list posted");
+                             }
+                         });
+    }
+
+private:
+    asio::io_context &io_context;
+    bp::child process;
+    bp::group group;
+    bp::async_pipe output_pipe;
+    asio::streambuf output_buffer;
+};

BuilderQueue/scripts/list-images.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+/usr/local/bin/docker-ls tags --progress-indicator=false --user ${DOCKERHUB_READONLY_USERNAME} --password ${DOCKERHUB_READONLY_TOKEN} olcf/titan
+
+/usr/local/bin/docker-ls tags --progress-indicator=false --user ${DOCKERHUB_READONLY_USERNAME} --password ${DOCKERHUB_READONLY_TOKEN} olcf/summitdev
+
+/usr/local/bin/docker-ls tags --progress-indicator=false --user ${DOCKERHUB_READONLY_USERNAME} --password ${DOCKERHUB_READONLY_TOKEN} olcf/summit

BuilderQueue/src/Connection.cpp

@@ -1,6 +1,7 @@
 #include "Connection.h"
 #include <boost/beast/core/buffers_to_string.hpp>
 #include <boost/archive/text_oarchive.hpp>
+#include "Docker.h"
 
 using namespace std::placeholders;
 
@@ -71,6 +72,23 @@ void Connection::request_queue_stats() {
     });
 }
 
+void Connection::request_image_list() {
+    // Persist this connection
+    auto self(shared_from_this());
+
+    // Get list of docker images
+    Logger::info("Request for image list");
+    std::make_shared<Docker>(stream.get_executor().context())->request_image_list([this, self](std::error_code ec, std::string image_list){
+        Logger::info("Writing image list");
+        stream.async_write(asio::buffer(image_list), [this, self, image_list](beast::error_code error, std::size_t bytes) {
+            boost::ignore_unused(bytes);
+            if (error) {
+                Logger::error("Wrote image list");
+            }
+        });
+    });
+}
+
 void Connection::read_request_string() {
     // Persist this connection
     auto self(shared_from_this());
@@ -85,6 +103,8 @@ void Connection::read_request_string() {
                 request_builder();
             } else if(request == "queue_status_request") {
                 request_queue_stats();
+            } else if(request == "image_list_request") {
+                request_image_list();
             } else {
                 Logger::error("Bad initial request string: " + request);
             }

CMakeLists.txt

@@ -20,7 +20,8 @@ set(SOURCE_FILES_QUEUE
         BuilderQueue/include/OpenStack.h
         Common/src/BuilderData.cpp
         Common/include/ClientData.h
-        BuilderQueue/include/Server.h)
+        BuilderQueue/include/Server.h
+        BuilderQueue/include/Docker.h)
 
 set(SOURCE_FILES_BUILDER
         Builder/src/main.cpp
@@ -47,19 +48,28 @@ set(SOURCE_FILES_STATUS
         Common/src/BuilderData.cpp
         Common/include/ClientData.h)
 
+set(SOURCE_FILES_IMAGES
+        Client/src/images.cpp
+        Client/include/WaitingAnimation.h
+        Common/src/Logger.cpp
+        Common/include/Logger.h
+        Common/include/ClientData.h)
+
 # Scripts required for runtime
 set(BUILDER_SCRIPTS
         Builder/scripts/singularity-builder-backend.sh
         Builder/scripts/docker-builder-backend.sh)
 set(QUEUE_SCRIPTS
         BuilderQueue/scripts/create-builder.sh
-        BuilderQueue/scripts/destroy-builder.sh)
+        BuilderQueue/scripts/destroy-builder.sh
+        BuilderQueue/scripts/list-images.sh)
 
 # Create executables
 add_executable(builder-queue ${SOURCE_FILES_QUEUE})
 add_executable(builder-server ${SOURCE_FILES_BUILDER})
 add_executable(container-builder ${SOURCE_FILES_CLIENT})
 add_executable(cb-status ${SOURCE_FILES_STATUS})
+add_executable(cb-images ${SOURCE_FILES_IMAGES})
 
 # Ignore system boost and use module system boost
 set(Boost_NO_BOOST_CMAKE TRUE)
@@ -80,13 +90,16 @@ set_target_properties(container-builder PROPERTIES COMPILE_FLAGS "${COMPILE_FLAG
 set_target_properties(container-builder PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}")
 set_target_properties(cb-status PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} ${HARDENING_FLAGS}")
 set_target_properties(cb-status PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}")
+set_target_properties(cb-images PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} ${HARDENING_FLAGS}")
+set_target_properties(cb-images PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}")
 
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 target_link_libraries(builder-queue ${CMAKE_THREAD_LIBS_INIT})
 target_link_libraries(builder-server ${CMAKE_THREAD_LIBS_INIT})
 target_link_libraries(container-builder ${CMAKE_THREAD_LIBS_INIT})
 target_link_libraries(cb-status ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(cb-images ${CMAKE_THREAD_LIBS_INIT})
 
 
 find_package(Boost 1.66.0 COMPONENTS system filesystem serialization regex thread program_options REQUIRED)
@@ -95,12 +108,15 @@ target_link_libraries(builder-queue ${Boost_LIBRARIES})
 target_link_libraries(builder-server ${Boost_LIBRARIES})
 target_link_libraries(container-builder ${Boost_LIBRARIES})
 target_link_libraries(cb-status ${Boost_LIBRARIES})
+target_link_libraries(cb-images ${Boost_LIBRARIES})
+
 
 # Install executables
 install(TARGETS builder-queue COMPONENT builder-queue DESTINATION bin OPTIONAL)
 install(TARGETS builder-server COMPONENT builder-server DESTINATION bin OPTIONAL)
 install(TARGETS container-builder COMPONENT client DESTINATION bin OPTIONAL)
 install(TARGETS cb-status COMPONENT client DESTINATION bin OPTIONAL)
+install(TARGETS cb-images COMPONENT client DESTINATION bin OPTIONAL)
 
 # Install scripts
 install(FILES ${BUILDER_SCRIPTS}

Client/src/images.cpp

@@ -0,0 +1,89 @@
+#include <fstream>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/asio/connect.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/beast/core/error.hpp>
+#include <boost/beast/websocket.hpp>
+#include <boost/exception/all.hpp>
+#include <boost/program_options.hpp>
+#include <regex>
+#include <csignal>
+#include "ClientData.h"
+#include "WaitingAnimation.h"
+#include "pwd.h"
+
+namespace asio = boost::asio;
+using asio::ip::tcp;
+namespace beast = boost::beast;
+namespace websocket = beast::websocket;
+
+void parse_environment(ClientData &client_data) {
+    Logger::debug("Parsing environment variables");
+
+    struct passwd *pws;
+    pws = getpwuid(getuid());
+    if (pws == NULL) {
+        throw std::runtime_error("Error get login name for client data!");
+    }
+    client_data.user_id = std::string(pws->pw_name);
+
+    auto host = std::getenv("QUEUE_HOST");
+    if (!host) {
+        throw std::runtime_error("QUEUE_HOST not set!");
+    }
+    client_data.queue_host = std::string(host);
+}
+
+void print_images(websocket::stream<tcp::socket> &queue_stream) {
+    Logger::debug("Writing image list request string");
+    std::string request_string("image_list_request");
+    queue_stream.write(asio::buffer(request_string));
+
+    Logger::debug("Read image list string");
+    std::string images_string;
+    auto images_buffer = boost::asio::dynamic_buffer(images_string);
+    queue_stream.read(images_buffer);
+
+    std::cout << images_string;
+}
+
+int main(int argc, char *argv[]) {
+    asio::io_context io_context;
+    websocket::stream<tcp::socket> queue_stream(io_context);
+
+    try {
+        ClientData client_data;
+
+        parse_environment(client_data);
+
+        WaitingAnimation wait_queue("Connecting to BuilderQueue");
+        // Open a WebSocket stream to the queue
+        tcp::resolver queue_resolver(io_context);
+        asio::connect(queue_stream.next_layer(), queue_resolver.resolve({client_data.queue_host, "8080"}));
+        queue_stream.handshake(client_data.queue_host + ":8080", "/");
+        wait_queue.stop_success("Connected to queue: " + client_data.queue_host);
+
+        // Request a build host from the queue
+        WaitingAnimation wait_builder("Requesting image list");
+        print_images(queue_stream);
+
+    } catch (const boost::exception &ex) {
+        auto diagnostics = diagnostic_information(ex);
+        Logger::error(std::string() + "Container Builder exception encountered: " + diagnostics);
+    } catch (const std::exception &ex) {
+        Logger::error(std::string() + "Container Builder exception encountered: " + ex.what());
+    } catch (...) {
+        Logger::error("Unknown exception caught!");
+    }
+
+    // Attempt to disconnect from builder and queue
+    try {
+        Logger::debug("Attempting normal close");
+        queue_stream.close(websocket::close_code::normal);
+    } catch (...) {
+        Logger::debug("Failed to cleanly close the WebSockets");
+    }
+
+    return 0;
+}

Deployment/create-builder-image.sh

@@ -66,13 +66,14 @@ echo "Provisioning the builder"
 ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/provision-builder.sh
 
 # Copy readonly credentials to the builder, these variables must be set in the gitlab runner that's running this script
-echo "GITLAB_READONLY_USERNAME=${GITLAB_READONLY_USERNAME}" > ./registry-credentials
-echo "GITLAB_READONLY_TOKEN=${GITLAB_READONLY_TOKEN}" >> ./registry-credentials
-echo "DOCKERHUB_READONLY_USERNAME=${DOCKERHUB_READONLY_USERNAME}" >> ./registry-credentials
-echo "DOCKERHUB_READONLY_TOKEN=${DOCKERHUB_READONLY_TOKEN}" >> ./registry-credentials
-scp -o StrictHostKeyChecking=no -i ${KEY_FILE} ./registry-credentials cades@${VM_IP}:/home/cades/registry-credentials
-ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mv /home/cades/registry-credentials /home/builder/registry-credentials'
-ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo chown builder /home/builder/registry-credentials'
+echo "GITLAB_READONLY_USERNAME=${GITLAB_READONLY_USERNAME}" > ./environment.sh
+echo "GITLAB_READONLY_TOKEN=${GITLAB_READONLY_TOKEN}" >> ./environment.sh
+echo "DOCKERHUB_READONLY_USERNAME=${DOCKERHUB_READONLY_USERNAME}" >> ./environment.sh
+echo "DOCKERHUB_READONLY_TOKEN=${DOCKERHUB_READONLY_TOKEN}" >> ./environment.sh
+echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib" >> ./environment
+scp -o StrictHostKeyChecking=no -i ${KEY_FILE} ./environment.sh cades@${VM_IP}:/home/cades/environment.sh
+ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mv /home/cades/environment.sh /home/builder/environment.sh'
+ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo chown builder /home/builder/environment.sh'
 
 echo "Reboot the server to ensure its in a clean state before creating the snapshot"
 openstack server reboot --wait ${VM_UUID}

Deployment/create-queue.sh

@@ -45,12 +45,15 @@ ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < $
 echo "Provisioning the queue"
 ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/provision-queue.sh
 
-# Copy OpenStack credentials to VM and then move to correct directory
-# These credentials are available as environment variables to the runners
-printenv | grep ^OS_ > ./openrc.sh # "Reconstruct" openrc.sh
-scp -o StrictHostKeyChecking=no -i ${KEY_FILE} ./openrc.sh cades@${VM_IP}:/home/cades/openrc.sh
-ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mv /home/cades/openrc.sh /home/queue/openrc.sh'
-ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo chown queue /home/queue/openrc.sh'
+# Copy OpenStack and Docker registry credentials to VM and then move to correct directory
+# These credentials are available as environment variables to the gitlab runners
+printenv | grep ^OS_ > ./environment.sh # "Reconstruct" openrc.sh
+echo "DOCKERHUB_READONLY_USERNAME=${DOCKERHUB_READONLY_USERNAME}" >> ./environment.sh
+echo "DOCKERHUB_READONLY_TOKEN=${DOCKERHUB_READONLY_TOKEN}" >> ./environment.sh
+echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib" >> ./environment.sh
+scp -o StrictHostKeyChecking=no -i ${KEY_FILE} ./openrc.sh cades@${VM_IP}:/home/cades/environment.sh
+ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mv /home/cades/environment.sh /home/queue/environment.sh'
+ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo chown queue /home/queue/environment.sh'
 
 # Reboot to start queue service added in provisioning
 openstack server reboot --wait ${VM_UUID}

Deployment/deploy-summit.sh

@@ -38,6 +38,7 @@ rm -rf build && mkdir build && cd build
 CC=gcc CXX=g++ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${SW_ROOT} ..
 make container-builder
 make cb-status
+make images
 cmake -DCOMPONENT=client -P cmake_install.cmake
 
 # Generate a public modulefile

Deployment/deploy-summitdev.sh

@@ -38,6 +38,7 @@ rm -rf build && mkdir build && cd build
 CC=gcc CXX=g++ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${SW_ROOT} ..
 make container-builder
 make cb-status
+make images
 cmake -DCOMPONENT=client -P cmake_install.cmake
 
 # Generate a public modulefile

Deployment/deploy-titan.sh

@@ -39,6 +39,7 @@ rm -rf build && mkdir build && cd build
 CC=gcc CXX=g++ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${SW_ROOT} ..
 make container-builder
 make cb-status
+make images
 cmake -DCOMPONENT=client -P cmake_install.cmake
 
 # Generate a public modulefile

Deployment/provision-builder.sh

@@ -112,7 +112,7 @@ After=network.target
 Type=simple
 User=builder
 WorkingDirectory=/home/builder
-Environment="LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib"
+EnvironmentFile=/home/queue/environment.sh
 ExecStart=/usr/local/bin/builder-server
 Restart=no